sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2013, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.5.15";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 #ifdef IXGBE_LEGACY_TX
 106 static void     ixgbe_start(struct ifnet *);
 107 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 108 #else /* ! IXGBE_LEGACY_TX */
 109 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 110 static int      ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 static void     ixgbe_deferred_mq_start(void *, int);
 113 #endif /* IXGBE_LEGACY_TX */
 114 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 115 static void     ixgbe_init(void *);
 116 static void     ixgbe_init_locked(struct adapter *);
 117 static void     ixgbe_stop(void *);
 118 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 119 static int      ixgbe_media_change(struct ifnet *);
 120 static void     ixgbe_identify_hardware(struct adapter *);
 121 static int      ixgbe_allocate_pci_resources(struct adapter *);
 122 static void     ixgbe_get_slot_info(struct ixgbe_hw *);
 123 static int      ixgbe_allocate_msix(struct adapter *);
 124 static int      ixgbe_allocate_legacy(struct adapter *);
 125 static int      ixgbe_allocate_queues(struct adapter *);
 126 static int      ixgbe_setup_msix(struct adapter *);
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static bool     ixgbe_rxeof(struct ix_queue *);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static int      ixgbe_tx_ctx_setup(struct tx_ring *,
 165                     struct mbuf *, u32 *, u32 *);
 166 static int      ixgbe_tso_setup(struct tx_ring *,
 167                     struct mbuf *, u32 *, u32 *);
 168 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 169 static void     ixgbe_configure_ivars(struct adapter *);
 170 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 171
 172 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 173 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 174 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 175
 176 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 177
 178 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 179 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 180                     struct mbuf *, u32);
 181
 182 static void     ixgbe_enable_rx_drop(struct adapter *);
 183 static void     ixgbe_disable_rx_drop(struct adapter *);
 184
 185 /* Support for pluggable optic modules */
 186 static bool     ixgbe_sfp_probe(struct adapter *);
 187 static void     ixgbe_setup_optics(struct adapter *);
 188
 189 /* Legacy (single vector interrupt handler */
 190 static void     ixgbe_legacy_irq(void *);
 191
 192 /* The MSI/X Interrupt handlers */
 193 static void     ixgbe_msix_que(void *);
 194 static void     ixgbe_msix_link(void *);
 195
 196 /* Deferred interrupt tasklets */
 197 static void     ixgbe_handle_que(void *, int);
 198 static void     ixgbe_handle_link(void *, int);
 199 static void     ixgbe_handle_msf(void *, int);
 200 static void     ixgbe_handle_mod(void *, int);
 201
 202 #ifdef IXGBE_FDIR
 203 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 204 static void     ixgbe_reinit_fdir(void *, int);
 205 #endif
 206
 207 /* Missing shared code prototype */
 208 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
 209
 210 /*********************************************************************
 211  *  FreeBSD Device Interface Entry Points
 212  *********************************************************************/
 213
 214 static device_method_t ixgbe_methods[] = {
 215         /* Device interface */
 216         DEVMETHOD(device_probe, ixgbe_probe),
 217         DEVMETHOD(device_attach, ixgbe_attach),
 218         DEVMETHOD(device_detach, ixgbe_detach),
 219         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 220         DEVMETHOD_END
 221 };
 222
 223 static driver_t ixgbe_driver = {
 224         "ix", ixgbe_methods, sizeof(struct adapter),
 225 };
 226
 227 devclass_t ixgbe_devclass;
 228 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 229
 230 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 231 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 232
 233 /*
 234 ** TUNEABLE PARAMETERS:
 235 */
 236
 237 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD, 0,
 238                    "IXGBE driver parameters");
 239
 240 /*
 241 ** AIM: Adaptive Interrupt Moderation
 242 ** which means that the interrupt rate
 243 ** is varied over time based on the
 244 ** traffic for that interrupt vector
 245 */
 246 static int ixgbe_enable_aim = TRUE;
 247 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 248 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RW, &ixgbe_enable_aim, 0,
 249     "Enable adaptive interrupt moderation");
 250
 251 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 252 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 253 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
 254     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
 255
 256 /* How many packets rxeof tries to clean at a time */
 257 static int ixgbe_rx_process_limit = 256;
 258 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 259 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
 260     &ixgbe_rx_process_limit, 0,
 261     "Maximum number of received packets to process at a time,"
 262     "-1 means unlimited");
 263
 264 /* How many packets txeof tries to clean at a time */
 265 static int ixgbe_tx_process_limit = 256;
 266 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
 267 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
 268     &ixgbe_tx_process_limit, 0,
 269     "Maximum number of sent packets to process at a time,"
 270     "-1 means unlimited");
 271
 272 /*
 273 ** Smart speed setting, default to on
 274 ** this only works as a compile option
 275 ** right now as its during attach, set
 276 ** this to 'ixgbe_smart_speed_off' to
 277 ** disable.
 278 */
 279 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 280
 281 /*
 282  * MSIX should be the default for best performance,
 283  * but this allows it to be forced off for testing.
 284  */
 285 static int ixgbe_enable_msix = 1;
 286 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 287 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
 288     "Enable MSI-X interrupts");
 289
 290 /*
 291  * Number of Queues, can be set to 0,
 292  * it then autoconfigures based on the
 293  * number of cpus with a max of 8. This
 294  * can be overriden manually here.
 295  */
 296 static int ixgbe_num_queues = 0;
 297 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 298 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
 299     "Number of queues to configure, 0 indicates autoconfigure");
 300
 301 /*
 302 ** Number of TX descriptors per ring,
 303 ** setting higher than RX as this seems
 304 ** the better performing choice.
 305 */
 306 static int ixgbe_txd = PERFORM_TXD;
 307 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 308 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
 309     "Number of receive descriptors per queue");
 310
 311 /* Number of RX descriptors per ring */
 312 static int ixgbe_rxd = PERFORM_RXD;
 313 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 314 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
 315     "Number of receive descriptors per queue");
 316
 317 /*
 318 ** Defining this on will allow the use
 319 ** of unsupported SFP+ modules, note that
 320 ** doing so you are on your own :)
 321 */
 322 static int allow_unsupported_sfp = FALSE;
 323 TUNABLE_INT("hw.ixgbe.unsupported_sfp", &allow_unsupported_sfp);
 324
 325 /*
 326 ** HW RSC control:
 327 **  this feature only works with
 328 **  IPv4, and only on 82599 and later.
 329 **  Also this will cause IP forwarding to
 330 **  fail and that can't be controlled by
 331 **  the stack as LRO can. For all these
 332 **  reasons I've deemed it best to leave
 333 **  this off and not bother with a tuneable
 334 **  interface, this would need to be compiled
 335 **  to enable.
 336 */
 337 static bool ixgbe_rsc_enable = FALSE;
 338
 339 /* Keep running tab on them for sanity check */
 340 static int ixgbe_total_ports;
 341
 342 #ifdef IXGBE_FDIR
 343 /*
 344 ** For Flow Director: this is the
 345 ** number of TX packets we sample
 346 ** for the filter pool, this means
 347 ** every 20th packet will be probed.
 348 **
 349 ** This feature can be disabled by
 350 ** setting this to 0.
 351 */
 352 static int atr_sample_rate = 20;
 353 /*
 354 ** Flow Director actually 'steals'
 355 ** part of the packet buffer as its
 356 ** filter pool, this variable controls
 357 ** how much it uses:
 358 **  0 = 64K, 1 = 128K, 2 = 256K
 359 */
 360 static int fdir_pballoc = 1;
 361 #endif
 362
 363 #ifdef DEV_NETMAP
 364 /*
 365  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 366  * be a reference on how to implement netmap support in a driver.
 367  * Additional comments are in ixgbe_netmap.h .
 368  *
 369  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 370  * that extend the standard driver.
 371  */
 372 #include <dev/netmap/ixgbe_netmap.h>
 373 #endif /* DEV_NETMAP */
 374
 375 /*********************************************************************
 376  *  Device identification routine
 377  *
 378  *  ixgbe_probe determines if the driver should be loaded on
 379  *  adapter based on PCI vendor/device id of the adapter.
 380  *
 381  *  return BUS_PROBE_DEFAULT on success, positive on failure
 382  *********************************************************************/
 383
 384 static int
 385 ixgbe_probe(device_t dev)
 386 {
 387         ixgbe_vendor_info_t *ent;
 388
 389         u16     pci_vendor_id = 0;
 390         u16     pci_device_id = 0;
 391         u16     pci_subvendor_id = 0;
 392         u16     pci_subdevice_id = 0;
 393         char    adapter_name[256];
 394
 395         INIT_DEBUGOUT("ixgbe_probe: begin");
 396
 397         pci_vendor_id = pci_get_vendor(dev);
 398         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 399                 return (ENXIO);
 400
 401         pci_device_id = pci_get_device(dev);
 402         pci_subvendor_id = pci_get_subvendor(dev);
 403         pci_subdevice_id = pci_get_subdevice(dev);
 404
 405         ent = ixgbe_vendor_info_array;
 406         while (ent->vendor_id != 0) {
 407                 if ((pci_vendor_id == ent->vendor_id) &&
 408                     (pci_device_id == ent->device_id) &&
 409
 410                     ((pci_subvendor_id == ent->subvendor_id) ||
 411                      (ent->subvendor_id == 0)) &&
 412
 413                     ((pci_subdevice_id == ent->subdevice_id) ||
 414                      (ent->subdevice_id == 0))) {
 415                         sprintf(adapter_name, "%s, Version - %s",
 416                                 ixgbe_strings[ent->index],
 417                                 ixgbe_driver_version);
 418                         device_set_desc_copy(dev, adapter_name);
 419                         ++ixgbe_total_ports;
 420                         return (BUS_PROBE_DEFAULT);
 421                 }
 422                 ent++;
 423         }
 424         return (ENXIO);
 425 }
 426
 427 /*********************************************************************
 428  *  Device initialization routine
 429  *
 430  *  The attach entry point is called when the driver is being loaded.
 431  *  This routine identifies the type of hardware, allocates all resources
 432  *  and initializes the hardware.
 433  *
 434  *  return 0 on success, positive on failure
 435  *********************************************************************/
 436
 437 static int
 438 ixgbe_attach(device_t dev)
 439 {
 440         struct adapter *adapter;
 441         struct ixgbe_hw *hw;
 442         int             error = 0;
 443         u16             csum;
 444         u32             ctrl_ext;
 445
 446         INIT_DEBUGOUT("ixgbe_attach: begin");
 447
 448         /* Allocate, clear, and link in our adapter structure */
 449         adapter = device_get_softc(dev);
 450         adapter->dev = adapter->osdep.dev = dev;
 451         hw = &adapter->hw;
 452
 453         /* Core Lock Init*/
 454         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 455
 456         /* SYSCTL APIs */
 457
 458         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 459                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 460                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 461                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 462
 463         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 464                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 465                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 466                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 467
 468         /*
 469         ** Allow a kind of speed control by forcing the autoneg
 470         ** advertised speed list to only a certain value, this
 471         ** supports 1G on 82599 devices, and 100Mb on x540.
 472         */
 473         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 474                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 475                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 476                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 477
 478         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 479                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 480                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 481                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 482
 483         /* Set up the timer callout */
 484         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 485
 486         /* Determine hardware revision */
 487         ixgbe_identify_hardware(adapter);
 488
 489         /* Do base PCI setup - map BAR0 */
 490         if (ixgbe_allocate_pci_resources(adapter)) {
 491                 device_printf(dev, "Allocation of PCI resources failed\n");
 492                 error = ENXIO;
 493                 goto err_out;
 494         }
 495
 496         /* Do descriptor calc and sanity checks */
 497         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 498             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 499                 device_printf(dev, "TXD config issue, using default!\n");
 500                 adapter->num_tx_desc = DEFAULT_TXD;
 501         } else
 502                 adapter->num_tx_desc = ixgbe_txd;
 503
 504         /*
 505         ** With many RX rings it is easy to exceed the
 506         ** system mbuf allocation. Tuning nmbclusters
 507         ** can alleviate this.
 508         */
 509         if (nmbclusters > 0 ) {
 510                 int s;
 511                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 512                 if (s > nmbclusters) {
 513                         device_printf(dev, "RX Descriptors exceed "
 514                             "system mbuf max, using default instead!\n");
 515                         ixgbe_rxd = DEFAULT_RXD;
 516                 }
 517         }
 518
 519         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 520             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 521                 device_printf(dev, "RXD config issue, using default!\n");
 522                 adapter->num_rx_desc = DEFAULT_RXD;
 523         } else
 524                 adapter->num_rx_desc = ixgbe_rxd;
 525
 526         /* Allocate our TX/RX Queues */
 527         if (ixgbe_allocate_queues(adapter)) {
 528                 error = ENOMEM;
 529                 goto err_out;
 530         }
 531
 532         /* Allocate multicast array memory. */
 533         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 534             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 535         if (adapter->mta == NULL) {
 536                 device_printf(dev, "Can not allocate multicast setup array\n");
 537                 error = ENOMEM;
 538                 goto err_late;
 539         }
 540
 541         /* Initialize the shared code */
 542         hw->allow_unsupported_sfp = allow_unsupported_sfp;
 543         error = ixgbe_init_shared_code(hw);
 544         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 545                 /*
 546                 ** No optics in this port, set up
 547                 ** so the timer routine will probe
 548                 ** for later insertion.
 549                 */
 550                 adapter->sfp_probe = TRUE;
 551                 error = 0;
 552         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 553                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 554                 error = EIO;
 555                 goto err_late;
 556         } else if (error) {
 557                 device_printf(dev,"Unable to initialize the shared code\n");
 558                 error = EIO;
 559                 goto err_late;
 560         }
 561
 562         /* Make sure we have a good EEPROM before we read from it */
 563         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 564                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 565                 error = EIO;
 566                 goto err_late;
 567         }
 568
 569         error = ixgbe_init_hw(hw);
 570         switch (error) {
 571         case IXGBE_ERR_EEPROM_VERSION:
 572                 device_printf(dev, "This device is a pre-production adapter/"
 573                     "LOM.  Please be aware there may be issues associated "
 574                     "with your hardware.\n If you are experiencing problems "
 575                     "please contact your Intel or hardware representative "
 576                     "who provided you with this hardware.\n");
 577                 break;
 578         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 579                 device_printf(dev,"Unsupported SFP+ Module\n");
 580                 error = EIO;
 581                 goto err_late;
 582         case IXGBE_ERR_SFP_NOT_PRESENT:
 583                 device_printf(dev,"No SFP+ Module found\n");
 584                 /* falls thru */
 585         default:
 586                 break;
 587         }
 588
 589         /* Detect and set physical type */
 590         ixgbe_setup_optics(adapter);
 591
 592         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 593                 error = ixgbe_allocate_msix(adapter);
 594         else
 595                 error = ixgbe_allocate_legacy(adapter);
 596         if (error)
 597                 goto err_late;
 598
 599         /* Setup OS specific network interface */
 600         if (ixgbe_setup_interface(dev, adapter) != 0)
 601                 goto err_late;
 602
 603         /* Initialize statistics */
 604         ixgbe_update_stats_counters(adapter);
 605
 606         /* Register for VLAN events */
 607         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 608             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 609         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 610             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 611
 612         /*
 613         ** Check PCIE slot type/speed/width
 614         */
 615         ixgbe_get_slot_info(hw);
 616
 617         /* Set an initial default flow control value */
 618         adapter->fc =  ixgbe_fc_full;
 619
 620         /* let hardware know driver is loaded */
 621         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 622         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 623         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 624
 625         ixgbe_add_hw_stats(adapter);
 626
 627 #ifdef DEV_NETMAP
 628         ixgbe_netmap_attach(adapter);
 629 #endif /* DEV_NETMAP */
 630         INIT_DEBUGOUT("ixgbe_attach: end");
 631         return (0);
 632 err_late:
 633         ixgbe_free_transmit_structures(adapter);
 634         ixgbe_free_receive_structures(adapter);
 635 err_out:
 636         if (adapter->ifp != NULL)
 637                 if_free(adapter->ifp);
 638         ixgbe_free_pci_resources(adapter);
 639         free(adapter->mta, M_DEVBUF);
 640         return (error);
 641
 642 }
 643
 644 /*********************************************************************
 645  *  Device removal routine
 646  *
 647  *  The detach entry point is called when the driver is being removed.
 648  *  This routine stops the adapter and deallocates all the resources
 649  *  that were allocated for driver operation.
 650  *
 651  *  return 0 on success, positive on failure
 652  *********************************************************************/
 653
 654 static int
 655 ixgbe_detach(device_t dev)
 656 {
 657         struct adapter *adapter = device_get_softc(dev);
 658         struct ix_queue *que = adapter->queues;
 659         struct tx_ring *txr = adapter->tx_rings;
 660         u32     ctrl_ext;
 661
 662         INIT_DEBUGOUT("ixgbe_detach: begin");
 663
 664         /* Make sure VLANS are not using driver */
 665         if (adapter->ifp->if_vlantrunk != NULL) {
 666                 device_printf(dev,"Vlan in use, detach first\n");
 667                 return (EBUSY);
 668         }
 669
 670         IXGBE_CORE_LOCK(adapter);
 671         ixgbe_stop(adapter);
 672         IXGBE_CORE_UNLOCK(adapter);
 673
 674         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 675                 if (que->tq) {
 676 #ifndef IXGBE_LEGACY_TX
 677                         taskqueue_drain(que->tq, &txr->txq_task);
 678 #endif
 679                         taskqueue_drain(que->tq, &que->que_task);
 680                         taskqueue_free(que->tq);
 681                 }
 682         }
 683
 684         /* Drain the Link queue */
 685         if (adapter->tq) {
 686                 taskqueue_drain(adapter->tq, &adapter->link_task);
 687                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 688                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 689 #ifdef IXGBE_FDIR
 690                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 691 #endif
 692                 taskqueue_free(adapter->tq);
 693         }
 694
 695         /* let hardware know driver is unloading */
 696         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 697         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 698         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 699
 700         /* Unregister VLAN events */
 701         if (adapter->vlan_attach != NULL)
 702                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 703         if (adapter->vlan_detach != NULL)
 704                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 705
 706         ether_ifdetach(adapter->ifp);
 707         callout_drain(&adapter->timer);
 708 #ifdef DEV_NETMAP
 709         netmap_detach(adapter->ifp);
 710 #endif /* DEV_NETMAP */
 711         ixgbe_free_pci_resources(adapter);
 712         bus_generic_detach(dev);
 713         if_free(adapter->ifp);
 714
 715         ixgbe_free_transmit_structures(adapter);
 716         ixgbe_free_receive_structures(adapter);
 717         free(adapter->mta, M_DEVBUF);
 718
 719         IXGBE_CORE_LOCK_DESTROY(adapter);
 720         return (0);
 721 }
 722
 723 /*********************************************************************
 724  *
 725  *  Shutdown entry point
 726  *
 727  **********************************************************************/
 728
 729 static int
 730 ixgbe_shutdown(device_t dev)
 731 {
 732         struct adapter *adapter = device_get_softc(dev);
 733         IXGBE_CORE_LOCK(adapter);
 734         ixgbe_stop(adapter);
 735         IXGBE_CORE_UNLOCK(adapter);
 736         return (0);
 737 }
 738
 739
 740 #ifdef IXGBE_LEGACY_TX
 741 /*********************************************************************
 742  *  Transmit entry point
 743  *
 744  *  ixgbe_start is called by the stack to initiate a transmit.
 745  *  The driver will remain in this routine as long as there are
 746  *  packets to transmit and transmit resources are available.
 747  *  In case resources are not available stack is notified and
 748  *  the packet is requeued.
 749  **********************************************************************/
 750
 751 static void
 752 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 753 {
 754         struct mbuf    *m_head;
 755         struct adapter *adapter = txr->adapter;
 756
 757         IXGBE_TX_LOCK_ASSERT(txr);
 758
 759         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 760                 return;
 761         if (!adapter->link_active)
 762                 return;
 763
 764         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 765                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 766                         break;
 767
 768                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 769                 if (m_head == NULL)
 770                         break;
 771
 772                 if (ixgbe_xmit(txr, &m_head)) {
 773                         if (m_head != NULL)
 774                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 775                         break;
 776                 }
 777                 /* Send a copy of the frame to the BPF listener */
 778                 ETHER_BPF_MTAP(ifp, m_head);
 779
 780                 /* Set watchdog on */
 781                 txr->watchdog_time = ticks;
 782                 txr->queue_status = IXGBE_QUEUE_WORKING;
 783
 784         }
 785         return;
 786 }
 787
 788 /*
 789  * Legacy TX start - called by the stack, this
 790  * always uses the first tx ring, and should
 791  * not be used with multiqueue tx enabled.
 792  */
 793 static void
 794 ixgbe_start(struct ifnet *ifp)
 795 {
 796         struct adapter *adapter = ifp->if_softc;
 797         struct tx_ring  *txr = adapter->tx_rings;
 798
 799         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 800                 IXGBE_TX_LOCK(txr);
 801                 ixgbe_start_locked(txr, ifp);
 802                 IXGBE_TX_UNLOCK(txr);
 803         }
 804         return;
 805 }
 806
 807 #else /* ! IXGBE_LEGACY_TX */
 808
 809 /*
 810 ** Multiqueue Transmit driver
 811 **
 812 */
 813 static int
 814 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 815 {
 816         struct adapter  *adapter = ifp->if_softc;
 817         struct ix_queue *que;
 818         struct tx_ring  *txr;
 819         int             i, err = 0;
 820
 821         /* Which queue to use */
 822         if ((m->m_flags & M_FLOWID) != 0)
 823                 i = m->m_pkthdr.flowid % adapter->num_queues;
 824         else
 825                 i = curcpu % adapter->num_queues;
 826
 827         txr = &adapter->tx_rings[i];
 828         que = &adapter->queues[i];
 829
 830         err = drbr_enqueue(ifp, txr->br, m);
 831         if (err)
 832                 return (err);
 833         if (IXGBE_TX_TRYLOCK(txr)) {
 834                 ixgbe_mq_start_locked(ifp, txr);
 835                 IXGBE_TX_UNLOCK(txr);
 836         } else
 837                 taskqueue_enqueue(que->tq, &txr->txq_task);
 838
 839         return (0);
 840 }
 841
 842 static int
 843 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 844 {
 845         struct adapter  *adapter = txr->adapter;
 846         struct mbuf     *next;
 847         int             enqueued = 0, err = 0;
 848
 849         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 850             adapter->link_active == 0)
 851                 return (ENETDOWN);
 852
 853         /* Process the queue */
 854 #if __FreeBSD_version < 901504
 855         next = drbr_dequeue(ifp, txr->br);
 856         while (next != NULL) {
 857                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 858                         if (next != NULL)
 859                                 err = drbr_enqueue(ifp, txr->br, next);
 860 #else
 861         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 862                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 863                         if (next == NULL) {
 864                                 drbr_advance(ifp, txr->br);
 865                         } else {
 866                                 drbr_putback(ifp, txr->br, next);
 867                         }
 868 #endif
 869                         break;
 870                 }
 871 #if __FreeBSD_version >= 901504
 872                 drbr_advance(ifp, txr->br);
 873 #endif
 874                 enqueued++;
 875                 /* Send a copy of the frame to the BPF listener */
 876                 ETHER_BPF_MTAP(ifp, next);
 877                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 878                         break;
 879 #if __FreeBSD_version < 901504
 880                 next = drbr_dequeue(ifp, txr->br);
 881 #endif
 882         }
 883
 884         if (enqueued > 0) {
 885                 /* Set watchdog on */
 886                 txr->queue_status = IXGBE_QUEUE_WORKING;
 887                 txr->watchdog_time = ticks;
 888         }
 889
 890         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 891                 ixgbe_txeof(txr);
 892
 893         return (err);
 894 }
 895
 896 /*
 897  * Called from a taskqueue to drain queued transmit packets.
 898  */
 899 static void
 900 ixgbe_deferred_mq_start(void *arg, int pending)
 901 {
 902         struct tx_ring *txr = arg;
 903         struct adapter *adapter = txr->adapter;
 904         struct ifnet *ifp = adapter->ifp;
 905
 906         IXGBE_TX_LOCK(txr);
 907         if (!drbr_empty(ifp, txr->br))
 908                 ixgbe_mq_start_locked(ifp, txr);
 909         IXGBE_TX_UNLOCK(txr);
 910 }
 911
 912 /*
 913 ** Flush all ring buffers
 914 */
 915 static void
 916 ixgbe_qflush(struct ifnet *ifp)
 917 {
 918         struct adapter  *adapter = ifp->if_softc;
 919         struct tx_ring  *txr = adapter->tx_rings;
 920         struct mbuf     *m;
 921
 922         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 923                 IXGBE_TX_LOCK(txr);
 924                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 925                         m_freem(m);
 926                 IXGBE_TX_UNLOCK(txr);
 927         }
 928         if_qflush(ifp);
 929 }
 930 #endif /* IXGBE_LEGACY_TX */
 931
 932 /*********************************************************************
 933  *  Ioctl entry point
 934  *
 935  *  ixgbe_ioctl is called when the user wants to configure the
 936  *  interface.
 937  *
 938  *  return 0 on success, positive on failure
 939  **********************************************************************/
 940
 941 static int
 942 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 943 {
 944         struct adapter  *adapter = ifp->if_softc;
 945         struct ixgbe_hw *hw = &adapter->hw;
 946         struct ifreq    *ifr = (struct ifreq *) data;
 947 #if defined(INET) || defined(INET6)
 948         struct ifaddr *ifa = (struct ifaddr *)data;
 949         bool            avoid_reset = FALSE;
 950 #endif
 951         int             error = 0;
 952
 953         switch (command) {
 954
 955         case SIOCSIFADDR:
 956 #ifdef INET
 957                 if (ifa->ifa_addr->sa_family == AF_INET)
 958                         avoid_reset = TRUE;
 959 #endif
 960 #ifdef INET6
 961                 if (ifa->ifa_addr->sa_family == AF_INET6)
 962                         avoid_reset = TRUE;
 963 #endif
 964 #if defined(INET) || defined(INET6)
 965                 /*
 966                 ** Calling init results in link renegotiation,
 967                 ** so we avoid doing it when possible.
 968                 */
 969                 if (avoid_reset) {
 970                         ifp->if_flags |= IFF_UP;
 971                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 972                                 ixgbe_init(adapter);
 973                         if (!(ifp->if_flags & IFF_NOARP))
 974                                 arp_ifinit(ifp, ifa);
 975                 } else
 976                         error = ether_ioctl(ifp, command, data);
 977 #endif
 978                 break;
 979         case SIOCSIFMTU:
 980                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 981                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 982                         error = EINVAL;
 983                 } else {
 984                         IXGBE_CORE_LOCK(adapter);
 985                         ifp->if_mtu = ifr->ifr_mtu;
 986                         adapter->max_frame_size =
 987                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 988                         ixgbe_init_locked(adapter);
 989                         IXGBE_CORE_UNLOCK(adapter);
 990                 }
 991                 break;
 992         case SIOCSIFFLAGS:
 993                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 994                 IXGBE_CORE_LOCK(adapter);
 995                 if (ifp->if_flags & IFF_UP) {
 996                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 997                                 if ((ifp->if_flags ^ adapter->if_flags) &
 998                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 999                                         ixgbe_set_promisc(adapter);
1000                                 }
1001                         } else
1002                                 ixgbe_init_locked(adapter);
1003                 } else
1004                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1005                                 ixgbe_stop(adapter);
1006                 adapter->if_flags = ifp->if_flags;
1007                 IXGBE_CORE_UNLOCK(adapter);
1008                 break;
1009         case SIOCADDMULTI:
1010         case SIOCDELMULTI:
1011                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1012                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1013                         IXGBE_CORE_LOCK(adapter);
1014                         ixgbe_disable_intr(adapter);
1015                         ixgbe_set_multi(adapter);
1016                         ixgbe_enable_intr(adapter);
1017                         IXGBE_CORE_UNLOCK(adapter);
1018                 }
1019                 break;
1020         case SIOCSIFMEDIA:
1021         case SIOCGIFMEDIA:
1022                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1023                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1024                 break;
1025         case SIOCSIFCAP:
1026         {
1027                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1028                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1029                 if (mask & IFCAP_HWCSUM)
1030                         ifp->if_capenable ^= IFCAP_HWCSUM;
1031                 if (mask & IFCAP_TSO4)
1032                         ifp->if_capenable ^= IFCAP_TSO4;
1033                 if (mask & IFCAP_TSO6)
1034                         ifp->if_capenable ^= IFCAP_TSO6;
1035                 if (mask & IFCAP_LRO)
1036                         ifp->if_capenable ^= IFCAP_LRO;
1037                 if (mask & IFCAP_VLAN_HWTAGGING)
1038                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1039                 if (mask & IFCAP_VLAN_HWFILTER)
1040                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1041                 if (mask & IFCAP_VLAN_HWTSO)
1042                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1043                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1044                         IXGBE_CORE_LOCK(adapter);
1045                         ixgbe_init_locked(adapter);
1046                         IXGBE_CORE_UNLOCK(adapter);
1047                 }
1048                 VLAN_CAPABILITIES(ifp);
1049                 break;
1050         }
1051         case SIOCGI2C:
1052         {
1053                 struct ixgbe_i2c_req    i2c;
1054                 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1055                 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1056                 if (error)
1057                         break;
1058                 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1059                         error = EINVAL;
1060                         break;
1061                 }
1062                 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1063                     i2c.dev_addr, i2c.data);
1064                 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1065                 break;
1066         }
1067         default:
1068                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1069                 error = ether_ioctl(ifp, command, data);
1070                 break;
1071         }
1072
1073         return (error);
1074 }
1075
1076 /*********************************************************************
1077  *  Init entry point
1078  *
1079  *  This routine is used in two ways. It is used by the stack as
1080  *  init entry point in network interface structure. It is also used
1081  *  by the driver as a hw/sw initialization routine to get to a
1082  *  consistent state.
1083  *
1084  *  return 0 on success, positive on failure
1085  **********************************************************************/
1086 #define IXGBE_MHADD_MFS_SHIFT 16
1087
1088 static void
1089 ixgbe_init_locked(struct adapter *adapter)
1090 {
1091         struct ifnet   *ifp = adapter->ifp;
1092         device_t        dev = adapter->dev;
1093         struct ixgbe_hw *hw = &adapter->hw;
1094         u32             k, txdctl, mhadd, gpie;
1095         u32             rxdctl, rxctrl;
1096
1097         mtx_assert(&adapter->core_mtx, MA_OWNED);
1098         INIT_DEBUGOUT("ixgbe_init_locked: begin");
1099         hw->adapter_stopped = FALSE;
1100         ixgbe_stop_adapter(hw);
1101         callout_stop(&adapter->timer);
1102
1103         /* reprogram the RAR[0] in case user changed it. */
1104         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1105
1106         /* Get the latest mac address, User can use a LAA */
1107         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1108               IXGBE_ETH_LENGTH_OF_ADDRESS);
1109         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1110         hw->addr_ctrl.rar_used_count = 1;
1111
1112         /* Set the various hardware offload abilities */
1113         ifp->if_hwassist = 0;
1114         if (ifp->if_capenable & IFCAP_TSO)
1115                 ifp->if_hwassist |= CSUM_TSO;
1116         if (ifp->if_capenable & IFCAP_TXCSUM) {
1117                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1118 #if __FreeBSD_version >= 800000
1119                 if (hw->mac.type != ixgbe_mac_82598EB)
1120                         ifp->if_hwassist |= CSUM_SCTP;
1121 #endif
1122         }
1123
1124         /* Prepare transmit descriptors and buffers */
1125         if (ixgbe_setup_transmit_structures(adapter)) {
1126                 device_printf(dev,"Could not setup transmit structures\n");
1127                 ixgbe_stop(adapter);
1128                 return;
1129         }
1130
1131         ixgbe_init_hw(hw);
1132         ixgbe_initialize_transmit_units(adapter);
1133
1134         /* Setup Multicast table */
1135         ixgbe_set_multi(adapter);
1136
1137         /*
1138         ** Determine the correct mbuf pool
1139         ** for doing jumbo frames
1140         */
1141         if (adapter->max_frame_size <= 2048)
1142                 adapter->rx_mbuf_sz = MCLBYTES;
1143         else if (adapter->max_frame_size <= 4096)
1144                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1145         else if (adapter->max_frame_size <= 9216)
1146                 adapter->rx_mbuf_sz = MJUM9BYTES;
1147         else
1148                 adapter->rx_mbuf_sz = MJUM16BYTES;
1149
1150         /* Prepare receive descriptors and buffers */
1151         if (ixgbe_setup_receive_structures(adapter)) {
1152                 device_printf(dev,"Could not setup receive structures\n");
1153                 ixgbe_stop(adapter);
1154                 return;
1155         }
1156
1157         /* Configure RX settings */
1158         ixgbe_initialize_receive_units(adapter);
1159
1160         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1161
1162         /* Enable Fan Failure Interrupt */
1163         gpie |= IXGBE_SDP1_GPIEN;
1164
1165         /* Add for Module detection */
1166         if (hw->mac.type == ixgbe_mac_82599EB)
1167                 gpie |= IXGBE_SDP2_GPIEN;
1168
1169         /* Thermal Failure Detection */
1170         if (hw->mac.type == ixgbe_mac_X540)
1171                 gpie |= IXGBE_SDP0_GPIEN;
1172
1173         if (adapter->msix > 1) {
1174                 /* Enable Enhanced MSIX mode */
1175                 gpie |= IXGBE_GPIE_MSIX_MODE;
1176                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1177                     IXGBE_GPIE_OCD;
1178         }
1179         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1180
1181         /* Set MTU size */
1182         if (ifp->if_mtu > ETHERMTU) {
1183                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1184                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1185                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1186                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1187         }
1188
1189         /* Now enable all the queues */
1190
1191         for (int i = 0; i < adapter->num_queues; i++) {
1192                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1193                 txdctl |= IXGBE_TXDCTL_ENABLE;
1194                 /* Set WTHRESH to 8, burst writeback */
1195                 txdctl |= (8 << 16);
1196                 /*
1197                  * When the internal queue falls below PTHRESH (32),
1198                  * start prefetching as long as there are at least
1199                  * HTHRESH (1) buffers ready. The values are taken
1200                  * from the Intel linux driver 3.8.21.
1201                  * Prefetching enables tx line rate even with 1 queue.
1202                  */
1203                 txdctl |= (32 << 0) | (1 << 8);
1204                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1205         }
1206
1207         for (int i = 0; i < adapter->num_queues; i++) {
1208                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1209                 if (hw->mac.type == ixgbe_mac_82598EB) {
1210                         /*
1211                         ** PTHRESH = 21
1212                         ** HTHRESH = 4
1213                         ** WTHRESH = 8
1214                         */
1215                         rxdctl &= ~0x3FFFFF;
1216                         rxdctl |= 0x080420;
1217                 }
1218                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1219                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1220                 for (k = 0; k < 10; k++) {
1221                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1222                             IXGBE_RXDCTL_ENABLE)
1223                                 break;
1224                         else
1225                                 msec_delay(1);
1226                 }
1227                 wmb();
1228 #ifdef DEV_NETMAP
1229                 /*
1230                  * In netmap mode, we must preserve the buffers made
1231                  * available to userspace before the if_init()
1232                  * (this is true by default on the TX side, because
1233                  * init makes all buffers available to userspace).
1234                  *
1235                  * netmap_reset() and the device specific routines
1236                  * (e.g. ixgbe_setup_receive_rings()) map these
1237                  * buffers at the end of the NIC ring, so here we
1238                  * must set the RDT (tail) register to make sure
1239                  * they are not overwritten.
1240                  *
1241                  * In this driver the NIC ring starts at RDH = 0,
1242                  * RDT points to the last slot available for reception (?),
1243                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1244                  */
1245                 if (ifp->if_capenable & IFCAP_NETMAP) {
1246                         struct netmap_adapter *na = NA(adapter->ifp);
1247                         struct netmap_kring *kring = &na->rx_rings[i];
1248                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1249
1250                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1251                 } else
1252 #endif /* DEV_NETMAP */
1253                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1254         }
1255
1256         /* Enable Receive engine */
1257         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1258         if (hw->mac.type == ixgbe_mac_82598EB)
1259                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1260         rxctrl |= IXGBE_RXCTRL_RXEN;
1261         ixgbe_enable_rx_dma(hw, rxctrl);
1262
1263         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1264
1265         /* Set up MSI/X routing */
1266         if (ixgbe_enable_msix)  {
1267                 ixgbe_configure_ivars(adapter);
1268                 /* Set up auto-mask */
1269                 if (hw->mac.type == ixgbe_mac_82598EB)
1270                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1271                 else {
1272                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1273                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1274                 }
1275         } else {  /* Simple settings for Legacy/MSI */
1276                 ixgbe_set_ivar(adapter, 0, 0, 0);
1277                 ixgbe_set_ivar(adapter, 0, 0, 1);
1278                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1279         }
1280
1281 #ifdef IXGBE_FDIR
1282         /* Init Flow director */
1283         if (hw->mac.type != ixgbe_mac_82598EB) {
1284                 u32 hdrm = 32 << fdir_pballoc;
1285
1286                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1287                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1288         }
1289 #endif
1290
1291         /*
1292         ** Check on any SFP devices that
1293         ** need to be kick-started
1294         */
1295         if (hw->phy.type == ixgbe_phy_none) {
1296                 int err = hw->phy.ops.identify(hw);
1297                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1298                         device_printf(dev,
1299                             "Unsupported SFP+ module type was detected.\n");
1300                         return;
1301                 }
1302         }
1303
1304         /* Set moderation on the Link interrupt */
1305         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1306
1307         /* Config/Enable Link */
1308         ixgbe_config_link(adapter);
1309
1310         /* Hardware Packet Buffer & Flow Control setup */
1311         {
1312                 u32 rxpb, frame, size, tmp;
1313
1314                 frame = adapter->max_frame_size;
1315
1316                 /* Calculate High Water */
1317                 if (hw->mac.type == ixgbe_mac_X540)
1318                         tmp = IXGBE_DV_X540(frame, frame);
1319                 else
1320                         tmp = IXGBE_DV(frame, frame);
1321                 size = IXGBE_BT2KB(tmp);
1322                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1323                 hw->fc.high_water[0] = rxpb - size;
1324
1325                 /* Now calculate Low Water */
1326                 if (hw->mac.type == ixgbe_mac_X540)
1327                         tmp = IXGBE_LOW_DV_X540(frame);
1328                 else
1329                         tmp = IXGBE_LOW_DV(frame);
1330                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1331
1332                 hw->fc.requested_mode = adapter->fc;
1333                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1334                 hw->fc.send_xon = TRUE;
1335         }
1336         /* Initialize the FC settings */
1337         ixgbe_start_hw(hw);
1338
1339         /* Set up VLAN support and filter */
1340         ixgbe_setup_vlan_hw_support(adapter);
1341
1342         /* And now turn on interrupts */
1343         ixgbe_enable_intr(adapter);
1344
1345         /* Now inform the stack we're ready */
1346         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1347
1348         return;
1349 }
1350
1351 static void
1352 ixgbe_init(void *arg)
1353 {
1354         struct adapter *adapter = arg;
1355
1356         IXGBE_CORE_LOCK(adapter);
1357         ixgbe_init_locked(adapter);
1358         IXGBE_CORE_UNLOCK(adapter);
1359         return;
1360 }
1361
1362
1363 /*
1364 **
1365 ** MSIX Interrupt Handlers and Tasklets
1366 **
1367 */
1368
1369 static inline void
1370 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1371 {
1372         struct ixgbe_hw *hw = &adapter->hw;
1373         u64     queue = (u64)(1 << vector);
1374         u32     mask;
1375
1376         if (hw->mac.type == ixgbe_mac_82598EB) {
1377                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1379         } else {
1380                 mask = (queue & 0xFFFFFFFF);
1381                 if (mask)
1382                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1383                 mask = (queue >> 32);
1384                 if (mask)
1385                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1386         }
1387 }
1388
1389 static inline void
1390 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1391 {
1392         struct ixgbe_hw *hw = &adapter->hw;
1393         u64     queue = (u64)(1 << vector);
1394         u32     mask;
1395
1396         if (hw->mac.type == ixgbe_mac_82598EB) {
1397                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1398                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1399         } else {
1400                 mask = (queue & 0xFFFFFFFF);
1401                 if (mask)
1402                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1403                 mask = (queue >> 32);
1404                 if (mask)
1405                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1406         }
1407 }
1408
1409 static void
1410 ixgbe_handle_que(void *context, int pending)
1411 {
1412         struct ix_queue *que = context;
1413         struct adapter  *adapter = que->adapter;
1414         struct tx_ring  *txr = que->txr;
1415         struct ifnet    *ifp = adapter->ifp;
1416         bool            more;
1417
1418         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1419                 more = ixgbe_rxeof(que);
1420                 IXGBE_TX_LOCK(txr);
1421                 ixgbe_txeof(txr);
1422 #ifndef IXGBE_LEGACY_TX
1423                 if (!drbr_empty(ifp, txr->br))
1424                         ixgbe_mq_start_locked(ifp, txr);
1425 #else
1426                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1427                         ixgbe_start_locked(txr, ifp);
1428 #endif
1429                 IXGBE_TX_UNLOCK(txr);
1430         }
1431
1432         /* Reenable this interrupt */
1433         if (que->res != NULL)
1434                 ixgbe_enable_queue(adapter, que->msix);
1435         else
1436                 ixgbe_enable_intr(adapter);
1437         return;
1438 }
1439
1440
1441 /*********************************************************************
1442  *
1443  *  Legacy Interrupt Service routine
1444  *
1445  **********************************************************************/
1446
1447 static void
1448 ixgbe_legacy_irq(void *arg)
1449 {
1450         struct ix_queue *que = arg;
1451         struct adapter  *adapter = que->adapter;
1452         struct ixgbe_hw *hw = &adapter->hw;
1453         struct ifnet    *ifp = adapter->ifp;
1454         struct          tx_ring *txr = adapter->tx_rings;
1455         bool            more;
1456         u32             reg_eicr;
1457
1458
1459         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1460
1461         ++que->irqs;
1462         if (reg_eicr == 0) {
1463                 ixgbe_enable_intr(adapter);
1464                 return;
1465         }
1466
1467         more = ixgbe_rxeof(que);
1468
1469         IXGBE_TX_LOCK(txr);
1470         ixgbe_txeof(txr);
1471 #ifdef IXGBE_LEGACY_TX
1472         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1473                 ixgbe_start_locked(txr, ifp);
1474 #else
1475         if (!drbr_empty(ifp, txr->br))
1476                 ixgbe_mq_start_locked(ifp, txr);
1477 #endif
1478         IXGBE_TX_UNLOCK(txr);
1479
1480         /* Check for fan failure */
1481         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1482             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1483                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1484                     "REPLACE IMMEDIATELY!!\n");
1485                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1486         }
1487
1488         /* Link status change */
1489         if (reg_eicr & IXGBE_EICR_LSC)
1490                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1491
1492         if (more)
1493                 taskqueue_enqueue(que->tq, &que->que_task);
1494         else
1495                 ixgbe_enable_intr(adapter);
1496         return;
1497 }
1498
1499
1500 /*********************************************************************
1501  *
1502  *  MSIX Queue Interrupt Service routine
1503  *
1504  **********************************************************************/
1505 void
1506 ixgbe_msix_que(void *arg)
1507 {
1508         struct ix_queue *que = arg;
1509         struct adapter  *adapter = que->adapter;
1510         struct ifnet    *ifp = adapter->ifp;
1511         struct tx_ring  *txr = que->txr;
1512         struct rx_ring  *rxr = que->rxr;
1513         bool            more;
1514         u32             newitr = 0;
1515
1516         /* Protect against spurious interrupts */
1517         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1518                 return;
1519
1520         ixgbe_disable_queue(adapter, que->msix);
1521         ++que->irqs;
1522
1523         more = ixgbe_rxeof(que);
1524
1525         IXGBE_TX_LOCK(txr);
1526         ixgbe_txeof(txr);
1527 #ifdef IXGBE_LEGACY_TX
1528         if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1529                 ixgbe_start_locked(txr, ifp);
1530 #else
1531         if (!drbr_empty(ifp, txr->br))
1532                 ixgbe_mq_start_locked(ifp, txr);
1533 #endif
1534         IXGBE_TX_UNLOCK(txr);
1535
1536         /* Do AIM now? */
1537
1538         if (ixgbe_enable_aim == FALSE)
1539                 goto no_calc;
1540         /*
1541         ** Do Adaptive Interrupt Moderation:
1542         **  - Write out last calculated setting
1543         **  - Calculate based on average size over
1544         **    the last interval.
1545         */
1546         if (que->eitr_setting)
1547                 IXGBE_WRITE_REG(&adapter->hw,
1548                     IXGBE_EITR(que->msix), que->eitr_setting);
1549
1550         que->eitr_setting = 0;
1551
1552         /* Idle, do nothing */
1553         if ((txr->bytes == 0) && (rxr->bytes == 0))
1554                 goto no_calc;
1555
1556         if ((txr->bytes) && (txr->packets))
1557                 newitr = txr->bytes/txr->packets;
1558         if ((rxr->bytes) && (rxr->packets))
1559                 newitr = max(newitr,
1560                     (rxr->bytes / rxr->packets));
1561         newitr += 24; /* account for hardware frame, crc */
1562
1563         /* set an upper boundary */
1564         newitr = min(newitr, 3000);
1565
1566         /* Be nice to the mid range */
1567         if ((newitr > 300) && (newitr < 1200))
1568                 newitr = (newitr / 3);
1569         else
1570                 newitr = (newitr / 2);
1571
1572         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1573                 newitr |= newitr << 16;
1574         else
1575                 newitr |= IXGBE_EITR_CNT_WDIS;
1576
1577         /* save for next interrupt */
1578         que->eitr_setting = newitr;
1579
1580         /* Reset state */
1581         txr->bytes = 0;
1582         txr->packets = 0;
1583         rxr->bytes = 0;
1584         rxr->packets = 0;
1585
1586 no_calc:
1587         if (more)
1588                 taskqueue_enqueue(que->tq, &que->que_task);
1589         else
1590                 ixgbe_enable_queue(adapter, que->msix);
1591         return;
1592 }
1593
1594
1595 static void
1596 ixgbe_msix_link(void *arg)
1597 {
1598         struct adapter  *adapter = arg;
1599         struct ixgbe_hw *hw = &adapter->hw;
1600         u32             reg_eicr;
1601
1602         ++adapter->link_irq;
1603
1604         /* First get the cause */
1605         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1606         /* Be sure the queue bits are not cleared */
1607         reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1608         /* Clear interrupt with write */
1609         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1610
1611         /* Link status change */
1612         if (reg_eicr & IXGBE_EICR_LSC)
1613                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1614
1615         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1616 #ifdef IXGBE_FDIR
1617                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1618                         /* This is probably overkill :) */
1619                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1620                                 return;
1621                         /* Disable the interrupt */
1622                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1623                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1624                 } else
1625 #endif
1626                 if (reg_eicr & IXGBE_EICR_ECC) {
1627                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1628                             "Please Reboot!!\n");
1629                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1630                 } else
1631
1632                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1633                         /* Clear the interrupt */
1634                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1635                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1636                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1637                         /* Clear the interrupt */
1638                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1639                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1640                 }
1641         }
1642
1643         /* Check for fan failure */
1644         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1645             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1646                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1647                     "REPLACE IMMEDIATELY!!\n");
1648                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1649         }
1650
1651         /* Check for over temp condition */
1652         if ((hw->mac.type == ixgbe_mac_X540) &&
1653             (reg_eicr & IXGBE_EICR_TS)) {
1654                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1655                     "PHY IS SHUT DOWN!!\n");
1656                 device_printf(adapter->dev, "System shutdown required\n");
1657                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1658         }
1659
1660         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1661         return;
1662 }
1663
1664 /*********************************************************************
1665  *
1666  *  Media Ioctl callback
1667  *
1668  *  This routine is called whenever the user queries the status of
1669  *  the interface using ifconfig.
1670  *
1671  **********************************************************************/
1672 static void
1673 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1674 {
1675         struct adapter *adapter = ifp->if_softc;
1676
1677         INIT_DEBUGOUT("ixgbe_media_status: begin");
1678         IXGBE_CORE_LOCK(adapter);
1679         ixgbe_update_link_status(adapter);
1680
1681         ifmr->ifm_status = IFM_AVALID;
1682         ifmr->ifm_active = IFM_ETHER;
1683
1684         if (!adapter->link_active) {
1685                 IXGBE_CORE_UNLOCK(adapter);
1686                 return;
1687         }
1688
1689         ifmr->ifm_status |= IFM_ACTIVE;
1690
1691         switch (adapter->link_speed) {
1692                 case IXGBE_LINK_SPEED_100_FULL:
1693                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1694                         break;
1695                 case IXGBE_LINK_SPEED_1GB_FULL:
1696                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1697                         break;
1698                 case IXGBE_LINK_SPEED_10GB_FULL:
1699                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1700                         break;
1701         }
1702
1703         IXGBE_CORE_UNLOCK(adapter);
1704
1705         return;
1706 }
1707
1708 /*********************************************************************
1709  *
1710  *  Media Ioctl callback
1711  *
1712  *  This routine is called when the user changes speed/duplex using
1713  *  media/mediopt option with ifconfig.
1714  *
1715  **********************************************************************/
1716 static int
1717 ixgbe_media_change(struct ifnet * ifp)
1718 {
1719         struct adapter *adapter = ifp->if_softc;
1720         struct ifmedia *ifm = &adapter->media;
1721
1722         INIT_DEBUGOUT("ixgbe_media_change: begin");
1723
1724         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1725                 return (EINVAL);
1726
1727         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1728         case IFM_AUTO:
1729                 adapter->hw.phy.autoneg_advertised =
1730                     IXGBE_LINK_SPEED_100_FULL |
1731                     IXGBE_LINK_SPEED_1GB_FULL |
1732                     IXGBE_LINK_SPEED_10GB_FULL;
1733                 break;
1734         default:
1735                 device_printf(adapter->dev, "Only auto media type\n");
1736                 return (EINVAL);
1737         }
1738
1739         return (0);
1740 }
1741
1742 /*********************************************************************
1743  *
1744  *  This routine maps the mbufs to tx descriptors, allowing the
1745  *  TX engine to transmit the packets.
1746  *      - return 0 on success, positive on failure
1747  *
1748  **********************************************************************/
1749
1750 static int
1751 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1752 {
1753         struct adapter  *adapter = txr->adapter;
1754         u32             olinfo_status = 0, cmd_type_len;
1755         int             i, j, error, nsegs;
1756         int             first;
1757         bool            remap = TRUE;
1758         struct mbuf     *m_head;
1759         bus_dma_segment_t segs[adapter->num_segs];
1760         bus_dmamap_t    map;
1761         struct ixgbe_tx_buf *txbuf;
1762         union ixgbe_adv_tx_desc *txd = NULL;
1763
1764         m_head = *m_headp;
1765
1766         /* Basic descriptor defines */
1767         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1768             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1769
1770         if (m_head->m_flags & M_VLANTAG)
1771                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1772
1773         /*
1774          * Important to capture the first descriptor
1775          * used because it will contain the index of
1776          * the one we tell the hardware to report back
1777          */
1778         first = txr->next_avail_desc;
1779         txbuf = &txr->tx_buffers[first];
1780         map = txbuf->map;
1781
1782         /*
1783          * Map the packet for DMA.
1784          */
1785 retry:
1786         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1787             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1788
1789         if (__predict_false(error)) {
1790                 struct mbuf *m;
1791
1792                 switch (error) {
1793                 case EFBIG:
1794                         /* Try it again? - one try */
1795                         if (remap == TRUE) {
1796                                 remap = FALSE;
1797                                 m = m_defrag(*m_headp, M_NOWAIT);
1798                                 if (m == NULL) {
1799                                         adapter->mbuf_defrag_failed++;
1800                                         m_freem(*m_headp);
1801                                         *m_headp = NULL;
1802                                         return (ENOBUFS);
1803                                 }
1804                                 *m_headp = m;
1805                                 goto retry;
1806                         } else
1807                                 return (error);
1808                 case ENOMEM:
1809                         txr->no_tx_dma_setup++;
1810                         return (error);
1811                 default:
1812                         txr->no_tx_dma_setup++;
1813                         m_freem(*m_headp);
1814                         *m_headp = NULL;
1815                         return (error);
1816                 }
1817         }
1818
1819         /* Make certain there are enough descriptors */
1820         if (nsegs > txr->tx_avail - 2) {
1821                 txr->no_desc_avail++;
1822                 bus_dmamap_unload(txr->txtag, map);
1823                 return (ENOBUFS);
1824         }
1825         m_head = *m_headp;
1826
1827         /*
1828         ** Set up the appropriate offload context
1829         ** this will consume the first descriptor
1830         */
1831         error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1832         if (__predict_false(error)) {
1833                 if (error == ENOBUFS)
1834                         *m_headp = NULL;
1835                 return (error);
1836         }
1837
1838 #ifdef IXGBE_FDIR
1839         /* Do the flow director magic */
1840         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1841                 ++txr->atr_count;
1842                 if (txr->atr_count >= atr_sample_rate) {
1843                         ixgbe_atr(txr, m_head);
1844                         txr->atr_count = 0;
1845                 }
1846         }
1847 #endif
1848
1849         i = txr->next_avail_desc;
1850         for (j = 0; j < nsegs; j++) {
1851                 bus_size_t seglen;
1852                 bus_addr_t segaddr;
1853
1854                 txbuf = &txr->tx_buffers[i];
1855                 txd = &txr->tx_base[i];
1856                 seglen = segs[j].ds_len;
1857                 segaddr = htole64(segs[j].ds_addr);
1858
1859                 txd->read.buffer_addr = segaddr;
1860                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1861                     cmd_type_len |seglen);
1862                 txd->read.olinfo_status = htole32(olinfo_status);
1863
1864                 if (++i == txr->num_desc)
1865                         i = 0;
1866         }
1867
1868         txd->read.cmd_type_len |=
1869             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1870         txr->tx_avail -= nsegs;
1871         txr->next_avail_desc = i;
1872
1873         txbuf->m_head = m_head;
1874         /*
1875         ** Here we swap the map so the last descriptor,
1876         ** which gets the completion interrupt has the
1877         ** real map, and the first descriptor gets the
1878         ** unused map from this descriptor.
1879         */
1880         txr->tx_buffers[first].map = txbuf->map;
1881         txbuf->map = map;
1882         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1883
1884         /* Set the EOP descriptor that will be marked done */
1885         txbuf = &txr->tx_buffers[first];
1886         txbuf->eop = txd;
1887
1888         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1889             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1890         /*
1891          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1892          * hardware that this frame is available to transmit.
1893          */
1894         ++txr->total_packets;
1895         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1896
1897         return (0);
1898
1899 }
1900
1901 static void
1902 ixgbe_set_promisc(struct adapter *adapter)
1903 {
1904         u_int32_t       reg_rctl;
1905         struct ifnet   *ifp = adapter->ifp;
1906         int             mcnt = 0;
1907
1908         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1909         reg_rctl &= (~IXGBE_FCTRL_UPE);
1910         if (ifp->if_flags & IFF_ALLMULTI)
1911                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1912         else {
1913                 struct  ifmultiaddr *ifma;
1914 #if __FreeBSD_version < 800000
1915                 IF_ADDR_LOCK(ifp);
1916 #else
1917                 if_maddr_rlock(ifp);
1918 #endif
1919                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1920                         if (ifma->ifma_addr->sa_family != AF_LINK)
1921                                 continue;
1922                         if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1923                                 break;
1924                         mcnt++;
1925                 }
1926 #if __FreeBSD_version < 800000
1927                 IF_ADDR_UNLOCK(ifp);
1928 #else
1929                 if_maddr_runlock(ifp);
1930 #endif
1931         }
1932         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1933                 reg_rctl &= (~IXGBE_FCTRL_MPE);
1934         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1935
1936         if (ifp->if_flags & IFF_PROMISC) {
1937                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1938                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1939         } else if (ifp->if_flags & IFF_ALLMULTI) {
1940                 reg_rctl |= IXGBE_FCTRL_MPE;
1941                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1942                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1943         }
1944         return;
1945 }
1946
1947
1948 /*********************************************************************
1949  *  Multicast Update
1950  *
1951  *  This routine is called whenever multicast address list is updated.
1952  *
1953  **********************************************************************/
1954 #define IXGBE_RAR_ENTRIES 16
1955
1956 static void
1957 ixgbe_set_multi(struct adapter *adapter)
1958 {
1959         u32     fctrl;
1960         u8      *mta;
1961         u8      *update_ptr;
1962         struct  ifmultiaddr *ifma;
1963         int     mcnt = 0;
1964         struct ifnet   *ifp = adapter->ifp;
1965
1966         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1967
1968         mta = adapter->mta;
1969         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1970             MAX_NUM_MULTICAST_ADDRESSES);
1971
1972 #if __FreeBSD_version < 800000
1973         IF_ADDR_LOCK(ifp);
1974 #else
1975         if_maddr_rlock(ifp);
1976 #endif
1977         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1978                 if (ifma->ifma_addr->sa_family != AF_LINK)
1979                         continue;
1980                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1981                         break;
1982                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1983                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1984                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1985                 mcnt++;
1986         }
1987 #if __FreeBSD_version < 800000
1988         IF_ADDR_UNLOCK(ifp);
1989 #else
1990         if_maddr_runlock(ifp);
1991 #endif
1992
1993         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1994         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1995         if (ifp->if_flags & IFF_PROMISC)
1996                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1997         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1998             ifp->if_flags & IFF_ALLMULTI) {
1999                 fctrl |= IXGBE_FCTRL_MPE;
2000                 fctrl &= ~IXGBE_FCTRL_UPE;
2001         } else
2002                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2003
2004         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
2005
2006         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
2007                 update_ptr = mta;
2008                 ixgbe_update_mc_addr_list(&adapter->hw,
2009                     update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
2010         }
2011
2012         return;
2013 }
2014
2015 /*
2016  * This is an iterator function now needed by the multicast
2017  * shared code. It simply feeds the shared code routine the
2018  * addresses in the array of ixgbe_set_multi() one by one.
2019  */
2020 static u8 *
2021 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2022 {
2023         u8 *addr = *update_ptr;
2024         u8 *newptr;
2025         *vmdq = 0;
2026
2027         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2028         *update_ptr = newptr;
2029         return addr;
2030 }
2031
2032
2033 /*********************************************************************
2034  *  Timer routine
2035  *
2036  *  This routine checks for link status,updates statistics,
2037  *  and runs the watchdog check.
2038  *
2039  **********************************************************************/
2040
2041 static void
2042 ixgbe_local_timer(void *arg)
2043 {
2044         struct adapter  *adapter = arg;
2045         device_t        dev = adapter->dev;
2046         struct ix_queue *que = adapter->queues;
2047         struct tx_ring  *txr = adapter->tx_rings;
2048         int             hung = 0, paused = 0;
2049
2050         mtx_assert(&adapter->core_mtx, MA_OWNED);
2051
2052         /* Check for pluggable optics */
2053         if (adapter->sfp_probe)
2054                 if (!ixgbe_sfp_probe(adapter))
2055                         goto out; /* Nothing to do */
2056
2057         ixgbe_update_link_status(adapter);
2058         ixgbe_update_stats_counters(adapter);
2059
2060         /*
2061          * If the interface has been paused
2062          * then don't do the watchdog check
2063          */
2064         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2065                 paused = 1;
2066
2067         /*
2068         ** Check the TX queues status
2069         **      - watchdog only if all queues show hung
2070         */
2071         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2072                 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2073                     (paused == 0))
2074                         ++hung;
2075                 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2076                         taskqueue_enqueue(que->tq, &txr->txq_task);
2077         }
2078         /* Only truely watchdog if all queues show hung */
2079         if (hung == adapter->num_queues)
2080                 goto watchdog;
2081
2082 out:
2083         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2084         return;
2085
2086 watchdog:
2087         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2088         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2089             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2090             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2091         device_printf(dev,"TX(%d) desc avail = %d,"
2092             "Next TX to Clean = %d\n",
2093             txr->me, txr->tx_avail, txr->next_to_clean);
2094         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2095         adapter->watchdog_events++;
2096         ixgbe_init_locked(adapter);
2097 }
2098
2099 /*
2100 ** Note: this routine updates the OS on the link state
2101 **      the real check of the hardware only happens with
2102 **      a link interrupt.
2103 */
2104 static void
2105 ixgbe_update_link_status(struct adapter *adapter)
2106 {
2107         struct ifnet    *ifp = adapter->ifp;
2108         device_t dev = adapter->dev;
2109
2110
2111         if (adapter->link_up){
2112                 if (adapter->link_active == FALSE) {
2113                         if (bootverbose)
2114                                 device_printf(dev,"Link is up %d Gbps %s \n",
2115                                     ((adapter->link_speed == 128)? 10:1),
2116                                     "Full Duplex");
2117                         adapter->link_active = TRUE;
2118                         /* Update any Flow Control changes */
2119                         ixgbe_fc_enable(&adapter->hw);
2120                         if_link_state_change(ifp, LINK_STATE_UP);
2121                 }
2122         } else { /* Link down */
2123                 if (adapter->link_active == TRUE) {
2124                         if (bootverbose)
2125                                 device_printf(dev,"Link is Down\n");
2126                         if_link_state_change(ifp, LINK_STATE_DOWN);
2127                         adapter->link_active = FALSE;
2128                 }
2129         }
2130
2131         return;
2132 }
2133
2134
2135 /*********************************************************************
2136  *
2137  *  This routine disables all traffic on the adapter by issuing a
2138  *  global reset on the MAC and deallocates TX/RX buffers.
2139  *
2140  **********************************************************************/
2141
2142 static void
2143 ixgbe_stop(void *arg)
2144 {
2145         struct ifnet   *ifp;
2146         struct adapter *adapter = arg;
2147         struct ixgbe_hw *hw = &adapter->hw;
2148         ifp = adapter->ifp;
2149
2150         mtx_assert(&adapter->core_mtx, MA_OWNED);
2151
2152         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2153         ixgbe_disable_intr(adapter);
2154         callout_stop(&adapter->timer);
2155
2156         /* Let the stack know...*/
2157         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2158
2159         ixgbe_reset_hw(hw);
2160         hw->adapter_stopped = FALSE;
2161         ixgbe_stop_adapter(hw);
2162         if (hw->mac.type == ixgbe_mac_82599EB)
2163                 ixgbe_stop_mac_link_on_d3_82599(hw);
2164         /* Turn off the laser - noop with no optics */
2165         ixgbe_disable_tx_laser(hw);
2166
2167         /* Update the stack */
2168         adapter->link_up = FALSE;
2169         ixgbe_update_link_status(adapter);
2170
2171         /* reprogram the RAR[0] in case user changed it. */
2172         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2173
2174         return;
2175 }
2176
2177
2178 /*********************************************************************
2179  *
2180  *  Determine hardware revision.
2181  *
2182  **********************************************************************/
2183 static void
2184 ixgbe_identify_hardware(struct adapter *adapter)
2185 {
2186         device_t        dev = adapter->dev;
2187         struct ixgbe_hw *hw = &adapter->hw;
2188
2189         /* Save off the information about this board */
2190         hw->vendor_id = pci_get_vendor(dev);
2191         hw->device_id = pci_get_device(dev);
2192         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2193         hw->subsystem_vendor_id =
2194             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2195         hw->subsystem_device_id =
2196             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2197
2198         /* We need this here to set the num_segs below */
2199         ixgbe_set_mac_type(hw);
2200
2201         /* Pick up the 82599 and VF settings */
2202         if (hw->mac.type != ixgbe_mac_82598EB) {
2203                 hw->phy.smart_speed = ixgbe_smart_speed;
2204                 adapter->num_segs = IXGBE_82599_SCATTER;
2205         } else
2206                 adapter->num_segs = IXGBE_82598_SCATTER;
2207
2208         return;
2209 }
2210
2211 /*********************************************************************
2212  *
2213  *  Determine optic type
2214  *
2215  **********************************************************************/
2216 static void
2217 ixgbe_setup_optics(struct adapter *adapter)
2218 {
2219         struct ixgbe_hw *hw = &adapter->hw;
2220         int             layer;
2221
2222         layer = ixgbe_get_supported_physical_layer(hw);
2223
2224         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2225                 adapter->optics = IFM_10G_T;
2226                 return;
2227         }
2228
2229         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2230                 adapter->optics = IFM_1000_T;
2231                 return;
2232         }
2233
2234         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2235                 adapter->optics = IFM_1000_SX;
2236                 return;
2237         }
2238
2239         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2240             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2241                 adapter->optics = IFM_10G_LR;
2242                 return;
2243         }
2244
2245         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2246                 adapter->optics = IFM_10G_SR;
2247                 return;
2248         }
2249
2250         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2251                 adapter->optics = IFM_10G_TWINAX;
2252                 return;
2253         }
2254
2255         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2256             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2257                 adapter->optics = IFM_10G_CX4;
2258                 return;
2259         }
2260
2261         /* If we get here just set the default */
2262         adapter->optics = IFM_ETHER | IFM_AUTO;
2263         return;
2264 }
2265
2266 /*********************************************************************
2267  *
2268  *  Setup the Legacy or MSI Interrupt handler
2269  *
2270  **********************************************************************/
2271 static int
2272 ixgbe_allocate_legacy(struct adapter *adapter)
2273 {
2274         device_t        dev = adapter->dev;
2275         struct          ix_queue *que = adapter->queues;
2276 #ifndef IXGBE_LEGACY_TX
2277         struct tx_ring          *txr = adapter->tx_rings;
2278 #endif
2279         int             error, rid = 0;
2280
2281         /* MSI RID at 1 */
2282         if (adapter->msix == 1)
2283                 rid = 1;
2284
2285         /* We allocate a single interrupt resource */
2286         adapter->res = bus_alloc_resource_any(dev,
2287             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2288         if (adapter->res == NULL) {
2289                 device_printf(dev, "Unable to allocate bus resource: "
2290                     "interrupt\n");
2291                 return (ENXIO);
2292         }
2293
2294         /*
2295          * Try allocating a fast interrupt and the associated deferred
2296          * processing contexts.
2297          */
2298 #ifndef IXGBE_LEGACY_TX
2299         TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2300 #endif
2301         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2302         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2303             taskqueue_thread_enqueue, &que->tq);
2304         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2305             device_get_nameunit(adapter->dev));
2306
2307         /* Tasklets for Link, SFP and Multispeed Fiber */
2308         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2309         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2310         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2311 #ifdef IXGBE_FDIR
2312         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2313 #endif
2314         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2315             taskqueue_thread_enqueue, &adapter->tq);
2316         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2317             device_get_nameunit(adapter->dev));
2318
2319         if ((error = bus_setup_intr(dev, adapter->res,
2320             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2321             que, &adapter->tag)) != 0) {
2322                 device_printf(dev, "Failed to register fast interrupt "
2323                     "handler: %d\n", error);
2324                 taskqueue_free(que->tq);
2325                 taskqueue_free(adapter->tq);
2326                 que->tq = NULL;
2327                 adapter->tq = NULL;
2328                 return (error);
2329         }
2330         /* For simplicity in the handlers */
2331         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2332
2333         return (0);
2334 }
2335
2336
2337 /*********************************************************************
2338  *
2339  *  Setup MSIX Interrupt resources and handlers
2340  *
2341  **********************************************************************/
2342 static int
2343 ixgbe_allocate_msix(struct adapter *adapter)
2344 {
2345         device_t        dev = adapter->dev;
2346         struct          ix_queue *que = adapter->queues;
2347         struct          tx_ring *txr = adapter->tx_rings;
2348         int             error, rid, vector = 0;
2349
2350         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2351                 rid = vector + 1;
2352                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2353                     RF_SHAREABLE | RF_ACTIVE);
2354                 if (que->res == NULL) {
2355                         device_printf(dev,"Unable to allocate"
2356                             " bus resource: que interrupt [%d]\n", vector);
2357                         return (ENXIO);
2358                 }
2359                 /* Set the handler function */
2360                 error = bus_setup_intr(dev, que->res,
2361                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2362                     ixgbe_msix_que, que, &que->tag);
2363                 if (error) {
2364                         que->res = NULL;
2365                         device_printf(dev, "Failed to register QUE handler");
2366                         return (error);
2367                 }
2368 #if __FreeBSD_version >= 800504
2369                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2370 #endif
2371                 que->msix = vector;
2372                 adapter->que_mask |= (u64)(1 << que->msix);
2373                 /*
2374                 ** Bind the msix vector, and thus the
2375                 ** ring to the corresponding cpu.
2376                 */
2377                 if (adapter->num_queues > 1)
2378                         bus_bind_intr(dev, que->res, i);
2379
2380 #ifndef IXGBE_LEGACY_TX
2381                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2382 #endif
2383                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2384                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2385                     taskqueue_thread_enqueue, &que->tq);
2386                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2387                     device_get_nameunit(adapter->dev));
2388         }
2389
2390         /* and Link */
2391         rid = vector + 1;
2392         adapter->res = bus_alloc_resource_any(dev,
2393             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2394         if (!adapter->res) {
2395                 device_printf(dev,"Unable to allocate"
2396             " bus resource: Link interrupt [%d]\n", rid);
2397                 return (ENXIO);
2398         }
2399         /* Set the link handler function */
2400         error = bus_setup_intr(dev, adapter->res,
2401             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2402             ixgbe_msix_link, adapter, &adapter->tag);
2403         if (error) {
2404                 adapter->res = NULL;
2405                 device_printf(dev, "Failed to register LINK handler");
2406                 return (error);
2407         }
2408 #if __FreeBSD_version >= 800504
2409         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2410 #endif
2411         adapter->linkvec = vector;
2412         /* Tasklets for Link, SFP and Multispeed Fiber */
2413         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2414         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2415         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2416 #ifdef IXGBE_FDIR
2417         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2418 #endif
2419         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2420             taskqueue_thread_enqueue, &adapter->tq);
2421         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2422             device_get_nameunit(adapter->dev));
2423
2424         return (0);
2425 }
2426
2427 /*
2428  * Setup Either MSI/X or MSI
2429  */
2430 static int
2431 ixgbe_setup_msix(struct adapter *adapter)
2432 {
2433         device_t dev = adapter->dev;
2434         int rid, want, queues, msgs;
2435
2436         /* Override by tuneable */
2437         if (ixgbe_enable_msix == 0)
2438                 goto msi;
2439
2440         /* First try MSI/X */
2441         msgs = pci_msix_count(dev);
2442         if (msgs == 0)
2443                 goto msi;
2444         rid = PCIR_BAR(MSIX_82598_BAR);
2445         adapter->msix_mem = bus_alloc_resource_any(dev,
2446             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2447         if (adapter->msix_mem == NULL) {
2448                 rid += 4;       /* 82599 maps in higher BAR */
2449                 adapter->msix_mem = bus_alloc_resource_any(dev,
2450                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2451         }
2452         if (adapter->msix_mem == NULL) {
2453                 /* May not be enabled */
2454                 device_printf(adapter->dev,
2455                     "Unable to map MSIX table \n");
2456                 goto msi;
2457         }
2458
2459         /* Figure out a reasonable auto config value */
2460         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2461
2462         if (ixgbe_num_queues != 0)
2463                 queues = ixgbe_num_queues;
2464         /* Set max queues to 8 when autoconfiguring */
2465         else if ((ixgbe_num_queues == 0) && (queues > 8))
2466                 queues = 8;
2467
2468         /* reflect correct sysctl value */
2469         ixgbe_num_queues = queues;
2470
2471         /*
2472         ** Want one vector (RX/TX pair) per queue
2473         ** plus an additional for Link.
2474         */
2475         want = queues + 1;
2476         if (msgs >= want)
2477                 msgs = want;
2478         else {
2479                 device_printf(adapter->dev,
2480                     "MSIX Configuration Problem, "
2481                     "%d vectors but %d queues wanted!\n",
2482                     msgs, want);
2483                 goto msi;
2484         }
2485         if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2486                 device_printf(adapter->dev,
2487                     "Using MSIX interrupts with %d vectors\n", msgs);
2488                 adapter->num_queues = queues;
2489                 return (msgs);
2490         }
2491         /*
2492         ** If MSIX alloc failed or provided us with
2493         ** less than needed, free and fall through to MSI
2494         */
2495         pci_release_msi(dev);
2496
2497 msi:
2498         if (adapter->msix_mem != NULL) {
2499                 bus_release_resource(dev, SYS_RES_MEMORY,
2500                     rid, adapter->msix_mem);
2501                 adapter->msix_mem = NULL;
2502         }
2503         msgs = 1;
2504         if (pci_alloc_msi(dev, &msgs) == 0) {
2505                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2506                 return (msgs);
2507         }
2508         device_printf(adapter->dev,"Using a Legacy interrupt\n");
2509         return (0);
2510 }
2511
2512
2513 static int
2514 ixgbe_allocate_pci_resources(struct adapter *adapter)
2515 {
2516         int             rid;
2517         device_t        dev = adapter->dev;
2518
2519         rid = PCIR_BAR(0);
2520         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2521             &rid, RF_ACTIVE);
2522
2523         if (!(adapter->pci_mem)) {
2524                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2525                 return (ENXIO);
2526         }
2527
2528         adapter->osdep.mem_bus_space_tag =
2529                 rman_get_bustag(adapter->pci_mem);
2530         adapter->osdep.mem_bus_space_handle =
2531                 rman_get_bushandle(adapter->pci_mem);
2532         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2533
2534         /* Legacy defaults */
2535         adapter->num_queues = 1;
2536         adapter->hw.back = &adapter->osdep;
2537
2538         /*
2539         ** Now setup MSI or MSI/X, should
2540         ** return us the number of supported
2541         ** vectors. (Will be 1 for MSI)
2542         */
2543         adapter->msix = ixgbe_setup_msix(adapter);
2544         return (0);
2545 }
2546
2547 static void
2548 ixgbe_free_pci_resources(struct adapter * adapter)
2549 {
2550         struct          ix_queue *que = adapter->queues;
2551         device_t        dev = adapter->dev;
2552         int             rid, memrid;
2553
2554         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2555                 memrid = PCIR_BAR(MSIX_82598_BAR);
2556         else
2557                 memrid = PCIR_BAR(MSIX_82599_BAR);
2558
2559         /*
2560         ** There is a slight possibility of a failure mode
2561         ** in attach that will result in entering this function
2562         ** before interrupt resources have been initialized, and
2563         ** in that case we do not want to execute the loops below
2564         ** We can detect this reliably by the state of the adapter
2565         ** res pointer.
2566         */
2567         if (adapter->res == NULL)
2568                 goto mem;
2569
2570         /*
2571         **  Release all msix queue resources:
2572         */
2573         for (int i = 0; i < adapter->num_queues; i++, que++) {
2574                 rid = que->msix + 1;
2575                 if (que->tag != NULL) {
2576                         bus_teardown_intr(dev, que->res, que->tag);
2577                         que->tag = NULL;
2578                 }
2579                 if (que->res != NULL)
2580                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2581         }
2582
2583
2584         /* Clean the Legacy or Link interrupt last */
2585         if (adapter->linkvec) /* we are doing MSIX */
2586                 rid = adapter->linkvec + 1;
2587         else
2588                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2589
2590         if (adapter->tag != NULL) {
2591                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2592                 adapter->tag = NULL;
2593         }
2594         if (adapter->res != NULL)
2595                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2596
2597 mem:
2598         if (adapter->msix)
2599                 pci_release_msi(dev);
2600
2601         if (adapter->msix_mem != NULL)
2602                 bus_release_resource(dev, SYS_RES_MEMORY,
2603                     memrid, adapter->msix_mem);
2604
2605         if (adapter->pci_mem != NULL)
2606                 bus_release_resource(dev, SYS_RES_MEMORY,
2607                     PCIR_BAR(0), adapter->pci_mem);
2608
2609         return;
2610 }
2611
2612 /*********************************************************************
2613  *
2614  *  Setup networking device structure and register an interface.
2615  *
2616  **********************************************************************/
2617 static int
2618 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2619 {
2620         struct ixgbe_hw *hw = &adapter->hw;
2621         struct ifnet   *ifp;
2622
2623         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2624
2625         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2626         if (ifp == NULL) {
2627                 device_printf(dev, "can not allocate ifnet structure\n");
2628                 return (-1);
2629         }
2630         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2631         ifp->if_baudrate = IF_Gbps(10);
2632         ifp->if_init = ixgbe_init;
2633         ifp->if_softc = adapter;
2634         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2635         ifp->if_ioctl = ixgbe_ioctl;
2636 #ifndef IXGBE_LEGACY_TX
2637         ifp->if_transmit = ixgbe_mq_start;
2638         ifp->if_qflush = ixgbe_qflush;
2639 #else
2640         ifp->if_start = ixgbe_start;
2641         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2642         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2643         IFQ_SET_READY(&ifp->if_snd);
2644 #endif
2645
2646         ether_ifattach(ifp, adapter->hw.mac.addr);
2647
2648         adapter->max_frame_size =
2649             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2650
2651         /*
2652          * Tell the upper layer(s) we support long frames.
2653          */
2654         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2655
2656         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2657         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2658         ifp->if_capabilities |= IFCAP_LRO;
2659         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2660                              |  IFCAP_VLAN_HWTSO
2661                              |  IFCAP_VLAN_MTU
2662                              |  IFCAP_HWSTATS;
2663         ifp->if_capenable = ifp->if_capabilities;
2664
2665         /*
2666         ** Don't turn this on by default, if vlans are
2667         ** created on another pseudo device (eg. lagg)
2668         ** then vlan events are not passed thru, breaking
2669         ** operation, but with HW FILTER off it works. If
2670         ** using vlans directly on the ixgbe driver you can
2671         ** enable this and get full hardware tag filtering.
2672         */
2673         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2674
2675         /*
2676          * Specify the media types supported by this adapter and register
2677          * callbacks to update media and link information
2678          */
2679         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2680                      ixgbe_media_status);
2681         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2682         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2683         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2684                 ifmedia_add(&adapter->media,
2685                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2686                 ifmedia_add(&adapter->media,
2687                     IFM_ETHER | IFM_1000_T, 0, NULL);
2688         }
2689         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2690         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2691
2692         return (0);
2693 }
2694
2695 static void
2696 ixgbe_config_link(struct adapter *adapter)
2697 {
2698         struct ixgbe_hw *hw = &adapter->hw;
2699         u32     autoneg, err = 0;
2700         bool    sfp, negotiate;
2701
2702         sfp = ixgbe_is_sfp(hw);
2703
2704         if (sfp) {
2705                 if (hw->phy.multispeed_fiber) {
2706                         hw->mac.ops.setup_sfp(hw);
2707                         ixgbe_enable_tx_laser(hw);
2708                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2709                 } else
2710                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2711         } else {
2712                 if (hw->mac.ops.check_link)
2713                         err = ixgbe_check_link(hw, &adapter->link_speed,
2714                             &adapter->link_up, FALSE);
2715                 if (err)
2716                         goto out;
2717                 autoneg = hw->phy.autoneg_advertised;
2718                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2719                         err  = hw->mac.ops.get_link_capabilities(hw,
2720                             &autoneg, &negotiate);
2721                 if (err)
2722                         goto out;
2723                 if (hw->mac.ops.setup_link)
2724                         err = hw->mac.ops.setup_link(hw,
2725                             autoneg, adapter->link_up);
2726         }
2727 out:
2728         return;
2729 }
2730
2731 /********************************************************************
2732  * Manage DMA'able memory.
2733  *******************************************************************/
2734 static void
2735 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2736 {
2737         if (error)
2738                 return;
2739         *(bus_addr_t *) arg = segs->ds_addr;
2740         return;
2741 }
2742
2743 static int
2744 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2745                 struct ixgbe_dma_alloc *dma, int mapflags)
2746 {
2747         device_t dev = adapter->dev;
2748         int             r;
2749
2750         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2751                                DBA_ALIGN, 0,    /* alignment, bounds */
2752                                BUS_SPACE_MAXADDR,       /* lowaddr */
2753                                BUS_SPACE_MAXADDR,       /* highaddr */
2754                                NULL, NULL,      /* filter, filterarg */
2755                                size,    /* maxsize */
2756                                1,       /* nsegments */
2757                                size,    /* maxsegsize */
2758                                BUS_DMA_ALLOCNOW,        /* flags */
2759                                NULL,    /* lockfunc */
2760                                NULL,    /* lockfuncarg */
2761                                &dma->dma_tag);
2762         if (r != 0) {
2763                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2764                        "error %u\n", r);
2765                 goto fail_0;
2766         }
2767         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2768                              BUS_DMA_NOWAIT, &dma->dma_map);
2769         if (r != 0) {
2770                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2771                        "error %u\n", r);
2772                 goto fail_1;
2773         }
2774         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2775                             size,
2776                             ixgbe_dmamap_cb,
2777                             &dma->dma_paddr,
2778                             mapflags | BUS_DMA_NOWAIT);
2779         if (r != 0) {
2780                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2781                        "error %u\n", r);
2782                 goto fail_2;
2783         }
2784         dma->dma_size = size;
2785         return (0);
2786 fail_2:
2787         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2788 fail_1:
2789         bus_dma_tag_destroy(dma->dma_tag);
2790 fail_0:
2791         dma->dma_tag = NULL;
2792         return (r);
2793 }
2794
2795 static void
2796 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2797 {
2798         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2799             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2800         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2801         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2802         bus_dma_tag_destroy(dma->dma_tag);
2803 }
2804
2805
2806 /*********************************************************************
2807  *
2808  *  Allocate memory for the transmit and receive rings, and then
2809  *  the descriptors associated with each, called only once at attach.
2810  *
2811  **********************************************************************/
2812 static int
2813 ixgbe_allocate_queues(struct adapter *adapter)
2814 {
2815         device_t        dev = adapter->dev;
2816         struct ix_queue *que;
2817         struct tx_ring  *txr;
2818         struct rx_ring  *rxr;
2819         int rsize, tsize, error = IXGBE_SUCCESS;
2820         int txconf = 0, rxconf = 0;
2821
2822         /* First allocate the top level queue structs */
2823         if (!(adapter->queues =
2824             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2825             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2826                 device_printf(dev, "Unable to allocate queue memory\n");
2827                 error = ENOMEM;
2828                 goto fail;
2829         }
2830
2831         /* First allocate the TX ring struct memory */
2832         if (!(adapter->tx_rings =
2833             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2834             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2835                 device_printf(dev, "Unable to allocate TX ring memory\n");
2836                 error = ENOMEM;
2837                 goto tx_fail;
2838         }
2839
2840         /* Next allocate the RX */
2841         if (!(adapter->rx_rings =
2842             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2843             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2844                 device_printf(dev, "Unable to allocate RX ring memory\n");
2845                 error = ENOMEM;
2846                 goto rx_fail;
2847         }
2848
2849         /* For the ring itself */
2850         tsize = roundup2(adapter->num_tx_desc *
2851             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2852
2853         /*
2854          * Now set up the TX queues, txconf is needed to handle the
2855          * possibility that things fail midcourse and we need to
2856          * undo memory gracefully
2857          */
2858         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2859                 /* Set up some basics */
2860                 txr = &adapter->tx_rings[i];
2861                 txr->adapter = adapter;
2862                 txr->me = i;
2863                 txr->num_desc = adapter->num_tx_desc;
2864
2865                 /* Initialize the TX side lock */
2866                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2867                     device_get_nameunit(dev), txr->me);
2868                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2869
2870                 if (ixgbe_dma_malloc(adapter, tsize,
2871                         &txr->txdma, BUS_DMA_NOWAIT)) {
2872                         device_printf(dev,
2873                             "Unable to allocate TX Descriptor memory\n");
2874                         error = ENOMEM;
2875                         goto err_tx_desc;
2876                 }
2877                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2878                 bzero((void *)txr->tx_base, tsize);
2879
2880                 /* Now allocate transmit buffers for the ring */
2881                 if (ixgbe_allocate_transmit_buffers(txr)) {
2882                         device_printf(dev,
2883                             "Critical Failure setting up transmit buffers\n");
2884                         error = ENOMEM;
2885                         goto err_tx_desc;
2886                 }
2887 #ifndef IXGBE_LEGACY_TX
2888                 /* Allocate a buf ring */
2889                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2890                     M_WAITOK, &txr->tx_mtx);
2891                 if (txr->br == NULL) {
2892                         device_printf(dev,
2893                             "Critical Failure setting up buf ring\n");
2894                         error = ENOMEM;
2895                         goto err_tx_desc;
2896                 }
2897 #endif
2898         }
2899
2900         /*
2901          * Next the RX queues...
2902          */
2903         rsize = roundup2(adapter->num_rx_desc *
2904             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2905         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2906                 rxr = &adapter->rx_rings[i];
2907                 /* Set up some basics */
2908                 rxr->adapter = adapter;
2909                 rxr->me = i;
2910                 rxr->num_desc = adapter->num_rx_desc;
2911
2912                 /* Initialize the RX side lock */
2913                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2914                     device_get_nameunit(dev), rxr->me);
2915                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2916
2917                 if (ixgbe_dma_malloc(adapter, rsize,
2918                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2919                         device_printf(dev,
2920                             "Unable to allocate RxDescriptor memory\n");
2921                         error = ENOMEM;
2922                         goto err_rx_desc;
2923                 }
2924                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2925                 bzero((void *)rxr->rx_base, rsize);
2926
2927                 /* Allocate receive buffers for the ring*/
2928                 if (ixgbe_allocate_receive_buffers(rxr)) {
2929                         device_printf(dev,
2930                             "Critical Failure setting up receive buffers\n");
2931                         error = ENOMEM;
2932                         goto err_rx_desc;
2933                 }
2934         }
2935
2936         /*
2937         ** Finally set up the queue holding structs
2938         */
2939         for (int i = 0; i < adapter->num_queues; i++) {
2940                 que = &adapter->queues[i];
2941                 que->adapter = adapter;
2942                 que->txr = &adapter->tx_rings[i];
2943                 que->rxr = &adapter->rx_rings[i];
2944         }
2945
2946         return (0);
2947
2948 err_rx_desc:
2949         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2950                 ixgbe_dma_free(adapter, &rxr->rxdma);
2951 err_tx_desc:
2952         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2953                 ixgbe_dma_free(adapter, &txr->txdma);
2954         free(adapter->rx_rings, M_DEVBUF);
2955 rx_fail:
2956         free(adapter->tx_rings, M_DEVBUF);
2957 tx_fail:
2958         free(adapter->queues, M_DEVBUF);
2959 fail:
2960         return (error);
2961 }
2962
2963 /*********************************************************************
2964  *
2965  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2966  *  the information needed to transmit a packet on the wire. This is
2967  *  called only once at attach, setup is done every reset.
2968  *
2969  **********************************************************************/
2970 static int
2971 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2972 {
2973         struct adapter *adapter = txr->adapter;
2974         device_t dev = adapter->dev;
2975         struct ixgbe_tx_buf *txbuf;
2976         int error, i;
2977
2978         /*
2979          * Setup DMA descriptor areas.
2980          */
2981         if ((error = bus_dma_tag_create(
2982                                bus_get_dma_tag(adapter->dev),   /* parent */
2983                                1, 0,            /* alignment, bounds */
2984                                BUS_SPACE_MAXADDR,       /* lowaddr */
2985                                BUS_SPACE_MAXADDR,       /* highaddr */
2986                                NULL, NULL,              /* filter, filterarg */
2987                                IXGBE_TSO_SIZE,          /* maxsize */
2988                                adapter->num_segs,       /* nsegments */
2989                                PAGE_SIZE,               /* maxsegsize */
2990                                0,                       /* flags */
2991                                NULL,                    /* lockfunc */
2992                                NULL,                    /* lockfuncarg */
2993                                &txr->txtag))) {
2994                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2995                 goto fail;
2996         }
2997
2998         if (!(txr->tx_buffers =
2999             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
3000             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3001                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3002                 error = ENOMEM;
3003                 goto fail;
3004         }
3005
3006         /* Create the descriptor buffer dma maps */
3007         txbuf = txr->tx_buffers;
3008         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3009                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3010                 if (error != 0) {
3011                         device_printf(dev, "Unable to create TX DMA map\n");
3012                         goto fail;
3013                 }
3014         }
3015
3016         return 0;
3017 fail:
3018         /* We free all, it handles case where we are in the middle */
3019         ixgbe_free_transmit_structures(adapter);
3020         return (error);
3021 }
3022
3023 /*********************************************************************
3024  *
3025  *  Initialize a transmit ring.
3026  *
3027  **********************************************************************/
3028 static void
3029 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3030 {
3031         struct adapter *adapter = txr->adapter;
3032         struct ixgbe_tx_buf *txbuf;
3033         int i;
3034 #ifdef DEV_NETMAP
3035         struct netmap_adapter *na = NA(adapter->ifp);
3036         struct netmap_slot *slot;
3037 #endif /* DEV_NETMAP */
3038
3039         /* Clear the old ring contents */
3040         IXGBE_TX_LOCK(txr);
3041 #ifdef DEV_NETMAP
3042         /*
3043          * (under lock): if in netmap mode, do some consistency
3044          * checks and set slot to entry 0 of the netmap ring.
3045          */
3046         slot = netmap_reset(na, NR_TX, txr->me, 0);
3047 #endif /* DEV_NETMAP */
3048         bzero((void *)txr->tx_base,
3049               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3050         /* Reset indices */
3051         txr->next_avail_desc = 0;
3052         txr->next_to_clean = 0;
3053
3054         /* Free any existing tx buffers. */
3055         txbuf = txr->tx_buffers;
3056         for (i = 0; i < txr->num_desc; i++, txbuf++) {
3057                 if (txbuf->m_head != NULL) {
3058                         bus_dmamap_sync(txr->txtag, txbuf->map,
3059                             BUS_DMASYNC_POSTWRITE);
3060                         bus_dmamap_unload(txr->txtag, txbuf->map);
3061                         m_freem(txbuf->m_head);
3062                         txbuf->m_head = NULL;
3063                 }
3064 #ifdef DEV_NETMAP
3065                 /*
3066                  * In netmap mode, set the map for the packet buffer.
3067                  * NOTE: Some drivers (not this one) also need to set
3068                  * the physical buffer address in the NIC ring.
3069                  * Slots in the netmap ring (indexed by "si") are
3070                  * kring->nkr_hwofs positions "ahead" wrt the
3071                  * corresponding slot in the NIC ring. In some drivers
3072                  * (not here) nkr_hwofs can be negative. Function
3073                  * netmap_idx_n2k() handles wraparounds properly.
3074                  */
3075                 if (slot) {
3076                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3077                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3078                 }
3079 #endif /* DEV_NETMAP */
3080                 /* Clear the EOP descriptor pointer */
3081                 txbuf->eop = NULL;
3082         }
3083
3084 #ifdef IXGBE_FDIR
3085         /* Set the rate at which we sample packets */
3086         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3087                 txr->atr_sample = atr_sample_rate;
3088 #endif
3089
3090         /* Set number of descriptors available */
3091         txr->tx_avail = adapter->num_tx_desc;
3092
3093         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3094             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3095         IXGBE_TX_UNLOCK(txr);
3096 }
3097
3098 /*********************************************************************
3099  *
3100  *  Initialize all transmit rings.
3101  *
3102  **********************************************************************/
3103 static int
3104 ixgbe_setup_transmit_structures(struct adapter *adapter)
3105 {
3106         struct tx_ring *txr = adapter->tx_rings;
3107
3108         for (int i = 0; i < adapter->num_queues; i++, txr++)
3109                 ixgbe_setup_transmit_ring(txr);
3110
3111         return (0);
3112 }
3113
3114 /*********************************************************************
3115  *
3116  *  Enable transmit unit.
3117  *
3118  **********************************************************************/
3119 static void
3120 ixgbe_initialize_transmit_units(struct adapter *adapter)
3121 {
3122         struct tx_ring  *txr = adapter->tx_rings;
3123         struct ixgbe_hw *hw = &adapter->hw;
3124
3125         /* Setup the Base and Length of the Tx Descriptor Ring */
3126
3127         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3128                 u64     tdba = txr->txdma.dma_paddr;
3129                 u32     txctrl;
3130
3131                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3132                        (tdba & 0x00000000ffffffffULL));
3133                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3134                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3135                     adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3136
3137                 /* Setup the HW Tx Head and Tail descriptor pointers */
3138                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3139                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3140
3141                 /* Setup Transmit Descriptor Cmd Settings */
3142                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3143                 txr->queue_status = IXGBE_QUEUE_IDLE;
3144
3145                 /* Set the processing limit */
3146                 txr->process_limit = ixgbe_tx_process_limit;
3147
3148                 /* Disable Head Writeback */
3149                 switch (hw->mac.type) {
3150                 case ixgbe_mac_82598EB:
3151                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3152                         break;
3153                 case ixgbe_mac_82599EB:
3154                 case ixgbe_mac_X540:
3155                 default:
3156                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3157                         break;
3158                 }
3159                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3160                 switch (hw->mac.type) {
3161                 case ixgbe_mac_82598EB:
3162                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3163                         break;
3164                 case ixgbe_mac_82599EB:
3165                 case ixgbe_mac_X540:
3166                 default:
3167                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3168                         break;
3169                 }
3170
3171         }
3172
3173         if (hw->mac.type != ixgbe_mac_82598EB) {
3174                 u32 dmatxctl, rttdcs;
3175                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3176                 dmatxctl |= IXGBE_DMATXCTL_TE;
3177                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3178                 /* Disable arbiter to set MTQC */
3179                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3180                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3181                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3182                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3183                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3184                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3185         }
3186
3187         return;
3188 }
3189
3190 /*********************************************************************
3191  *
3192  *  Free all transmit rings.
3193  *
3194  **********************************************************************/
3195 static void
3196 ixgbe_free_transmit_structures(struct adapter *adapter)
3197 {
3198         struct tx_ring *txr = adapter->tx_rings;
3199
3200         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3201                 IXGBE_TX_LOCK(txr);
3202                 ixgbe_free_transmit_buffers(txr);
3203                 ixgbe_dma_free(adapter, &txr->txdma);
3204                 IXGBE_TX_UNLOCK(txr);
3205                 IXGBE_TX_LOCK_DESTROY(txr);
3206         }
3207         free(adapter->tx_rings, M_DEVBUF);
3208 }
3209
3210 /*********************************************************************
3211  *
3212  *  Free transmit ring related data structures.
3213  *
3214  **********************************************************************/
3215 static void
3216 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3217 {
3218         struct adapter *adapter = txr->adapter;
3219         struct ixgbe_tx_buf *tx_buffer;
3220         int             i;
3221
3222         INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin");
3223
3224         if (txr->tx_buffers == NULL)
3225                 return;
3226
3227         tx_buffer = txr->tx_buffers;
3228         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3229                 if (tx_buffer->m_head != NULL) {
3230                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3231                             BUS_DMASYNC_POSTWRITE);
3232                         bus_dmamap_unload(txr->txtag,
3233                             tx_buffer->map);
3234                         m_freem(tx_buffer->m_head);
3235                         tx_buffer->m_head = NULL;
3236                         if (tx_buffer->map != NULL) {
3237                                 bus_dmamap_destroy(txr->txtag,
3238                                     tx_buffer->map);
3239                                 tx_buffer->map = NULL;
3240                         }
3241                 } else if (tx_buffer->map != NULL) {
3242                         bus_dmamap_unload(txr->txtag,
3243                             tx_buffer->map);
3244                         bus_dmamap_destroy(txr->txtag,
3245                             tx_buffer->map);
3246                         tx_buffer->map = NULL;
3247                 }
3248         }
3249 #ifdef IXGBE_LEGACY_TX
3250         if (txr->br != NULL)
3251                 buf_ring_free(txr->br, M_DEVBUF);
3252 #endif
3253         if (txr->tx_buffers != NULL) {
3254                 free(txr->tx_buffers, M_DEVBUF);
3255                 txr->tx_buffers = NULL;
3256         }
3257         if (txr->txtag != NULL) {
3258                 bus_dma_tag_destroy(txr->txtag);
3259                 txr->txtag = NULL;
3260         }
3261         return;
3262 }
3263
3264 /*********************************************************************
3265  *
3266  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
3267  *
3268  **********************************************************************/
3269
3270 static int
3271 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3272     u32 *cmd_type_len, u32 *olinfo_status)
3273 {
3274         struct ixgbe_adv_tx_context_desc *TXD;
3275         struct ether_vlan_header *eh;
3276         struct ip *ip;
3277         struct ip6_hdr *ip6;
3278         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3279         int     ehdrlen, ip_hlen = 0;
3280         u16     etype;
3281         u8      ipproto = 0;
3282         int     offload = TRUE;
3283         int     ctxd = txr->next_avail_desc;
3284         u16     vtag = 0;
3285
3286         /* First check if TSO is to be used */
3287         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3288                 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3289
3290         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3291                 offload = FALSE;
3292
3293         /* Indicate the whole packet as payload when not doing TSO */
3294         *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3295
3296         /* Now ready a context descriptor */
3297         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3298
3299         /*
3300         ** In advanced descriptors the vlan tag must
3301         ** be placed into the context descriptor. Hence
3302         ** we need to make one even if not doing offloads.
3303         */
3304         if (mp->m_flags & M_VLANTAG) {
3305                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3306                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3307         } else if (offload == FALSE) /* ... no offload to do */
3308                 return (0);
3309
3310         /*
3311          * Determine where frame payload starts.
3312          * Jump over vlan headers if already present,
3313          * helpful for QinQ too.
3314          */
3315         eh = mtod(mp, struct ether_vlan_header *);
3316         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3317                 etype = ntohs(eh->evl_proto);
3318                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3319         } else {
3320                 etype = ntohs(eh->evl_encap_proto);
3321                 ehdrlen = ETHER_HDR_LEN;
3322         }
3323
3324         /* Set the ether header length */
3325         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3326
3327         switch (etype) {
3328                 case ETHERTYPE_IP:
3329                         ip = (struct ip *)(mp->m_data + ehdrlen);
3330                         ip_hlen = ip->ip_hl << 2;
3331                         ipproto = ip->ip_p;
3332                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3333                         break;
3334                 case ETHERTYPE_IPV6:
3335                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3336                         ip_hlen = sizeof(struct ip6_hdr);
3337                         /* XXX-BZ this will go badly in case of ext hdrs. */
3338                         ipproto = ip6->ip6_nxt;
3339                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3340                         break;
3341                 default:
3342                         offload = FALSE;
3343                         break;
3344         }
3345
3346         vlan_macip_lens |= ip_hlen;
3347         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3348
3349         switch (ipproto) {
3350                 case IPPROTO_TCP:
3351                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3352                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3353                         break;
3354
3355                 case IPPROTO_UDP:
3356                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3357                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3358                         break;
3359
3360 #if __FreeBSD_version >= 800000
3361                 case IPPROTO_SCTP:
3362                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3363                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3364                         break;
3365 #endif
3366                 default:
3367                         offload = FALSE;
3368                         break;
3369         }
3370
3371         if (offload) /* For the TX descriptor setup */
3372                 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3373
3374         /* Now copy bits into descriptor */
3375         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3376         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3377         TXD->seqnum_seed = htole32(0);
3378         TXD->mss_l4len_idx = htole32(0);
3379
3380         /* We've consumed the first desc, adjust counters */
3381         if (++ctxd == txr->num_desc)
3382                 ctxd = 0;
3383         txr->next_avail_desc = ctxd;
3384         --txr->tx_avail;
3385
3386         return (0);
3387 }
3388
3389 /**********************************************************************
3390  *
3391  *  Setup work for hardware segmentation offload (TSO) on
3392  *  adapters using advanced tx descriptors
3393  *
3394  **********************************************************************/
3395 static int
3396 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3397     u32 *cmd_type_len, u32 *olinfo_status)
3398 {
3399         struct ixgbe_adv_tx_context_desc *TXD;
3400         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3401         u32 mss_l4len_idx = 0, paylen;
3402         u16 vtag = 0, eh_type;
3403         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3404         struct ether_vlan_header *eh;
3405 #ifdef INET6
3406         struct ip6_hdr *ip6;
3407 #endif
3408 #ifdef INET
3409         struct ip *ip;
3410 #endif
3411         struct tcphdr *th;
3412
3413
3414         /*
3415          * Determine where frame payload starts.
3416          * Jump over vlan headers if already present
3417          */
3418         eh = mtod(mp, struct ether_vlan_header *);
3419         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3420                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3421                 eh_type = eh->evl_proto;
3422         } else {
3423                 ehdrlen = ETHER_HDR_LEN;
3424                 eh_type = eh->evl_encap_proto;
3425         }
3426
3427         switch (ntohs(eh_type)) {
3428 #ifdef INET6
3429         case ETHERTYPE_IPV6:
3430                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3431                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3432                 if (ip6->ip6_nxt != IPPROTO_TCP)
3433                         return (ENXIO);
3434                 ip_hlen = sizeof(struct ip6_hdr);
3435                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3436                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3437                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3438                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3439                 break;
3440 #endif
3441 #ifdef INET
3442         case ETHERTYPE_IP:
3443                 ip = (struct ip *)(mp->m_data + ehdrlen);
3444                 if (ip->ip_p != IPPROTO_TCP)
3445                         return (ENXIO);
3446                 ip->ip_sum = 0;
3447                 ip_hlen = ip->ip_hl << 2;
3448                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3449                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3450                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3451                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3452                 /* Tell transmit desc to also do IPv4 checksum. */
3453                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3454                 break;
3455 #endif
3456         default:
3457                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3458                     __func__, ntohs(eh_type));
3459                 break;
3460         }
3461
3462         ctxd = txr->next_avail_desc;
3463         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3464
3465         tcp_hlen = th->th_off << 2;
3466
3467         /* This is used in the transmit desc in encap */
3468         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3469
3470         /* VLAN MACLEN IPLEN */
3471         if (mp->m_flags & M_VLANTAG) {
3472                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3473                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3474         }
3475
3476         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3477         vlan_macip_lens |= ip_hlen;
3478         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3479
3480         /* ADV DTYPE TUCMD */
3481         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3482         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3483         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3484
3485         /* MSS L4LEN IDX */
3486         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3487         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3488         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3489
3490         TXD->seqnum_seed = htole32(0);
3491
3492         if (++ctxd == txr->num_desc)
3493                 ctxd = 0;
3494
3495         txr->tx_avail--;
3496         txr->next_avail_desc = ctxd;
3497         *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3498         *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3499         *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3500         ++txr->tso_tx;
3501         return (0);
3502 }
3503
3504 #ifdef IXGBE_FDIR
3505 /*
3506 ** This routine parses packet headers so that Flow
3507 ** Director can make a hashed filter table entry
3508 ** allowing traffic flows to be identified and kept
3509 ** on the same cpu.  This would be a performance
3510 ** hit, but we only do it at IXGBE_FDIR_RATE of
3511 ** packets.
3512 */
3513 static void
3514 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3515 {
3516         struct adapter                  *adapter = txr->adapter;
3517         struct ix_queue                 *que;
3518         struct ip                       *ip;
3519         struct tcphdr                   *th;
3520         struct udphdr                   *uh;
3521         struct ether_vlan_header        *eh;
3522         union ixgbe_atr_hash_dword      input = {.dword = 0};
3523         union ixgbe_atr_hash_dword      common = {.dword = 0};
3524         int                             ehdrlen, ip_hlen;
3525         u16                             etype;
3526
3527         eh = mtod(mp, struct ether_vlan_header *);
3528         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3529                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3530                 etype = eh->evl_proto;
3531         } else {
3532                 ehdrlen = ETHER_HDR_LEN;
3533                 etype = eh->evl_encap_proto;
3534         }
3535
3536         /* Only handling IPv4 */
3537         if (etype != htons(ETHERTYPE_IP))
3538                 return;
3539
3540         ip = (struct ip *)(mp->m_data + ehdrlen);
3541         ip_hlen = ip->ip_hl << 2;
3542
3543         /* check if we're UDP or TCP */
3544         switch (ip->ip_p) {
3545         case IPPROTO_TCP:
3546                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3547                 /* src and dst are inverted */
3548                 common.port.dst ^= th->th_sport;
3549                 common.port.src ^= th->th_dport;
3550                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3551                 break;
3552         case IPPROTO_UDP:
3553                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3554                 /* src and dst are inverted */
3555                 common.port.dst ^= uh->uh_sport;
3556                 common.port.src ^= uh->uh_dport;
3557                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3558                 break;
3559         default:
3560                 return;
3561         }
3562
3563         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3564         if (mp->m_pkthdr.ether_vtag)
3565                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3566         else
3567                 common.flex_bytes ^= etype;
3568         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3569
3570         que = &adapter->queues[txr->me];
3571         /*
3572         ** This assumes the Rx queue and Tx
3573         ** queue are bound to the same CPU
3574         */
3575         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3576             input, common, que->msix);
3577 }
3578 #endif /* IXGBE_FDIR */
3579
3580 /**********************************************************************
3581  *
3582  *  Examine each tx_buffer in the used queue. If the hardware is done
3583  *  processing the packet then free associated resources. The
3584  *  tx_buffer is put back on the free queue.
3585  *
3586  **********************************************************************/
3587 static void
3588 ixgbe_txeof(struct tx_ring *txr)
3589 {
3590 #ifdef DEV_NETMAP
3591         struct adapter          *adapter = txr->adapter;
3592         struct ifnet            *ifp = adapter->ifp;
3593 #endif
3594         u32                     work, processed = 0;
3595         u16                     limit = txr->process_limit;
3596         struct ixgbe_tx_buf     *buf;
3597         union ixgbe_adv_tx_desc *txd;
3598
3599         mtx_assert(&txr->tx_mtx, MA_OWNED);
3600
3601 #ifdef DEV_NETMAP
3602         if (ifp->if_capenable & IFCAP_NETMAP) {
3603                 struct netmap_adapter *na = NA(ifp);
3604                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3605                 txd = txr->tx_base;
3606                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3607                     BUS_DMASYNC_POSTREAD);
3608                 /*
3609                  * In netmap mode, all the work is done in the context
3610                  * of the client thread. Interrupt handlers only wake up
3611                  * clients, which may be sleeping on individual rings
3612                  * or on a global resource for all rings.
3613                  * To implement tx interrupt mitigation, we wake up the client
3614                  * thread roughly every half ring, even if the NIC interrupts
3615                  * more frequently. This is implemented as follows:
3616                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3617                  *   the slot that should wake up the thread (nkr_num_slots
3618                  *   means the user thread should not be woken up);
3619                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3620                  *   or the slot has the DD bit set.
3621                  */
3622                 if (!netmap_mitigate ||
3623                     (kring->nr_kflags < kring->nkr_num_slots &&
3624                     txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3625                         netmap_tx_irq(ifp, txr->me);
3626                 }
3627                 return;
3628         }
3629 #endif /* DEV_NETMAP */
3630
3631         if (txr->tx_avail == txr->num_desc) {
3632                 txr->queue_status = IXGBE_QUEUE_IDLE;
3633                 return;
3634         }
3635
3636         /* Get work starting point */
3637         work = txr->next_to_clean;
3638         buf = &txr->tx_buffers[work];
3639         txd = &txr->tx_base[work];
3640         work -= txr->num_desc; /* The distance to ring end */
3641         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3642             BUS_DMASYNC_POSTREAD);
3643
3644         do {
3645                 union ixgbe_adv_tx_desc *eop= buf->eop;
3646                 if (eop == NULL) /* No work */
3647                         break;
3648
3649                 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3650                         break;  /* I/O not complete */
3651
3652                 if (buf->m_head) {
3653                         txr->bytes +=
3654                             buf->m_head->m_pkthdr.len;
3655                         bus_dmamap_sync(txr->txtag,
3656                             buf->map,
3657                             BUS_DMASYNC_POSTWRITE);
3658                         bus_dmamap_unload(txr->txtag,
3659                             buf->map);
3660                         m_freem(buf->m_head);
3661                         buf->m_head = NULL;
3662                         buf->map = NULL;
3663                 }
3664                 buf->eop = NULL;
3665                 ++txr->tx_avail;
3666
3667                 /* We clean the range if multi segment */
3668                 while (txd != eop) {
3669                         ++txd;
3670                         ++buf;
3671                         ++work;
3672                         /* wrap the ring? */
3673                         if (__predict_false(!work)) {
3674                                 work -= txr->num_desc;
3675                                 buf = txr->tx_buffers;
3676                                 txd = txr->tx_base;
3677                         }
3678                         if (buf->m_head) {
3679                                 txr->bytes +=
3680                                     buf->m_head->m_pkthdr.len;
3681                                 bus_dmamap_sync(txr->txtag,
3682                                     buf->map,
3683                                     BUS_DMASYNC_POSTWRITE);
3684                                 bus_dmamap_unload(txr->txtag,
3685                                     buf->map);
3686                                 m_freem(buf->m_head);
3687                                 buf->m_head = NULL;
3688                                 buf->map = NULL;
3689                         }
3690                         ++txr->tx_avail;
3691                         buf->eop = NULL;
3692
3693                 }
3694                 ++txr->packets;
3695                 ++processed;
3696                 txr->watchdog_time = ticks;
3697
3698                 /* Try the next packet */
3699                 ++txd;
3700                 ++buf;
3701                 ++work;
3702                 /* reset with a wrap */
3703                 if (__predict_false(!work)) {
3704                         work -= txr->num_desc;
3705                         buf = txr->tx_buffers;
3706                         txd = txr->tx_base;
3707                 }
3708                 prefetch(txd);
3709         } while (__predict_true(--limit));
3710
3711         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3712             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3713
3714         work += txr->num_desc;
3715         txr->next_to_clean = work;
3716
3717         /*
3718         ** Watchdog calculation, we know there's
3719         ** work outstanding or the first return
3720         ** would have been taken, so none processed
3721         ** for too long indicates a hang.
3722         */
3723         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3724                 txr->queue_status = IXGBE_QUEUE_HUNG;
3725
3726         if (txr->tx_avail == txr->num_desc)
3727                 txr->queue_status = IXGBE_QUEUE_IDLE;
3728
3729         return;
3730 }
3731
3732 /*********************************************************************
3733  *
3734  *  Refresh mbuf buffers for RX descriptor rings
3735  *   - now keeps its own state so discards due to resource
3736  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3737  *     it just returns, keeping its placeholder, thus it can simply
3738  *     be recalled to try again.
3739  *
3740  **********************************************************************/
3741 static void
3742 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3743 {
3744         struct adapter          *adapter = rxr->adapter;
3745         bus_dma_segment_t       seg[1];
3746         struct ixgbe_rx_buf     *rxbuf;
3747         struct mbuf             *mp;
3748         int                     i, j, nsegs, error;
3749         bool                    refreshed = FALSE;
3750
3751         i = j = rxr->next_to_refresh;
3752         /* Control the loop with one beyond */
3753         if (++j == rxr->num_desc)
3754                 j = 0;
3755
3756         while (j != limit) {
3757                 rxbuf = &rxr->rx_buffers[i];
3758                 if (rxbuf->buf == NULL) {
3759                         mp = m_getjcl(M_NOWAIT, MT_DATA,
3760                             M_PKTHDR, rxr->mbuf_sz);
3761                         if (mp == NULL)
3762                                 goto update;
3763                         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3764                                 m_adj(mp, ETHER_ALIGN);
3765                 } else
3766                         mp = rxbuf->buf;
3767
3768                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3769
3770                 /* If we're dealing with an mbuf that was copied rather
3771                  * than replaced, there's no need to go through busdma.
3772                  */
3773                 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3774                         /* Get the memory mapping */
3775                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3776                             rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3777                         if (error != 0) {
3778                                 printf("Refresh mbufs: payload dmamap load"
3779                                     " failure - %d\n", error);
3780                                 m_free(mp);
3781                                 rxbuf->buf = NULL;
3782                                 goto update;
3783                         }
3784                         rxbuf->buf = mp;
3785                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3786                             BUS_DMASYNC_PREREAD);
3787                         rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3788                             htole64(seg[0].ds_addr);
3789                 } else {
3790                         rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3791                         rxbuf->flags &= ~IXGBE_RX_COPY;
3792                 }
3793
3794                 refreshed = TRUE;
3795                 /* Next is precalculated */
3796                 i = j;
3797                 rxr->next_to_refresh = i;
3798                 if (++j == rxr->num_desc)
3799                         j = 0;
3800         }
3801 update:
3802         if (refreshed) /* Update hardware tail index */
3803                 IXGBE_WRITE_REG(&adapter->hw,
3804                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3805         return;
3806 }
3807
3808 /*********************************************************************
3809  *
3810  *  Allocate memory for rx_buffer structures. Since we use one
3811  *  rx_buffer per received packet, the maximum number of rx_buffer's
3812  *  that we'll need is equal to the number of receive descriptors
3813  *  that we've allocated.
3814  *
3815  **********************************************************************/
3816 static int
3817 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3818 {
3819         struct  adapter         *adapter = rxr->adapter;
3820         device_t                dev = adapter->dev;
3821         struct ixgbe_rx_buf     *rxbuf;
3822         int                     i, bsize, error;
3823
3824         bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3825         if (!(rxr->rx_buffers =
3826             (struct ixgbe_rx_buf *) malloc(bsize,
3827             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3828                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3829                 error = ENOMEM;
3830                 goto fail;
3831         }
3832
3833         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3834                                    1, 0,        /* alignment, bounds */
3835                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3836                                    BUS_SPACE_MAXADDR,   /* highaddr */
3837                                    NULL, NULL,          /* filter, filterarg */
3838                                    MJUM16BYTES,         /* maxsize */
3839                                    1,                   /* nsegments */
3840                                    MJUM16BYTES,         /* maxsegsize */
3841                                    0,                   /* flags */
3842                                    NULL,                /* lockfunc */
3843                                    NULL,                /* lockfuncarg */
3844                                    &rxr->ptag))) {
3845                 device_printf(dev, "Unable to create RX DMA tag\n");
3846                 goto fail;
3847         }
3848
3849         for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3850                 rxbuf = &rxr->rx_buffers[i];
3851                 error = bus_dmamap_create(rxr->ptag,
3852                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3853                 if (error) {
3854                         device_printf(dev, "Unable to create RX dma map\n");
3855                         goto fail;
3856                 }
3857         }
3858
3859         return (0);
3860
3861 fail:
3862         /* Frees all, but can handle partial completion */
3863         ixgbe_free_receive_structures(adapter);
3864         return (error);
3865 }
3866
3867 /*
3868 ** Used to detect a descriptor that has
3869 ** been merged by Hardware RSC.
3870 */
3871 static inline u32
3872 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3873 {
3874         return (le32toh(rx->wb.lower.lo_dword.data) &
3875             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3876 }
3877
3878 /*********************************************************************
3879  *
3880  *  Initialize Hardware RSC (LRO) feature on 82599
3881  *  for an RX ring, this is toggled by the LRO capability
3882  *  even though it is transparent to the stack.
3883  *
3884  *  NOTE: since this HW feature only works with IPV4 and
3885  *        our testing has shown soft LRO to be as effective
3886  *        I have decided to disable this by default.
3887  *
3888  **********************************************************************/
3889 static void
3890 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3891 {
3892         struct  adapter         *adapter = rxr->adapter;
3893         struct  ixgbe_hw        *hw = &adapter->hw;
3894         u32                     rscctrl, rdrxctl;
3895
3896         /* If turning LRO/RSC off we need to disable it */
3897         if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3898                 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3899                 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3900                 return;
3901         }
3902
3903         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3904         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3905 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3906         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3907 #endif /* DEV_NETMAP */
3908         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3909         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3910         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3911
3912         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3913         rscctrl |= IXGBE_RSCCTL_RSCEN;
3914         /*
3915         ** Limit the total number of descriptors that
3916         ** can be combined, so it does not exceed 64K
3917         */
3918         if (rxr->mbuf_sz == MCLBYTES)
3919                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3920         else if (rxr->mbuf_sz == MJUMPAGESIZE)
3921                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3922         else if (rxr->mbuf_sz == MJUM9BYTES)
3923                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3924         else  /* Using 16K cluster */
3925                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3926
3927         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3928
3929         /* Enable TCP header recognition */
3930         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3931             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3932             IXGBE_PSRTYPE_TCPHDR));
3933
3934         /* Disable RSC for ACK packets */
3935         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3936             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3937
3938         rxr->hw_rsc = TRUE;
3939 }
3940
3941
3942 static void
3943 ixgbe_free_receive_ring(struct rx_ring *rxr)
3944 {
3945         struct ixgbe_rx_buf       *rxbuf;
3946         int i;
3947
3948         for (i = 0; i < rxr->num_desc; i++) {
3949                 rxbuf = &rxr->rx_buffers[i];
3950                 if (rxbuf->buf != NULL) {
3951                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3952                             BUS_DMASYNC_POSTREAD);
3953                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3954                         rxbuf->buf->m_flags |= M_PKTHDR;
3955                         m_freem(rxbuf->buf);
3956                         rxbuf->buf = NULL;
3957                         rxbuf->flags = 0;
3958                 }
3959         }
3960 }
3961
3962
3963 /*********************************************************************
3964  *
3965  *  Initialize a receive ring and its buffers.
3966  *
3967  **********************************************************************/
3968 static int
3969 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3970 {
3971         struct  adapter         *adapter;
3972         struct ifnet            *ifp;
3973         device_t                dev;
3974         struct ixgbe_rx_buf     *rxbuf;
3975         bus_dma_segment_t       seg[1];
3976         struct lro_ctrl         *lro = &rxr->lro;
3977         int                     rsize, nsegs, error = 0;
3978 #ifdef DEV_NETMAP
3979         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3980         struct netmap_slot *slot;
3981 #endif /* DEV_NETMAP */
3982
3983         adapter = rxr->adapter;
3984         ifp = adapter->ifp;
3985         dev = adapter->dev;
3986
3987         /* Clear the ring contents */
3988         IXGBE_RX_LOCK(rxr);
3989 #ifdef DEV_NETMAP
3990         /* same as in ixgbe_setup_transmit_ring() */
3991         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3992 #endif /* DEV_NETMAP */
3993         rsize = roundup2(adapter->num_rx_desc *
3994             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3995         bzero((void *)rxr->rx_base, rsize);
3996         /* Cache the size */
3997         rxr->mbuf_sz = adapter->rx_mbuf_sz;
3998
3999         /* Free current RX buffer structs and their mbufs */
4000         ixgbe_free_receive_ring(rxr);
4001
4002         /* Now replenish the mbufs */
4003         for (int j = 0; j != rxr->num_desc; ++j) {
4004                 struct mbuf     *mp;
4005
4006                 rxbuf = &rxr->rx_buffers[j];
4007 #ifdef DEV_NETMAP
4008                 /*
4009                  * In netmap mode, fill the map and set the buffer
4010                  * address in the NIC ring, considering the offset
4011                  * between the netmap and NIC rings (see comment in
4012                  * ixgbe_setup_transmit_ring() ). No need to allocate
4013                  * an mbuf, so end the block with a continue;
4014                  */
4015                 if (slot) {
4016                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4017                         uint64_t paddr;
4018                         void *addr;
4019
4020                         addr = PNMB(slot + sj, &paddr);
4021                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4022                         /* Update descriptor and the cached value */
4023                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4024                         rxbuf->addr = htole64(paddr);
4025                         continue;
4026                 }
4027 #endif /* DEV_NETMAP */
4028                 rxbuf->flags = 0;
4029                 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
4030                     M_PKTHDR, adapter->rx_mbuf_sz);
4031                 if (rxbuf->buf == NULL) {
4032                         error = ENOBUFS;
4033                         goto fail;
4034                 }
4035                 mp = rxbuf->buf;
4036                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4037                 /* Get the memory mapping */
4038                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4039                     rxbuf->pmap, mp, seg,
4040                     &nsegs, BUS_DMA_NOWAIT);
4041                 if (error != 0)
4042                         goto fail;
4043                 bus_dmamap_sync(rxr->ptag,
4044                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4045                 /* Update the descriptor and the cached value */
4046                 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4047                 rxbuf->addr = htole64(seg[0].ds_addr);
4048         }
4049
4050
4051         /* Setup our descriptor indices */
4052         rxr->next_to_check = 0;
4053         rxr->next_to_refresh = 0;
4054         rxr->lro_enabled = FALSE;
4055         rxr->rx_copies = 0;
4056         rxr->rx_bytes = 0;
4057         rxr->discard = FALSE;
4058         rxr->vtag_strip = FALSE;
4059
4060         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4061             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4062
4063         /*
4064         ** Now set up the LRO interface:
4065         */
4066         if (ixgbe_rsc_enable)
4067                 ixgbe_setup_hw_rsc(rxr);
4068         else if (ifp->if_capenable & IFCAP_LRO) {
4069                 int err = tcp_lro_init(lro);
4070                 if (err) {
4071                         device_printf(dev, "LRO Initialization failed!\n");
4072                         goto fail;
4073                 }
4074                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4075                 rxr->lro_enabled = TRUE;
4076                 lro->ifp = adapter->ifp;
4077         }
4078
4079         IXGBE_RX_UNLOCK(rxr);
4080         return (0);
4081
4082 fail:
4083         ixgbe_free_receive_ring(rxr);
4084         IXGBE_RX_UNLOCK(rxr);
4085         return (error);
4086 }
4087
4088 /*********************************************************************
4089  *
4090  *  Initialize all receive rings.
4091  *
4092  **********************************************************************/
4093 static int
4094 ixgbe_setup_receive_structures(struct adapter *adapter)
4095 {
4096         struct rx_ring *rxr = adapter->rx_rings;
4097         int j;
4098
4099         for (j = 0; j < adapter->num_queues; j++, rxr++)
4100                 if (ixgbe_setup_receive_ring(rxr))
4101                         goto fail;
4102
4103         return (0);
4104 fail:
4105         /*
4106          * Free RX buffers allocated so far, we will only handle
4107          * the rings that completed, the failing case will have
4108          * cleaned up for itself. 'j' failed, so its the terminus.
4109          */
4110         for (int i = 0; i < j; ++i) {
4111                 rxr = &adapter->rx_rings[i];
4112                 ixgbe_free_receive_ring(rxr);
4113         }
4114
4115         return (ENOBUFS);
4116 }
4117
4118 /*********************************************************************
4119  *
4120  *  Setup receive registers and features.
4121  *
4122  **********************************************************************/
4123 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4124
4125 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4126
4127 static void
4128 ixgbe_initialize_receive_units(struct adapter *adapter)
4129 {
4130         struct  rx_ring *rxr = adapter->rx_rings;
4131         struct ixgbe_hw *hw = &adapter->hw;
4132         struct ifnet   *ifp = adapter->ifp;
4133         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4134         u32             reta, mrqc = 0, hlreg, random[10];
4135
4136
4137         /*
4138          * Make sure receives are disabled while
4139          * setting up the descriptor ring
4140          */
4141         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4142         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4143             rxctrl & ~IXGBE_RXCTRL_RXEN);
4144
4145         /* Enable broadcasts */
4146         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4147         fctrl |= IXGBE_FCTRL_BAM;
4148         fctrl |= IXGBE_FCTRL_DPF;
4149         fctrl |= IXGBE_FCTRL_PMCF;
4150         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4151
4152         /* Set for Jumbo Frames? */
4153         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4154         if (ifp->if_mtu > ETHERMTU)
4155                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4156         else
4157                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4158 #ifdef DEV_NETMAP
4159         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4160         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4161                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4162         else
4163                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4164 #endif /* DEV_NETMAP */
4165         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4166
4167         bufsz = (adapter->rx_mbuf_sz +
4168             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4169
4170         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4171                 u64 rdba = rxr->rxdma.dma_paddr;
4172
4173                 /* Setup the Base and Length of the Rx Descriptor Ring */
4174                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4175                                (rdba & 0x00000000ffffffffULL));
4176                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4177                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4178                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4179
4180                 /* Set up the SRRCTL register */
4181                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4182                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4183                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4184                 srrctl |= bufsz;
4185                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4186                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4187
4188                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4189                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4190                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4191
4192                 /* Set the processing limit */
4193                 rxr->process_limit = ixgbe_rx_process_limit;
4194         }
4195
4196         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4197                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4198                               IXGBE_PSRTYPE_UDPHDR |
4199                               IXGBE_PSRTYPE_IPV4HDR |
4200                               IXGBE_PSRTYPE_IPV6HDR;
4201                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4202         }
4203
4204         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4205
4206         /* Setup RSS */
4207         if (adapter->num_queues > 1) {
4208                 int i, j;
4209                 reta = 0;
4210
4211                 /* set up random bits */
4212                 arc4rand(&random, sizeof(random), 0);
4213
4214                 /* Set up the redirection table */
4215                 for (i = 0, j = 0; i < 128; i++, j++) {
4216                         if (j == adapter->num_queues) j = 0;
4217                         reta = (reta << 8) | (j * 0x11);
4218                         if ((i & 3) == 3)
4219                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4220                 }
4221
4222                 /* Now fill our hash function seeds */
4223                 for (int i = 0; i < 10; i++)
4224                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4225
4226                 /* Perform hash on these packet types */
4227                 mrqc = IXGBE_MRQC_RSSEN
4228                      | IXGBE_MRQC_RSS_FIELD_IPV4
4229                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4230                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4231                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4232                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4233                      | IXGBE_MRQC_RSS_FIELD_IPV6
4234                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4235                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4236                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4237                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4238
4239                 /* RSS and RX IPP Checksum are mutually exclusive */
4240                 rxcsum |= IXGBE_RXCSUM_PCSD;
4241         }
4242
4243         if (ifp->if_capenable & IFCAP_RXCSUM)
4244                 rxcsum |= IXGBE_RXCSUM_PCSD;
4245
4246         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4247                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4248
4249         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4250
4251         return;
4252 }
4253
4254 /*********************************************************************
4255  *
4256  *  Free all receive rings.
4257  *
4258  **********************************************************************/
4259 static void
4260 ixgbe_free_receive_structures(struct adapter *adapter)
4261 {
4262         struct rx_ring *rxr = adapter->rx_rings;
4263
4264         INIT_DEBUGOUT("ixgbe_free_receive_structures: begin");
4265
4266         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4267                 struct lro_ctrl         *lro = &rxr->lro;
4268                 ixgbe_free_receive_buffers(rxr);
4269                 /* Free LRO memory */
4270                 tcp_lro_free(lro);
4271                 /* Free the ring memory as well */
4272                 ixgbe_dma_free(adapter, &rxr->rxdma);
4273         }
4274
4275         free(adapter->rx_rings, M_DEVBUF);
4276 }
4277
4278
4279 /*********************************************************************
4280  *
4281  *  Free receive ring data structures
4282  *
4283  **********************************************************************/
4284 static void
4285 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4286 {
4287         struct adapter          *adapter = rxr->adapter;
4288         struct ixgbe_rx_buf     *rxbuf;
4289
4290         INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin");
4291
4292         /* Cleanup any existing buffers */
4293         if (rxr->rx_buffers != NULL) {
4294                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4295                         rxbuf = &rxr->rx_buffers[i];
4296                         if (rxbuf->buf != NULL) {
4297                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4298                                     BUS_DMASYNC_POSTREAD);
4299                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4300                                 rxbuf->buf->m_flags |= M_PKTHDR;
4301                                 m_freem(rxbuf->buf);
4302                         }
4303                         rxbuf->buf = NULL;
4304                         if (rxbuf->pmap != NULL) {
4305                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4306                                 rxbuf->pmap = NULL;
4307                         }
4308                 }
4309                 if (rxr->rx_buffers != NULL) {
4310                         free(rxr->rx_buffers, M_DEVBUF);
4311                         rxr->rx_buffers = NULL;
4312                 }
4313         }
4314
4315         if (rxr->ptag != NULL) {
4316                 bus_dma_tag_destroy(rxr->ptag);
4317                 rxr->ptag = NULL;
4318         }
4319
4320         return;
4321 }
4322
4323 static __inline void
4324 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4325 {
4326
4327         /*
4328          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4329          * should be computed by hardware. Also it should not have VLAN tag in
4330          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4331          */
4332         if (rxr->lro_enabled &&
4333             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4334             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4335             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4336             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4337             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4338             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4339             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4340             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4341                 /*
4342                  * Send to the stack if:
4343                  **  - LRO not enabled, or
4344                  **  - no LRO resources, or
4345                  **  - lro enqueue fails
4346                  */
4347                 if (rxr->lro.lro_cnt != 0)
4348                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4349                                 return;
4350         }
4351         IXGBE_RX_UNLOCK(rxr);
4352         (*ifp->if_input)(ifp, m);
4353         IXGBE_RX_LOCK(rxr);
4354 }
4355
4356 static __inline void
4357 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4358 {
4359         struct ixgbe_rx_buf     *rbuf;
4360
4361         rbuf = &rxr->rx_buffers[i];
4362
4363         if (rbuf->fmp != NULL) {/* Partial chain ? */
4364                 rbuf->fmp->m_flags |= M_PKTHDR;
4365                 m_freem(rbuf->fmp);
4366                 rbuf->fmp = NULL;
4367         }
4368
4369         /*
4370         ** With advanced descriptors the writeback
4371         ** clobbers the buffer addrs, so its easier
4372         ** to just free the existing mbufs and take
4373         ** the normal refresh path to get new buffers
4374         ** and mapping.
4375         */
4376         if (rbuf->buf) {
4377                 m_free(rbuf->buf);
4378                 rbuf->buf = NULL;
4379         }
4380
4381         rbuf->flags = 0;
4382
4383         return;
4384 }
4385
4386
4387 /*********************************************************************
4388  *
4389  *  This routine executes in interrupt context. It replenishes
4390  *  the mbufs in the descriptor and sends data which has been
4391  *  dma'ed into host memory to upper layer.
4392  *
4393  *  We loop at most count times if count is > 0, or until done if
4394  *  count < 0.
4395  *
4396  *  Return TRUE for more work, FALSE for all clean.
4397  *********************************************************************/
4398 static bool
4399 ixgbe_rxeof(struct ix_queue *que)
4400 {
4401         struct adapter          *adapter = que->adapter;
4402         struct rx_ring          *rxr = que->rxr;
4403         struct ifnet            *ifp = adapter->ifp;
4404         struct lro_ctrl         *lro = &rxr->lro;
4405         struct lro_entry        *queued;
4406         int                     i, nextp, processed = 0;
4407         u32                     staterr = 0;
4408         u16                     count = rxr->process_limit;
4409         union ixgbe_adv_rx_desc *cur;
4410         struct ixgbe_rx_buf     *rbuf, *nbuf;
4411
4412         IXGBE_RX_LOCK(rxr);
4413
4414 #ifdef DEV_NETMAP
4415         /* Same as the txeof routine: wakeup clients on intr. */
4416         if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4417                 IXGBE_RX_UNLOCK(rxr);
4418                 return (FALSE);
4419         }
4420 #endif /* DEV_NETMAP */
4421
4422         for (i = rxr->next_to_check; count != 0;) {
4423                 struct mbuf     *sendmp, *mp;
4424                 u32             rsc, ptype;
4425                 u16             len;
4426                 u16             vtag = 0;
4427                 bool            eop;
4428
4429                 /* Sync the ring. */
4430                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4431                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4432
4433                 cur = &rxr->rx_base[i];
4434                 staterr = le32toh(cur->wb.upper.status_error);
4435
4436                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4437                         break;
4438                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4439                         break;
4440
4441                 count--;
4442                 sendmp = NULL;
4443                 nbuf = NULL;
4444                 rsc = 0;
4445                 cur->wb.upper.status_error = 0;
4446                 rbuf = &rxr->rx_buffers[i];
4447                 mp = rbuf->buf;
4448
4449                 len = le16toh(cur->wb.upper.length);
4450                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4451                     IXGBE_RXDADV_PKTTYPE_MASK;
4452                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4453
4454                 /* Make sure bad packets are discarded */
4455                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4456                     (rxr->discard)) {
4457                         rxr->rx_discarded++;
4458                         if (eop)
4459                                 rxr->discard = FALSE;
4460                         else
4461                                 rxr->discard = TRUE;
4462                         ixgbe_rx_discard(rxr, i);
4463                         goto next_desc;
4464                 }
4465
4466                 /*
4467                 ** On 82599 which supports a hardware
4468                 ** LRO (called HW RSC), packets need
4469                 ** not be fragmented across sequential
4470                 ** descriptors, rather the next descriptor
4471                 ** is indicated in bits of the descriptor.
4472                 ** This also means that we might proceses
4473                 ** more than one packet at a time, something
4474                 ** that has never been true before, it
4475                 ** required eliminating global chain pointers
4476                 ** in favor of what we are doing here.  -jfv
4477                 */
4478                 if (!eop) {
4479                         /*
4480                         ** Figure out the next descriptor
4481                         ** of this frame.
4482                         */
4483                         if (rxr->hw_rsc == TRUE) {
4484                                 rsc = ixgbe_rsc_count(cur);
4485                                 rxr->rsc_num += (rsc - 1);
4486                         }
4487                         if (rsc) { /* Get hardware index */
4488                                 nextp = ((staterr &
4489                                     IXGBE_RXDADV_NEXTP_MASK) >>
4490                                     IXGBE_RXDADV_NEXTP_SHIFT);
4491                         } else { /* Just sequential */
4492                                 nextp = i + 1;
4493                                 if (nextp == adapter->num_rx_desc)
4494                                         nextp = 0;
4495                         }
4496                         nbuf = &rxr->rx_buffers[nextp];
4497                         prefetch(nbuf);
4498                 }
4499                 /*
4500                 ** Rather than using the fmp/lmp global pointers
4501                 ** we now keep the head of a packet chain in the
4502                 ** buffer struct and pass this along from one
4503                 ** descriptor to the next, until we get EOP.
4504                 */
4505                 mp->m_len = len;
4506                 /*
4507                 ** See if there is a stored head
4508                 ** that determines what we are
4509                 */
4510                 sendmp = rbuf->fmp;
4511                 if (sendmp != NULL) {  /* secondary frag */
4512                         rbuf->buf = rbuf->fmp = NULL;
4513                         mp->m_flags &= ~M_PKTHDR;
4514                         sendmp->m_pkthdr.len += mp->m_len;
4515                 } else {
4516                         /*
4517                          * Optimize.  This might be a small packet,
4518                          * maybe just a TCP ACK.  Do a fast copy that
4519                          * is cache aligned into a new mbuf, and
4520                          * leave the old mbuf+cluster for re-use.
4521                          */
4522                         if (eop && len <= IXGBE_RX_COPY_LEN) {
4523                                 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4524                                 if (sendmp != NULL) {
4525                                         sendmp->m_data +=
4526                                             IXGBE_RX_COPY_ALIGN;
4527                                         ixgbe_bcopy(mp->m_data,
4528                                             sendmp->m_data, len);
4529                                         sendmp->m_len = len;
4530                                         rxr->rx_copies++;
4531                                         rbuf->flags |= IXGBE_RX_COPY;
4532                                 }
4533                         }
4534                         if (sendmp == NULL) {
4535                                 rbuf->buf = rbuf->fmp = NULL;
4536                                 sendmp = mp;
4537                         }
4538
4539                         /* first desc of a non-ps chain */
4540                         sendmp->m_flags |= M_PKTHDR;
4541                         sendmp->m_pkthdr.len = mp->m_len;
4542                 }
4543                 ++processed;
4544
4545                 /* Pass the head pointer on */
4546                 if (eop == 0) {
4547                         nbuf->fmp = sendmp;
4548                         sendmp = NULL;
4549                         mp->m_next = nbuf->buf;
4550                 } else { /* Sending this frame */
4551                         sendmp->m_pkthdr.rcvif = ifp;
4552                         rxr->rx_packets++;
4553                         /* capture data for AIM */
4554                         rxr->bytes += sendmp->m_pkthdr.len;
4555                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4556                         /* Process vlan info */
4557                         if ((rxr->vtag_strip) &&
4558                             (staterr & IXGBE_RXD_STAT_VP))
4559                                 vtag = le16toh(cur->wb.upper.vlan);
4560                         if (vtag) {
4561                                 sendmp->m_pkthdr.ether_vtag = vtag;
4562                                 sendmp->m_flags |= M_VLANTAG;
4563                         }
4564                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4565                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4566 #if __FreeBSD_version >= 800000
4567                         sendmp->m_pkthdr.flowid = que->msix;
4568                         sendmp->m_flags |= M_FLOWID;
4569 #endif
4570                 }
4571 next_desc:
4572                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4573                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4574
4575                 /* Advance our pointers to the next descriptor. */
4576                 if (++i == rxr->num_desc)
4577                         i = 0;
4578
4579                 /* Now send to the stack or do LRO */
4580                 if (sendmp != NULL) {
4581                         rxr->next_to_check = i;
4582                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4583                         i = rxr->next_to_check;
4584                 }
4585
4586                /* Every 8 descriptors we go to refresh mbufs */
4587                 if (processed == 8) {
4588                         ixgbe_refresh_mbufs(rxr, i);
4589                         processed = 0;
4590                 }
4591         }
4592
4593         /* Refresh any remaining buf structs */
4594         if (ixgbe_rx_unrefreshed(rxr))
4595                 ixgbe_refresh_mbufs(rxr, i);
4596
4597         rxr->next_to_check = i;
4598
4599         /*
4600          * Flush any outstanding LRO work
4601          */
4602         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4603                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4604                 tcp_lro_flush(lro, queued);
4605         }
4606
4607         IXGBE_RX_UNLOCK(rxr);
4608
4609         /*
4610         ** Still have cleaning to do?
4611         */
4612         if ((staterr & IXGBE_RXD_STAT_DD) != 0)
4613                 return (TRUE);
4614         else
4615                 return (FALSE);
4616 }
4617
4618
4619 /*********************************************************************
4620  *
4621  *  Verify that the hardware indicated that the checksum is valid.
4622  *  Inform the stack about the status of checksum so that stack
4623  *  doesn't spend time verifying the checksum.
4624  *
4625  *********************************************************************/
4626 static void
4627 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4628 {
4629         u16     status = (u16) staterr;
4630         u8      errors = (u8) (staterr >> 24);
4631         bool    sctp = FALSE;
4632
4633         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4634             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4635                 sctp = TRUE;
4636
4637         if (status & IXGBE_RXD_STAT_IPCS) {
4638                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4639                         /* IP Checksum Good */
4640                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4641                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4642
4643                 } else
4644                         mp->m_pkthdr.csum_flags = 0;
4645         }
4646         if (status & IXGBE_RXD_STAT_L4CS) {
4647                 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4648 #if __FreeBSD_version >= 800000
4649                 if (sctp)
4650                         type = CSUM_SCTP_VALID;
4651 #endif
4652                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4653                         mp->m_pkthdr.csum_flags |= type;
4654                         if (!sctp)
4655                                 mp->m_pkthdr.csum_data = htons(0xffff);
4656                 }
4657         }
4658         return;
4659 }
4660
4661
4662 /*
4663 ** This routine is run via an vlan config EVENT,
4664 ** it enables us to use the HW Filter table since
4665 ** we can get the vlan id. This just creates the
4666 ** entry in the soft version of the VFTA, init will
4667 ** repopulate the real table.
4668 */
4669 static void
4670 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4671 {
4672         struct adapter  *adapter = ifp->if_softc;
4673         u16             index, bit;
4674
4675         if (ifp->if_softc !=  arg)   /* Not our event */
4676                 return;
4677
4678         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4679                 return;
4680
4681         IXGBE_CORE_LOCK(adapter);
4682         index = (vtag >> 5) & 0x7F;
4683         bit = vtag & 0x1F;
4684         adapter->shadow_vfta[index] |= (1 << bit);
4685         ++adapter->num_vlans;
4686         ixgbe_setup_vlan_hw_support(adapter);
4687         IXGBE_CORE_UNLOCK(adapter);
4688 }
4689
4690 /*
4691 ** This routine is run via an vlan
4692 ** unconfig EVENT, remove our entry
4693 ** in the soft vfta.
4694 */
4695 static void
4696 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4697 {
4698         struct adapter  *adapter = ifp->if_softc;
4699         u16             index, bit;
4700
4701         if (ifp->if_softc !=  arg)
4702                 return;
4703
4704         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4705                 return;
4706
4707         IXGBE_CORE_LOCK(adapter);
4708         index = (vtag >> 5) & 0x7F;
4709         bit = vtag & 0x1F;
4710         adapter->shadow_vfta[index] &= ~(1 << bit);
4711         --adapter->num_vlans;
4712         /* Re-init to load the changes */
4713         ixgbe_setup_vlan_hw_support(adapter);
4714         IXGBE_CORE_UNLOCK(adapter);
4715 }
4716
4717 static void
4718 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4719 {
4720         struct ifnet    *ifp = adapter->ifp;
4721         struct ixgbe_hw *hw = &adapter->hw;
4722         struct rx_ring  *rxr;
4723         u32             ctrl;
4724
4725
4726         /*
4727         ** We get here thru init_locked, meaning
4728         ** a soft reset, this has already cleared
4729         ** the VFTA and other state, so if there
4730         ** have been no vlan's registered do nothing.
4731         */
4732         if (adapter->num_vlans == 0)
4733                 return;
4734
4735         /* Setup the queues for vlans */
4736         for (int i = 0; i < adapter->num_queues; i++) {
4737                 rxr = &adapter->rx_rings[i];
4738                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4739                 if (hw->mac.type != ixgbe_mac_82598EB) {
4740                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4741                         ctrl |= IXGBE_RXDCTL_VME;
4742                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4743                 }
4744                 rxr->vtag_strip = TRUE;
4745         }
4746
4747         if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
4748                 return;
4749         /*
4750         ** A soft reset zero's out the VFTA, so
4751         ** we need to repopulate it now.
4752         */
4753         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4754                 if (adapter->shadow_vfta[i] != 0)
4755                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4756                             adapter->shadow_vfta[i]);
4757
4758         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4759         /* Enable the Filter Table if enabled */
4760         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4761                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4762                 ctrl |= IXGBE_VLNCTRL_VFE;
4763         }
4764         if (hw->mac.type == ixgbe_mac_82598EB)
4765                 ctrl |= IXGBE_VLNCTRL_VME;
4766         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4767 }
4768
4769 static void
4770 ixgbe_enable_intr(struct adapter *adapter)
4771 {
4772         struct ixgbe_hw *hw = &adapter->hw;
4773         struct ix_queue *que = adapter->queues;
4774         u32             mask, fwsm;
4775
4776         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4777         /* Enable Fan Failure detection */
4778         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4779                     mask |= IXGBE_EIMS_GPI_SDP1;
4780
4781         switch (adapter->hw.mac.type) {
4782                 case ixgbe_mac_82599EB:
4783                         mask |= IXGBE_EIMS_ECC;
4784                         mask |= IXGBE_EIMS_GPI_SDP0;
4785                         mask |= IXGBE_EIMS_GPI_SDP1;
4786                         mask |= IXGBE_EIMS_GPI_SDP2;
4787 #ifdef IXGBE_FDIR
4788                         mask |= IXGBE_EIMS_FLOW_DIR;
4789 #endif
4790                         break;
4791                 case ixgbe_mac_X540:
4792                         mask |= IXGBE_EIMS_ECC;
4793                         /* Detect if Thermal Sensor is enabled */
4794                         fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
4795                         if (fwsm & IXGBE_FWSM_TS_ENABLED)
4796                                 mask |= IXGBE_EIMS_TS;
4797 #ifdef IXGBE_FDIR
4798                         mask |= IXGBE_EIMS_FLOW_DIR;
4799 #endif
4800                 /* falls through */
4801                 default:
4802                         break;
4803         }
4804
4805         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4806
4807         /* With RSS we use auto clear */
4808         if (adapter->msix_mem) {
4809                 mask = IXGBE_EIMS_ENABLE_MASK;
4810                 /* Don't autoclear Link */
4811                 mask &= ~IXGBE_EIMS_OTHER;
4812                 mask &= ~IXGBE_EIMS_LSC;
4813                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4814         }
4815
4816         /*
4817         ** Now enable all queues, this is done separately to
4818         ** allow for handling the extended (beyond 32) MSIX
4819         ** vectors that can be used by 82599
4820         */
4821         for (int i = 0; i < adapter->num_queues; i++, que++)
4822                 ixgbe_enable_queue(adapter, que->msix);
4823
4824         IXGBE_WRITE_FLUSH(hw);
4825
4826         return;
4827 }
4828
4829 static void
4830 ixgbe_disable_intr(struct adapter *adapter)
4831 {
4832         if (adapter->msix_mem)
4833                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4834         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4835                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4836         } else {
4837                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4838                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4839                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4840         }
4841         IXGBE_WRITE_FLUSH(&adapter->hw);
4842         return;
4843 }
4844
4845 u16
4846 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4847 {
4848         u16 value;
4849
4850         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4851             reg, 2);
4852
4853         return (value);
4854 }
4855
4856 void
4857 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4858 {
4859         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4860             reg, value, 2);
4861
4862         return;
4863 }
4864
4865 /*
4866 ** Get the width and transaction speed of
4867 ** the slot this adapter is plugged into.
4868 */
4869 static void
4870 ixgbe_get_slot_info(struct ixgbe_hw *hw)
4871 {
4872         device_t                dev = ((struct ixgbe_osdep *)hw->back)->dev;
4873         struct ixgbe_mac_info   *mac = &hw->mac;
4874         u16                     link;
4875         u32                     offset;
4876
4877         /* For most devices simply call the shared code routine */
4878         if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
4879                 ixgbe_get_bus_info(hw);
4880                 goto display;
4881         }
4882
4883         /*
4884         ** For the Quad port adapter we need to parse back
4885         ** up the PCI tree to find the speed of the expansion
4886         ** slot into which this adapter is plugged. A bit more work.
4887         */
4888         dev = device_get_parent(device_get_parent(dev));
4889 #ifdef IXGBE_DEBUG
4890         device_printf(dev, "parent pcib = %x,%x,%x\n",
4891             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4892 #endif
4893         dev = device_get_parent(device_get_parent(dev));
4894 #ifdef IXGBE_DEBUG
4895         device_printf(dev, "slot pcib = %x,%x,%x\n",
4896             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4897 #endif
4898         /* Now get the PCI Express Capabilities offset */
4899         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4900         /* ...and read the Link Status Register */
4901         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4902         switch (link & IXGBE_PCI_LINK_WIDTH) {
4903         case IXGBE_PCI_LINK_WIDTH_1:
4904                 hw->bus.width = ixgbe_bus_width_pcie_x1;
4905                 break;
4906         case IXGBE_PCI_LINK_WIDTH_2:
4907                 hw->bus.width = ixgbe_bus_width_pcie_x2;
4908                 break;
4909         case IXGBE_PCI_LINK_WIDTH_4:
4910                 hw->bus.width = ixgbe_bus_width_pcie_x4;
4911                 break;
4912         case IXGBE_PCI_LINK_WIDTH_8:
4913                 hw->bus.width = ixgbe_bus_width_pcie_x8;
4914                 break;
4915         default:
4916                 hw->bus.width = ixgbe_bus_width_unknown;
4917                 break;
4918         }
4919
4920         switch (link & IXGBE_PCI_LINK_SPEED) {
4921         case IXGBE_PCI_LINK_SPEED_2500:
4922                 hw->bus.speed = ixgbe_bus_speed_2500;
4923                 break;
4924         case IXGBE_PCI_LINK_SPEED_5000:
4925                 hw->bus.speed = ixgbe_bus_speed_5000;
4926                 break;
4927         case IXGBE_PCI_LINK_SPEED_8000:
4928                 hw->bus.speed = ixgbe_bus_speed_8000;
4929                 break;
4930         default:
4931                 hw->bus.speed = ixgbe_bus_speed_unknown;
4932                 break;
4933         }
4934
4935         mac->ops.set_lan_id(hw);
4936
4937 display:
4938         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4939             ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
4940             (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
4941             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4942             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
4943             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
4944             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
4945             ("Unknown"));
4946
4947         if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4948             ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
4949             (hw->bus.speed == ixgbe_bus_speed_2500))) {
4950                 device_printf(dev, "PCI-Express bandwidth available"
4951                     " for this card\n     is not sufficient for"
4952                     " optimal performance.\n");
4953                 device_printf(dev, "For optimal performance a x8 "
4954                     "PCIE, or x4 PCIE Gen2 slot is required.\n");
4955         }
4956         if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4957             ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
4958             (hw->bus.speed < ixgbe_bus_speed_8000))) {
4959                 device_printf(dev, "PCI-Express bandwidth available"
4960                     " for this card\n     is not sufficient for"
4961                     " optimal performance.\n");
4962                 device_printf(dev, "For optimal performance a x8 "
4963                     "PCIE Gen3 slot is required.\n");
4964         }
4965
4966         return;
4967 }
4968
4969
4970 /*
4971 ** Setup the correct IVAR register for a particular MSIX interrupt
4972 **   (yes this is all very magic and confusing :)
4973 **  - entry is the register array entry
4974 **  - vector is the MSIX vector for this queue
4975 **  - type is RX/TX/MISC
4976 */
4977 static void
4978 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4979 {
4980         struct ixgbe_hw *hw = &adapter->hw;
4981         u32 ivar, index;
4982
4983         vector |= IXGBE_IVAR_ALLOC_VAL;
4984
4985         switch (hw->mac.type) {
4986
4987         case ixgbe_mac_82598EB:
4988                 if (type == -1)
4989                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4990                 else
4991                         entry += (type * 64);
4992                 index = (entry >> 2) & 0x1F;
4993                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4994                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4995                 ivar |= (vector << (8 * (entry & 0x3)));
4996                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4997                 break;
4998
4999         case ixgbe_mac_82599EB:
5000         case ixgbe_mac_X540:
5001                 if (type == -1) { /* MISC IVAR */
5002                         index = (entry & 1) * 8;
5003                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5004                         ivar &= ~(0xFF << index);
5005                         ivar |= (vector << index);
5006                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5007                 } else {        /* RX/TX IVARS */
5008                         index = (16 * (entry & 1)) + (8 * type);
5009                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5010                         ivar &= ~(0xFF << index);
5011                         ivar |= (vector << index);
5012                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5013                 }
5014
5015         default:
5016                 break;
5017         }
5018 }
5019
5020 static void
5021 ixgbe_configure_ivars(struct adapter *adapter)
5022 {
5023         struct  ix_queue *que = adapter->queues;
5024         u32 newitr;
5025
5026         if (ixgbe_max_interrupt_rate > 0)
5027                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5028         else
5029                 newitr = 0;
5030
5031         for (int i = 0; i < adapter->num_queues; i++, que++) {
5032                 /* First the RX queue entry */
5033                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5034                 /* ... and the TX */
5035                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5036                 /* Set an Initial EITR value */
5037                 IXGBE_WRITE_REG(&adapter->hw,
5038                     IXGBE_EITR(que->msix), newitr);
5039         }
5040
5041         /* For the Link interrupt */
5042         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5043 }
5044
5045 /*
5046 ** ixgbe_sfp_probe - called in the local timer to
5047 ** determine if a port had optics inserted.
5048 */
5049 static bool ixgbe_sfp_probe(struct adapter *adapter)
5050 {
5051         struct ixgbe_hw *hw = &adapter->hw;
5052         device_t        dev = adapter->dev;
5053         bool            result = FALSE;
5054
5055         if ((hw->phy.type == ixgbe_phy_nl) &&
5056             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5057                 s32 ret = hw->phy.ops.identify_sfp(hw);
5058                 if (ret)
5059                         goto out;
5060                 ret = hw->phy.ops.reset(hw);
5061                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5062                         device_printf(dev,"Unsupported SFP+ module detected!");
5063                         printf(" Reload driver with supported module.\n");
5064                         adapter->sfp_probe = FALSE;
5065                         goto out;
5066                 } else
5067                         device_printf(dev,"SFP+ module detected!\n");
5068                 /* We now have supported optics */
5069                 adapter->sfp_probe = FALSE;
5070                 /* Set the optics type so system reports correctly */
5071                 ixgbe_setup_optics(adapter);
5072                 result = TRUE;
5073         }
5074 out:
5075         return (result);
5076 }
5077
5078 /*
5079 ** Tasklet handler for MSIX Link interrupts
5080 **  - do outside interrupt since it might sleep
5081 */
5082 static void
5083 ixgbe_handle_link(void *context, int pending)
5084 {
5085         struct adapter  *adapter = context;
5086
5087         ixgbe_check_link(&adapter->hw,
5088             &adapter->link_speed, &adapter->link_up, 0);
5089         ixgbe_update_link_status(adapter);
5090 }
5091
5092 /*
5093 ** Tasklet for handling SFP module interrupts
5094 */
5095 static void
5096 ixgbe_handle_mod(void *context, int pending)
5097 {
5098         struct adapter  *adapter = context;
5099         struct ixgbe_hw *hw = &adapter->hw;
5100         device_t        dev = adapter->dev;
5101         u32 err;
5102
5103         err = hw->phy.ops.identify_sfp(hw);
5104         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5105                 device_printf(dev,
5106                     "Unsupported SFP+ module type was detected.\n");
5107                 return;
5108         }
5109         err = hw->mac.ops.setup_sfp(hw);
5110         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5111                 device_printf(dev,
5112                     "Setup failure - unsupported SFP+ module type.\n");
5113                 return;
5114         }
5115         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5116         return;
5117 }
5118
5119
5120 /*
5121 ** Tasklet for handling MSF (multispeed fiber) interrupts
5122 */
5123 static void
5124 ixgbe_handle_msf(void *context, int pending)
5125 {
5126         struct adapter  *adapter = context;
5127         struct ixgbe_hw *hw = &adapter->hw;
5128         u32 autoneg;
5129         bool negotiate;
5130
5131         autoneg = hw->phy.autoneg_advertised;
5132         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5133                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5134         if (hw->mac.ops.setup_link)
5135                 hw->mac.ops.setup_link(hw, autoneg, TRUE);
5136         return;
5137 }
5138
5139 #ifdef IXGBE_FDIR
5140 /*
5141 ** Tasklet for reinitializing the Flow Director filter table
5142 */
5143 static void
5144 ixgbe_reinit_fdir(void *context, int pending)
5145 {
5146         struct adapter  *adapter = context;
5147         struct ifnet   *ifp = adapter->ifp;
5148
5149         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5150                 return;
5151         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5152         adapter->fdir_reinit = 0;
5153         /* re-enable flow director interrupts */
5154         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5155         /* Restart the interface */
5156         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5157         return;
5158 }
5159 #endif
5160
5161 /**********************************************************************
5162  *
5163  *  Update the board statistics counters.
5164  *
5165  **********************************************************************/
5166 static void
5167 ixgbe_update_stats_counters(struct adapter *adapter)
5168 {
5169         struct ifnet   *ifp = adapter->ifp;
5170         struct ixgbe_hw *hw = &adapter->hw;
5171         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5172         u64  total_missed_rx = 0;
5173
5174         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5175         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5176         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5177         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5178
5179         /*
5180         ** Note: these are for the 8 possible traffic classes,
5181         **       which in current implementation is unused,
5182         **       therefore only 0 should read real data.
5183         */
5184         for (int i = 0; i < 8; i++) {
5185                 u32 mp;
5186                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5187                 /* missed_rx tallies misses for the gprc workaround */
5188                 missed_rx += mp;
5189                 /* global total per queue */
5190                 adapter->stats.mpc[i] += mp;
5191                 /* Running comprehensive total for stats display */
5192                 total_missed_rx += adapter->stats.mpc[i];
5193                 if (hw->mac.type == ixgbe_mac_82598EB) {
5194                         adapter->stats.rnbc[i] +=
5195                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5196                         adapter->stats.qbtc[i] +=
5197                             IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5198                         adapter->stats.qbrc[i] +=
5199                             IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5200                         adapter->stats.pxonrxc[i] +=
5201                             IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5202                 } else
5203                         adapter->stats.pxonrxc[i] +=
5204                             IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5205                 adapter->stats.pxontxc[i] +=
5206                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5207                 adapter->stats.pxofftxc[i] +=
5208                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5209                 adapter->stats.pxoffrxc[i] +=
5210                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5211                 adapter->stats.pxon2offc[i] +=
5212                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5213         }
5214         for (int i = 0; i < 16; i++) {
5215                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5216                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5217                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5218         }
5219         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5220         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5221         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5222
5223         /* Hardware workaround, gprc counts missed packets */
5224         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5225         adapter->stats.gprc -= missed_rx;
5226
5227         if (hw->mac.type != ixgbe_mac_82598EB) {
5228                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5229                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5230                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5231                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5232                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5233                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5234                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5235                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5236         } else {
5237                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5238                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5239                 /* 82598 only has a counter in the high register */
5240                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5241                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5242                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5243         }
5244
5245         /*
5246          * Workaround: mprc hardware is incorrectly counting
5247          * broadcasts, so for now we subtract those.
5248          */
5249         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5250         adapter->stats.bprc += bprc;
5251         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5252         if (hw->mac.type == ixgbe_mac_82598EB)
5253                 adapter->stats.mprc -= bprc;
5254
5255         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5256         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5257         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5258         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5259         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5260         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5261
5262         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5263         adapter->stats.lxontxc += lxon;
5264         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5265         adapter->stats.lxofftxc += lxoff;
5266         total = lxon + lxoff;
5267
5268         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5269         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5270         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5271         adapter->stats.gptc -= total;
5272         adapter->stats.mptc -= total;
5273         adapter->stats.ptc64 -= total;
5274         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5275
5276         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5277         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5278         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5279         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5280         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5281         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5282         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5283         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5284         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5285         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5286         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5287         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5288         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5289         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5290         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5291         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5292         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5293         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5294         /* Only read FCOE on 82599 */
5295         if (hw->mac.type != ixgbe_mac_82598EB) {
5296                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5297                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5298                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5299                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5300                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5301         }
5302
5303         /* Fill out the OS statistics structure */
5304         ifp->if_ipackets = adapter->stats.gprc;
5305         ifp->if_opackets = adapter->stats.gptc;
5306         ifp->if_ibytes = adapter->stats.gorc;
5307         ifp->if_obytes = adapter->stats.gotc;
5308         ifp->if_imcasts = adapter->stats.mprc;
5309         ifp->if_omcasts = adapter->stats.mptc;
5310         ifp->if_collisions = 0;
5311
5312         /* Rx Errors */
5313         ifp->if_iqdrops = total_missed_rx;
5314         ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec;
5315 }
5316
5317 /** ixgbe_sysctl_tdh_handler - Handler function
5318  *  Retrieves the TDH value from the hardware
5319  */
5320 static int
5321 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5322 {
5323         int error;
5324
5325         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5326         if (!txr) return 0;
5327
5328         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5329         error = sysctl_handle_int(oidp, &val, 0, req);
5330         if (error || !req->newptr)
5331                 return error;
5332         return 0;
5333 }
5334
5335 /** ixgbe_sysctl_tdt_handler - Handler function
5336  *  Retrieves the TDT value from the hardware
5337  */
5338 static int
5339 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5340 {
5341         int error;
5342
5343         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5344         if (!txr) return 0;
5345
5346         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5347         error = sysctl_handle_int(oidp, &val, 0, req);
5348         if (error || !req->newptr)
5349                 return error;
5350         return 0;
5351 }
5352
5353 /** ixgbe_sysctl_rdh_handler - Handler function
5354  *  Retrieves the RDH value from the hardware
5355  */
5356 static int
5357 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5358 {
5359         int error;
5360
5361         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5362         if (!rxr) return 0;
5363
5364         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5365         error = sysctl_handle_int(oidp, &val, 0, req);
5366         if (error || !req->newptr)
5367                 return error;
5368         return 0;
5369 }
5370
5371 /** ixgbe_sysctl_rdt_handler - Handler function
5372  *  Retrieves the RDT value from the hardware
5373  */
5374 static int
5375 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5376 {
5377         int error;
5378
5379         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5380         if (!rxr) return 0;
5381
5382         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5383         error = sysctl_handle_int(oidp, &val, 0, req);
5384         if (error || !req->newptr)
5385                 return error;
5386         return 0;
5387 }
5388
5389 static int
5390 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5391 {
5392         int error;
5393         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5394         unsigned int reg, usec, rate;
5395
5396         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5397         usec = ((reg & 0x0FF8) >> 3);
5398         if (usec > 0)
5399                 rate = 500000 / usec;
5400         else
5401                 rate = 0;
5402         error = sysctl_handle_int(oidp, &rate, 0, req);
5403         if (error || !req->newptr)
5404                 return error;
5405         reg &= ~0xfff; /* default, no limitation */
5406         ixgbe_max_interrupt_rate = 0;
5407         if (rate > 0 && rate < 500000) {
5408                 if (rate < 1000)
5409                         rate = 1000;
5410                 ixgbe_max_interrupt_rate = rate;
5411                 reg |= ((4000000/rate) & 0xff8 );
5412         }
5413         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5414         return 0;
5415 }
5416
5417 /*
5418  * Add sysctl variables, one per statistic, to the system.
5419  */
5420 static void
5421 ixgbe_add_hw_stats(struct adapter *adapter)
5422 {
5423
5424         device_t dev = adapter->dev;
5425
5426         struct tx_ring *txr = adapter->tx_rings;
5427         struct rx_ring *rxr = adapter->rx_rings;
5428
5429         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5430         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5431         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5432         struct ixgbe_hw_stats *stats = &adapter->stats;
5433
5434         struct sysctl_oid *stat_node, *queue_node;
5435         struct sysctl_oid_list *stat_list, *queue_list;
5436
5437 #define QUEUE_NAME_LEN 32
5438         char namebuf[QUEUE_NAME_LEN];
5439
5440         /* Driver Statistics */
5441         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5442                         CTLFLAG_RD, &adapter->dropped_pkts,
5443                         "Driver dropped packets");
5444         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5445                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5446                         "m_defrag() failed");
5447         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5448                         CTLFLAG_RD, &adapter->watchdog_events,
5449                         "Watchdog timeouts");
5450         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5451                         CTLFLAG_RD, &adapter->link_irq,
5452                         "Link MSIX IRQ Handled");
5453
5454         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5455                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5456                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5457                                             CTLFLAG_RD, NULL, "Queue Name");
5458                 queue_list = SYSCTL_CHILDREN(queue_node);
5459
5460                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5461                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5462                                 sizeof(&adapter->queues[i]),
5463                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5464                                 "Interrupt Rate");
5465                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5466                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
5467                                 "irqs on this queue");
5468                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5469                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5470                                 ixgbe_sysctl_tdh_handler, "IU",
5471                                 "Transmit Descriptor Head");
5472                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5473                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5474                                 ixgbe_sysctl_tdt_handler, "IU",
5475                                 "Transmit Descriptor Tail");
5476                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5477                                 CTLFLAG_RD, &txr->tso_tx,
5478                                 "TSO");
5479                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5480                                 CTLFLAG_RD, &txr->no_tx_dma_setup,
5481                                 "Driver tx dma failure in xmit");
5482                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5483                                 CTLFLAG_RD, &txr->no_desc_avail,
5484                                 "Queue No Descriptor Available");
5485                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5486                                 CTLFLAG_RD, &txr->total_packets,
5487                                 "Queue Packets Transmitted");
5488         }
5489
5490         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5491                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5492                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5493                                             CTLFLAG_RD, NULL, "Queue Name");
5494                 queue_list = SYSCTL_CHILDREN(queue_node);
5495
5496                 struct lro_ctrl *lro = &rxr->lro;
5497
5498                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5499                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5500                                             CTLFLAG_RD, NULL, "Queue Name");
5501                 queue_list = SYSCTL_CHILDREN(queue_node);
5502
5503                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5504                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5505                                 ixgbe_sysctl_rdh_handler, "IU",
5506                                 "Receive Descriptor Head");
5507                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5508                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5509                                 ixgbe_sysctl_rdt_handler, "IU",
5510                                 "Receive Descriptor Tail");
5511                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5512                                 CTLFLAG_RD, &rxr->rx_packets,
5513                                 "Queue Packets Received");
5514                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5515                                 CTLFLAG_RD, &rxr->rx_bytes,
5516                                 "Queue Bytes Received");
5517                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5518                                 CTLFLAG_RD, &rxr->rx_copies,
5519                                 "Copied RX Frames");
5520                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5521                                 CTLFLAG_RD, &lro->lro_queued, 0,
5522                                 "LRO Queued");
5523                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5524                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5525                                 "LRO Flushed");
5526         }
5527
5528         /* MAC stats get the own sub node */
5529
5530         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5531                                     CTLFLAG_RD, NULL, "MAC Statistics");
5532         stat_list = SYSCTL_CHILDREN(stat_node);
5533
5534         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5535                         CTLFLAG_RD, &stats->crcerrs,
5536                         "CRC Errors");
5537         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5538                         CTLFLAG_RD, &stats->illerrc,
5539                         "Illegal Byte Errors");
5540         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5541                         CTLFLAG_RD, &stats->errbc,
5542                         "Byte Errors");
5543         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5544                         CTLFLAG_RD, &stats->mspdc,
5545                         "MAC Short Packets Discarded");
5546         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5547                         CTLFLAG_RD, &stats->mlfc,
5548                         "MAC Local Faults");
5549         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5550                         CTLFLAG_RD, &stats->mrfc,
5551                         "MAC Remote Faults");
5552         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5553                         CTLFLAG_RD, &stats->rlec,
5554                         "Receive Length Errors");
5555
5556         /* Flow Control stats */
5557         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5558                         CTLFLAG_RD, &stats->lxontxc,
5559                         "Link XON Transmitted");
5560         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5561                         CTLFLAG_RD, &stats->lxonrxc,
5562                         "Link XON Received");
5563         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5564                         CTLFLAG_RD, &stats->lxofftxc,
5565                         "Link XOFF Transmitted");
5566         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5567                         CTLFLAG_RD, &stats->lxoffrxc,
5568                         "Link XOFF Received");
5569
5570         /* Packet Reception Stats */
5571         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5572                         CTLFLAG_RD, &stats->tor,
5573                         "Total Octets Received");
5574         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5575                         CTLFLAG_RD, &stats->gorc,
5576                         "Good Octets Received");
5577         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5578                         CTLFLAG_RD, &stats->tpr,
5579                         "Total Packets Received");
5580         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5581                         CTLFLAG_RD, &stats->gprc,
5582                         "Good Packets Received");
5583         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5584                         CTLFLAG_RD, &stats->mprc,
5585                         "Multicast Packets Received");
5586         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5587                         CTLFLAG_RD, &stats->bprc,
5588                         "Broadcast Packets Received");
5589         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5590                         CTLFLAG_RD, &stats->prc64,
5591                         "64 byte frames received ");
5592         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5593                         CTLFLAG_RD, &stats->prc127,
5594                         "65-127 byte frames received");
5595         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5596                         CTLFLAG_RD, &stats->prc255,
5597                         "128-255 byte frames received");
5598         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5599                         CTLFLAG_RD, &stats->prc511,
5600                         "256-511 byte frames received");
5601         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5602                         CTLFLAG_RD, &stats->prc1023,
5603                         "512-1023 byte frames received");
5604         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5605                         CTLFLAG_RD, &stats->prc1522,
5606                         "1023-1522 byte frames received");
5607         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5608                         CTLFLAG_RD, &stats->ruc,
5609                         "Receive Undersized");
5610         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5611                         CTLFLAG_RD, &stats->rfc,
5612                         "Fragmented Packets Received ");
5613         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5614                         CTLFLAG_RD, &stats->roc,
5615                         "Oversized Packets Received");
5616         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5617                         CTLFLAG_RD, &stats->rjc,
5618                         "Received Jabber");
5619         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5620                         CTLFLAG_RD, &stats->mngprc,
5621                         "Management Packets Received");
5622         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5623                         CTLFLAG_RD, &stats->mngptc,
5624                         "Management Packets Dropped");
5625         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5626                         CTLFLAG_RD, &stats->xec,
5627                         "Checksum Errors");
5628
5629         /* Packet Transmission Stats */
5630         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5631                         CTLFLAG_RD, &stats->gotc,
5632                         "Good Octets Transmitted");
5633         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5634                         CTLFLAG_RD, &stats->tpt,
5635                         "Total Packets Transmitted");
5636         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5637                         CTLFLAG_RD, &stats->gptc,
5638                         "Good Packets Transmitted");
5639         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5640                         CTLFLAG_RD, &stats->bptc,
5641                         "Broadcast Packets Transmitted");
5642         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5643                         CTLFLAG_RD, &stats->mptc,
5644                         "Multicast Packets Transmitted");
5645         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5646                         CTLFLAG_RD, &stats->mngptc,
5647                         "Management Packets Transmitted");
5648         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5649                         CTLFLAG_RD, &stats->ptc64,
5650                         "64 byte frames transmitted ");
5651         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5652                         CTLFLAG_RD, &stats->ptc127,
5653                         "65-127 byte frames transmitted");
5654         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5655                         CTLFLAG_RD, &stats->ptc255,
5656                         "128-255 byte frames transmitted");
5657         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5658                         CTLFLAG_RD, &stats->ptc511,
5659                         "256-511 byte frames transmitted");
5660         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5661                         CTLFLAG_RD, &stats->ptc1023,
5662                         "512-1023 byte frames transmitted");
5663         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5664                         CTLFLAG_RD, &stats->ptc1522,
5665                         "1024-1522 byte frames transmitted");
5666 }
5667
5668 /*
5669 ** Set flow control using sysctl:
5670 ** Flow control values:
5671 **      0 - off
5672 **      1 - rx pause
5673 **      2 - tx pause
5674 **      3 - full
5675 */
5676 static int
5677 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5678 {
5679         int error, last;
5680         struct adapter *adapter = (struct adapter *) arg1;
5681
5682         last = adapter->fc;
5683         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5684         if ((error) || (req->newptr == NULL))
5685                 return (error);
5686
5687         /* Don't bother if it's not changed */
5688         if (adapter->fc == last)
5689                 return (0);
5690
5691         switch (adapter->fc) {
5692                 case ixgbe_fc_rx_pause:
5693                 case ixgbe_fc_tx_pause:
5694                 case ixgbe_fc_full:
5695                         adapter->hw.fc.requested_mode = adapter->fc;
5696                         if (adapter->num_queues > 1)
5697                                 ixgbe_disable_rx_drop(adapter);
5698                         break;
5699                 case ixgbe_fc_none:
5700                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5701                         if (adapter->num_queues > 1)
5702                                 ixgbe_enable_rx_drop(adapter);
5703                         break;
5704                 default:
5705                         adapter->fc = last;
5706                         return (EINVAL);
5707         }
5708         /* Don't autoneg if forcing a value */
5709         adapter->hw.fc.disable_fc_autoneg = TRUE;
5710         ixgbe_fc_enable(&adapter->hw);
5711         return error;
5712 }
5713
5714 /*
5715 ** Control link advertise speed:
5716 **      1 - advertise only 1G
5717 **      2 - advertise 100Mb
5718 **      3 - advertise normal
5719 */
5720 static int
5721 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5722 {
5723         int                     error = 0;
5724         struct adapter          *adapter;
5725         device_t                dev;
5726         struct ixgbe_hw         *hw;
5727         ixgbe_link_speed        speed, last;
5728
5729         adapter = (struct adapter *) arg1;
5730         dev = adapter->dev;
5731         hw = &adapter->hw;
5732         last = adapter->advertise;
5733
5734         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5735         if ((error) || (req->newptr == NULL))
5736                 return (error);
5737
5738         if (adapter->advertise == last) /* no change */
5739                 return (0);
5740
5741         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5742             (hw->phy.multispeed_fiber)))
5743                 return (EINVAL);
5744
5745         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5746                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5747                 return (EINVAL);
5748         }
5749
5750         if (adapter->advertise == 1)
5751                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5752         else if (adapter->advertise == 2)
5753                 speed = IXGBE_LINK_SPEED_100_FULL;
5754         else if (adapter->advertise == 3)
5755                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5756                         IXGBE_LINK_SPEED_10GB_FULL;
5757         else {  /* bogus value */
5758                 adapter->advertise = last;
5759                 return (EINVAL);
5760         }
5761
5762         hw->mac.autotry_restart = TRUE;
5763         hw->mac.ops.setup_link(hw, speed, TRUE);
5764
5765         return (error);
5766 }
5767
5768 /*
5769 ** Thermal Shutdown Trigger
5770 **   - cause a Thermal Overtemp IRQ
5771 **   - this now requires firmware enabling
5772 */
5773 static int
5774 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5775 {
5776         int             error, fire = 0;
5777         struct adapter  *adapter = (struct adapter *) arg1;
5778         struct ixgbe_hw *hw = &adapter->hw;
5779
5780
5781         if (hw->mac.type != ixgbe_mac_X540)
5782                 return (0);
5783
5784         error = sysctl_handle_int(oidp, &fire, 0, req);
5785         if ((error) || (req->newptr == NULL))
5786                 return (error);
5787
5788         if (fire) {
5789                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5790                 reg |= IXGBE_EICR_TS;
5791                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5792         }
5793
5794         return (0);
5795 }
5796
5797 /*
5798 ** Enable the hardware to drop packets when the buffer is
5799 ** full. This is useful when multiqueue,so that no single
5800 ** queue being full stalls the entire RX engine. We only
5801 ** enable this when Multiqueue AND when Flow Control is
5802 ** disabled.
5803 */
5804 static void
5805 ixgbe_enable_rx_drop(struct adapter *adapter)
5806 {
5807         struct ixgbe_hw *hw = &adapter->hw;
5808
5809         for (int i = 0; i < adapter->num_queues; i++) {
5810                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5811                 srrctl |= IXGBE_SRRCTL_DROP_EN;
5812                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5813         }
5814 }
5815
5816 static void
5817 ixgbe_disable_rx_drop(struct adapter *adapter)
5818 {
5819         struct ixgbe_hw *hw = &adapter->hw;
5820
5821         for (int i = 0; i < adapter->num_queues; i++) {
5822                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5823                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5824                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5825         }
5826 }