sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2013, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.5.15";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 #ifdef IXGBE_LEGACY_TX
 106 static void     ixgbe_start(struct ifnet *);
 107 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 108 #else /* ! IXGBE_LEGACY_TX */
 109 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 110 static int      ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 static void     ixgbe_deferred_mq_start(void *, int);
 113 #endif /* IXGBE_LEGACY_TX */
 114 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 115 static void     ixgbe_init(void *);
 116 static void     ixgbe_init_locked(struct adapter *);
 117 static void     ixgbe_stop(void *);
 118 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 119 static int      ixgbe_media_change(struct ifnet *);
 120 static void     ixgbe_identify_hardware(struct adapter *);
 121 static int      ixgbe_allocate_pci_resources(struct adapter *);
 122 static void     ixgbe_get_slot_info(struct ixgbe_hw *);
 123 static int      ixgbe_allocate_msix(struct adapter *);
 124 static int      ixgbe_allocate_legacy(struct adapter *);
 125 static int      ixgbe_allocate_queues(struct adapter *);
 126 static int      ixgbe_setup_msix(struct adapter *);
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static bool     ixgbe_rxeof(struct ix_queue *);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static int      ixgbe_tx_ctx_setup(struct tx_ring *,
 165                     struct mbuf *, u32 *, u32 *);
 166 static int      ixgbe_tso_setup(struct tx_ring *,
 167                     struct mbuf *, u32 *, u32 *);
 168 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 169 static void     ixgbe_configure_ivars(struct adapter *);
 170 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 171
 172 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 173 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 174 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 175
 176 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 177
 178 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 179 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 180                     struct mbuf *, u32);
 181
 182 static void     ixgbe_enable_rx_drop(struct adapter *);
 183 static void     ixgbe_disable_rx_drop(struct adapter *);
 184
 185 /* Support for pluggable optic modules */
 186 static bool     ixgbe_sfp_probe(struct adapter *);
 187 static void     ixgbe_setup_optics(struct adapter *);
 188
 189 /* Legacy (single vector interrupt handler */
 190 static void     ixgbe_legacy_irq(void *);
 191
 192 /* The MSI/X Interrupt handlers */
 193 static void     ixgbe_msix_que(void *);
 194 static void     ixgbe_msix_link(void *);
 195
 196 /* Deferred interrupt tasklets */
 197 static void     ixgbe_handle_que(void *, int);
 198 static void     ixgbe_handle_link(void *, int);
 199 static void     ixgbe_handle_msf(void *, int);
 200 static void     ixgbe_handle_mod(void *, int);
 201
 202 #ifdef IXGBE_FDIR
 203 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 204 static void     ixgbe_reinit_fdir(void *, int);
 205 #endif
 206
 207 /* Missing shared code prototype */
 208 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
 209
 210 /*********************************************************************
 211  *  FreeBSD Device Interface Entry Points
 212  *********************************************************************/
 213
 214 static device_method_t ixgbe_methods[] = {
 215         /* Device interface */
 216         DEVMETHOD(device_probe, ixgbe_probe),
 217         DEVMETHOD(device_attach, ixgbe_attach),
 218         DEVMETHOD(device_detach, ixgbe_detach),
 219         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 220         DEVMETHOD_END
 221 };
 222
 223 static driver_t ixgbe_driver = {
 224         "ix", ixgbe_methods, sizeof(struct adapter),
 225 };
 226
 227 devclass_t ixgbe_devclass;
 228 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 229
 230 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 231 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 232
 233 /*
 234 ** TUNEABLE PARAMETERS:
 235 */
 236
 237 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD, 0,
 238                    "IXGBE driver parameters");
 239
 240 /*
 241 ** AIM: Adaptive Interrupt Moderation
 242 ** which means that the interrupt rate
 243 ** is varied over time based on the
 244 ** traffic for that interrupt vector
 245 */
 246 static int ixgbe_enable_aim = TRUE;
 247 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 248 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RW, &ixgbe_enable_aim, 0,
 249     "Enable adaptive interrupt moderation");
 250
 251 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 252 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 253 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
 254     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
 255
 256 /* How many packets rxeof tries to clean at a time */
 257 static int ixgbe_rx_process_limit = 256;
 258 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 259 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
 260     &ixgbe_rx_process_limit, 0,
 261     "Maximum number of received packets to process at a time,"
 262     "-1 means unlimited");
 263
 264 /* How many packets txeof tries to clean at a time */
 265 static int ixgbe_tx_process_limit = 256;
 266 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
 267 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
 268     &ixgbe_tx_process_limit, 0,
 269     "Maximum number of sent packets to process at a time,"
 270     "-1 means unlimited");
 271
 272 /*
 273 ** Smart speed setting, default to on
 274 ** this only works as a compile option
 275 ** right now as its during attach, set
 276 ** this to 'ixgbe_smart_speed_off' to
 277 ** disable.
 278 */
 279 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 280
 281 /*
 282  * MSIX should be the default for best performance,
 283  * but this allows it to be forced off for testing.
 284  */
 285 static int ixgbe_enable_msix = 1;
 286 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 287 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
 288     "Enable MSI-X interrupts");
 289
 290 /*
 291  * Number of Queues, can be set to 0,
 292  * it then autoconfigures based on the
 293  * number of cpus with a max of 8. This
 294  * can be overriden manually here.
 295  */
 296 static int ixgbe_num_queues = 0;
 297 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 298 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
 299     "Number of queues to configure, 0 indicates autoconfigure");
 300
 301 /*
 302 ** Number of TX descriptors per ring,
 303 ** setting higher than RX as this seems
 304 ** the better performing choice.
 305 */
 306 static int ixgbe_txd = PERFORM_TXD;
 307 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 308 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
 309     "Number of receive descriptors per queue");
 310
 311 /* Number of RX descriptors per ring */
 312 static int ixgbe_rxd = PERFORM_RXD;
 313 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 314 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
 315     "Number of receive descriptors per queue");
 316
 317 /*
 318 ** Defining this on will allow the use
 319 ** of unsupported SFP+ modules, note that
 320 ** doing so you are on your own :)
 321 */
 322 static int allow_unsupported_sfp = FALSE;
 323 TUNABLE_INT("hw.ixgbe.unsupported_sfp", &allow_unsupported_sfp);
 324
 325 /*
 326 ** HW RSC control:
 327 **  this feature only works with
 328 **  IPv4, and only on 82599 and later.
 329 **  Also this will cause IP forwarding to
 330 **  fail and that can't be controlled by
 331 **  the stack as LRO can. For all these
 332 **  reasons I've deemed it best to leave
 333 **  this off and not bother with a tuneable
 334 **  interface, this would need to be compiled
 335 **  to enable.
 336 */
 337 static bool ixgbe_rsc_enable = FALSE;
 338
 339 /* Keep running tab on them for sanity check */
 340 static int ixgbe_total_ports;
 341
 342 #ifdef IXGBE_FDIR
 343 /*
 344 ** For Flow Director: this is the
 345 ** number of TX packets we sample
 346 ** for the filter pool, this means
 347 ** every 20th packet will be probed.
 348 **
 349 ** This feature can be disabled by
 350 ** setting this to 0.
 351 */
 352 static int atr_sample_rate = 20;
 353 /*
 354 ** Flow Director actually 'steals'
 355 ** part of the packet buffer as its
 356 ** filter pool, this variable controls
 357 ** how much it uses:
 358 **  0 = 64K, 1 = 128K, 2 = 256K
 359 */
 360 static int fdir_pballoc = 1;
 361 #endif
 362
 363 #ifdef DEV_NETMAP
 364 /*
 365  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 366  * be a reference on how to implement netmap support in a driver.
 367  * Additional comments are in ixgbe_netmap.h .
 368  *
 369  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 370  * that extend the standard driver.
 371  */
 372 #include <dev/netmap/ixgbe_netmap.h>
 373 #endif /* DEV_NETMAP */
 374
 375 /*********************************************************************
 376  *  Device identification routine
 377  *
 378  *  ixgbe_probe determines if the driver should be loaded on
 379  *  adapter based on PCI vendor/device id of the adapter.
 380  *
 381  *  return BUS_PROBE_DEFAULT on success, positive on failure
 382  *********************************************************************/
 383
 384 static int
 385 ixgbe_probe(device_t dev)
 386 {
 387         ixgbe_vendor_info_t *ent;
 388
 389         u16     pci_vendor_id = 0;
 390         u16     pci_device_id = 0;
 391         u16     pci_subvendor_id = 0;
 392         u16     pci_subdevice_id = 0;
 393         char    adapter_name[256];
 394
 395         INIT_DEBUGOUT("ixgbe_probe: begin");
 396
 397         pci_vendor_id = pci_get_vendor(dev);
 398         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 399                 return (ENXIO);
 400
 401         pci_device_id = pci_get_device(dev);
 402         pci_subvendor_id = pci_get_subvendor(dev);
 403         pci_subdevice_id = pci_get_subdevice(dev);
 404
 405         ent = ixgbe_vendor_info_array;
 406         while (ent->vendor_id != 0) {
 407                 if ((pci_vendor_id == ent->vendor_id) &&
 408                     (pci_device_id == ent->device_id) &&
 409
 410                     ((pci_subvendor_id == ent->subvendor_id) ||
 411                      (ent->subvendor_id == 0)) &&
 412
 413                     ((pci_subdevice_id == ent->subdevice_id) ||
 414                      (ent->subdevice_id == 0))) {
 415                         sprintf(adapter_name, "%s, Version - %s",
 416                                 ixgbe_strings[ent->index],
 417                                 ixgbe_driver_version);
 418                         device_set_desc_copy(dev, adapter_name);
 419                         ++ixgbe_total_ports;
 420                         return (BUS_PROBE_DEFAULT);
 421                 }
 422                 ent++;
 423         }
 424         return (ENXIO);
 425 }
 426
 427 /*********************************************************************
 428  *  Device initialization routine
 429  *
 430  *  The attach entry point is called when the driver is being loaded.
 431  *  This routine identifies the type of hardware, allocates all resources
 432  *  and initializes the hardware.
 433  *
 434  *  return 0 on success, positive on failure
 435  *********************************************************************/
 436
 437 static int
 438 ixgbe_attach(device_t dev)
 439 {
 440         struct adapter *adapter;
 441         struct ixgbe_hw *hw;
 442         int             error = 0;
 443         u16             csum;
 444         u32             ctrl_ext;
 445
 446         INIT_DEBUGOUT("ixgbe_attach: begin");
 447
 448         /* Allocate, clear, and link in our adapter structure */
 449         adapter = device_get_softc(dev);
 450         adapter->dev = adapter->osdep.dev = dev;
 451         hw = &adapter->hw;
 452
 453         /* Core Lock Init*/
 454         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 455
 456         /* SYSCTL APIs */
 457
 458         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 459                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 460                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 461                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 462
 463         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 464                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 465                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 466                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 467
 468         /*
 469         ** Allow a kind of speed control by forcing the autoneg
 470         ** advertised speed list to only a certain value, this
 471         ** supports 1G on 82599 devices, and 100Mb on x540.
 472         */
 473         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 474                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 475                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 476                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 477
 478         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 479                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 480                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 481                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 482
 483         /* Set up the timer callout */
 484         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 485
 486         /* Determine hardware revision */
 487         ixgbe_identify_hardware(adapter);
 488
 489         /* Do base PCI setup - map BAR0 */
 490         if (ixgbe_allocate_pci_resources(adapter)) {
 491                 device_printf(dev, "Allocation of PCI resources failed\n");
 492                 error = ENXIO;
 493                 goto err_out;
 494         }
 495
 496         /* Do descriptor calc and sanity checks */
 497         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 498             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 499                 device_printf(dev, "TXD config issue, using default!\n");
 500                 adapter->num_tx_desc = DEFAULT_TXD;
 501         } else
 502                 adapter->num_tx_desc = ixgbe_txd;
 503
 504         /*
 505         ** With many RX rings it is easy to exceed the
 506         ** system mbuf allocation. Tuning nmbclusters
 507         ** can alleviate this.
 508         */
 509         if (nmbclusters > 0 ) {
 510                 int s;
 511                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 512                 if (s > nmbclusters) {
 513                         device_printf(dev, "RX Descriptors exceed "
 514                             "system mbuf max, using default instead!\n");
 515                         ixgbe_rxd = DEFAULT_RXD;
 516                 }
 517         }
 518
 519         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 520             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 521                 device_printf(dev, "RXD config issue, using default!\n");
 522                 adapter->num_rx_desc = DEFAULT_RXD;
 523         } else
 524                 adapter->num_rx_desc = ixgbe_rxd;
 525
 526         /* Allocate our TX/RX Queues */
 527         if (ixgbe_allocate_queues(adapter)) {
 528                 error = ENOMEM;
 529                 goto err_out;
 530         }
 531
 532         /* Allocate multicast array memory. */
 533         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 534             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 535         if (adapter->mta == NULL) {
 536                 device_printf(dev, "Can not allocate multicast setup array\n");
 537                 error = ENOMEM;
 538                 goto err_late;
 539         }
 540
 541         /* Initialize the shared code */
 542         hw->allow_unsupported_sfp = allow_unsupported_sfp;
 543         error = ixgbe_init_shared_code(hw);
 544         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 545                 /*
 546                 ** No optics in this port, set up
 547                 ** so the timer routine will probe
 548                 ** for later insertion.
 549                 */
 550                 adapter->sfp_probe = TRUE;
 551                 error = 0;
 552         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 553                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 554                 error = EIO;
 555                 goto err_late;
 556         } else if (error) {
 557                 device_printf(dev,"Unable to initialize the shared code\n");
 558                 error = EIO;
 559                 goto err_late;
 560         }
 561
 562         /* Make sure we have a good EEPROM before we read from it */
 563         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 564                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 565                 error = EIO;
 566                 goto err_late;
 567         }
 568
 569         error = ixgbe_init_hw(hw);
 570         switch (error) {
 571         case IXGBE_ERR_EEPROM_VERSION:
 572                 device_printf(dev, "This device is a pre-production adapter/"
 573                     "LOM.  Please be aware there may be issues associated "
 574                     "with your hardware.\n If you are experiencing problems "
 575                     "please contact your Intel or hardware representative "
 576                     "who provided you with this hardware.\n");
 577                 break;
 578         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 579                 device_printf(dev,"Unsupported SFP+ Module\n");
 580                 error = EIO;
 581                 goto err_late;
 582         case IXGBE_ERR_SFP_NOT_PRESENT:
 583                 device_printf(dev,"No SFP+ Module found\n");
 584                 /* falls thru */
 585         default:
 586                 break;
 587         }
 588
 589         /* Detect and set physical type */
 590         ixgbe_setup_optics(adapter);
 591
 592         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 593                 error = ixgbe_allocate_msix(adapter);
 594         else
 595                 error = ixgbe_allocate_legacy(adapter);
 596         if (error)
 597                 goto err_late;
 598
 599         /* Setup OS specific network interface */
 600         if (ixgbe_setup_interface(dev, adapter) != 0)
 601                 goto err_late;
 602
 603         /* Initialize statistics */
 604         ixgbe_update_stats_counters(adapter);
 605
 606         /* Register for VLAN events */
 607         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 608             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 609         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 610             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 611
 612         /*
 613         ** Check PCIE slot type/speed/width
 614         */
 615         ixgbe_get_slot_info(hw);
 616
 617         /* Set an initial default flow control value */
 618         adapter->fc =  ixgbe_fc_full;
 619
 620         /* let hardware know driver is loaded */
 621         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 622         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 623         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 624
 625         ixgbe_add_hw_stats(adapter);
 626
 627 #ifdef DEV_NETMAP
 628         ixgbe_netmap_attach(adapter);
 629 #endif /* DEV_NETMAP */
 630         INIT_DEBUGOUT("ixgbe_attach: end");
 631         return (0);
 632 err_late:
 633         ixgbe_free_transmit_structures(adapter);
 634         ixgbe_free_receive_structures(adapter);
 635 err_out:
 636         if (adapter->ifp != NULL)
 637                 if_free(adapter->ifp);
 638         ixgbe_free_pci_resources(adapter);
 639         free(adapter->mta, M_DEVBUF);
 640         return (error);
 641
 642 }
 643
 644 /*********************************************************************
 645  *  Device removal routine
 646  *
 647  *  The detach entry point is called when the driver is being removed.
 648  *  This routine stops the adapter and deallocates all the resources
 649  *  that were allocated for driver operation.
 650  *
 651  *  return 0 on success, positive on failure
 652  *********************************************************************/
 653
 654 static int
 655 ixgbe_detach(device_t dev)
 656 {
 657         struct adapter *adapter = device_get_softc(dev);
 658         struct ix_queue *que = adapter->queues;
 659         struct tx_ring *txr = adapter->tx_rings;
 660         u32     ctrl_ext;
 661
 662         INIT_DEBUGOUT("ixgbe_detach: begin");
 663
 664         /* Make sure VLANS are not using driver */
 665         if (adapter->ifp->if_vlantrunk != NULL) {
 666                 device_printf(dev,"Vlan in use, detach first\n");
 667                 return (EBUSY);
 668         }
 669
 670         IXGBE_CORE_LOCK(adapter);
 671         ixgbe_stop(adapter);
 672         IXGBE_CORE_UNLOCK(adapter);
 673
 674         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 675                 if (que->tq) {
 676 #ifndef IXGBE_LEGACY_TX
 677                         taskqueue_drain(que->tq, &txr->txq_task);
 678 #endif
 679                         taskqueue_drain(que->tq, &que->que_task);
 680                         taskqueue_free(que->tq);
 681                 }
 682         }
 683
 684         /* Drain the Link queue */
 685         if (adapter->tq) {
 686                 taskqueue_drain(adapter->tq, &adapter->link_task);
 687                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 688                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 689 #ifdef IXGBE_FDIR
 690                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 691 #endif
 692                 taskqueue_free(adapter->tq);
 693         }
 694
 695         /* let hardware know driver is unloading */
 696         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 697         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 698         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 699
 700         /* Unregister VLAN events */
 701         if (adapter->vlan_attach != NULL)
 702                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 703         if (adapter->vlan_detach != NULL)
 704                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 705
 706         ether_ifdetach(adapter->ifp);
 707         callout_drain(&adapter->timer);
 708 #ifdef DEV_NETMAP
 709         netmap_detach(adapter->ifp);
 710 #endif /* DEV_NETMAP */
 711         ixgbe_free_pci_resources(adapter);
 712         bus_generic_detach(dev);
 713         if_free(adapter->ifp);
 714
 715         ixgbe_free_transmit_structures(adapter);
 716         ixgbe_free_receive_structures(adapter);
 717         free(adapter->mta, M_DEVBUF);
 718
 719         IXGBE_CORE_LOCK_DESTROY(adapter);
 720         return (0);
 721 }
 722
 723 /*********************************************************************
 724  *
 725  *  Shutdown entry point
 726  *
 727  **********************************************************************/
 728
 729 static int
 730 ixgbe_shutdown(device_t dev)
 731 {
 732         struct adapter *adapter = device_get_softc(dev);
 733         IXGBE_CORE_LOCK(adapter);
 734         ixgbe_stop(adapter);
 735         IXGBE_CORE_UNLOCK(adapter);
 736         return (0);
 737 }
 738
 739
 740 #ifdef IXGBE_LEGACY_TX
 741 /*********************************************************************
 742  *  Transmit entry point
 743  *
 744  *  ixgbe_start is called by the stack to initiate a transmit.
 745  *  The driver will remain in this routine as long as there are
 746  *  packets to transmit and transmit resources are available.
 747  *  In case resources are not available stack is notified and
 748  *  the packet is requeued.
 749  **********************************************************************/
 750
 751 static void
 752 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 753 {
 754         struct mbuf    *m_head;
 755         struct adapter *adapter = txr->adapter;
 756
 757         IXGBE_TX_LOCK_ASSERT(txr);
 758
 759         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 760                 return;
 761         if (!adapter->link_active)
 762                 return;
 763
 764         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 765                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 766                         break;
 767
 768                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 769                 if (m_head == NULL)
 770                         break;
 771
 772                 if (ixgbe_xmit(txr, &m_head)) {
 773                         if (m_head != NULL)
 774                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 775                         break;
 776                 }
 777                 /* Send a copy of the frame to the BPF listener */
 778                 ETHER_BPF_MTAP(ifp, m_head);
 779
 780                 /* Set watchdog on */
 781                 txr->watchdog_time = ticks;
 782                 txr->queue_status = IXGBE_QUEUE_WORKING;
 783
 784         }
 785         return;
 786 }
 787
 788 /*
 789  * Legacy TX start - called by the stack, this
 790  * always uses the first tx ring, and should
 791  * not be used with multiqueue tx enabled.
 792  */
 793 static void
 794 ixgbe_start(struct ifnet *ifp)
 795 {
 796         struct adapter *adapter = ifp->if_softc;
 797         struct tx_ring  *txr = adapter->tx_rings;
 798
 799         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 800                 IXGBE_TX_LOCK(txr);
 801                 ixgbe_start_locked(txr, ifp);
 802                 IXGBE_TX_UNLOCK(txr);
 803         }
 804         return;
 805 }
 806
 807 #else /* ! IXGBE_LEGACY_TX */
 808
 809 /*
 810 ** Multiqueue Transmit driver
 811 **
 812 */
 813 static int
 814 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 815 {
 816         struct adapter  *adapter = ifp->if_softc;
 817         struct ix_queue *que;
 818         struct tx_ring  *txr;
 819         int             i, err = 0;
 820
 821         /* Which queue to use */
 822         if ((m->m_flags & M_FLOWID) != 0)
 823                 i = m->m_pkthdr.flowid % adapter->num_queues;
 824         else
 825                 i = curcpu % adapter->num_queues;
 826
 827         txr = &adapter->tx_rings[i];
 828         que = &adapter->queues[i];
 829
 830         err = drbr_enqueue(ifp, txr->br, m);
 831         if (err)
 832                 return (err);
 833         if (IXGBE_TX_TRYLOCK(txr)) {
 834                 err = ixgbe_mq_start_locked(ifp, txr);
 835                 IXGBE_TX_UNLOCK(txr);
 836         } else
 837                 taskqueue_enqueue(que->tq, &txr->txq_task);
 838
 839         return (err);
 840 }
 841
 842 static int
 843 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 844 {
 845         struct adapter  *adapter = txr->adapter;
 846         struct mbuf     *next;
 847         int             enqueued = 0, err = 0;
 848
 849         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 850             adapter->link_active == 0)
 851                 return (ENETDOWN);
 852
 853         /* Process the queue */
 854 #if __FreeBSD_version < 901504
 855         next = drbr_dequeue(ifp, txr->br);
 856         while (next != NULL) {
 857                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 858                         if (next != NULL)
 859                                 err = drbr_enqueue(ifp, txr->br, next);
 860 #else
 861         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 862                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 863                         if (next == NULL) {
 864                                 drbr_advance(ifp, txr->br);
 865                         } else {
 866                                 drbr_putback(ifp, txr->br, next);
 867                         }
 868 #endif
 869                         break;
 870                 }
 871 #if __FreeBSD_version >= 901504
 872                 drbr_advance(ifp, txr->br);
 873 #endif
 874                 enqueued++;
 875                 /* Send a copy of the frame to the BPF listener */
 876                 ETHER_BPF_MTAP(ifp, next);
 877                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 878                         break;
 879 #if __FreeBSD_version < 901504
 880                 next = drbr_dequeue(ifp, txr->br);
 881 #endif
 882         }
 883
 884         if (enqueued > 0) {
 885                 /* Set watchdog on */
 886                 txr->queue_status = IXGBE_QUEUE_WORKING;
 887                 txr->watchdog_time = ticks;
 888         }
 889
 890         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 891                 ixgbe_txeof(txr);
 892
 893         return (err);
 894 }
 895
 896 /*
 897  * Called from a taskqueue to drain queued transmit packets.
 898  */
 899 static void
 900 ixgbe_deferred_mq_start(void *arg, int pending)
 901 {
 902         struct tx_ring *txr = arg;
 903         struct adapter *adapter = txr->adapter;
 904         struct ifnet *ifp = adapter->ifp;
 905
 906         IXGBE_TX_LOCK(txr);
 907         if (!drbr_empty(ifp, txr->br))
 908                 ixgbe_mq_start_locked(ifp, txr);
 909         IXGBE_TX_UNLOCK(txr);
 910 }
 911
 912 /*
 913 ** Flush all ring buffers
 914 */
 915 static void
 916 ixgbe_qflush(struct ifnet *ifp)
 917 {
 918         struct adapter  *adapter = ifp->if_softc;
 919         struct tx_ring  *txr = adapter->tx_rings;
 920         struct mbuf     *m;
 921
 922         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 923                 IXGBE_TX_LOCK(txr);
 924                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 925                         m_freem(m);
 926                 IXGBE_TX_UNLOCK(txr);
 927         }
 928         if_qflush(ifp);
 929 }
 930 #endif /* IXGBE_LEGACY_TX */
 931
 932 /*********************************************************************
 933  *  Ioctl entry point
 934  *
 935  *  ixgbe_ioctl is called when the user wants to configure the
 936  *  interface.
 937  *
 938  *  return 0 on success, positive on failure
 939  **********************************************************************/
 940
 941 static int
 942 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 943 {
 944         struct adapter  *adapter = ifp->if_softc;
 945         struct ixgbe_hw *hw = &adapter->hw;
 946         struct ifreq    *ifr = (struct ifreq *) data;
 947 #if defined(INET) || defined(INET6)
 948         struct ifaddr *ifa = (struct ifaddr *)data;
 949         bool            avoid_reset = FALSE;
 950 #endif
 951         int             error = 0;
 952
 953         switch (command) {
 954
 955         case SIOCSIFADDR:
 956 #ifdef INET
 957                 if (ifa->ifa_addr->sa_family == AF_INET)
 958                         avoid_reset = TRUE;
 959 #endif
 960 #ifdef INET6
 961                 if (ifa->ifa_addr->sa_family == AF_INET6)
 962                         avoid_reset = TRUE;
 963 #endif
 964 #if defined(INET) || defined(INET6)
 965                 /*
 966                 ** Calling init results in link renegotiation,
 967                 ** so we avoid doing it when possible.
 968                 */
 969                 if (avoid_reset) {
 970                         ifp->if_flags |= IFF_UP;
 971                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 972                                 ixgbe_init(adapter);
 973                         if (!(ifp->if_flags & IFF_NOARP))
 974                                 arp_ifinit(ifp, ifa);
 975                 } else
 976                         error = ether_ioctl(ifp, command, data);
 977 #endif
 978                 break;
 979         case SIOCSIFMTU:
 980                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 981                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 982                         error = EINVAL;
 983                 } else {
 984                         IXGBE_CORE_LOCK(adapter);
 985                         ifp->if_mtu = ifr->ifr_mtu;
 986                         adapter->max_frame_size =
 987                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 988                         ixgbe_init_locked(adapter);
 989                         IXGBE_CORE_UNLOCK(adapter);
 990                 }
 991                 break;
 992         case SIOCSIFFLAGS:
 993                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 994                 IXGBE_CORE_LOCK(adapter);
 995                 if (ifp->if_flags & IFF_UP) {
 996                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 997                                 if ((ifp->if_flags ^ adapter->if_flags) &
 998                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 999                                         ixgbe_set_promisc(adapter);
1000                                 }
1001                         } else
1002                                 ixgbe_init_locked(adapter);
1003                 } else
1004                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1005                                 ixgbe_stop(adapter);
1006                 adapter->if_flags = ifp->if_flags;
1007                 IXGBE_CORE_UNLOCK(adapter);
1008                 break;
1009         case SIOCADDMULTI:
1010         case SIOCDELMULTI:
1011                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1012                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1013                         IXGBE_CORE_LOCK(adapter);
1014                         ixgbe_disable_intr(adapter);
1015                         ixgbe_set_multi(adapter);
1016                         ixgbe_enable_intr(adapter);
1017                         IXGBE_CORE_UNLOCK(adapter);
1018                 }
1019                 break;
1020         case SIOCSIFMEDIA:
1021         case SIOCGIFMEDIA:
1022                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1023                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1024                 break;
1025         case SIOCSIFCAP:
1026         {
1027                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1028                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1029                 if (mask & IFCAP_HWCSUM)
1030                         ifp->if_capenable ^= IFCAP_HWCSUM;
1031                 if (mask & IFCAP_TSO4)
1032                         ifp->if_capenable ^= IFCAP_TSO4;
1033                 if (mask & IFCAP_TSO6)
1034                         ifp->if_capenable ^= IFCAP_TSO6;
1035                 if (mask & IFCAP_LRO)
1036                         ifp->if_capenable ^= IFCAP_LRO;
1037                 if (mask & IFCAP_VLAN_HWTAGGING)
1038                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1039                 if (mask & IFCAP_VLAN_HWFILTER)
1040                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1041                 if (mask & IFCAP_VLAN_HWTSO)
1042                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1043                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1044                         IXGBE_CORE_LOCK(adapter);
1045                         ixgbe_init_locked(adapter);
1046                         IXGBE_CORE_UNLOCK(adapter);
1047                 }
1048                 VLAN_CAPABILITIES(ifp);
1049                 break;
1050         }
1051         case SIOCGI2C:
1052         {
1053                 struct ixgbe_i2c_req    i2c;
1054                 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1055                 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1056                 if (error)
1057                         break;
1058                 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1059                         error = EINVAL;
1060                         break;
1061                 }
1062                 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1063                     i2c.dev_addr, i2c.data);
1064                 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1065                 break;
1066         }
1067         default:
1068                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1069                 error = ether_ioctl(ifp, command, data);
1070                 break;
1071         }
1072
1073         return (error);
1074 }
1075
1076 /*********************************************************************
1077  *  Init entry point
1078  *
1079  *  This routine is used in two ways. It is used by the stack as
1080  *  init entry point in network interface structure. It is also used
1081  *  by the driver as a hw/sw initialization routine to get to a
1082  *  consistent state.
1083  *
1084  *  return 0 on success, positive on failure
1085  **********************************************************************/
1086 #define IXGBE_MHADD_MFS_SHIFT 16
1087
1088 static void
1089 ixgbe_init_locked(struct adapter *adapter)
1090 {
1091         struct ifnet   *ifp = adapter->ifp;
1092         device_t        dev = adapter->dev;
1093         struct ixgbe_hw *hw = &adapter->hw;
1094         u32             k, txdctl, mhadd, gpie;
1095         u32             rxdctl, rxctrl;
1096
1097         mtx_assert(&adapter->core_mtx, MA_OWNED);
1098         INIT_DEBUGOUT("ixgbe_init_locked: begin");
1099         hw->adapter_stopped = FALSE;
1100         ixgbe_stop_adapter(hw);
1101         callout_stop(&adapter->timer);
1102
1103         /* reprogram the RAR[0] in case user changed it. */
1104         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1105
1106         /* Get the latest mac address, User can use a LAA */
1107         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1108               IXGBE_ETH_LENGTH_OF_ADDRESS);
1109         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1110         hw->addr_ctrl.rar_used_count = 1;
1111
1112         /* Set the various hardware offload abilities */
1113         ifp->if_hwassist = 0;
1114         if (ifp->if_capenable & IFCAP_TSO)
1115                 ifp->if_hwassist |= CSUM_TSO;
1116         if (ifp->if_capenable & IFCAP_TXCSUM) {
1117                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1118 #if __FreeBSD_version >= 800000
1119                 if (hw->mac.type != ixgbe_mac_82598EB)
1120                         ifp->if_hwassist |= CSUM_SCTP;
1121 #endif
1122         }
1123
1124         /* Prepare transmit descriptors and buffers */
1125         if (ixgbe_setup_transmit_structures(adapter)) {
1126                 device_printf(dev,"Could not setup transmit structures\n");
1127                 ixgbe_stop(adapter);
1128                 return;
1129         }
1130
1131         ixgbe_init_hw(hw);
1132         ixgbe_initialize_transmit_units(adapter);
1133
1134         /* Setup Multicast table */
1135         ixgbe_set_multi(adapter);
1136
1137         /*
1138         ** Determine the correct mbuf pool
1139         ** for doing jumbo frames
1140         */
1141         if (adapter->max_frame_size <= 2048)
1142                 adapter->rx_mbuf_sz = MCLBYTES;
1143         else if (adapter->max_frame_size <= 4096)
1144                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1145         else if (adapter->max_frame_size <= 9216)
1146                 adapter->rx_mbuf_sz = MJUM9BYTES;
1147         else
1148                 adapter->rx_mbuf_sz = MJUM16BYTES;
1149
1150         /* Prepare receive descriptors and buffers */
1151         if (ixgbe_setup_receive_structures(adapter)) {
1152                 device_printf(dev,"Could not setup receive structures\n");
1153                 ixgbe_stop(adapter);
1154                 return;
1155         }
1156
1157         /* Configure RX settings */
1158         ixgbe_initialize_receive_units(adapter);
1159
1160         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1161
1162         /* Enable Fan Failure Interrupt */
1163         gpie |= IXGBE_SDP1_GPIEN;
1164
1165         /* Add for Module detection */
1166         if (hw->mac.type == ixgbe_mac_82599EB)
1167                 gpie |= IXGBE_SDP2_GPIEN;
1168
1169         /* Thermal Failure Detection */
1170         if (hw->mac.type == ixgbe_mac_X540)
1171                 gpie |= IXGBE_SDP0_GPIEN;
1172
1173         if (adapter->msix > 1) {
1174                 /* Enable Enhanced MSIX mode */
1175                 gpie |= IXGBE_GPIE_MSIX_MODE;
1176                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1177                     IXGBE_GPIE_OCD;
1178         }
1179         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1180
1181         /* Set MTU size */
1182         if (ifp->if_mtu > ETHERMTU) {
1183                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1184                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1185                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1186                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1187         }
1188
1189         /* Now enable all the queues */
1190
1191         for (int i = 0; i < adapter->num_queues; i++) {
1192                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1193                 txdctl |= IXGBE_TXDCTL_ENABLE;
1194                 /* Set WTHRESH to 8, burst writeback */
1195                 txdctl |= (8 << 16);
1196                 /*
1197                  * When the internal queue falls below PTHRESH (32),
1198                  * start prefetching as long as there are at least
1199                  * HTHRESH (1) buffers ready. The values are taken
1200                  * from the Intel linux driver 3.8.21.
1201                  * Prefetching enables tx line rate even with 1 queue.
1202                  */
1203                 txdctl |= (32 << 0) | (1 << 8);
1204                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1205         }
1206
1207         for (int i = 0; i < adapter->num_queues; i++) {
1208                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1209                 if (hw->mac.type == ixgbe_mac_82598EB) {
1210                         /*
1211                         ** PTHRESH = 21
1212                         ** HTHRESH = 4
1213                         ** WTHRESH = 8
1214                         */
1215                         rxdctl &= ~0x3FFFFF;
1216                         rxdctl |= 0x080420;
1217                 }
1218                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1219                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1220                 for (k = 0; k < 10; k++) {
1221                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1222                             IXGBE_RXDCTL_ENABLE)
1223                                 break;
1224                         else
1225                                 msec_delay(1);
1226                 }
1227                 wmb();
1228 #ifdef DEV_NETMAP
1229                 /*
1230                  * In netmap mode, we must preserve the buffers made
1231                  * available to userspace before the if_init()
1232                  * (this is true by default on the TX side, because
1233                  * init makes all buffers available to userspace).
1234                  *
1235                  * netmap_reset() and the device specific routines
1236                  * (e.g. ixgbe_setup_receive_rings()) map these
1237                  * buffers at the end of the NIC ring, so here we
1238                  * must set the RDT (tail) register to make sure
1239                  * they are not overwritten.
1240                  *
1241                  * In this driver the NIC ring starts at RDH = 0,
1242                  * RDT points to the last slot available for reception (?),
1243                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1244                  */
1245                 if (ifp->if_capenable & IFCAP_NETMAP) {
1246                         struct netmap_adapter *na = NA(adapter->ifp);
1247                         struct netmap_kring *kring = &na->rx_rings[i];
1248                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1249
1250                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1251                 } else
1252 #endif /* DEV_NETMAP */
1253                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1254         }
1255
1256         /* Enable Receive engine */
1257         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1258         if (hw->mac.type == ixgbe_mac_82598EB)
1259                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1260         rxctrl |= IXGBE_RXCTRL_RXEN;
1261         ixgbe_enable_rx_dma(hw, rxctrl);
1262
1263         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1264
1265         /* Set up MSI/X routing */
1266         if (ixgbe_enable_msix)  {
1267                 ixgbe_configure_ivars(adapter);
1268                 /* Set up auto-mask */
1269                 if (hw->mac.type == ixgbe_mac_82598EB)
1270                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1271                 else {
1272                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1273                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1274                 }
1275         } else {  /* Simple settings for Legacy/MSI */
1276                 ixgbe_set_ivar(adapter, 0, 0, 0);
1277                 ixgbe_set_ivar(adapter, 0, 0, 1);
1278                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1279         }
1280
1281 #ifdef IXGBE_FDIR
1282         /* Init Flow director */
1283         if (hw->mac.type != ixgbe_mac_82598EB) {
1284                 u32 hdrm = 32 << fdir_pballoc;
1285
1286                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1287                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1288         }
1289 #endif
1290
1291         /*
1292         ** Check on any SFP devices that
1293         ** need to be kick-started
1294         */
1295         if (hw->phy.type == ixgbe_phy_none) {
1296                 int err = hw->phy.ops.identify(hw);
1297                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1298                         device_printf(dev,
1299                             "Unsupported SFP+ module type was detected.\n");
1300                         return;
1301                 }
1302         }
1303
1304         /* Set moderation on the Link interrupt */
1305         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1306
1307         /* Config/Enable Link */
1308         ixgbe_config_link(adapter);
1309
1310         /* Hardware Packet Buffer & Flow Control setup */
1311         {
1312                 u32 rxpb, frame, size, tmp;
1313
1314                 frame = adapter->max_frame_size;
1315
1316                 /* Calculate High Water */
1317                 if (hw->mac.type == ixgbe_mac_X540)
1318                         tmp = IXGBE_DV_X540(frame, frame);
1319                 else
1320                         tmp = IXGBE_DV(frame, frame);
1321                 size = IXGBE_BT2KB(tmp);
1322                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1323                 hw->fc.high_water[0] = rxpb - size;
1324
1325                 /* Now calculate Low Water */
1326                 if (hw->mac.type == ixgbe_mac_X540)
1327                         tmp = IXGBE_LOW_DV_X540(frame);
1328                 else
1329                         tmp = IXGBE_LOW_DV(frame);
1330                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1331
1332                 hw->fc.requested_mode = adapter->fc;
1333                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1334                 hw->fc.send_xon = TRUE;
1335         }
1336         /* Initialize the FC settings */
1337         ixgbe_start_hw(hw);
1338
1339         /* Set up VLAN support and filter */
1340         ixgbe_setup_vlan_hw_support(adapter);
1341
1342         /* And now turn on interrupts */
1343         ixgbe_enable_intr(adapter);
1344
1345         /* Now inform the stack we're ready */
1346         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1347
1348         return;
1349 }
1350
1351 static void
1352 ixgbe_init(void *arg)
1353 {
1354         struct adapter *adapter = arg;
1355
1356         IXGBE_CORE_LOCK(adapter);
1357         ixgbe_init_locked(adapter);
1358         IXGBE_CORE_UNLOCK(adapter);
1359         return;
1360 }
1361
1362
1363 /*
1364 **
1365 ** MSIX Interrupt Handlers and Tasklets
1366 **
1367 */
1368
1369 static inline void
1370 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1371 {
1372         struct ixgbe_hw *hw = &adapter->hw;
1373         u64     queue = (u64)(1 << vector);
1374         u32     mask;
1375
1376         if (hw->mac.type == ixgbe_mac_82598EB) {
1377                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1379         } else {
1380                 mask = (queue & 0xFFFFFFFF);
1381                 if (mask)
1382                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1383                 mask = (queue >> 32);
1384                 if (mask)
1385                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1386         }
1387 }
1388
1389 static inline void
1390 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1391 {
1392         struct ixgbe_hw *hw = &adapter->hw;
1393         u64     queue = (u64)(1 << vector);
1394         u32     mask;
1395
1396         if (hw->mac.type == ixgbe_mac_82598EB) {
1397                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1398                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1399         } else {
1400                 mask = (queue & 0xFFFFFFFF);
1401                 if (mask)
1402                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1403                 mask = (queue >> 32);
1404                 if (mask)
1405                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1406         }
1407 }
1408
1409 static void
1410 ixgbe_handle_que(void *context, int pending)
1411 {
1412         struct ix_queue *que = context;
1413         struct adapter  *adapter = que->adapter;
1414         struct tx_ring  *txr = que->txr;
1415         struct ifnet    *ifp = adapter->ifp;
1416         bool            more;
1417
1418         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1419                 more = ixgbe_rxeof(que);
1420                 IXGBE_TX_LOCK(txr);
1421                 ixgbe_txeof(txr);
1422 #ifndef IXGBE_LEGACY_TX
1423                 if (!drbr_empty(ifp, txr->br))
1424                         ixgbe_mq_start_locked(ifp, txr);
1425 #else
1426                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1427                         ixgbe_start_locked(txr, ifp);
1428 #endif
1429                 IXGBE_TX_UNLOCK(txr);
1430         }
1431
1432         /* Reenable this interrupt */
1433         if (que->res != NULL)
1434                 ixgbe_enable_queue(adapter, que->msix);
1435         else
1436                 ixgbe_enable_intr(adapter);
1437         return;
1438 }
1439
1440
1441 /*********************************************************************
1442  *
1443  *  Legacy Interrupt Service routine
1444  *
1445  **********************************************************************/
1446
1447 static void
1448 ixgbe_legacy_irq(void *arg)
1449 {
1450         struct ix_queue *que = arg;
1451         struct adapter  *adapter = que->adapter;
1452         struct ixgbe_hw *hw = &adapter->hw;
1453         struct ifnet    *ifp = adapter->ifp;
1454         struct          tx_ring *txr = adapter->tx_rings;
1455         bool            more;
1456         u32             reg_eicr;
1457
1458
1459         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1460
1461         ++que->irqs;
1462         if (reg_eicr == 0) {
1463                 ixgbe_enable_intr(adapter);
1464                 return;
1465         }
1466
1467         more = ixgbe_rxeof(que);
1468
1469         IXGBE_TX_LOCK(txr);
1470         ixgbe_txeof(txr);
1471 #ifdef IXGBE_LEGACY_TX
1472         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1473                 ixgbe_start_locked(txr, ifp);
1474 #else
1475         if (!drbr_empty(ifp, txr->br))
1476                 ixgbe_mq_start_locked(ifp, txr);
1477 #endif
1478         IXGBE_TX_UNLOCK(txr);
1479
1480         /* Check for fan failure */
1481         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1482             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1483                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1484                     "REPLACE IMMEDIATELY!!\n");
1485                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1486         }
1487
1488         /* Link status change */
1489         if (reg_eicr & IXGBE_EICR_LSC)
1490                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1491
1492         if (more)
1493                 taskqueue_enqueue(que->tq, &que->que_task);
1494         else
1495                 ixgbe_enable_intr(adapter);
1496         return;
1497 }
1498
1499
1500 /*********************************************************************
1501  *
1502  *  MSIX Queue Interrupt Service routine
1503  *
1504  **********************************************************************/
1505 void
1506 ixgbe_msix_que(void *arg)
1507 {
1508         struct ix_queue *que = arg;
1509         struct adapter  *adapter = que->adapter;
1510         struct ifnet    *ifp = adapter->ifp;
1511         struct tx_ring  *txr = que->txr;
1512         struct rx_ring  *rxr = que->rxr;
1513         bool            more;
1514         u32             newitr = 0;
1515
1516         /* Protect against spurious interrupts */
1517         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1518                 return;
1519
1520         ixgbe_disable_queue(adapter, que->msix);
1521         ++que->irqs;
1522
1523         more = ixgbe_rxeof(que);
1524
1525         IXGBE_TX_LOCK(txr);
1526         ixgbe_txeof(txr);
1527 #ifdef IXGBE_LEGACY_TX
1528         if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1529                 ixgbe_start_locked(txr, ifp);
1530 #else
1531         if (!drbr_empty(ifp, txr->br))
1532                 ixgbe_mq_start_locked(ifp, txr);
1533 #endif
1534         IXGBE_TX_UNLOCK(txr);
1535
1536         /* Do AIM now? */
1537
1538         if (ixgbe_enable_aim == FALSE)
1539                 goto no_calc;
1540         /*
1541         ** Do Adaptive Interrupt Moderation:
1542         **  - Write out last calculated setting
1543         **  - Calculate based on average size over
1544         **    the last interval.
1545         */
1546         if (que->eitr_setting)
1547                 IXGBE_WRITE_REG(&adapter->hw,
1548                     IXGBE_EITR(que->msix), que->eitr_setting);
1549
1550         que->eitr_setting = 0;
1551
1552         /* Idle, do nothing */
1553         if ((txr->bytes == 0) && (rxr->bytes == 0))
1554                 goto no_calc;
1555
1556         if ((txr->bytes) && (txr->packets))
1557                 newitr = txr->bytes/txr->packets;
1558         if ((rxr->bytes) && (rxr->packets))
1559                 newitr = max(newitr,
1560                     (rxr->bytes / rxr->packets));
1561         newitr += 24; /* account for hardware frame, crc */
1562
1563         /* set an upper boundary */
1564         newitr = min(newitr, 3000);
1565
1566         /* Be nice to the mid range */
1567         if ((newitr > 300) && (newitr < 1200))
1568                 newitr = (newitr / 3);
1569         else
1570                 newitr = (newitr / 2);
1571
1572         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1573                 newitr |= newitr << 16;
1574         else
1575                 newitr |= IXGBE_EITR_CNT_WDIS;
1576
1577         /* save for next interrupt */
1578         que->eitr_setting = newitr;
1579
1580         /* Reset state */
1581         txr->bytes = 0;
1582         txr->packets = 0;
1583         rxr->bytes = 0;
1584         rxr->packets = 0;
1585
1586 no_calc:
1587         if (more)
1588                 taskqueue_enqueue(que->tq, &que->que_task);
1589         else
1590                 ixgbe_enable_queue(adapter, que->msix);
1591         return;
1592 }
1593
1594
1595 static void
1596 ixgbe_msix_link(void *arg)
1597 {
1598         struct adapter  *adapter = arg;
1599         struct ixgbe_hw *hw = &adapter->hw;
1600         u32             reg_eicr;
1601
1602         ++adapter->link_irq;
1603
1604         /* First get the cause */
1605         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1606         /* Be sure the queue bits are not cleared */
1607         reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1608         /* Clear interrupt with write */
1609         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1610
1611         /* Link status change */
1612         if (reg_eicr & IXGBE_EICR_LSC)
1613                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1614
1615         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1616 #ifdef IXGBE_FDIR
1617                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1618                         /* This is probably overkill :) */
1619                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1620                                 return;
1621                         /* Disable the interrupt */
1622                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1623                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1624                 } else
1625 #endif
1626                 if (reg_eicr & IXGBE_EICR_ECC) {
1627                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1628                             "Please Reboot!!\n");
1629                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1630                 } else
1631
1632                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1633                         /* Clear the interrupt */
1634                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1635                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1636                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1637                         /* Clear the interrupt */
1638                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1639                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1640                 }
1641         }
1642
1643         /* Check for fan failure */
1644         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1645             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1646                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1647                     "REPLACE IMMEDIATELY!!\n");
1648                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1649         }
1650
1651         /* Check for over temp condition */
1652         if ((hw->mac.type == ixgbe_mac_X540) &&
1653             (reg_eicr & IXGBE_EICR_TS)) {
1654                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1655                     "PHY IS SHUT DOWN!!\n");
1656                 device_printf(adapter->dev, "System shutdown required\n");
1657                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1658         }
1659
1660         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1661         return;
1662 }
1663
1664 /*********************************************************************
1665  *
1666  *  Media Ioctl callback
1667  *
1668  *  This routine is called whenever the user queries the status of
1669  *  the interface using ifconfig.
1670  *
1671  **********************************************************************/
1672 static void
1673 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1674 {
1675         struct adapter *adapter = ifp->if_softc;
1676
1677         INIT_DEBUGOUT("ixgbe_media_status: begin");
1678         IXGBE_CORE_LOCK(adapter);
1679         ixgbe_update_link_status(adapter);
1680
1681         ifmr->ifm_status = IFM_AVALID;
1682         ifmr->ifm_active = IFM_ETHER;
1683
1684         if (!adapter->link_active) {
1685                 IXGBE_CORE_UNLOCK(adapter);
1686                 return;
1687         }
1688
1689         ifmr->ifm_status |= IFM_ACTIVE;
1690
1691         switch (adapter->link_speed) {
1692                 case IXGBE_LINK_SPEED_100_FULL:
1693                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1694                         break;
1695                 case IXGBE_LINK_SPEED_1GB_FULL:
1696                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1697                         break;
1698                 case IXGBE_LINK_SPEED_10GB_FULL:
1699                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1700                         break;
1701         }
1702
1703         IXGBE_CORE_UNLOCK(adapter);
1704
1705         return;
1706 }
1707
1708 /*********************************************************************
1709  *
1710  *  Media Ioctl callback
1711  *
1712  *  This routine is called when the user changes speed/duplex using
1713  *  media/mediopt option with ifconfig.
1714  *
1715  **********************************************************************/
1716 static int
1717 ixgbe_media_change(struct ifnet * ifp)
1718 {
1719         struct adapter *adapter = ifp->if_softc;
1720         struct ifmedia *ifm = &adapter->media;
1721
1722         INIT_DEBUGOUT("ixgbe_media_change: begin");
1723
1724         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1725                 return (EINVAL);
1726
1727         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1728         case IFM_AUTO:
1729                 adapter->hw.phy.autoneg_advertised =
1730                     IXGBE_LINK_SPEED_100_FULL |
1731                     IXGBE_LINK_SPEED_1GB_FULL |
1732                     IXGBE_LINK_SPEED_10GB_FULL;
1733                 break;
1734         default:
1735                 device_printf(adapter->dev, "Only auto media type\n");
1736                 return (EINVAL);
1737         }
1738
1739         return (0);
1740 }
1741
1742 /*********************************************************************
1743  *
1744  *  This routine maps the mbufs to tx descriptors, allowing the
1745  *  TX engine to transmit the packets.
1746  *      - return 0 on success, positive on failure
1747  *
1748  **********************************************************************/
1749
1750 static int
1751 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1752 {
1753         struct adapter  *adapter = txr->adapter;
1754         u32             olinfo_status = 0, cmd_type_len;
1755         int             i, j, error, nsegs;
1756         int             first;
1757         bool            remap = TRUE;
1758         struct mbuf     *m_head;
1759         bus_dma_segment_t segs[adapter->num_segs];
1760         bus_dmamap_t    map;
1761         struct ixgbe_tx_buf *txbuf;
1762         union ixgbe_adv_tx_desc *txd = NULL;
1763
1764         m_head = *m_headp;
1765
1766         /* Basic descriptor defines */
1767         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1768             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1769
1770         if (m_head->m_flags & M_VLANTAG)
1771                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1772
1773         /*
1774          * Important to capture the first descriptor
1775          * used because it will contain the index of
1776          * the one we tell the hardware to report back
1777          */
1778         first = txr->next_avail_desc;
1779         txbuf = &txr->tx_buffers[first];
1780         map = txbuf->map;
1781
1782         /*
1783          * Map the packet for DMA.
1784          */
1785 retry:
1786         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1787             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1788
1789         if (__predict_false(error)) {
1790                 struct mbuf *m;
1791
1792                 switch (error) {
1793                 case EFBIG:
1794                         /* Try it again? - one try */
1795                         if (remap == TRUE) {
1796                                 remap = FALSE;
1797                                 m = m_defrag(*m_headp, M_NOWAIT);
1798                                 if (m == NULL) {
1799                                         adapter->mbuf_defrag_failed++;
1800                                         m_freem(*m_headp);
1801                                         *m_headp = NULL;
1802                                         return (ENOBUFS);
1803                                 }
1804                                 *m_headp = m;
1805                                 goto retry;
1806                         } else
1807                                 return (error);
1808                 case ENOMEM:
1809                         txr->no_tx_dma_setup++;
1810                         return (error);
1811                 default:
1812                         txr->no_tx_dma_setup++;
1813                         m_freem(*m_headp);
1814                         *m_headp = NULL;
1815                         return (error);
1816                 }
1817         }
1818
1819         /* Make certain there are enough descriptors */
1820         if (nsegs > txr->tx_avail - 2) {
1821                 txr->no_desc_avail++;
1822                 bus_dmamap_unload(txr->txtag, map);
1823                 return (ENOBUFS);
1824         }
1825         m_head = *m_headp;
1826
1827         /*
1828         ** Set up the appropriate offload context
1829         ** this will consume the first descriptor
1830         */
1831         error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1832         if (__predict_false(error)) {
1833                 if (error == ENOBUFS)
1834                         *m_headp = NULL;
1835                 return (error);
1836         }
1837
1838 #ifdef IXGBE_FDIR
1839         /* Do the flow director magic */
1840         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1841                 ++txr->atr_count;
1842                 if (txr->atr_count >= atr_sample_rate) {
1843                         ixgbe_atr(txr, m_head);
1844                         txr->atr_count = 0;
1845                 }
1846         }
1847 #endif
1848
1849         i = txr->next_avail_desc;
1850         for (j = 0; j < nsegs; j++) {
1851                 bus_size_t seglen;
1852                 bus_addr_t segaddr;
1853
1854                 txbuf = &txr->tx_buffers[i];
1855                 txd = &txr->tx_base[i];
1856                 seglen = segs[j].ds_len;
1857                 segaddr = htole64(segs[j].ds_addr);
1858
1859                 txd->read.buffer_addr = segaddr;
1860                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1861                     cmd_type_len |seglen);
1862                 txd->read.olinfo_status = htole32(olinfo_status);
1863
1864                 if (++i == txr->num_desc)
1865                         i = 0;
1866         }
1867
1868         txd->read.cmd_type_len |=
1869             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1870         txr->tx_avail -= nsegs;
1871         txr->next_avail_desc = i;
1872
1873         txbuf->m_head = m_head;
1874         /*
1875         ** Here we swap the map so the last descriptor,
1876         ** which gets the completion interrupt has the
1877         ** real map, and the first descriptor gets the
1878         ** unused map from this descriptor.
1879         */
1880         txr->tx_buffers[first].map = txbuf->map;
1881         txbuf->map = map;
1882         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1883
1884         /* Set the EOP descriptor that will be marked done */
1885         txbuf = &txr->tx_buffers[first];
1886         txbuf->eop = txd;
1887
1888         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1889             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1890         /*
1891          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1892          * hardware that this frame is available to transmit.
1893          */
1894         ++txr->total_packets;
1895         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1896
1897         return (0);
1898
1899 }
1900
1901 static void
1902 ixgbe_set_promisc(struct adapter *adapter)
1903 {
1904         u_int32_t       reg_rctl;
1905         struct ifnet   *ifp = adapter->ifp;
1906         int             mcnt = 0;
1907
1908         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1909         reg_rctl &= (~IXGBE_FCTRL_UPE);
1910         if (ifp->if_flags & IFF_ALLMULTI)
1911                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1912         else {
1913                 struct  ifmultiaddr *ifma;
1914 #if __FreeBSD_version < 800000
1915                 IF_ADDR_LOCK(ifp);
1916 #else
1917                 if_maddr_rlock(ifp);
1918 #endif
1919                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1920                         if (ifma->ifma_addr->sa_family != AF_LINK)
1921                                 continue;
1922                         if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1923                                 break;
1924                         mcnt++;
1925                 }
1926 #if __FreeBSD_version < 800000
1927                 IF_ADDR_UNLOCK(ifp);
1928 #else
1929                 if_maddr_runlock(ifp);
1930 #endif
1931         }
1932         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1933                 reg_rctl &= (~IXGBE_FCTRL_MPE);
1934         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1935
1936         if (ifp->if_flags & IFF_PROMISC) {
1937                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1938                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1939         } else if (ifp->if_flags & IFF_ALLMULTI) {
1940                 reg_rctl |= IXGBE_FCTRL_MPE;
1941                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1942                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1943         }
1944         return;
1945 }
1946
1947
1948 /*********************************************************************
1949  *  Multicast Update
1950  *
1951  *  This routine is called whenever multicast address list is updated.
1952  *
1953  **********************************************************************/
1954 #define IXGBE_RAR_ENTRIES 16
1955
1956 static void
1957 ixgbe_set_multi(struct adapter *adapter)
1958 {
1959         u32     fctrl;
1960         u8      *mta;
1961         u8      *update_ptr;
1962         struct  ifmultiaddr *ifma;
1963         int     mcnt = 0;
1964         struct ifnet   *ifp = adapter->ifp;
1965
1966         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1967
1968         mta = adapter->mta;
1969         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1970             MAX_NUM_MULTICAST_ADDRESSES);
1971
1972 #if __FreeBSD_version < 800000
1973         IF_ADDR_LOCK(ifp);
1974 #else
1975         if_maddr_rlock(ifp);
1976 #endif
1977         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1978                 if (ifma->ifma_addr->sa_family != AF_LINK)
1979                         continue;
1980                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1981                         break;
1982                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1983                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1984                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1985                 mcnt++;
1986         }
1987 #if __FreeBSD_version < 800000
1988         IF_ADDR_UNLOCK(ifp);
1989 #else
1990         if_maddr_runlock(ifp);
1991 #endif
1992
1993         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1994         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1995         if (ifp->if_flags & IFF_PROMISC)
1996                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1997         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1998             ifp->if_flags & IFF_ALLMULTI) {
1999                 fctrl |= IXGBE_FCTRL_MPE;
2000                 fctrl &= ~IXGBE_FCTRL_UPE;
2001         } else
2002                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2003
2004         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
2005
2006         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
2007                 update_ptr = mta;
2008                 ixgbe_update_mc_addr_list(&adapter->hw,
2009                     update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
2010         }
2011
2012         return;
2013 }
2014
2015 /*
2016  * This is an iterator function now needed by the multicast
2017  * shared code. It simply feeds the shared code routine the
2018  * addresses in the array of ixgbe_set_multi() one by one.
2019  */
2020 static u8 *
2021 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2022 {
2023         u8 *addr = *update_ptr;
2024         u8 *newptr;
2025         *vmdq = 0;
2026
2027         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2028         *update_ptr = newptr;
2029         return addr;
2030 }
2031
2032
2033 /*********************************************************************
2034  *  Timer routine
2035  *
2036  *  This routine checks for link status,updates statistics,
2037  *  and runs the watchdog check.
2038  *
2039  **********************************************************************/
2040
2041 static void
2042 ixgbe_local_timer(void *arg)
2043 {
2044         struct adapter  *adapter = arg;
2045         device_t        dev = adapter->dev;
2046         struct ix_queue *que = adapter->queues;
2047         struct tx_ring  *txr = adapter->tx_rings;
2048         int             hung = 0, paused = 0;
2049
2050         mtx_assert(&adapter->core_mtx, MA_OWNED);
2051
2052         /* Check for pluggable optics */
2053         if (adapter->sfp_probe)
2054                 if (!ixgbe_sfp_probe(adapter))
2055                         goto out; /* Nothing to do */
2056
2057         ixgbe_update_link_status(adapter);
2058         ixgbe_update_stats_counters(adapter);
2059
2060         /*
2061          * If the interface has been paused
2062          * then don't do the watchdog check
2063          */
2064         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2065                 paused = 1;
2066
2067         /*
2068         ** Check the TX queues status
2069         **      - watchdog only if all queues show hung
2070         */
2071         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2072                 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2073                     (paused == 0))
2074                         ++hung;
2075                 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2076                         taskqueue_enqueue(que->tq, &txr->txq_task);
2077         }
2078         /* Only truely watchdog if all queues show hung */
2079         if (hung == adapter->num_queues)
2080                 goto watchdog;
2081
2082 out:
2083         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2084         return;
2085
2086 watchdog:
2087         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2088         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2089             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2090             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2091         device_printf(dev,"TX(%d) desc avail = %d,"
2092             "Next TX to Clean = %d\n",
2093             txr->me, txr->tx_avail, txr->next_to_clean);
2094         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2095         adapter->watchdog_events++;
2096         ixgbe_init_locked(adapter);
2097 }
2098
2099 /*
2100 ** Note: this routine updates the OS on the link state
2101 **      the real check of the hardware only happens with
2102 **      a link interrupt.
2103 */
2104 static void
2105 ixgbe_update_link_status(struct adapter *adapter)
2106 {
2107         struct ifnet    *ifp = adapter->ifp;
2108         device_t dev = adapter->dev;
2109
2110
2111         if (adapter->link_up){
2112                 if (adapter->link_active == FALSE) {
2113                         if (bootverbose)
2114                                 device_printf(dev,"Link is up %d Gbps %s \n",
2115                                     ((adapter->link_speed == 128)? 10:1),
2116                                     "Full Duplex");
2117                         adapter->link_active = TRUE;
2118                         /* Update any Flow Control changes */
2119                         ixgbe_fc_enable(&adapter->hw);
2120                         if_link_state_change(ifp, LINK_STATE_UP);
2121                 }
2122         } else { /* Link down */
2123                 if (adapter->link_active == TRUE) {
2124                         if (bootverbose)
2125                                 device_printf(dev,"Link is Down\n");
2126                         if_link_state_change(ifp, LINK_STATE_DOWN);
2127                         adapter->link_active = FALSE;
2128                 }
2129         }
2130
2131         return;
2132 }
2133
2134
2135 /*********************************************************************
2136  *
2137  *  This routine disables all traffic on the adapter by issuing a
2138  *  global reset on the MAC and deallocates TX/RX buffers.
2139  *
2140  **********************************************************************/
2141
2142 static void
2143 ixgbe_stop(void *arg)
2144 {
2145         struct ifnet   *ifp;
2146         struct adapter *adapter = arg;
2147         struct ixgbe_hw *hw = &adapter->hw;
2148         ifp = adapter->ifp;
2149
2150         mtx_assert(&adapter->core_mtx, MA_OWNED);
2151
2152         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2153         ixgbe_disable_intr(adapter);
2154         callout_stop(&adapter->timer);
2155
2156         /* Let the stack know...*/
2157         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2158
2159         ixgbe_reset_hw(hw);
2160         hw->adapter_stopped = FALSE;
2161         ixgbe_stop_adapter(hw);
2162         if (hw->mac.type == ixgbe_mac_82599EB)
2163                 ixgbe_stop_mac_link_on_d3_82599(hw);
2164         /* Turn off the laser - noop with no optics */
2165         ixgbe_disable_tx_laser(hw);
2166
2167         /* Update the stack */
2168         adapter->link_up = FALSE;
2169         ixgbe_update_link_status(adapter);
2170
2171         /* reprogram the RAR[0] in case user changed it. */
2172         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2173
2174         return;
2175 }
2176
2177
2178 /*********************************************************************
2179  *
2180  *  Determine hardware revision.
2181  *
2182  **********************************************************************/
2183 static void
2184 ixgbe_identify_hardware(struct adapter *adapter)
2185 {
2186         device_t        dev = adapter->dev;
2187         struct ixgbe_hw *hw = &adapter->hw;
2188
2189         /* Save off the information about this board */
2190         hw->vendor_id = pci_get_vendor(dev);
2191         hw->device_id = pci_get_device(dev);
2192         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2193         hw->subsystem_vendor_id =
2194             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2195         hw->subsystem_device_id =
2196             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2197
2198         /* We need this here to set the num_segs below */
2199         ixgbe_set_mac_type(hw);
2200
2201         /* Pick up the 82599 and VF settings */
2202         if (hw->mac.type != ixgbe_mac_82598EB) {
2203                 hw->phy.smart_speed = ixgbe_smart_speed;
2204                 adapter->num_segs = IXGBE_82599_SCATTER;
2205         } else
2206                 adapter->num_segs = IXGBE_82598_SCATTER;
2207
2208         return;
2209 }
2210
2211 /*********************************************************************
2212  *
2213  *  Determine optic type
2214  *
2215  **********************************************************************/
2216 static void
2217 ixgbe_setup_optics(struct adapter *adapter)
2218 {
2219         struct ixgbe_hw *hw = &adapter->hw;
2220         int             layer;
2221
2222         layer = ixgbe_get_supported_physical_layer(hw);
2223
2224         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2225                 adapter->optics = IFM_10G_T;
2226                 return;
2227         }
2228
2229         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2230                 adapter->optics = IFM_1000_T;
2231                 return;
2232         }
2233
2234         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2235                 adapter->optics = IFM_1000_SX;
2236                 return;
2237         }
2238
2239         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2240             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2241                 adapter->optics = IFM_10G_LR;
2242                 return;
2243         }
2244
2245         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2246                 adapter->optics = IFM_10G_SR;
2247                 return;
2248         }
2249
2250         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2251                 adapter->optics = IFM_10G_TWINAX;
2252                 return;
2253         }
2254
2255         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2256             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2257                 adapter->optics = IFM_10G_CX4;
2258                 return;
2259         }
2260
2261         /* If we get here just set the default */
2262         adapter->optics = IFM_ETHER | IFM_AUTO;
2263         return;
2264 }
2265
2266 /*********************************************************************
2267  *
2268  *  Setup the Legacy or MSI Interrupt handler
2269  *
2270  **********************************************************************/
2271 static int
2272 ixgbe_allocate_legacy(struct adapter *adapter)
2273 {
2274         device_t        dev = adapter->dev;
2275         struct          ix_queue *que = adapter->queues;
2276 #ifndef IXGBE_LEGACY_TX
2277         struct tx_ring          *txr = adapter->tx_rings;
2278 #endif
2279         int             error, rid = 0;
2280
2281         /* MSI RID at 1 */
2282         if (adapter->msix == 1)
2283                 rid = 1;
2284
2285         /* We allocate a single interrupt resource */
2286         adapter->res = bus_alloc_resource_any(dev,
2287             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2288         if (adapter->res == NULL) {
2289                 device_printf(dev, "Unable to allocate bus resource: "
2290                     "interrupt\n");
2291                 return (ENXIO);
2292         }
2293
2294         /*
2295          * Try allocating a fast interrupt and the associated deferred
2296          * processing contexts.
2297          */
2298 #ifndef IXGBE_LEGACY_TX
2299         TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2300 #endif
2301         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2302         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2303             taskqueue_thread_enqueue, &que->tq);
2304         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2305             device_get_nameunit(adapter->dev));
2306
2307         /* Tasklets for Link, SFP and Multispeed Fiber */
2308         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2309         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2310         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2311 #ifdef IXGBE_FDIR
2312         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2313 #endif
2314         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2315             taskqueue_thread_enqueue, &adapter->tq);
2316         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2317             device_get_nameunit(adapter->dev));
2318
2319         if ((error = bus_setup_intr(dev, adapter->res,
2320             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2321             que, &adapter->tag)) != 0) {
2322                 device_printf(dev, "Failed to register fast interrupt "
2323                     "handler: %d\n", error);
2324                 taskqueue_free(que->tq);
2325                 taskqueue_free(adapter->tq);
2326                 que->tq = NULL;
2327                 adapter->tq = NULL;
2328                 return (error);
2329         }
2330         /* For simplicity in the handlers */
2331         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2332
2333         return (0);
2334 }
2335
2336
2337 /*********************************************************************
2338  *
2339  *  Setup MSIX Interrupt resources and handlers
2340  *
2341  **********************************************************************/
2342 static int
2343 ixgbe_allocate_msix(struct adapter *adapter)
2344 {
2345         device_t        dev = adapter->dev;
2346         struct          ix_queue *que = adapter->queues;
2347         struct          tx_ring *txr = adapter->tx_rings;
2348         int             error, rid, vector = 0;
2349
2350         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2351                 rid = vector + 1;
2352                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2353                     RF_SHAREABLE | RF_ACTIVE);
2354                 if (que->res == NULL) {
2355                         device_printf(dev,"Unable to allocate"
2356                             " bus resource: que interrupt [%d]\n", vector);
2357                         return (ENXIO);
2358                 }
2359                 /* Set the handler function */
2360                 error = bus_setup_intr(dev, que->res,
2361                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2362                     ixgbe_msix_que, que, &que->tag);
2363                 if (error) {
2364                         que->res = NULL;
2365                         device_printf(dev, "Failed to register QUE handler");
2366                         return (error);
2367                 }
2368 #if __FreeBSD_version >= 800504
2369                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2370 #endif
2371                 que->msix = vector;
2372                 adapter->que_mask |= (u64)(1 << que->msix);
2373                 /*
2374                 ** Bind the msix vector, and thus the
2375                 ** ring to the corresponding cpu.
2376                 */
2377                 if (adapter->num_queues > 1)
2378                         bus_bind_intr(dev, que->res, i);
2379
2380 #ifndef IXGBE_LEGACY_TX
2381                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2382 #endif
2383                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2384                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2385                     taskqueue_thread_enqueue, &que->tq);
2386                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2387                     device_get_nameunit(adapter->dev));
2388         }
2389
2390         /* and Link */
2391         rid = vector + 1;
2392         adapter->res = bus_alloc_resource_any(dev,
2393             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2394         if (!adapter->res) {
2395                 device_printf(dev,"Unable to allocate"
2396             " bus resource: Link interrupt [%d]\n", rid);
2397                 return (ENXIO);
2398         }
2399         /* Set the link handler function */
2400         error = bus_setup_intr(dev, adapter->res,
2401             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2402             ixgbe_msix_link, adapter, &adapter->tag);
2403         if (error) {
2404                 adapter->res = NULL;
2405                 device_printf(dev, "Failed to register LINK handler");
2406                 return (error);
2407         }
2408 #if __FreeBSD_version >= 800504
2409         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2410 #endif
2411         adapter->linkvec = vector;
2412         /* Tasklets for Link, SFP and Multispeed Fiber */
2413         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2414         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2415         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2416 #ifdef IXGBE_FDIR
2417         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2418 #endif
2419         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2420             taskqueue_thread_enqueue, &adapter->tq);
2421         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2422             device_get_nameunit(adapter->dev));
2423
2424         return (0);
2425 }
2426
2427 /*
2428  * Setup Either MSI/X or MSI
2429  */
2430 static int
2431 ixgbe_setup_msix(struct adapter *adapter)
2432 {
2433         device_t dev = adapter->dev;
2434         int rid, want, queues, msgs;
2435
2436         /* Override by tuneable */
2437         if (ixgbe_enable_msix == 0)
2438                 goto msi;
2439
2440         /* First try MSI/X */
2441         msgs = pci_msix_count(dev);
2442         if (msgs == 0)
2443                 goto msi;
2444         rid = PCIR_BAR(MSIX_82598_BAR);
2445         adapter->msix_mem = bus_alloc_resource_any(dev,
2446             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2447         if (adapter->msix_mem == NULL) {
2448                 rid += 4;       /* 82599 maps in higher BAR */
2449                 adapter->msix_mem = bus_alloc_resource_any(dev,
2450                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2451         }
2452         if (adapter->msix_mem == NULL) {
2453                 /* May not be enabled */
2454                 device_printf(adapter->dev,
2455                     "Unable to map MSIX table \n");
2456                 goto msi;
2457         }
2458
2459         /* Figure out a reasonable auto config value */
2460         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2461
2462         if (ixgbe_num_queues != 0)
2463                 queues = ixgbe_num_queues;
2464         /* Set max queues to 8 when autoconfiguring */
2465         else if ((ixgbe_num_queues == 0) && (queues > 8))
2466                 queues = 8;
2467
2468         /* reflect correct sysctl value */
2469         ixgbe_num_queues = queues;
2470
2471         /*
2472         ** Want one vector (RX/TX pair) per queue
2473         ** plus an additional for Link.
2474         */
2475         want = queues + 1;
2476         if (msgs >= want)
2477                 msgs = want;
2478         else {
2479                 device_printf(adapter->dev,
2480                     "MSIX Configuration Problem, "
2481                     "%d vectors but %d queues wanted!\n",
2482                     msgs, want);
2483                 goto msi;
2484         }
2485         if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2486                 device_printf(adapter->dev,
2487                     "Using MSIX interrupts with %d vectors\n", msgs);
2488                 adapter->num_queues = queues;
2489                 return (msgs);
2490         }
2491         /*
2492         ** If MSIX alloc failed or provided us with
2493         ** less than needed, free and fall through to MSI
2494         */
2495         pci_release_msi(dev);
2496
2497 msi:
2498         if (adapter->msix_mem != NULL) {
2499                 bus_release_resource(dev, SYS_RES_MEMORY,
2500                     rid, adapter->msix_mem);
2501                 adapter->msix_mem = NULL;
2502         }
2503         msgs = 1;
2504         if (pci_alloc_msi(dev, &msgs) == 0) {
2505                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2506                 return (msgs);
2507         }
2508         device_printf(adapter->dev,"Using a Legacy interrupt\n");
2509         return (0);
2510 }
2511
2512
2513 static int
2514 ixgbe_allocate_pci_resources(struct adapter *adapter)
2515 {
2516         int             rid;
2517         device_t        dev = adapter->dev;
2518
2519         rid = PCIR_BAR(0);
2520         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2521             &rid, RF_ACTIVE);
2522
2523         if (!(adapter->pci_mem)) {
2524                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2525                 return (ENXIO);
2526         }
2527
2528         adapter->osdep.mem_bus_space_tag =
2529                 rman_get_bustag(adapter->pci_mem);
2530         adapter->osdep.mem_bus_space_handle =
2531                 rman_get_bushandle(adapter->pci_mem);
2532         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2533
2534         /* Legacy defaults */
2535         adapter->num_queues = 1;
2536         adapter->hw.back = &adapter->osdep;
2537
2538         /*
2539         ** Now setup MSI or MSI/X, should
2540         ** return us the number of supported
2541         ** vectors. (Will be 1 for MSI)
2542         */
2543         adapter->msix = ixgbe_setup_msix(adapter);
2544         return (0);
2545 }
2546
2547 static void
2548 ixgbe_free_pci_resources(struct adapter * adapter)
2549 {
2550         struct          ix_queue *que = adapter->queues;
2551         device_t        dev = adapter->dev;
2552         int             rid, memrid;
2553
2554         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2555                 memrid = PCIR_BAR(MSIX_82598_BAR);
2556         else
2557                 memrid = PCIR_BAR(MSIX_82599_BAR);
2558
2559         /*
2560         ** There is a slight possibility of a failure mode
2561         ** in attach that will result in entering this function
2562         ** before interrupt resources have been initialized, and
2563         ** in that case we do not want to execute the loops below
2564         ** We can detect this reliably by the state of the adapter
2565         ** res pointer.
2566         */
2567         if (adapter->res == NULL)
2568                 goto mem;
2569
2570         /*
2571         **  Release all msix queue resources:
2572         */
2573         for (int i = 0; i < adapter->num_queues; i++, que++) {
2574                 rid = que->msix + 1;
2575                 if (que->tag != NULL) {
2576                         bus_teardown_intr(dev, que->res, que->tag);
2577                         que->tag = NULL;
2578                 }
2579                 if (que->res != NULL)
2580                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2581         }
2582
2583
2584         /* Clean the Legacy or Link interrupt last */
2585         if (adapter->linkvec) /* we are doing MSIX */
2586                 rid = adapter->linkvec + 1;
2587         else
2588                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2589
2590         if (adapter->tag != NULL) {
2591                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2592                 adapter->tag = NULL;
2593         }
2594         if (adapter->res != NULL)
2595                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2596
2597 mem:
2598         if (adapter->msix)
2599                 pci_release_msi(dev);
2600
2601         if (adapter->msix_mem != NULL)
2602                 bus_release_resource(dev, SYS_RES_MEMORY,
2603                     memrid, adapter->msix_mem);
2604
2605         if (adapter->pci_mem != NULL)
2606                 bus_release_resource(dev, SYS_RES_MEMORY,
2607                     PCIR_BAR(0), adapter->pci_mem);
2608
2609         return;
2610 }
2611
2612 /*********************************************************************
2613  *
2614  *  Setup networking device structure and register an interface.
2615  *
2616  **********************************************************************/
2617 static int
2618 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2619 {
2620         struct ixgbe_hw *hw = &adapter->hw;
2621         struct ifnet   *ifp;
2622
2623         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2624
2625         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2626         if (ifp == NULL) {
2627                 device_printf(dev, "can not allocate ifnet structure\n");
2628                 return (-1);
2629         }
2630         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2631 #if __FreeBSD_version < 1000025
2632         ifp->if_baudrate = 1000000000;
2633 #else
2634         if_initbaudrate(ifp, IF_Gbps(10));
2635 #endif
2636         ifp->if_init = ixgbe_init;
2637         ifp->if_softc = adapter;
2638         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2639         ifp->if_ioctl = ixgbe_ioctl;
2640 #ifndef IXGBE_LEGACY_TX
2641         ifp->if_transmit = ixgbe_mq_start;
2642         ifp->if_qflush = ixgbe_qflush;
2643 #else
2644         ifp->if_start = ixgbe_start;
2645         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2646         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2647         IFQ_SET_READY(&ifp->if_snd);
2648 #endif
2649
2650         ether_ifattach(ifp, adapter->hw.mac.addr);
2651
2652         adapter->max_frame_size =
2653             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2654
2655         /*
2656          * Tell the upper layer(s) we support long frames.
2657          */
2658         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2659
2660         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2661         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2662         ifp->if_capabilities |= IFCAP_LRO;
2663         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2664                              |  IFCAP_VLAN_HWTSO
2665                              |  IFCAP_VLAN_MTU
2666                              |  IFCAP_HWSTATS;
2667         ifp->if_capenable = ifp->if_capabilities;
2668
2669         /*
2670         ** Don't turn this on by default, if vlans are
2671         ** created on another pseudo device (eg. lagg)
2672         ** then vlan events are not passed thru, breaking
2673         ** operation, but with HW FILTER off it works. If
2674         ** using vlans directly on the ixgbe driver you can
2675         ** enable this and get full hardware tag filtering.
2676         */
2677         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2678
2679         /*
2680          * Specify the media types supported by this adapter and register
2681          * callbacks to update media and link information
2682          */
2683         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2684                      ixgbe_media_status);
2685         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2686         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2687         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2688                 ifmedia_add(&adapter->media,
2689                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2690                 ifmedia_add(&adapter->media,
2691                     IFM_ETHER | IFM_1000_T, 0, NULL);
2692         }
2693         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2694         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2695
2696         return (0);
2697 }
2698
2699 static void
2700 ixgbe_config_link(struct adapter *adapter)
2701 {
2702         struct ixgbe_hw *hw = &adapter->hw;
2703         u32     autoneg, err = 0;
2704         bool    sfp, negotiate;
2705
2706         sfp = ixgbe_is_sfp(hw);
2707
2708         if (sfp) {
2709                 if (hw->phy.multispeed_fiber) {
2710                         hw->mac.ops.setup_sfp(hw);
2711                         ixgbe_enable_tx_laser(hw);
2712                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2713                 } else
2714                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2715         } else {
2716                 if (hw->mac.ops.check_link)
2717                         err = ixgbe_check_link(hw, &adapter->link_speed,
2718                             &adapter->link_up, FALSE);
2719                 if (err)
2720                         goto out;
2721                 autoneg = hw->phy.autoneg_advertised;
2722                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2723                         err  = hw->mac.ops.get_link_capabilities(hw,
2724                             &autoneg, &negotiate);
2725                 if (err)
2726                         goto out;
2727                 if (hw->mac.ops.setup_link)
2728                         err = hw->mac.ops.setup_link(hw,
2729                             autoneg, adapter->link_up);
2730         }
2731 out:
2732         return;
2733 }
2734
2735 /********************************************************************
2736  * Manage DMA'able memory.
2737  *******************************************************************/
2738 static void
2739 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2740 {
2741         if (error)
2742                 return;
2743         *(bus_addr_t *) arg = segs->ds_addr;
2744         return;
2745 }
2746
2747 static int
2748 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2749                 struct ixgbe_dma_alloc *dma, int mapflags)
2750 {
2751         device_t dev = adapter->dev;
2752         int             r;
2753
2754         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2755                                DBA_ALIGN, 0,    /* alignment, bounds */
2756                                BUS_SPACE_MAXADDR,       /* lowaddr */
2757                                BUS_SPACE_MAXADDR,       /* highaddr */
2758                                NULL, NULL,      /* filter, filterarg */
2759                                size,    /* maxsize */
2760                                1,       /* nsegments */
2761                                size,    /* maxsegsize */
2762                                BUS_DMA_ALLOCNOW,        /* flags */
2763                                NULL,    /* lockfunc */
2764                                NULL,    /* lockfuncarg */
2765                                &dma->dma_tag);
2766         if (r != 0) {
2767                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2768                        "error %u\n", r);
2769                 goto fail_0;
2770         }
2771         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2772                              BUS_DMA_NOWAIT, &dma->dma_map);
2773         if (r != 0) {
2774                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2775                        "error %u\n", r);
2776                 goto fail_1;
2777         }
2778         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2779                             size,
2780                             ixgbe_dmamap_cb,
2781                             &dma->dma_paddr,
2782                             mapflags | BUS_DMA_NOWAIT);
2783         if (r != 0) {
2784                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2785                        "error %u\n", r);
2786                 goto fail_2;
2787         }
2788         dma->dma_size = size;
2789         return (0);
2790 fail_2:
2791         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2792 fail_1:
2793         bus_dma_tag_destroy(dma->dma_tag);
2794 fail_0:
2795         dma->dma_map = NULL;
2796         dma->dma_tag = NULL;
2797         return (r);
2798 }
2799
2800 static void
2801 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2802 {
2803         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2804             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2805         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2806         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2807         bus_dma_tag_destroy(dma->dma_tag);
2808 }
2809
2810
2811 /*********************************************************************
2812  *
2813  *  Allocate memory for the transmit and receive rings, and then
2814  *  the descriptors associated with each, called only once at attach.
2815  *
2816  **********************************************************************/
2817 static int
2818 ixgbe_allocate_queues(struct adapter *adapter)
2819 {
2820         device_t        dev = adapter->dev;
2821         struct ix_queue *que;
2822         struct tx_ring  *txr;
2823         struct rx_ring  *rxr;
2824         int rsize, tsize, error = IXGBE_SUCCESS;
2825         int txconf = 0, rxconf = 0;
2826
2827         /* First allocate the top level queue structs */
2828         if (!(adapter->queues =
2829             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2830             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2831                 device_printf(dev, "Unable to allocate queue memory\n");
2832                 error = ENOMEM;
2833                 goto fail;
2834         }
2835
2836         /* First allocate the TX ring struct memory */
2837         if (!(adapter->tx_rings =
2838             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2839             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2840                 device_printf(dev, "Unable to allocate TX ring memory\n");
2841                 error = ENOMEM;
2842                 goto tx_fail;
2843         }
2844
2845         /* Next allocate the RX */
2846         if (!(adapter->rx_rings =
2847             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2848             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2849                 device_printf(dev, "Unable to allocate RX ring memory\n");
2850                 error = ENOMEM;
2851                 goto rx_fail;
2852         }
2853
2854         /* For the ring itself */
2855         tsize = roundup2(adapter->num_tx_desc *
2856             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2857
2858         /*
2859          * Now set up the TX queues, txconf is needed to handle the
2860          * possibility that things fail midcourse and we need to
2861          * undo memory gracefully
2862          */
2863         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2864                 /* Set up some basics */
2865                 txr = &adapter->tx_rings[i];
2866                 txr->adapter = adapter;
2867                 txr->me = i;
2868                 txr->num_desc = adapter->num_tx_desc;
2869
2870                 /* Initialize the TX side lock */
2871                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2872                     device_get_nameunit(dev), txr->me);
2873                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2874
2875                 if (ixgbe_dma_malloc(adapter, tsize,
2876                         &txr->txdma, BUS_DMA_NOWAIT)) {
2877                         device_printf(dev,
2878                             "Unable to allocate TX Descriptor memory\n");
2879                         error = ENOMEM;
2880                         goto err_tx_desc;
2881                 }
2882                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2883                 bzero((void *)txr->tx_base, tsize);
2884
2885                 /* Now allocate transmit buffers for the ring */
2886                 if (ixgbe_allocate_transmit_buffers(txr)) {
2887                         device_printf(dev,
2888                             "Critical Failure setting up transmit buffers\n");
2889                         error = ENOMEM;
2890                         goto err_tx_desc;
2891                 }
2892 #ifndef IXGBE_LEGACY_TX
2893                 /* Allocate a buf ring */
2894                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2895                     M_WAITOK, &txr->tx_mtx);
2896                 if (txr->br == NULL) {
2897                         device_printf(dev,
2898                             "Critical Failure setting up buf ring\n");
2899                         error = ENOMEM;
2900                         goto err_tx_desc;
2901                 }
2902 #endif
2903         }
2904
2905         /*
2906          * Next the RX queues...
2907          */
2908         rsize = roundup2(adapter->num_rx_desc *
2909             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2910         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2911                 rxr = &adapter->rx_rings[i];
2912                 /* Set up some basics */
2913                 rxr->adapter = adapter;
2914                 rxr->me = i;
2915                 rxr->num_desc = adapter->num_rx_desc;
2916
2917                 /* Initialize the RX side lock */
2918                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2919                     device_get_nameunit(dev), rxr->me);
2920                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2921
2922                 if (ixgbe_dma_malloc(adapter, rsize,
2923                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2924                         device_printf(dev,
2925                             "Unable to allocate RxDescriptor memory\n");
2926                         error = ENOMEM;
2927                         goto err_rx_desc;
2928                 }
2929                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2930                 bzero((void *)rxr->rx_base, rsize);
2931
2932                 /* Allocate receive buffers for the ring*/
2933                 if (ixgbe_allocate_receive_buffers(rxr)) {
2934                         device_printf(dev,
2935                             "Critical Failure setting up receive buffers\n");
2936                         error = ENOMEM;
2937                         goto err_rx_desc;
2938                 }
2939         }
2940
2941         /*
2942         ** Finally set up the queue holding structs
2943         */
2944         for (int i = 0; i < adapter->num_queues; i++) {
2945                 que = &adapter->queues[i];
2946                 que->adapter = adapter;
2947                 que->txr = &adapter->tx_rings[i];
2948                 que->rxr = &adapter->rx_rings[i];
2949         }
2950
2951         return (0);
2952
2953 err_rx_desc:
2954         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2955                 ixgbe_dma_free(adapter, &rxr->rxdma);
2956 err_tx_desc:
2957         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2958                 ixgbe_dma_free(adapter, &txr->txdma);
2959         free(adapter->rx_rings, M_DEVBUF);
2960 rx_fail:
2961         free(adapter->tx_rings, M_DEVBUF);
2962 tx_fail:
2963         free(adapter->queues, M_DEVBUF);
2964 fail:
2965         return (error);
2966 }
2967
2968 /*********************************************************************
2969  *
2970  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2971  *  the information needed to transmit a packet on the wire. This is
2972  *  called only once at attach, setup is done every reset.
2973  *
2974  **********************************************************************/
2975 static int
2976 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2977 {
2978         struct adapter *adapter = txr->adapter;
2979         device_t dev = adapter->dev;
2980         struct ixgbe_tx_buf *txbuf;
2981         int error, i;
2982
2983         /*
2984          * Setup DMA descriptor areas.
2985          */
2986         if ((error = bus_dma_tag_create(
2987                                bus_get_dma_tag(adapter->dev),   /* parent */
2988                                1, 0,            /* alignment, bounds */
2989                                BUS_SPACE_MAXADDR,       /* lowaddr */
2990                                BUS_SPACE_MAXADDR,       /* highaddr */
2991                                NULL, NULL,              /* filter, filterarg */
2992                                IXGBE_TSO_SIZE,          /* maxsize */
2993                                adapter->num_segs,       /* nsegments */
2994                                PAGE_SIZE,               /* maxsegsize */
2995                                0,                       /* flags */
2996                                NULL,                    /* lockfunc */
2997                                NULL,                    /* lockfuncarg */
2998                                &txr->txtag))) {
2999                 device_printf(dev,"Unable to allocate TX DMA tag\n");
3000                 goto fail;
3001         }
3002
3003         if (!(txr->tx_buffers =
3004             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
3005             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3006                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3007                 error = ENOMEM;
3008                 goto fail;
3009         }
3010
3011         /* Create the descriptor buffer dma maps */
3012         txbuf = txr->tx_buffers;
3013         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3014                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3015                 if (error != 0) {
3016                         device_printf(dev, "Unable to create TX DMA map\n");
3017                         goto fail;
3018                 }
3019         }
3020
3021         return 0;
3022 fail:
3023         /* We free all, it handles case where we are in the middle */
3024         ixgbe_free_transmit_structures(adapter);
3025         return (error);
3026 }
3027
3028 /*********************************************************************
3029  *
3030  *  Initialize a transmit ring.
3031  *
3032  **********************************************************************/
3033 static void
3034 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3035 {
3036         struct adapter *adapter = txr->adapter;
3037         struct ixgbe_tx_buf *txbuf;
3038         int i;
3039 #ifdef DEV_NETMAP
3040         struct netmap_adapter *na = NA(adapter->ifp);
3041         struct netmap_slot *slot;
3042 #endif /* DEV_NETMAP */
3043
3044         /* Clear the old ring contents */
3045         IXGBE_TX_LOCK(txr);
3046 #ifdef DEV_NETMAP
3047         /*
3048          * (under lock): if in netmap mode, do some consistency
3049          * checks and set slot to entry 0 of the netmap ring.
3050          */
3051         slot = netmap_reset(na, NR_TX, txr->me, 0);
3052 #endif /* DEV_NETMAP */
3053         bzero((void *)txr->tx_base,
3054               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3055         /* Reset indices */
3056         txr->next_avail_desc = 0;
3057         txr->next_to_clean = 0;
3058
3059         /* Free any existing tx buffers. */
3060         txbuf = txr->tx_buffers;
3061         for (i = 0; i < txr->num_desc; i++, txbuf++) {
3062                 if (txbuf->m_head != NULL) {
3063                         bus_dmamap_sync(txr->txtag, txbuf->map,
3064                             BUS_DMASYNC_POSTWRITE);
3065                         bus_dmamap_unload(txr->txtag, txbuf->map);
3066                         m_freem(txbuf->m_head);
3067                         txbuf->m_head = NULL;
3068                 }
3069 #ifdef DEV_NETMAP
3070                 /*
3071                  * In netmap mode, set the map for the packet buffer.
3072                  * NOTE: Some drivers (not this one) also need to set
3073                  * the physical buffer address in the NIC ring.
3074                  * Slots in the netmap ring (indexed by "si") are
3075                  * kring->nkr_hwofs positions "ahead" wrt the
3076                  * corresponding slot in the NIC ring. In some drivers
3077                  * (not here) nkr_hwofs can be negative. Function
3078                  * netmap_idx_n2k() handles wraparounds properly.
3079                  */
3080                 if (slot) {
3081                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3082                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3083                 }
3084 #endif /* DEV_NETMAP */
3085                 /* Clear the EOP descriptor pointer */
3086                 txbuf->eop = NULL;
3087         }
3088
3089 #ifdef IXGBE_FDIR
3090         /* Set the rate at which we sample packets */
3091         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3092                 txr->atr_sample = atr_sample_rate;
3093 #endif
3094
3095         /* Set number of descriptors available */
3096         txr->tx_avail = adapter->num_tx_desc;
3097
3098         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3099             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3100         IXGBE_TX_UNLOCK(txr);
3101 }
3102
3103 /*********************************************************************
3104  *
3105  *  Initialize all transmit rings.
3106  *
3107  **********************************************************************/
3108 static int
3109 ixgbe_setup_transmit_structures(struct adapter *adapter)
3110 {
3111         struct tx_ring *txr = adapter->tx_rings;
3112
3113         for (int i = 0; i < adapter->num_queues; i++, txr++)
3114                 ixgbe_setup_transmit_ring(txr);
3115
3116         return (0);
3117 }
3118
3119 /*********************************************************************
3120  *
3121  *  Enable transmit unit.
3122  *
3123  **********************************************************************/
3124 static void
3125 ixgbe_initialize_transmit_units(struct adapter *adapter)
3126 {
3127         struct tx_ring  *txr = adapter->tx_rings;
3128         struct ixgbe_hw *hw = &adapter->hw;
3129
3130         /* Setup the Base and Length of the Tx Descriptor Ring */
3131
3132         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3133                 u64     tdba = txr->txdma.dma_paddr;
3134                 u32     txctrl;
3135
3136                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3137                        (tdba & 0x00000000ffffffffULL));
3138                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3139                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3140                     adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3141
3142                 /* Setup the HW Tx Head and Tail descriptor pointers */
3143                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3144                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3145
3146                 /* Setup Transmit Descriptor Cmd Settings */
3147                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3148                 txr->queue_status = IXGBE_QUEUE_IDLE;
3149
3150                 /* Set the processing limit */
3151                 txr->process_limit = ixgbe_tx_process_limit;
3152
3153                 /* Disable Head Writeback */
3154                 switch (hw->mac.type) {
3155                 case ixgbe_mac_82598EB:
3156                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3157                         break;
3158                 case ixgbe_mac_82599EB:
3159                 case ixgbe_mac_X540:
3160                 default:
3161                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3162                         break;
3163                 }
3164                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3165                 switch (hw->mac.type) {
3166                 case ixgbe_mac_82598EB:
3167                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3168                         break;
3169                 case ixgbe_mac_82599EB:
3170                 case ixgbe_mac_X540:
3171                 default:
3172                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3173                         break;
3174                 }
3175
3176         }
3177
3178         if (hw->mac.type != ixgbe_mac_82598EB) {
3179                 u32 dmatxctl, rttdcs;
3180                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3181                 dmatxctl |= IXGBE_DMATXCTL_TE;
3182                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3183                 /* Disable arbiter to set MTQC */
3184                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3185                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3186                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3187                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3188                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3189                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3190         }
3191
3192         return;
3193 }
3194
3195 /*********************************************************************
3196  *
3197  *  Free all transmit rings.
3198  *
3199  **********************************************************************/
3200 static void
3201 ixgbe_free_transmit_structures(struct adapter *adapter)
3202 {
3203         struct tx_ring *txr = adapter->tx_rings;
3204
3205         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3206                 IXGBE_TX_LOCK(txr);
3207                 ixgbe_free_transmit_buffers(txr);
3208                 ixgbe_dma_free(adapter, &txr->txdma);
3209                 IXGBE_TX_UNLOCK(txr);
3210                 IXGBE_TX_LOCK_DESTROY(txr);
3211         }
3212         free(adapter->tx_rings, M_DEVBUF);
3213 }
3214
3215 /*********************************************************************
3216  *
3217  *  Free transmit ring related data structures.
3218  *
3219  **********************************************************************/
3220 static void
3221 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3222 {
3223         struct adapter *adapter = txr->adapter;
3224         struct ixgbe_tx_buf *tx_buffer;
3225         int             i;
3226
3227         INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin");
3228
3229         if (txr->tx_buffers == NULL)
3230                 return;
3231
3232         tx_buffer = txr->tx_buffers;
3233         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3234                 if (tx_buffer->m_head != NULL) {
3235                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3236                             BUS_DMASYNC_POSTWRITE);
3237                         bus_dmamap_unload(txr->txtag,
3238                             tx_buffer->map);
3239                         m_freem(tx_buffer->m_head);
3240                         tx_buffer->m_head = NULL;
3241                         if (tx_buffer->map != NULL) {
3242                                 bus_dmamap_destroy(txr->txtag,
3243                                     tx_buffer->map);
3244                                 tx_buffer->map = NULL;
3245                         }
3246                 } else if (tx_buffer->map != NULL) {
3247                         bus_dmamap_unload(txr->txtag,
3248                             tx_buffer->map);
3249                         bus_dmamap_destroy(txr->txtag,
3250                             tx_buffer->map);
3251                         tx_buffer->map = NULL;
3252                 }
3253         }
3254 #ifdef IXGBE_LEGACY_TX
3255         if (txr->br != NULL)
3256                 buf_ring_free(txr->br, M_DEVBUF);
3257 #endif
3258         if (txr->tx_buffers != NULL) {
3259                 free(txr->tx_buffers, M_DEVBUF);
3260                 txr->tx_buffers = NULL;
3261         }
3262         if (txr->txtag != NULL) {
3263                 bus_dma_tag_destroy(txr->txtag);
3264                 txr->txtag = NULL;
3265         }
3266         return;
3267 }
3268
3269 /*********************************************************************
3270  *
3271  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
3272  *
3273  **********************************************************************/
3274
3275 static int
3276 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3277     u32 *cmd_type_len, u32 *olinfo_status)
3278 {
3279         struct ixgbe_adv_tx_context_desc *TXD;
3280         struct ether_vlan_header *eh;
3281         struct ip *ip;
3282         struct ip6_hdr *ip6;
3283         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3284         int     ehdrlen, ip_hlen = 0;
3285         u16     etype;
3286         u8      ipproto = 0;
3287         int     offload = TRUE;
3288         int     ctxd = txr->next_avail_desc;
3289         u16     vtag = 0;
3290
3291         /* First check if TSO is to be used */
3292         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3293                 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3294
3295         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3296                 offload = FALSE;
3297
3298         /* Indicate the whole packet as payload when not doing TSO */
3299         *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3300
3301         /* Now ready a context descriptor */
3302         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3303
3304         /*
3305         ** In advanced descriptors the vlan tag must
3306         ** be placed into the context descriptor. Hence
3307         ** we need to make one even if not doing offloads.
3308         */
3309         if (mp->m_flags & M_VLANTAG) {
3310                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3311                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3312         } else if (offload == FALSE) /* ... no offload to do */
3313                 return (0);
3314
3315         /*
3316          * Determine where frame payload starts.
3317          * Jump over vlan headers if already present,
3318          * helpful for QinQ too.
3319          */
3320         eh = mtod(mp, struct ether_vlan_header *);
3321         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3322                 etype = ntohs(eh->evl_proto);
3323                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3324         } else {
3325                 etype = ntohs(eh->evl_encap_proto);
3326                 ehdrlen = ETHER_HDR_LEN;
3327         }
3328
3329         /* Set the ether header length */
3330         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3331
3332         switch (etype) {
3333                 case ETHERTYPE_IP:
3334                         ip = (struct ip *)(mp->m_data + ehdrlen);
3335                         ip_hlen = ip->ip_hl << 2;
3336                         ipproto = ip->ip_p;
3337                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3338                         break;
3339                 case ETHERTYPE_IPV6:
3340                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3341                         ip_hlen = sizeof(struct ip6_hdr);
3342                         /* XXX-BZ this will go badly in case of ext hdrs. */
3343                         ipproto = ip6->ip6_nxt;
3344                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3345                         break;
3346                 default:
3347                         offload = FALSE;
3348                         break;
3349         }
3350
3351         vlan_macip_lens |= ip_hlen;
3352         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3353
3354         switch (ipproto) {
3355                 case IPPROTO_TCP:
3356                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3357                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3358                         break;
3359
3360                 case IPPROTO_UDP:
3361                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3362                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3363                         break;
3364
3365 #if __FreeBSD_version >= 800000
3366                 case IPPROTO_SCTP:
3367                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3368                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3369                         break;
3370 #endif
3371                 default:
3372                         offload = FALSE;
3373                         break;
3374         }
3375
3376         if (offload) /* For the TX descriptor setup */
3377                 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3378
3379         /* Now copy bits into descriptor */
3380         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3381         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3382         TXD->seqnum_seed = htole32(0);
3383         TXD->mss_l4len_idx = htole32(0);
3384
3385         /* We've consumed the first desc, adjust counters */
3386         if (++ctxd == txr->num_desc)
3387                 ctxd = 0;
3388         txr->next_avail_desc = ctxd;
3389         --txr->tx_avail;
3390
3391         return (0);
3392 }
3393
3394 /**********************************************************************
3395  *
3396  *  Setup work for hardware segmentation offload (TSO) on
3397  *  adapters using advanced tx descriptors
3398  *
3399  **********************************************************************/
3400 static int
3401 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3402     u32 *cmd_type_len, u32 *olinfo_status)
3403 {
3404         struct ixgbe_adv_tx_context_desc *TXD;
3405         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3406         u32 mss_l4len_idx = 0, paylen;
3407         u16 vtag = 0, eh_type;
3408         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3409         struct ether_vlan_header *eh;
3410 #ifdef INET6
3411         struct ip6_hdr *ip6;
3412 #endif
3413 #ifdef INET
3414         struct ip *ip;
3415 #endif
3416         struct tcphdr *th;
3417
3418
3419         /*
3420          * Determine where frame payload starts.
3421          * Jump over vlan headers if already present
3422          */
3423         eh = mtod(mp, struct ether_vlan_header *);
3424         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3425                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3426                 eh_type = eh->evl_proto;
3427         } else {
3428                 ehdrlen = ETHER_HDR_LEN;
3429                 eh_type = eh->evl_encap_proto;
3430         }
3431
3432         switch (ntohs(eh_type)) {
3433 #ifdef INET6
3434         case ETHERTYPE_IPV6:
3435                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3436                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3437                 if (ip6->ip6_nxt != IPPROTO_TCP)
3438                         return (ENXIO);
3439                 ip_hlen = sizeof(struct ip6_hdr);
3440                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3441                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3442                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3443                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3444                 break;
3445 #endif
3446 #ifdef INET
3447         case ETHERTYPE_IP:
3448                 ip = (struct ip *)(mp->m_data + ehdrlen);
3449                 if (ip->ip_p != IPPROTO_TCP)
3450                         return (ENXIO);
3451                 ip->ip_sum = 0;
3452                 ip_hlen = ip->ip_hl << 2;
3453                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3454                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3455                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3456                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3457                 /* Tell transmit desc to also do IPv4 checksum. */
3458                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3459                 break;
3460 #endif
3461         default:
3462                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3463                     __func__, ntohs(eh_type));
3464                 break;
3465         }
3466
3467         ctxd = txr->next_avail_desc;
3468         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3469
3470         tcp_hlen = th->th_off << 2;
3471
3472         /* This is used in the transmit desc in encap */
3473         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3474
3475         /* VLAN MACLEN IPLEN */
3476         if (mp->m_flags & M_VLANTAG) {
3477                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3478                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3479         }
3480
3481         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3482         vlan_macip_lens |= ip_hlen;
3483         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3484
3485         /* ADV DTYPE TUCMD */
3486         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3487         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3488         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3489
3490         /* MSS L4LEN IDX */
3491         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3492         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3493         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3494
3495         TXD->seqnum_seed = htole32(0);
3496
3497         if (++ctxd == txr->num_desc)
3498                 ctxd = 0;
3499
3500         txr->tx_avail--;
3501         txr->next_avail_desc = ctxd;
3502         *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3503         *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3504         *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3505         ++txr->tso_tx;
3506         return (0);
3507 }
3508
3509 #ifdef IXGBE_FDIR
3510 /*
3511 ** This routine parses packet headers so that Flow
3512 ** Director can make a hashed filter table entry
3513 ** allowing traffic flows to be identified and kept
3514 ** on the same cpu.  This would be a performance
3515 ** hit, but we only do it at IXGBE_FDIR_RATE of
3516 ** packets.
3517 */
3518 static void
3519 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3520 {
3521         struct adapter                  *adapter = txr->adapter;
3522         struct ix_queue                 *que;
3523         struct ip                       *ip;
3524         struct tcphdr                   *th;
3525         struct udphdr                   *uh;
3526         struct ether_vlan_header        *eh;
3527         union ixgbe_atr_hash_dword      input = {.dword = 0};
3528         union ixgbe_atr_hash_dword      common = {.dword = 0};
3529         int                             ehdrlen, ip_hlen;
3530         u16                             etype;
3531
3532         eh = mtod(mp, struct ether_vlan_header *);
3533         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3534                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3535                 etype = eh->evl_proto;
3536         } else {
3537                 ehdrlen = ETHER_HDR_LEN;
3538                 etype = eh->evl_encap_proto;
3539         }
3540
3541         /* Only handling IPv4 */
3542         if (etype != htons(ETHERTYPE_IP))
3543                 return;
3544
3545         ip = (struct ip *)(mp->m_data + ehdrlen);
3546         ip_hlen = ip->ip_hl << 2;
3547
3548         /* check if we're UDP or TCP */
3549         switch (ip->ip_p) {
3550         case IPPROTO_TCP:
3551                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3552                 /* src and dst are inverted */
3553                 common.port.dst ^= th->th_sport;
3554                 common.port.src ^= th->th_dport;
3555                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3556                 break;
3557         case IPPROTO_UDP:
3558                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3559                 /* src and dst are inverted */
3560                 common.port.dst ^= uh->uh_sport;
3561                 common.port.src ^= uh->uh_dport;
3562                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3563                 break;
3564         default:
3565                 return;
3566         }
3567
3568         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3569         if (mp->m_pkthdr.ether_vtag)
3570                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3571         else
3572                 common.flex_bytes ^= etype;
3573         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3574
3575         que = &adapter->queues[txr->me];
3576         /*
3577         ** This assumes the Rx queue and Tx
3578         ** queue are bound to the same CPU
3579         */
3580         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3581             input, common, que->msix);
3582 }
3583 #endif /* IXGBE_FDIR */
3584
3585 /**********************************************************************
3586  *
3587  *  Examine each tx_buffer in the used queue. If the hardware is done
3588  *  processing the packet then free associated resources. The
3589  *  tx_buffer is put back on the free queue.
3590  *
3591  **********************************************************************/
3592 static void
3593 ixgbe_txeof(struct tx_ring *txr)
3594 {
3595 #ifdef DEV_NETMAP
3596         struct adapter          *adapter = txr->adapter;
3597         struct ifnet            *ifp = adapter->ifp;
3598 #endif
3599         u32                     work, processed = 0;
3600         u16                     limit = txr->process_limit;
3601         struct ixgbe_tx_buf     *buf;
3602         union ixgbe_adv_tx_desc *txd;
3603
3604         mtx_assert(&txr->tx_mtx, MA_OWNED);
3605
3606 #ifdef DEV_NETMAP
3607         if (ifp->if_capenable & IFCAP_NETMAP) {
3608                 struct netmap_adapter *na = NA(ifp);
3609                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3610                 txd = txr->tx_base;
3611                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3612                     BUS_DMASYNC_POSTREAD);
3613                 /*
3614                  * In netmap mode, all the work is done in the context
3615                  * of the client thread. Interrupt handlers only wake up
3616                  * clients, which may be sleeping on individual rings
3617                  * or on a global resource for all rings.
3618                  * To implement tx interrupt mitigation, we wake up the client
3619                  * thread roughly every half ring, even if the NIC interrupts
3620                  * more frequently. This is implemented as follows:
3621                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3622                  *   the slot that should wake up the thread (nkr_num_slots
3623                  *   means the user thread should not be woken up);
3624                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3625                  *   or the slot has the DD bit set.
3626                  */
3627                 if (!netmap_mitigate ||
3628                     (kring->nr_kflags < kring->nkr_num_slots &&
3629                     txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3630                         netmap_tx_irq(ifp, txr->me);
3631                 }
3632                 return;
3633         }
3634 #endif /* DEV_NETMAP */
3635
3636         if (txr->tx_avail == txr->num_desc) {
3637                 txr->queue_status = IXGBE_QUEUE_IDLE;
3638                 return;
3639         }
3640
3641         /* Get work starting point */
3642         work = txr->next_to_clean;
3643         buf = &txr->tx_buffers[work];
3644         txd = &txr->tx_base[work];
3645         work -= txr->num_desc; /* The distance to ring end */
3646         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3647             BUS_DMASYNC_POSTREAD);
3648
3649         do {
3650                 union ixgbe_adv_tx_desc *eop= buf->eop;
3651                 if (eop == NULL) /* No work */
3652                         break;
3653
3654                 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3655                         break;  /* I/O not complete */
3656
3657                 if (buf->m_head) {
3658                         txr->bytes +=
3659                             buf->m_head->m_pkthdr.len;
3660                         bus_dmamap_sync(txr->txtag,
3661                             buf->map,
3662                             BUS_DMASYNC_POSTWRITE);
3663                         bus_dmamap_unload(txr->txtag,
3664                             buf->map);
3665                         m_freem(buf->m_head);
3666                         buf->m_head = NULL;
3667                         buf->map = NULL;
3668                 }
3669                 buf->eop = NULL;
3670                 ++txr->tx_avail;
3671
3672                 /* We clean the range if multi segment */
3673                 while (txd != eop) {
3674                         ++txd;
3675                         ++buf;
3676                         ++work;
3677                         /* wrap the ring? */
3678                         if (__predict_false(!work)) {
3679                                 work -= txr->num_desc;
3680                                 buf = txr->tx_buffers;
3681                                 txd = txr->tx_base;
3682                         }
3683                         if (buf->m_head) {
3684                                 txr->bytes +=
3685                                     buf->m_head->m_pkthdr.len;
3686                                 bus_dmamap_sync(txr->txtag,
3687                                     buf->map,
3688                                     BUS_DMASYNC_POSTWRITE);
3689                                 bus_dmamap_unload(txr->txtag,
3690                                     buf->map);
3691                                 m_freem(buf->m_head);
3692                                 buf->m_head = NULL;
3693                                 buf->map = NULL;
3694                         }
3695                         ++txr->tx_avail;
3696                         buf->eop = NULL;
3697
3698                 }
3699                 ++txr->packets;
3700                 ++processed;
3701                 txr->watchdog_time = ticks;
3702
3703                 /* Try the next packet */
3704                 ++txd;
3705                 ++buf;
3706                 ++work;
3707                 /* reset with a wrap */
3708                 if (__predict_false(!work)) {
3709                         work -= txr->num_desc;
3710                         buf = txr->tx_buffers;
3711                         txd = txr->tx_base;
3712                 }
3713                 prefetch(txd);
3714         } while (__predict_true(--limit));
3715
3716         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3717             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3718
3719         work += txr->num_desc;
3720         txr->next_to_clean = work;
3721
3722         /*
3723         ** Watchdog calculation, we know there's
3724         ** work outstanding or the first return
3725         ** would have been taken, so none processed
3726         ** for too long indicates a hang.
3727         */
3728         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3729                 txr->queue_status = IXGBE_QUEUE_HUNG;
3730
3731         if (txr->tx_avail == txr->num_desc)
3732                 txr->queue_status = IXGBE_QUEUE_IDLE;
3733
3734         return;
3735 }
3736
3737 /*********************************************************************
3738  *
3739  *  Refresh mbuf buffers for RX descriptor rings
3740  *   - now keeps its own state so discards due to resource
3741  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3742  *     it just returns, keeping its placeholder, thus it can simply
3743  *     be recalled to try again.
3744  *
3745  **********************************************************************/
3746 static void
3747 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3748 {
3749         struct adapter          *adapter = rxr->adapter;
3750         bus_dma_segment_t       seg[1];
3751         struct ixgbe_rx_buf     *rxbuf;
3752         struct mbuf             *mp;
3753         int                     i, j, nsegs, error;
3754         bool                    refreshed = FALSE;
3755
3756         i = j = rxr->next_to_refresh;
3757         /* Control the loop with one beyond */
3758         if (++j == rxr->num_desc)
3759                 j = 0;
3760
3761         while (j != limit) {
3762                 rxbuf = &rxr->rx_buffers[i];
3763                 if (rxbuf->buf == NULL) {
3764                         mp = m_getjcl(M_NOWAIT, MT_DATA,
3765                             M_PKTHDR, rxr->mbuf_sz);
3766                         if (mp == NULL)
3767                                 goto update;
3768                         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3769                                 m_adj(mp, ETHER_ALIGN);
3770                 } else
3771                         mp = rxbuf->buf;
3772
3773                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3774
3775                 /* If we're dealing with an mbuf that was copied rather
3776                  * than replaced, there's no need to go through busdma.
3777                  */
3778                 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3779                         /* Get the memory mapping */
3780                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3781                             rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3782                         if (error != 0) {
3783                                 printf("Refresh mbufs: payload dmamap load"
3784                                     " failure - %d\n", error);
3785                                 m_free(mp);
3786                                 rxbuf->buf = NULL;
3787                                 goto update;
3788                         }
3789                         rxbuf->buf = mp;
3790                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3791                             BUS_DMASYNC_PREREAD);
3792                         rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3793                             htole64(seg[0].ds_addr);
3794                 } else {
3795                         rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3796                         rxbuf->flags &= ~IXGBE_RX_COPY;
3797                 }
3798
3799                 refreshed = TRUE;
3800                 /* Next is precalculated */
3801                 i = j;
3802                 rxr->next_to_refresh = i;
3803                 if (++j == rxr->num_desc)
3804                         j = 0;
3805         }
3806 update:
3807         if (refreshed) /* Update hardware tail index */
3808                 IXGBE_WRITE_REG(&adapter->hw,
3809                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3810         return;
3811 }
3812
3813 /*********************************************************************
3814  *
3815  *  Allocate memory for rx_buffer structures. Since we use one
3816  *  rx_buffer per received packet, the maximum number of rx_buffer's
3817  *  that we'll need is equal to the number of receive descriptors
3818  *  that we've allocated.
3819  *
3820  **********************************************************************/
3821 static int
3822 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3823 {
3824         struct  adapter         *adapter = rxr->adapter;
3825         device_t                dev = adapter->dev;
3826         struct ixgbe_rx_buf     *rxbuf;
3827         int                     i, bsize, error;
3828
3829         bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3830         if (!(rxr->rx_buffers =
3831             (struct ixgbe_rx_buf *) malloc(bsize,
3832             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3833                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3834                 error = ENOMEM;
3835                 goto fail;
3836         }
3837
3838         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3839                                    1, 0,        /* alignment, bounds */
3840                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3841                                    BUS_SPACE_MAXADDR,   /* highaddr */
3842                                    NULL, NULL,          /* filter, filterarg */
3843                                    MJUM16BYTES,         /* maxsize */
3844                                    1,                   /* nsegments */
3845                                    MJUM16BYTES,         /* maxsegsize */
3846                                    0,                   /* flags */
3847                                    NULL,                /* lockfunc */
3848                                    NULL,                /* lockfuncarg */
3849                                    &rxr->ptag))) {
3850                 device_printf(dev, "Unable to create RX DMA tag\n");
3851                 goto fail;
3852         }
3853
3854         for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3855                 rxbuf = &rxr->rx_buffers[i];
3856                 error = bus_dmamap_create(rxr->ptag,
3857                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3858                 if (error) {
3859                         device_printf(dev, "Unable to create RX dma map\n");
3860                         goto fail;
3861                 }
3862         }
3863
3864         return (0);
3865
3866 fail:
3867         /* Frees all, but can handle partial completion */
3868         ixgbe_free_receive_structures(adapter);
3869         return (error);
3870 }
3871
3872 /*
3873 ** Used to detect a descriptor that has
3874 ** been merged by Hardware RSC.
3875 */
3876 static inline u32
3877 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3878 {
3879         return (le32toh(rx->wb.lower.lo_dword.data) &
3880             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3881 }
3882
3883 /*********************************************************************
3884  *
3885  *  Initialize Hardware RSC (LRO) feature on 82599
3886  *  for an RX ring, this is toggled by the LRO capability
3887  *  even though it is transparent to the stack.
3888  *
3889  *  NOTE: since this HW feature only works with IPV4 and
3890  *        our testing has shown soft LRO to be as effective
3891  *        I have decided to disable this by default.
3892  *
3893  **********************************************************************/
3894 static void
3895 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3896 {
3897         struct  adapter         *adapter = rxr->adapter;
3898         struct  ixgbe_hw        *hw = &adapter->hw;
3899         u32                     rscctrl, rdrxctl;
3900
3901         /* If turning LRO/RSC off we need to disable it */
3902         if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3903                 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3904                 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3905                 return;
3906         }
3907
3908         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3909         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3910 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3911         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3912 #endif /* DEV_NETMAP */
3913         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3914         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3915         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3916
3917         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3918         rscctrl |= IXGBE_RSCCTL_RSCEN;
3919         /*
3920         ** Limit the total number of descriptors that
3921         ** can be combined, so it does not exceed 64K
3922         */
3923         if (rxr->mbuf_sz == MCLBYTES)
3924                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3925         else if (rxr->mbuf_sz == MJUMPAGESIZE)
3926                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3927         else if (rxr->mbuf_sz == MJUM9BYTES)
3928                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3929         else  /* Using 16K cluster */
3930                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3931
3932         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3933
3934         /* Enable TCP header recognition */
3935         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3936             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3937             IXGBE_PSRTYPE_TCPHDR));
3938
3939         /* Disable RSC for ACK packets */
3940         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3941             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3942
3943         rxr->hw_rsc = TRUE;
3944 }
3945
3946
3947 static void
3948 ixgbe_free_receive_ring(struct rx_ring *rxr)
3949 {
3950         struct ixgbe_rx_buf       *rxbuf;
3951         int i;
3952
3953         for (i = 0; i < rxr->num_desc; i++) {
3954                 rxbuf = &rxr->rx_buffers[i];
3955                 if (rxbuf->buf != NULL) {
3956                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3957                             BUS_DMASYNC_POSTREAD);
3958                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3959                         rxbuf->buf->m_flags |= M_PKTHDR;
3960                         m_freem(rxbuf->buf);
3961                         rxbuf->buf = NULL;
3962                         rxbuf->flags = 0;
3963                 }
3964         }
3965 }
3966
3967
3968 /*********************************************************************
3969  *
3970  *  Initialize a receive ring and its buffers.
3971  *
3972  **********************************************************************/
3973 static int
3974 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3975 {
3976         struct  adapter         *adapter;
3977         struct ifnet            *ifp;
3978         device_t                dev;
3979         struct ixgbe_rx_buf     *rxbuf;
3980         bus_dma_segment_t       seg[1];
3981         struct lro_ctrl         *lro = &rxr->lro;
3982         int                     rsize, nsegs, error = 0;
3983 #ifdef DEV_NETMAP
3984         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3985         struct netmap_slot *slot;
3986 #endif /* DEV_NETMAP */
3987
3988         adapter = rxr->adapter;
3989         ifp = adapter->ifp;
3990         dev = adapter->dev;
3991
3992         /* Clear the ring contents */
3993         IXGBE_RX_LOCK(rxr);
3994 #ifdef DEV_NETMAP
3995         /* same as in ixgbe_setup_transmit_ring() */
3996         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3997 #endif /* DEV_NETMAP */
3998         rsize = roundup2(adapter->num_rx_desc *
3999             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4000         bzero((void *)rxr->rx_base, rsize);
4001         /* Cache the size */
4002         rxr->mbuf_sz = adapter->rx_mbuf_sz;
4003
4004         /* Free current RX buffer structs and their mbufs */
4005         ixgbe_free_receive_ring(rxr);
4006
4007         /* Now replenish the mbufs */
4008         for (int j = 0; j != rxr->num_desc; ++j) {
4009                 struct mbuf     *mp;
4010
4011                 rxbuf = &rxr->rx_buffers[j];
4012 #ifdef DEV_NETMAP
4013                 /*
4014                  * In netmap mode, fill the map and set the buffer
4015                  * address in the NIC ring, considering the offset
4016                  * between the netmap and NIC rings (see comment in
4017                  * ixgbe_setup_transmit_ring() ). No need to allocate
4018                  * an mbuf, so end the block with a continue;
4019                  */
4020                 if (slot) {
4021                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4022                         uint64_t paddr;
4023                         void *addr;
4024
4025                         addr = PNMB(slot + sj, &paddr);
4026                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4027                         /* Update descriptor and the cached value */
4028                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4029                         rxbuf->addr = htole64(paddr);
4030                         continue;
4031                 }
4032 #endif /* DEV_NETMAP */
4033                 rxbuf->flags = 0;
4034                 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
4035                     M_PKTHDR, adapter->rx_mbuf_sz);
4036                 if (rxbuf->buf == NULL) {
4037                         error = ENOBUFS;
4038                         goto fail;
4039                 }
4040                 mp = rxbuf->buf;
4041                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4042                 /* Get the memory mapping */
4043                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4044                     rxbuf->pmap, mp, seg,
4045                     &nsegs, BUS_DMA_NOWAIT);
4046                 if (error != 0)
4047                         goto fail;
4048                 bus_dmamap_sync(rxr->ptag,
4049                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4050                 /* Update the descriptor and the cached value */
4051                 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4052                 rxbuf->addr = htole64(seg[0].ds_addr);
4053         }
4054
4055
4056         /* Setup our descriptor indices */
4057         rxr->next_to_check = 0;
4058         rxr->next_to_refresh = 0;
4059         rxr->lro_enabled = FALSE;
4060         rxr->rx_copies = 0;
4061         rxr->rx_bytes = 0;
4062         rxr->discard = FALSE;
4063         rxr->vtag_strip = FALSE;
4064
4065         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4066             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4067
4068         /*
4069         ** Now set up the LRO interface:
4070         */
4071         if (ixgbe_rsc_enable)
4072                 ixgbe_setup_hw_rsc(rxr);
4073         else if (ifp->if_capenable & IFCAP_LRO) {
4074                 int err = tcp_lro_init(lro);
4075                 if (err) {
4076                         device_printf(dev, "LRO Initialization failed!\n");
4077                         goto fail;
4078                 }
4079                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4080                 rxr->lro_enabled = TRUE;
4081                 lro->ifp = adapter->ifp;
4082         }
4083
4084         IXGBE_RX_UNLOCK(rxr);
4085         return (0);
4086
4087 fail:
4088         ixgbe_free_receive_ring(rxr);
4089         IXGBE_RX_UNLOCK(rxr);
4090         return (error);
4091 }
4092
4093 /*********************************************************************
4094  *
4095  *  Initialize all receive rings.
4096  *
4097  **********************************************************************/
4098 static int
4099 ixgbe_setup_receive_structures(struct adapter *adapter)
4100 {
4101         struct rx_ring *rxr = adapter->rx_rings;
4102         int j;
4103
4104         for (j = 0; j < adapter->num_queues; j++, rxr++)
4105                 if (ixgbe_setup_receive_ring(rxr))
4106                         goto fail;
4107
4108         return (0);
4109 fail:
4110         /*
4111          * Free RX buffers allocated so far, we will only handle
4112          * the rings that completed, the failing case will have
4113          * cleaned up for itself. 'j' failed, so its the terminus.
4114          */
4115         for (int i = 0; i < j; ++i) {
4116                 rxr = &adapter->rx_rings[i];
4117                 ixgbe_free_receive_ring(rxr);
4118         }
4119
4120         return (ENOBUFS);
4121 }
4122
4123 /*********************************************************************
4124  *
4125  *  Setup receive registers and features.
4126  *
4127  **********************************************************************/
4128 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4129
4130 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4131
4132 static void
4133 ixgbe_initialize_receive_units(struct adapter *adapter)
4134 {
4135         struct  rx_ring *rxr = adapter->rx_rings;
4136         struct ixgbe_hw *hw = &adapter->hw;
4137         struct ifnet   *ifp = adapter->ifp;
4138         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4139         u32             reta, mrqc = 0, hlreg, random[10];
4140
4141
4142         /*
4143          * Make sure receives are disabled while
4144          * setting up the descriptor ring
4145          */
4146         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4147         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4148             rxctrl & ~IXGBE_RXCTRL_RXEN);
4149
4150         /* Enable broadcasts */
4151         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4152         fctrl |= IXGBE_FCTRL_BAM;
4153         fctrl |= IXGBE_FCTRL_DPF;
4154         fctrl |= IXGBE_FCTRL_PMCF;
4155         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4156
4157         /* Set for Jumbo Frames? */
4158         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4159         if (ifp->if_mtu > ETHERMTU)
4160                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4161         else
4162                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4163 #ifdef DEV_NETMAP
4164         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4165         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4166                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4167         else
4168                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4169 #endif /* DEV_NETMAP */
4170         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4171
4172         bufsz = (adapter->rx_mbuf_sz +
4173             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4174
4175         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4176                 u64 rdba = rxr->rxdma.dma_paddr;
4177
4178                 /* Setup the Base and Length of the Rx Descriptor Ring */
4179                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4180                                (rdba & 0x00000000ffffffffULL));
4181                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4182                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4183                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4184
4185                 /* Set up the SRRCTL register */
4186                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4187                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4188                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4189                 srrctl |= bufsz;
4190                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4191                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4192
4193                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4194                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4195                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4196
4197                 /* Set the processing limit */
4198                 rxr->process_limit = ixgbe_rx_process_limit;
4199         }
4200
4201         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4202                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4203                               IXGBE_PSRTYPE_UDPHDR |
4204                               IXGBE_PSRTYPE_IPV4HDR |
4205                               IXGBE_PSRTYPE_IPV6HDR;
4206                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4207         }
4208
4209         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4210
4211         /* Setup RSS */
4212         if (adapter->num_queues > 1) {
4213                 int i, j;
4214                 reta = 0;
4215
4216                 /* set up random bits */
4217                 arc4rand(&random, sizeof(random), 0);
4218
4219                 /* Set up the redirection table */
4220                 for (i = 0, j = 0; i < 128; i++, j++) {
4221                         if (j == adapter->num_queues) j = 0;
4222                         reta = (reta << 8) | (j * 0x11);
4223                         if ((i & 3) == 3)
4224                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4225                 }
4226
4227                 /* Now fill our hash function seeds */
4228                 for (int i = 0; i < 10; i++)
4229                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4230
4231                 /* Perform hash on these packet types */
4232                 mrqc = IXGBE_MRQC_RSSEN
4233                      | IXGBE_MRQC_RSS_FIELD_IPV4
4234                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4235                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4236                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4237                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4238                      | IXGBE_MRQC_RSS_FIELD_IPV6
4239                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4240                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4241                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4242                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4243
4244                 /* RSS and RX IPP Checksum are mutually exclusive */
4245                 rxcsum |= IXGBE_RXCSUM_PCSD;
4246         }
4247
4248         if (ifp->if_capenable & IFCAP_RXCSUM)
4249                 rxcsum |= IXGBE_RXCSUM_PCSD;
4250
4251         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4252                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4253
4254         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4255
4256         return;
4257 }
4258
4259 /*********************************************************************
4260  *
4261  *  Free all receive rings.
4262  *
4263  **********************************************************************/
4264 static void
4265 ixgbe_free_receive_structures(struct adapter *adapter)
4266 {
4267         struct rx_ring *rxr = adapter->rx_rings;
4268
4269         INIT_DEBUGOUT("ixgbe_free_receive_structures: begin");
4270
4271         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4272                 struct lro_ctrl         *lro = &rxr->lro;
4273                 ixgbe_free_receive_buffers(rxr);
4274                 /* Free LRO memory */
4275                 tcp_lro_free(lro);
4276                 /* Free the ring memory as well */
4277                 ixgbe_dma_free(adapter, &rxr->rxdma);
4278         }
4279
4280         free(adapter->rx_rings, M_DEVBUF);
4281 }
4282
4283
4284 /*********************************************************************
4285  *
4286  *  Free receive ring data structures
4287  *
4288  **********************************************************************/
4289 static void
4290 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4291 {
4292         struct adapter          *adapter = rxr->adapter;
4293         struct ixgbe_rx_buf     *rxbuf;
4294
4295         INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin");
4296
4297         /* Cleanup any existing buffers */
4298         if (rxr->rx_buffers != NULL) {
4299                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4300                         rxbuf = &rxr->rx_buffers[i];
4301                         if (rxbuf->buf != NULL) {
4302                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4303                                     BUS_DMASYNC_POSTREAD);
4304                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4305                                 rxbuf->buf->m_flags |= M_PKTHDR;
4306                                 m_freem(rxbuf->buf);
4307                         }
4308                         rxbuf->buf = NULL;
4309                         if (rxbuf->pmap != NULL) {
4310                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4311                                 rxbuf->pmap = NULL;
4312                         }
4313                 }
4314                 if (rxr->rx_buffers != NULL) {
4315                         free(rxr->rx_buffers, M_DEVBUF);
4316                         rxr->rx_buffers = NULL;
4317                 }
4318         }
4319
4320         if (rxr->ptag != NULL) {
4321                 bus_dma_tag_destroy(rxr->ptag);
4322                 rxr->ptag = NULL;
4323         }
4324
4325         return;
4326 }
4327
4328 static __inline void
4329 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4330 {
4331
4332         /*
4333          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4334          * should be computed by hardware. Also it should not have VLAN tag in
4335          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4336          */
4337         if (rxr->lro_enabled &&
4338             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4339             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4340             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4341             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4342             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4343             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4344             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4345             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4346                 /*
4347                  * Send to the stack if:
4348                  **  - LRO not enabled, or
4349                  **  - no LRO resources, or
4350                  **  - lro enqueue fails
4351                  */
4352                 if (rxr->lro.lro_cnt != 0)
4353                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4354                                 return;
4355         }
4356         IXGBE_RX_UNLOCK(rxr);
4357         (*ifp->if_input)(ifp, m);
4358         IXGBE_RX_LOCK(rxr);
4359 }
4360
4361 static __inline void
4362 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4363 {
4364         struct ixgbe_rx_buf     *rbuf;
4365
4366         rbuf = &rxr->rx_buffers[i];
4367
4368         if (rbuf->fmp != NULL) {/* Partial chain ? */
4369                 rbuf->fmp->m_flags |= M_PKTHDR;
4370                 m_freem(rbuf->fmp);
4371                 rbuf->fmp = NULL;
4372         }
4373
4374         /*
4375         ** With advanced descriptors the writeback
4376         ** clobbers the buffer addrs, so its easier
4377         ** to just free the existing mbufs and take
4378         ** the normal refresh path to get new buffers
4379         ** and mapping.
4380         */
4381         if (rbuf->buf) {
4382                 m_free(rbuf->buf);
4383                 rbuf->buf = NULL;
4384         }
4385
4386         rbuf->flags = 0;
4387
4388         return;
4389 }
4390
4391
4392 /*********************************************************************
4393  *
4394  *  This routine executes in interrupt context. It replenishes
4395  *  the mbufs in the descriptor and sends data which has been
4396  *  dma'ed into host memory to upper layer.
4397  *
4398  *  We loop at most count times if count is > 0, or until done if
4399  *  count < 0.
4400  *
4401  *  Return TRUE for more work, FALSE for all clean.
4402  *********************************************************************/
4403 static bool
4404 ixgbe_rxeof(struct ix_queue *que)
4405 {
4406         struct adapter          *adapter = que->adapter;
4407         struct rx_ring          *rxr = que->rxr;
4408         struct ifnet            *ifp = adapter->ifp;
4409         struct lro_ctrl         *lro = &rxr->lro;
4410         struct lro_entry        *queued;
4411         int                     i, nextp, processed = 0;
4412         u32                     staterr = 0;
4413         u16                     count = rxr->process_limit;
4414         union ixgbe_adv_rx_desc *cur;
4415         struct ixgbe_rx_buf     *rbuf, *nbuf;
4416
4417         IXGBE_RX_LOCK(rxr);
4418
4419 #ifdef DEV_NETMAP
4420         /* Same as the txeof routine: wakeup clients on intr. */
4421         if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4422                 IXGBE_RX_UNLOCK(rxr);
4423                 return (FALSE);
4424         }
4425 #endif /* DEV_NETMAP */
4426
4427         for (i = rxr->next_to_check; count != 0;) {
4428                 struct mbuf     *sendmp, *mp;
4429                 u32             rsc, ptype;
4430                 u16             len;
4431                 u16             vtag = 0;
4432                 bool            eop;
4433
4434                 /* Sync the ring. */
4435                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4436                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4437
4438                 cur = &rxr->rx_base[i];
4439                 staterr = le32toh(cur->wb.upper.status_error);
4440
4441                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4442                         break;
4443                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4444                         break;
4445
4446                 count--;
4447                 sendmp = NULL;
4448                 nbuf = NULL;
4449                 rsc = 0;
4450                 cur->wb.upper.status_error = 0;
4451                 rbuf = &rxr->rx_buffers[i];
4452                 mp = rbuf->buf;
4453
4454                 len = le16toh(cur->wb.upper.length);
4455                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4456                     IXGBE_RXDADV_PKTTYPE_MASK;
4457                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4458
4459                 /* Make sure bad packets are discarded */
4460                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4461                     (rxr->discard)) {
4462                         rxr->rx_discarded++;
4463                         if (eop)
4464                                 rxr->discard = FALSE;
4465                         else
4466                                 rxr->discard = TRUE;
4467                         ixgbe_rx_discard(rxr, i);
4468                         goto next_desc;
4469                 }
4470
4471                 /*
4472                 ** On 82599 which supports a hardware
4473                 ** LRO (called HW RSC), packets need
4474                 ** not be fragmented across sequential
4475                 ** descriptors, rather the next descriptor
4476                 ** is indicated in bits of the descriptor.
4477                 ** This also means that we might proceses
4478                 ** more than one packet at a time, something
4479                 ** that has never been true before, it
4480                 ** required eliminating global chain pointers
4481                 ** in favor of what we are doing here.  -jfv
4482                 */
4483                 if (!eop) {
4484                         /*
4485                         ** Figure out the next descriptor
4486                         ** of this frame.
4487                         */
4488                         if (rxr->hw_rsc == TRUE) {
4489                                 rsc = ixgbe_rsc_count(cur);
4490                                 rxr->rsc_num += (rsc - 1);
4491                         }
4492                         if (rsc) { /* Get hardware index */
4493                                 nextp = ((staterr &
4494                                     IXGBE_RXDADV_NEXTP_MASK) >>
4495                                     IXGBE_RXDADV_NEXTP_SHIFT);
4496                         } else { /* Just sequential */
4497                                 nextp = i + 1;
4498                                 if (nextp == adapter->num_rx_desc)
4499                                         nextp = 0;
4500                         }
4501                         nbuf = &rxr->rx_buffers[nextp];
4502                         prefetch(nbuf);
4503                 }
4504                 /*
4505                 ** Rather than using the fmp/lmp global pointers
4506                 ** we now keep the head of a packet chain in the
4507                 ** buffer struct and pass this along from one
4508                 ** descriptor to the next, until we get EOP.
4509                 */
4510                 mp->m_len = len;
4511                 /*
4512                 ** See if there is a stored head
4513                 ** that determines what we are
4514                 */
4515                 sendmp = rbuf->fmp;
4516                 if (sendmp != NULL) {  /* secondary frag */
4517                         rbuf->buf = rbuf->fmp = NULL;
4518                         mp->m_flags &= ~M_PKTHDR;
4519                         sendmp->m_pkthdr.len += mp->m_len;
4520                 } else {
4521                         /*
4522                          * Optimize.  This might be a small packet,
4523                          * maybe just a TCP ACK.  Do a fast copy that
4524                          * is cache aligned into a new mbuf, and
4525                          * leave the old mbuf+cluster for re-use.
4526                          */
4527                         if (eop && len <= IXGBE_RX_COPY_LEN) {
4528                                 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4529                                 if (sendmp != NULL) {
4530                                         sendmp->m_data +=
4531                                             IXGBE_RX_COPY_ALIGN;
4532                                         ixgbe_bcopy(mp->m_data,
4533                                             sendmp->m_data, len);
4534                                         sendmp->m_len = len;
4535                                         rxr->rx_copies++;
4536                                         rbuf->flags |= IXGBE_RX_COPY;
4537                                 }
4538                         }
4539                         if (sendmp == NULL) {
4540                                 rbuf->buf = rbuf->fmp = NULL;
4541                                 sendmp = mp;
4542                         }
4543
4544                         /* first desc of a non-ps chain */
4545                         sendmp->m_flags |= M_PKTHDR;
4546                         sendmp->m_pkthdr.len = mp->m_len;
4547                 }
4548                 ++processed;
4549
4550                 /* Pass the head pointer on */
4551                 if (eop == 0) {
4552                         nbuf->fmp = sendmp;
4553                         sendmp = NULL;
4554                         mp->m_next = nbuf->buf;
4555                 } else { /* Sending this frame */
4556                         sendmp->m_pkthdr.rcvif = ifp;
4557                         rxr->rx_packets++;
4558                         /* capture data for AIM */
4559                         rxr->bytes += sendmp->m_pkthdr.len;
4560                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4561                         /* Process vlan info */
4562                         if ((rxr->vtag_strip) &&
4563                             (staterr & IXGBE_RXD_STAT_VP))
4564                                 vtag = le16toh(cur->wb.upper.vlan);
4565                         if (vtag) {
4566                                 sendmp->m_pkthdr.ether_vtag = vtag;
4567                                 sendmp->m_flags |= M_VLANTAG;
4568                         }
4569                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4570                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4571 #if __FreeBSD_version >= 800000
4572                         sendmp->m_pkthdr.flowid = que->msix;
4573                         sendmp->m_flags |= M_FLOWID;
4574 #endif
4575                 }
4576 next_desc:
4577                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4578                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4579
4580                 /* Advance our pointers to the next descriptor. */
4581                 if (++i == rxr->num_desc)
4582                         i = 0;
4583
4584                 /* Now send to the stack or do LRO */
4585                 if (sendmp != NULL) {
4586                         rxr->next_to_check = i;
4587                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4588                         i = rxr->next_to_check;
4589                 }
4590
4591                /* Every 8 descriptors we go to refresh mbufs */
4592                 if (processed == 8) {
4593                         ixgbe_refresh_mbufs(rxr, i);
4594                         processed = 0;
4595                 }
4596         }
4597
4598         /* Refresh any remaining buf structs */
4599         if (ixgbe_rx_unrefreshed(rxr))
4600                 ixgbe_refresh_mbufs(rxr, i);
4601
4602         rxr->next_to_check = i;
4603
4604         /*
4605          * Flush any outstanding LRO work
4606          */
4607         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4608                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4609                 tcp_lro_flush(lro, queued);
4610         }
4611
4612         IXGBE_RX_UNLOCK(rxr);
4613
4614         /*
4615         ** Still have cleaning to do?
4616         */
4617         if ((staterr & IXGBE_RXD_STAT_DD) != 0)
4618                 return (TRUE);
4619         else
4620                 return (FALSE);
4621 }
4622
4623
4624 /*********************************************************************
4625  *
4626  *  Verify that the hardware indicated that the checksum is valid.
4627  *  Inform the stack about the status of checksum so that stack
4628  *  doesn't spend time verifying the checksum.
4629  *
4630  *********************************************************************/
4631 static void
4632 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4633 {
4634         u16     status = (u16) staterr;
4635         u8      errors = (u8) (staterr >> 24);
4636         bool    sctp = FALSE;
4637
4638         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4639             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4640                 sctp = TRUE;
4641
4642         if (status & IXGBE_RXD_STAT_IPCS) {
4643                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4644                         /* IP Checksum Good */
4645                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4646                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4647
4648                 } else
4649                         mp->m_pkthdr.csum_flags = 0;
4650         }
4651         if (status & IXGBE_RXD_STAT_L4CS) {
4652                 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4653 #if __FreeBSD_version >= 800000
4654                 if (sctp)
4655                         type = CSUM_SCTP_VALID;
4656 #endif
4657                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4658                         mp->m_pkthdr.csum_flags |= type;
4659                         if (!sctp)
4660                                 mp->m_pkthdr.csum_data = htons(0xffff);
4661                 }
4662         }
4663         return;
4664 }
4665
4666
4667 /*
4668 ** This routine is run via an vlan config EVENT,
4669 ** it enables us to use the HW Filter table since
4670 ** we can get the vlan id. This just creates the
4671 ** entry in the soft version of the VFTA, init will
4672 ** repopulate the real table.
4673 */
4674 static void
4675 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4676 {
4677         struct adapter  *adapter = ifp->if_softc;
4678         u16             index, bit;
4679
4680         if (ifp->if_softc !=  arg)   /* Not our event */
4681                 return;
4682
4683         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4684                 return;
4685
4686         IXGBE_CORE_LOCK(adapter);
4687         index = (vtag >> 5) & 0x7F;
4688         bit = vtag & 0x1F;
4689         adapter->shadow_vfta[index] |= (1 << bit);
4690         ++adapter->num_vlans;
4691         ixgbe_setup_vlan_hw_support(adapter);
4692         IXGBE_CORE_UNLOCK(adapter);
4693 }
4694
4695 /*
4696 ** This routine is run via an vlan
4697 ** unconfig EVENT, remove our entry
4698 ** in the soft vfta.
4699 */
4700 static void
4701 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4702 {
4703         struct adapter  *adapter = ifp->if_softc;
4704         u16             index, bit;
4705
4706         if (ifp->if_softc !=  arg)
4707                 return;
4708
4709         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4710                 return;
4711
4712         IXGBE_CORE_LOCK(adapter);
4713         index = (vtag >> 5) & 0x7F;
4714         bit = vtag & 0x1F;
4715         adapter->shadow_vfta[index] &= ~(1 << bit);
4716         --adapter->num_vlans;
4717         /* Re-init to load the changes */
4718         ixgbe_setup_vlan_hw_support(adapter);
4719         IXGBE_CORE_UNLOCK(adapter);
4720 }
4721
4722 static void
4723 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4724 {
4725         struct ifnet    *ifp = adapter->ifp;
4726         struct ixgbe_hw *hw = &adapter->hw;
4727         struct rx_ring  *rxr;
4728         u32             ctrl;
4729
4730
4731         /*
4732         ** We get here thru init_locked, meaning
4733         ** a soft reset, this has already cleared
4734         ** the VFTA and other state, so if there
4735         ** have been no vlan's registered do nothing.
4736         */
4737         if (adapter->num_vlans == 0)
4738                 return;
4739
4740         /* Setup the queues for vlans */
4741         for (int i = 0; i < adapter->num_queues; i++) {
4742                 rxr = &adapter->rx_rings[i];
4743                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4744                 if (hw->mac.type != ixgbe_mac_82598EB) {
4745                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4746                         ctrl |= IXGBE_RXDCTL_VME;
4747                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4748                 }
4749                 rxr->vtag_strip = TRUE;
4750         }
4751
4752         if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
4753                 return;
4754         /*
4755         ** A soft reset zero's out the VFTA, so
4756         ** we need to repopulate it now.
4757         */
4758         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4759                 if (adapter->shadow_vfta[i] != 0)
4760                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4761                             adapter->shadow_vfta[i]);
4762
4763         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4764         /* Enable the Filter Table if enabled */
4765         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4766                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4767                 ctrl |= IXGBE_VLNCTRL_VFE;
4768         }
4769         if (hw->mac.type == ixgbe_mac_82598EB)
4770                 ctrl |= IXGBE_VLNCTRL_VME;
4771         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4772 }
4773
4774 static void
4775 ixgbe_enable_intr(struct adapter *adapter)
4776 {
4777         struct ixgbe_hw *hw = &adapter->hw;
4778         struct ix_queue *que = adapter->queues;
4779         u32             mask, fwsm;
4780
4781         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4782         /* Enable Fan Failure detection */
4783         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4784                     mask |= IXGBE_EIMS_GPI_SDP1;
4785
4786         switch (adapter->hw.mac.type) {
4787                 case ixgbe_mac_82599EB:
4788                         mask |= IXGBE_EIMS_ECC;
4789                         mask |= IXGBE_EIMS_GPI_SDP0;
4790                         mask |= IXGBE_EIMS_GPI_SDP1;
4791                         mask |= IXGBE_EIMS_GPI_SDP2;
4792 #ifdef IXGBE_FDIR
4793                         mask |= IXGBE_EIMS_FLOW_DIR;
4794 #endif
4795                         break;
4796                 case ixgbe_mac_X540:
4797                         mask |= IXGBE_EIMS_ECC;
4798                         /* Detect if Thermal Sensor is enabled */
4799                         fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
4800                         if (fwsm & IXGBE_FWSM_TS_ENABLED)
4801                                 mask |= IXGBE_EIMS_TS;
4802 #ifdef IXGBE_FDIR
4803                         mask |= IXGBE_EIMS_FLOW_DIR;
4804 #endif
4805                 /* falls through */
4806                 default:
4807                         break;
4808         }
4809
4810         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4811
4812         /* With RSS we use auto clear */
4813         if (adapter->msix_mem) {
4814                 mask = IXGBE_EIMS_ENABLE_MASK;
4815                 /* Don't autoclear Link */
4816                 mask &= ~IXGBE_EIMS_OTHER;
4817                 mask &= ~IXGBE_EIMS_LSC;
4818                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4819         }
4820
4821         /*
4822         ** Now enable all queues, this is done separately to
4823         ** allow for handling the extended (beyond 32) MSIX
4824         ** vectors that can be used by 82599
4825         */
4826         for (int i = 0; i < adapter->num_queues; i++, que++)
4827                 ixgbe_enable_queue(adapter, que->msix);
4828
4829         IXGBE_WRITE_FLUSH(hw);
4830
4831         return;
4832 }
4833
4834 static void
4835 ixgbe_disable_intr(struct adapter *adapter)
4836 {
4837         if (adapter->msix_mem)
4838                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4839         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4840                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4841         } else {
4842                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4843                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4844                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4845         }
4846         IXGBE_WRITE_FLUSH(&adapter->hw);
4847         return;
4848 }
4849
4850 u16
4851 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4852 {
4853         u16 value;
4854
4855         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4856             reg, 2);
4857
4858         return (value);
4859 }
4860
4861 void
4862 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4863 {
4864         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4865             reg, value, 2);
4866
4867         return;
4868 }
4869
4870 /*
4871 ** Get the width and transaction speed of
4872 ** the slot this adapter is plugged into.
4873 */
4874 static void
4875 ixgbe_get_slot_info(struct ixgbe_hw *hw)
4876 {
4877         device_t                dev = ((struct ixgbe_osdep *)hw->back)->dev;
4878         struct ixgbe_mac_info   *mac = &hw->mac;
4879         u16                     link;
4880         u32                     offset;
4881
4882         /* For most devices simply call the shared code routine */
4883         if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
4884                 ixgbe_get_bus_info(hw);
4885                 goto display;
4886         }
4887
4888         /*
4889         ** For the Quad port adapter we need to parse back
4890         ** up the PCI tree to find the speed of the expansion
4891         ** slot into which this adapter is plugged. A bit more work.
4892         */
4893         dev = device_get_parent(device_get_parent(dev));
4894 #ifdef IXGBE_DEBUG
4895         device_printf(dev, "parent pcib = %x,%x,%x\n",
4896             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4897 #endif
4898         dev = device_get_parent(device_get_parent(dev));
4899 #ifdef IXGBE_DEBUG
4900         device_printf(dev, "slot pcib = %x,%x,%x\n",
4901             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4902 #endif
4903         /* Now get the PCI Express Capabilities offset */
4904         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4905         /* ...and read the Link Status Register */
4906         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4907         switch (link & IXGBE_PCI_LINK_WIDTH) {
4908         case IXGBE_PCI_LINK_WIDTH_1:
4909                 hw->bus.width = ixgbe_bus_width_pcie_x1;
4910                 break;
4911         case IXGBE_PCI_LINK_WIDTH_2:
4912                 hw->bus.width = ixgbe_bus_width_pcie_x2;
4913                 break;
4914         case IXGBE_PCI_LINK_WIDTH_4:
4915                 hw->bus.width = ixgbe_bus_width_pcie_x4;
4916                 break;
4917         case IXGBE_PCI_LINK_WIDTH_8:
4918                 hw->bus.width = ixgbe_bus_width_pcie_x8;
4919                 break;
4920         default:
4921                 hw->bus.width = ixgbe_bus_width_unknown;
4922                 break;
4923         }
4924
4925         switch (link & IXGBE_PCI_LINK_SPEED) {
4926         case IXGBE_PCI_LINK_SPEED_2500:
4927                 hw->bus.speed = ixgbe_bus_speed_2500;
4928                 break;
4929         case IXGBE_PCI_LINK_SPEED_5000:
4930                 hw->bus.speed = ixgbe_bus_speed_5000;
4931                 break;
4932         case IXGBE_PCI_LINK_SPEED_8000:
4933                 hw->bus.speed = ixgbe_bus_speed_8000;
4934                 break;
4935         default:
4936                 hw->bus.speed = ixgbe_bus_speed_unknown;
4937                 break;
4938         }
4939
4940         mac->ops.set_lan_id(hw);
4941
4942 display:
4943         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4944             ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
4945             (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
4946             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4947             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
4948             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
4949             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
4950             ("Unknown"));
4951
4952         if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4953             ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
4954             (hw->bus.speed == ixgbe_bus_speed_2500))) {
4955                 device_printf(dev, "PCI-Express bandwidth available"
4956                     " for this card\n     is not sufficient for"
4957                     " optimal performance.\n");
4958                 device_printf(dev, "For optimal performance a x8 "
4959                     "PCIE, or x4 PCIE Gen2 slot is required.\n");
4960         }
4961         if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4962             ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
4963             (hw->bus.speed < ixgbe_bus_speed_8000))) {
4964                 device_printf(dev, "PCI-Express bandwidth available"
4965                     " for this card\n     is not sufficient for"
4966                     " optimal performance.\n");
4967                 device_printf(dev, "For optimal performance a x8 "
4968                     "PCIE Gen3 slot is required.\n");
4969         }
4970
4971         return;
4972 }
4973
4974
4975 /*
4976 ** Setup the correct IVAR register for a particular MSIX interrupt
4977 **   (yes this is all very magic and confusing :)
4978 **  - entry is the register array entry
4979 **  - vector is the MSIX vector for this queue
4980 **  - type is RX/TX/MISC
4981 */
4982 static void
4983 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4984 {
4985         struct ixgbe_hw *hw = &adapter->hw;
4986         u32 ivar, index;
4987
4988         vector |= IXGBE_IVAR_ALLOC_VAL;
4989
4990         switch (hw->mac.type) {
4991
4992         case ixgbe_mac_82598EB:
4993                 if (type == -1)
4994                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4995                 else
4996                         entry += (type * 64);
4997                 index = (entry >> 2) & 0x1F;
4998                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4999                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5000                 ivar |= (vector << (8 * (entry & 0x3)));
5001                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5002                 break;
5003
5004         case ixgbe_mac_82599EB:
5005         case ixgbe_mac_X540:
5006                 if (type == -1) { /* MISC IVAR */
5007                         index = (entry & 1) * 8;
5008                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5009                         ivar &= ~(0xFF << index);
5010                         ivar |= (vector << index);
5011                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5012                 } else {        /* RX/TX IVARS */
5013                         index = (16 * (entry & 1)) + (8 * type);
5014                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5015                         ivar &= ~(0xFF << index);
5016                         ivar |= (vector << index);
5017                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5018                 }
5019
5020         default:
5021                 break;
5022         }
5023 }
5024
5025 static void
5026 ixgbe_configure_ivars(struct adapter *adapter)
5027 {
5028         struct  ix_queue *que = adapter->queues;
5029         u32 newitr;
5030
5031         if (ixgbe_max_interrupt_rate > 0)
5032                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5033         else
5034                 newitr = 0;
5035
5036         for (int i = 0; i < adapter->num_queues; i++, que++) {
5037                 /* First the RX queue entry */
5038                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5039                 /* ... and the TX */
5040                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5041                 /* Set an Initial EITR value */
5042                 IXGBE_WRITE_REG(&adapter->hw,
5043                     IXGBE_EITR(que->msix), newitr);
5044         }
5045
5046         /* For the Link interrupt */
5047         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5048 }
5049
5050 /*
5051 ** ixgbe_sfp_probe - called in the local timer to
5052 ** determine if a port had optics inserted.
5053 */
5054 static bool ixgbe_sfp_probe(struct adapter *adapter)
5055 {
5056         struct ixgbe_hw *hw = &adapter->hw;
5057         device_t        dev = adapter->dev;
5058         bool            result = FALSE;
5059
5060         if ((hw->phy.type == ixgbe_phy_nl) &&
5061             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5062                 s32 ret = hw->phy.ops.identify_sfp(hw);
5063                 if (ret)
5064                         goto out;
5065                 ret = hw->phy.ops.reset(hw);
5066                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5067                         device_printf(dev,"Unsupported SFP+ module detected!");
5068                         printf(" Reload driver with supported module.\n");
5069                         adapter->sfp_probe = FALSE;
5070                         goto out;
5071                 } else
5072                         device_printf(dev,"SFP+ module detected!\n");
5073                 /* We now have supported optics */
5074                 adapter->sfp_probe = FALSE;
5075                 /* Set the optics type so system reports correctly */
5076                 ixgbe_setup_optics(adapter);
5077                 result = TRUE;
5078         }
5079 out:
5080         return (result);
5081 }
5082
5083 /*
5084 ** Tasklet handler for MSIX Link interrupts
5085 **  - do outside interrupt since it might sleep
5086 */
5087 static void
5088 ixgbe_handle_link(void *context, int pending)
5089 {
5090         struct adapter  *adapter = context;
5091
5092         ixgbe_check_link(&adapter->hw,
5093             &adapter->link_speed, &adapter->link_up, 0);
5094         ixgbe_update_link_status(adapter);
5095 }
5096
5097 /*
5098 ** Tasklet for handling SFP module interrupts
5099 */
5100 static void
5101 ixgbe_handle_mod(void *context, int pending)
5102 {
5103         struct adapter  *adapter = context;
5104         struct ixgbe_hw *hw = &adapter->hw;
5105         device_t        dev = adapter->dev;
5106         u32 err;
5107
5108         err = hw->phy.ops.identify_sfp(hw);
5109         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5110                 device_printf(dev,
5111                     "Unsupported SFP+ module type was detected.\n");
5112                 return;
5113         }
5114         err = hw->mac.ops.setup_sfp(hw);
5115         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5116                 device_printf(dev,
5117                     "Setup failure - unsupported SFP+ module type.\n");
5118                 return;
5119         }
5120         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5121         return;
5122 }
5123
5124
5125 /*
5126 ** Tasklet for handling MSF (multispeed fiber) interrupts
5127 */
5128 static void
5129 ixgbe_handle_msf(void *context, int pending)
5130 {
5131         struct adapter  *adapter = context;
5132         struct ixgbe_hw *hw = &adapter->hw;
5133         u32 autoneg;
5134         bool negotiate;
5135
5136         autoneg = hw->phy.autoneg_advertised;
5137         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5138                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5139         if (hw->mac.ops.setup_link)
5140                 hw->mac.ops.setup_link(hw, autoneg, TRUE);
5141         return;
5142 }
5143
5144 #ifdef IXGBE_FDIR
5145 /*
5146 ** Tasklet for reinitializing the Flow Director filter table
5147 */
5148 static void
5149 ixgbe_reinit_fdir(void *context, int pending)
5150 {
5151         struct adapter  *adapter = context;
5152         struct ifnet   *ifp = adapter->ifp;
5153
5154         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5155                 return;
5156         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5157         adapter->fdir_reinit = 0;
5158         /* re-enable flow director interrupts */
5159         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5160         /* Restart the interface */
5161         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5162         return;
5163 }
5164 #endif
5165
5166 /**********************************************************************
5167  *
5168  *  Update the board statistics counters.
5169  *
5170  **********************************************************************/
5171 static void
5172 ixgbe_update_stats_counters(struct adapter *adapter)
5173 {
5174         struct ifnet   *ifp = adapter->ifp;
5175         struct ixgbe_hw *hw = &adapter->hw;
5176         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5177         u64  total_missed_rx = 0;
5178
5179         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5180         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5181         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5182         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5183
5184         /*
5185         ** Note: these are for the 8 possible traffic classes,
5186         **       which in current implementation is unused,
5187         **       therefore only 0 should read real data.
5188         */
5189         for (int i = 0; i < 8; i++) {
5190                 u32 mp;
5191                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5192                 /* missed_rx tallies misses for the gprc workaround */
5193                 missed_rx += mp;
5194                 /* global total per queue */
5195                 adapter->stats.mpc[i] += mp;
5196                 /* Running comprehensive total for stats display */
5197                 total_missed_rx += adapter->stats.mpc[i];
5198                 if (hw->mac.type == ixgbe_mac_82598EB) {
5199                         adapter->stats.rnbc[i] +=
5200                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5201                         adapter->stats.qbtc[i] +=
5202                             IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5203                         adapter->stats.qbrc[i] +=
5204                             IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5205                         adapter->stats.pxonrxc[i] +=
5206                             IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5207                 } else
5208                         adapter->stats.pxonrxc[i] +=
5209                             IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5210                 adapter->stats.pxontxc[i] +=
5211                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5212                 adapter->stats.pxofftxc[i] +=
5213                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5214                 adapter->stats.pxoffrxc[i] +=
5215                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5216                 adapter->stats.pxon2offc[i] +=
5217                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5218         }
5219         for (int i = 0; i < 16; i++) {
5220                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5221                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5222                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5223         }
5224         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5225         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5226         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5227
5228         /* Hardware workaround, gprc counts missed packets */
5229         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5230         adapter->stats.gprc -= missed_rx;
5231
5232         if (hw->mac.type != ixgbe_mac_82598EB) {
5233                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5234                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5235                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5236                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5237                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5238                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5239                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5240                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5241         } else {
5242                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5243                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5244                 /* 82598 only has a counter in the high register */
5245                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5246                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5247                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5248         }
5249
5250         /*
5251          * Workaround: mprc hardware is incorrectly counting
5252          * broadcasts, so for now we subtract those.
5253          */
5254         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5255         adapter->stats.bprc += bprc;
5256         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5257         if (hw->mac.type == ixgbe_mac_82598EB)
5258                 adapter->stats.mprc -= bprc;
5259
5260         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5261         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5262         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5263         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5264         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5265         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5266
5267         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5268         adapter->stats.lxontxc += lxon;
5269         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5270         adapter->stats.lxofftxc += lxoff;
5271         total = lxon + lxoff;
5272
5273         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5274         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5275         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5276         adapter->stats.gptc -= total;
5277         adapter->stats.mptc -= total;
5278         adapter->stats.ptc64 -= total;
5279         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5280
5281         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5282         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5283         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5284         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5285         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5286         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5287         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5288         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5289         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5290         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5291         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5292         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5293         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5294         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5295         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5296         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5297         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5298         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5299         /* Only read FCOE on 82599 */
5300         if (hw->mac.type != ixgbe_mac_82598EB) {
5301                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5302                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5303                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5304                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5305                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5306         }
5307
5308         /* Fill out the OS statistics structure */
5309         ifp->if_ipackets = adapter->stats.gprc;
5310         ifp->if_opackets = adapter->stats.gptc;
5311         ifp->if_ibytes = adapter->stats.gorc;
5312         ifp->if_obytes = adapter->stats.gotc;
5313         ifp->if_imcasts = adapter->stats.mprc;
5314         ifp->if_omcasts = adapter->stats.mptc;
5315         ifp->if_collisions = 0;
5316
5317         /* Rx Errors */
5318         ifp->if_iqdrops = total_missed_rx;
5319         ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec;
5320 }
5321
5322 /** ixgbe_sysctl_tdh_handler - Handler function
5323  *  Retrieves the TDH value from the hardware
5324  */
5325 static int
5326 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5327 {
5328         int error;
5329
5330         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5331         if (!txr) return 0;
5332
5333         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5334         error = sysctl_handle_int(oidp, &val, 0, req);
5335         if (error || !req->newptr)
5336                 return error;
5337         return 0;
5338 }
5339
5340 /** ixgbe_sysctl_tdt_handler - Handler function
5341  *  Retrieves the TDT value from the hardware
5342  */
5343 static int
5344 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5345 {
5346         int error;
5347
5348         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5349         if (!txr) return 0;
5350
5351         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5352         error = sysctl_handle_int(oidp, &val, 0, req);
5353         if (error || !req->newptr)
5354                 return error;
5355         return 0;
5356 }
5357
5358 /** ixgbe_sysctl_rdh_handler - Handler function
5359  *  Retrieves the RDH value from the hardware
5360  */
5361 static int
5362 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5363 {
5364         int error;
5365
5366         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5367         if (!rxr) return 0;
5368
5369         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5370         error = sysctl_handle_int(oidp, &val, 0, req);
5371         if (error || !req->newptr)
5372                 return error;
5373         return 0;
5374 }
5375
5376 /** ixgbe_sysctl_rdt_handler - Handler function
5377  *  Retrieves the RDT value from the hardware
5378  */
5379 static int
5380 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5381 {
5382         int error;
5383
5384         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5385         if (!rxr) return 0;
5386
5387         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5388         error = sysctl_handle_int(oidp, &val, 0, req);
5389         if (error || !req->newptr)
5390                 return error;
5391         return 0;
5392 }
5393
5394 static int
5395 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5396 {
5397         int error;
5398         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5399         unsigned int reg, usec, rate;
5400
5401         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5402         usec = ((reg & 0x0FF8) >> 3);
5403         if (usec > 0)
5404                 rate = 500000 / usec;
5405         else
5406                 rate = 0;
5407         error = sysctl_handle_int(oidp, &rate, 0, req);
5408         if (error || !req->newptr)
5409                 return error;
5410         reg &= ~0xfff; /* default, no limitation */
5411         ixgbe_max_interrupt_rate = 0;
5412         if (rate > 0 && rate < 500000) {
5413                 if (rate < 1000)
5414                         rate = 1000;
5415                 ixgbe_max_interrupt_rate = rate;
5416                 reg |= ((4000000/rate) & 0xff8 );
5417         }
5418         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5419         return 0;
5420 }
5421
5422 /*
5423  * Add sysctl variables, one per statistic, to the system.
5424  */
5425 static void
5426 ixgbe_add_hw_stats(struct adapter *adapter)
5427 {
5428
5429         device_t dev = adapter->dev;
5430
5431         struct tx_ring *txr = adapter->tx_rings;
5432         struct rx_ring *rxr = adapter->rx_rings;
5433
5434         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5435         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5436         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5437         struct ixgbe_hw_stats *stats = &adapter->stats;
5438
5439         struct sysctl_oid *stat_node, *queue_node;
5440         struct sysctl_oid_list *stat_list, *queue_list;
5441
5442 #define QUEUE_NAME_LEN 32
5443         char namebuf[QUEUE_NAME_LEN];
5444
5445         /* Driver Statistics */
5446         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5447                         CTLFLAG_RD, &adapter->dropped_pkts,
5448                         "Driver dropped packets");
5449         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5450                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5451                         "m_defrag() failed");
5452         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5453                         CTLFLAG_RD, &adapter->watchdog_events,
5454                         "Watchdog timeouts");
5455         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5456                         CTLFLAG_RD, &adapter->link_irq,
5457                         "Link MSIX IRQ Handled");
5458
5459         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5460                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5461                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5462                                             CTLFLAG_RD, NULL, "Queue Name");
5463                 queue_list = SYSCTL_CHILDREN(queue_node);
5464
5465                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5466                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5467                                 sizeof(&adapter->queues[i]),
5468                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5469                                 "Interrupt Rate");
5470                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5471                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
5472                                 "irqs on this queue");
5473                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5474                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5475                                 ixgbe_sysctl_tdh_handler, "IU",
5476                                 "Transmit Descriptor Head");
5477                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5478                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5479                                 ixgbe_sysctl_tdt_handler, "IU",
5480                                 "Transmit Descriptor Tail");
5481                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5482                                 CTLFLAG_RD, &txr->tso_tx,
5483                                 "TSO");
5484                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5485                                 CTLFLAG_RD, &txr->no_tx_dma_setup,
5486                                 "Driver tx dma failure in xmit");
5487                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5488                                 CTLFLAG_RD, &txr->no_desc_avail,
5489                                 "Queue No Descriptor Available");
5490                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5491                                 CTLFLAG_RD, &txr->total_packets,
5492                                 "Queue Packets Transmitted");
5493         }
5494
5495         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5496                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5497                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5498                                             CTLFLAG_RD, NULL, "Queue Name");
5499                 queue_list = SYSCTL_CHILDREN(queue_node);
5500
5501                 struct lro_ctrl *lro = &rxr->lro;
5502
5503                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5504                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5505                                             CTLFLAG_RD, NULL, "Queue Name");
5506                 queue_list = SYSCTL_CHILDREN(queue_node);
5507
5508                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5509                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5510                                 ixgbe_sysctl_rdh_handler, "IU",
5511                                 "Receive Descriptor Head");
5512                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5513                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5514                                 ixgbe_sysctl_rdt_handler, "IU",
5515                                 "Receive Descriptor Tail");
5516                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5517                                 CTLFLAG_RD, &rxr->rx_packets,
5518                                 "Queue Packets Received");
5519                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5520                                 CTLFLAG_RD, &rxr->rx_bytes,
5521                                 "Queue Bytes Received");
5522                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5523                                 CTLFLAG_RD, &rxr->rx_copies,
5524                                 "Copied RX Frames");
5525                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5526                                 CTLFLAG_RD, &lro->lro_queued, 0,
5527                                 "LRO Queued");
5528                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5529                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5530                                 "LRO Flushed");
5531         }
5532
5533         /* MAC stats get the own sub node */
5534
5535         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5536                                     CTLFLAG_RD, NULL, "MAC Statistics");
5537         stat_list = SYSCTL_CHILDREN(stat_node);
5538
5539         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5540                         CTLFLAG_RD, &stats->crcerrs,
5541                         "CRC Errors");
5542         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5543                         CTLFLAG_RD, &stats->illerrc,
5544                         "Illegal Byte Errors");
5545         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5546                         CTLFLAG_RD, &stats->errbc,
5547                         "Byte Errors");
5548         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5549                         CTLFLAG_RD, &stats->mspdc,
5550                         "MAC Short Packets Discarded");
5551         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5552                         CTLFLAG_RD, &stats->mlfc,
5553                         "MAC Local Faults");
5554         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5555                         CTLFLAG_RD, &stats->mrfc,
5556                         "MAC Remote Faults");
5557         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5558                         CTLFLAG_RD, &stats->rlec,
5559                         "Receive Length Errors");
5560
5561         /* Flow Control stats */
5562         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5563                         CTLFLAG_RD, &stats->lxontxc,
5564                         "Link XON Transmitted");
5565         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5566                         CTLFLAG_RD, &stats->lxonrxc,
5567                         "Link XON Received");
5568         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5569                         CTLFLAG_RD, &stats->lxofftxc,
5570                         "Link XOFF Transmitted");
5571         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5572                         CTLFLAG_RD, &stats->lxoffrxc,
5573                         "Link XOFF Received");
5574
5575         /* Packet Reception Stats */
5576         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5577                         CTLFLAG_RD, &stats->tor,
5578                         "Total Octets Received");
5579         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5580                         CTLFLAG_RD, &stats->gorc,
5581                         "Good Octets Received");
5582         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5583                         CTLFLAG_RD, &stats->tpr,
5584                         "Total Packets Received");
5585         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5586                         CTLFLAG_RD, &stats->gprc,
5587                         "Good Packets Received");
5588         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5589                         CTLFLAG_RD, &stats->mprc,
5590                         "Multicast Packets Received");
5591         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5592                         CTLFLAG_RD, &stats->bprc,
5593                         "Broadcast Packets Received");
5594         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5595                         CTLFLAG_RD, &stats->prc64,
5596                         "64 byte frames received ");
5597         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5598                         CTLFLAG_RD, &stats->prc127,
5599                         "65-127 byte frames received");
5600         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5601                         CTLFLAG_RD, &stats->prc255,
5602                         "128-255 byte frames received");
5603         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5604                         CTLFLAG_RD, &stats->prc511,
5605                         "256-511 byte frames received");
5606         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5607                         CTLFLAG_RD, &stats->prc1023,
5608                         "512-1023 byte frames received");
5609         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5610                         CTLFLAG_RD, &stats->prc1522,
5611                         "1023-1522 byte frames received");
5612         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5613                         CTLFLAG_RD, &stats->ruc,
5614                         "Receive Undersized");
5615         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5616                         CTLFLAG_RD, &stats->rfc,
5617                         "Fragmented Packets Received ");
5618         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5619                         CTLFLAG_RD, &stats->roc,
5620                         "Oversized Packets Received");
5621         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5622                         CTLFLAG_RD, &stats->rjc,
5623                         "Received Jabber");
5624         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5625                         CTLFLAG_RD, &stats->mngprc,
5626                         "Management Packets Received");
5627         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5628                         CTLFLAG_RD, &stats->mngptc,
5629                         "Management Packets Dropped");
5630         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5631                         CTLFLAG_RD, &stats->xec,
5632                         "Checksum Errors");
5633
5634         /* Packet Transmission Stats */
5635         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5636                         CTLFLAG_RD, &stats->gotc,
5637                         "Good Octets Transmitted");
5638         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5639                         CTLFLAG_RD, &stats->tpt,
5640                         "Total Packets Transmitted");
5641         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5642                         CTLFLAG_RD, &stats->gptc,
5643                         "Good Packets Transmitted");
5644         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5645                         CTLFLAG_RD, &stats->bptc,
5646                         "Broadcast Packets Transmitted");
5647         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5648                         CTLFLAG_RD, &stats->mptc,
5649                         "Multicast Packets Transmitted");
5650         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5651                         CTLFLAG_RD, &stats->mngptc,
5652                         "Management Packets Transmitted");
5653         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5654                         CTLFLAG_RD, &stats->ptc64,
5655                         "64 byte frames transmitted ");
5656         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5657                         CTLFLAG_RD, &stats->ptc127,
5658                         "65-127 byte frames transmitted");
5659         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5660                         CTLFLAG_RD, &stats->ptc255,
5661                         "128-255 byte frames transmitted");
5662         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5663                         CTLFLAG_RD, &stats->ptc511,
5664                         "256-511 byte frames transmitted");
5665         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5666                         CTLFLAG_RD, &stats->ptc1023,
5667                         "512-1023 byte frames transmitted");
5668         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5669                         CTLFLAG_RD, &stats->ptc1522,
5670                         "1024-1522 byte frames transmitted");
5671 }
5672
5673 /*
5674 ** Set flow control using sysctl:
5675 ** Flow control values:
5676 **      0 - off
5677 **      1 - rx pause
5678 **      2 - tx pause
5679 **      3 - full
5680 */
5681 static int
5682 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5683 {
5684         int error, last;
5685         struct adapter *adapter = (struct adapter *) arg1;
5686
5687         last = adapter->fc;
5688         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5689         if ((error) || (req->newptr == NULL))
5690                 return (error);
5691
5692         /* Don't bother if it's not changed */
5693         if (adapter->fc == last)
5694                 return (0);
5695
5696         switch (adapter->fc) {
5697                 case ixgbe_fc_rx_pause:
5698                 case ixgbe_fc_tx_pause:
5699                 case ixgbe_fc_full:
5700                         adapter->hw.fc.requested_mode = adapter->fc;
5701                         if (adapter->num_queues > 1)
5702                                 ixgbe_disable_rx_drop(adapter);
5703                         break;
5704                 case ixgbe_fc_none:
5705                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5706                         if (adapter->num_queues > 1)
5707                                 ixgbe_enable_rx_drop(adapter);
5708                         break;
5709                 default:
5710                         adapter->fc = last;
5711                         return (EINVAL);
5712         }
5713         /* Don't autoneg if forcing a value */
5714         adapter->hw.fc.disable_fc_autoneg = TRUE;
5715         ixgbe_fc_enable(&adapter->hw);
5716         return error;
5717 }
5718
5719 /*
5720 ** Control link advertise speed:
5721 **      1 - advertise only 1G
5722 **      2 - advertise 100Mb
5723 **      3 - advertise normal
5724 */
5725 static int
5726 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5727 {
5728         int                     error = 0;
5729         struct adapter          *adapter;
5730         device_t                dev;
5731         struct ixgbe_hw         *hw;
5732         ixgbe_link_speed        speed, last;
5733
5734         adapter = (struct adapter *) arg1;
5735         dev = adapter->dev;
5736         hw = &adapter->hw;
5737         last = adapter->advertise;
5738
5739         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5740         if ((error) || (req->newptr == NULL))
5741                 return (error);
5742
5743         if (adapter->advertise == last) /* no change */
5744                 return (0);
5745
5746         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5747             (hw->phy.multispeed_fiber)))
5748                 return (EINVAL);
5749
5750         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5751                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5752                 return (EINVAL);
5753         }
5754
5755         if (adapter->advertise == 1)
5756                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5757         else if (adapter->advertise == 2)
5758                 speed = IXGBE_LINK_SPEED_100_FULL;
5759         else if (adapter->advertise == 3)
5760                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5761                         IXGBE_LINK_SPEED_10GB_FULL;
5762         else {  /* bogus value */
5763                 adapter->advertise = last;
5764                 return (EINVAL);
5765         }
5766
5767         hw->mac.autotry_restart = TRUE;
5768         hw->mac.ops.setup_link(hw, speed, TRUE);
5769
5770         return (error);
5771 }
5772
5773 /*
5774 ** Thermal Shutdown Trigger
5775 **   - cause a Thermal Overtemp IRQ
5776 **   - this now requires firmware enabling
5777 */
5778 static int
5779 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5780 {
5781         int             error, fire = 0;
5782         struct adapter  *adapter = (struct adapter *) arg1;
5783         struct ixgbe_hw *hw = &adapter->hw;
5784
5785
5786         if (hw->mac.type != ixgbe_mac_X540)
5787                 return (0);
5788
5789         error = sysctl_handle_int(oidp, &fire, 0, req);
5790         if ((error) || (req->newptr == NULL))
5791                 return (error);
5792
5793         if (fire) {
5794                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5795                 reg |= IXGBE_EICR_TS;
5796                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5797         }
5798
5799         return (0);
5800 }
5801
5802 /*
5803 ** Enable the hardware to drop packets when the buffer is
5804 ** full. This is useful when multiqueue,so that no single
5805 ** queue being full stalls the entire RX engine. We only
5806 ** enable this when Multiqueue AND when Flow Control is
5807 ** disabled.
5808 */
5809 static void
5810 ixgbe_enable_rx_drop(struct adapter *adapter)
5811 {
5812         struct ixgbe_hw *hw = &adapter->hw;
5813
5814         for (int i = 0; i < adapter->num_queues; i++) {
5815                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5816                 srrctl |= IXGBE_SRRCTL_DROP_EN;
5817                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5818         }
5819 }
5820
5821 static void
5822 ixgbe_disable_rx_drop(struct adapter *adapter)
5823 {
5824         struct ixgbe_hw *hw = &adapter->hw;
5825
5826         for (int i = 0; i < adapter->num_queues; i++) {
5827                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5828                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5829                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5830         }
5831 }