sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2013, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.5.15";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 #ifdef IXGBE_LEGACY_TX
 106 static void     ixgbe_start(struct ifnet *);
 107 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 108 #else /* ! IXGBE_LEGACY_TX */
 109 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 110 static int      ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 static void     ixgbe_deferred_mq_start(void *, int);
 113 #endif /* IXGBE_LEGACY_TX */
 114 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 115 static void     ixgbe_init(void *);
 116 static void     ixgbe_init_locked(struct adapter *);
 117 static void     ixgbe_stop(void *);
 118 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 119 static int      ixgbe_media_change(struct ifnet *);
 120 static void     ixgbe_identify_hardware(struct adapter *);
 121 static int      ixgbe_allocate_pci_resources(struct adapter *);
 122 static void     ixgbe_get_slot_info(struct ixgbe_hw *);
 123 static int      ixgbe_allocate_msix(struct adapter *);
 124 static int      ixgbe_allocate_legacy(struct adapter *);
 125 static int      ixgbe_allocate_queues(struct adapter *);
 126 static int      ixgbe_setup_msix(struct adapter *);
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static bool     ixgbe_rxeof(struct ix_queue *);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static int      ixgbe_tx_ctx_setup(struct tx_ring *,
 165                     struct mbuf *, u32 *, u32 *);
 166 static int      ixgbe_tso_setup(struct tx_ring *,
 167                     struct mbuf *, u32 *, u32 *);
 168 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 169 static void     ixgbe_configure_ivars(struct adapter *);
 170 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 171
 172 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 173 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 174 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 175
 176 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 177
 178 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 179 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 180                     struct mbuf *, u32);
 181
 182 static void     ixgbe_enable_rx_drop(struct adapter *);
 183 static void     ixgbe_disable_rx_drop(struct adapter *);
 184
 185 /* Support for pluggable optic modules */
 186 static bool     ixgbe_sfp_probe(struct adapter *);
 187 static void     ixgbe_setup_optics(struct adapter *);
 188
 189 /* Legacy (single vector interrupt handler */
 190 static void     ixgbe_legacy_irq(void *);
 191
 192 /* The MSI/X Interrupt handlers */
 193 static void     ixgbe_msix_que(void *);
 194 static void     ixgbe_msix_link(void *);
 195
 196 /* Deferred interrupt tasklets */
 197 static void     ixgbe_handle_que(void *, int);
 198 static void     ixgbe_handle_link(void *, int);
 199 static void     ixgbe_handle_msf(void *, int);
 200 static void     ixgbe_handle_mod(void *, int);
 201
 202 #ifdef IXGBE_FDIR
 203 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 204 static void     ixgbe_reinit_fdir(void *, int);
 205 #endif
 206
 207 /* Missing shared code prototype */
 208 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
 209
 210 /*********************************************************************
 211  *  FreeBSD Device Interface Entry Points
 212  *********************************************************************/
 213
 214 static device_method_t ixgbe_methods[] = {
 215         /* Device interface */
 216         DEVMETHOD(device_probe, ixgbe_probe),
 217         DEVMETHOD(device_attach, ixgbe_attach),
 218         DEVMETHOD(device_detach, ixgbe_detach),
 219         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 220         DEVMETHOD_END
 221 };
 222
 223 static driver_t ixgbe_driver = {
 224         "ix", ixgbe_methods, sizeof(struct adapter),
 225 };
 226
 227 devclass_t ixgbe_devclass;
 228 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 229
 230 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 231 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 232
 233 /*
 234 ** TUNEABLE PARAMETERS:
 235 */
 236
 237 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD, 0,
 238                    "IXGBE driver parameters");
 239
 240 /*
 241 ** AIM: Adaptive Interrupt Moderation
 242 ** which means that the interrupt rate
 243 ** is varied over time based on the
 244 ** traffic for that interrupt vector
 245 */
 246 static int ixgbe_enable_aim = TRUE;
 247 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 248 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RW, &ixgbe_enable_aim, 0,
 249     "Enable adaptive interrupt moderation");
 250
 251 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 252 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 253 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
 254     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
 255
 256 /* How many packets rxeof tries to clean at a time */
 257 static int ixgbe_rx_process_limit = 256;
 258 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 259 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
 260     &ixgbe_rx_process_limit, 0,
 261     "Maximum number of received packets to process at a time,"
 262     "-1 means unlimited");
 263
 264 /* How many packets txeof tries to clean at a time */
 265 static int ixgbe_tx_process_limit = 256;
 266 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
 267 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
 268     &ixgbe_tx_process_limit, 0,
 269     "Maximum number of sent packets to process at a time,"
 270     "-1 means unlimited");
 271
 272 /*
 273 ** Smart speed setting, default to on
 274 ** this only works as a compile option
 275 ** right now as its during attach, set
 276 ** this to 'ixgbe_smart_speed_off' to
 277 ** disable.
 278 */
 279 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 280
 281 /*
 282  * MSIX should be the default for best performance,
 283  * but this allows it to be forced off for testing.
 284  */
 285 static int ixgbe_enable_msix = 1;
 286 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 287 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
 288     "Enable MSI-X interrupts");
 289
 290 /*
 291  * Number of Queues, can be set to 0,
 292  * it then autoconfigures based on the
 293  * number of cpus with a max of 8. This
 294  * can be overriden manually here.
 295  */
 296 static int ixgbe_num_queues = 0;
 297 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 298 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
 299     "Number of queues to configure, 0 indicates autoconfigure");
 300
 301 /*
 302 ** Number of TX descriptors per ring,
 303 ** setting higher than RX as this seems
 304 ** the better performing choice.
 305 */
 306 static int ixgbe_txd = PERFORM_TXD;
 307 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 308 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
 309     "Number of receive descriptors per queue");
 310
 311 /* Number of RX descriptors per ring */
 312 static int ixgbe_rxd = PERFORM_RXD;
 313 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 314 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
 315     "Number of receive descriptors per queue");
 316
 317 /*
 318 ** Defining this on will allow the use
 319 ** of unsupported SFP+ modules, note that
 320 ** doing so you are on your own :)
 321 */
 322 static int allow_unsupported_sfp = FALSE;
 323 TUNABLE_INT("hw.ixgbe.unsupported_sfp", &allow_unsupported_sfp);
 324
 325 /*
 326 ** HW RSC control:
 327 **  this feature only works with
 328 **  IPv4, and only on 82599 and later.
 329 **  Also this will cause IP forwarding to
 330 **  fail and that can't be controlled by
 331 **  the stack as LRO can. For all these
 332 **  reasons I've deemed it best to leave
 333 **  this off and not bother with a tuneable
 334 **  interface, this would need to be compiled
 335 **  to enable.
 336 */
 337 static bool ixgbe_rsc_enable = FALSE;
 338
 339 /* Keep running tab on them for sanity check */
 340 static int ixgbe_total_ports;
 341
 342 #ifdef IXGBE_FDIR
 343 /*
 344 ** For Flow Director: this is the
 345 ** number of TX packets we sample
 346 ** for the filter pool, this means
 347 ** every 20th packet will be probed.
 348 **
 349 ** This feature can be disabled by
 350 ** setting this to 0.
 351 */
 352 static int atr_sample_rate = 20;
 353 /*
 354 ** Flow Director actually 'steals'
 355 ** part of the packet buffer as its
 356 ** filter pool, this variable controls
 357 ** how much it uses:
 358 **  0 = 64K, 1 = 128K, 2 = 256K
 359 */
 360 static int fdir_pballoc = 1;
 361 #endif
 362
 363 #ifdef DEV_NETMAP
 364 /*
 365  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 366  * be a reference on how to implement netmap support in a driver.
 367  * Additional comments are in ixgbe_netmap.h .
 368  *
 369  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 370  * that extend the standard driver.
 371  */
 372 #include <dev/netmap/ixgbe_netmap.h>
 373 #endif /* DEV_NETMAP */
 374
 375 /*********************************************************************
 376  *  Device identification routine
 377  *
 378  *  ixgbe_probe determines if the driver should be loaded on
 379  *  adapter based on PCI vendor/device id of the adapter.
 380  *
 381  *  return BUS_PROBE_DEFAULT on success, positive on failure
 382  *********************************************************************/
 383
 384 static int
 385 ixgbe_probe(device_t dev)
 386 {
 387         ixgbe_vendor_info_t *ent;
 388
 389         u16     pci_vendor_id = 0;
 390         u16     pci_device_id = 0;
 391         u16     pci_subvendor_id = 0;
 392         u16     pci_subdevice_id = 0;
 393         char    adapter_name[256];
 394
 395         INIT_DEBUGOUT("ixgbe_probe: begin");
 396
 397         pci_vendor_id = pci_get_vendor(dev);
 398         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 399                 return (ENXIO);
 400
 401         pci_device_id = pci_get_device(dev);
 402         pci_subvendor_id = pci_get_subvendor(dev);
 403         pci_subdevice_id = pci_get_subdevice(dev);
 404
 405         ent = ixgbe_vendor_info_array;
 406         while (ent->vendor_id != 0) {
 407                 if ((pci_vendor_id == ent->vendor_id) &&
 408                     (pci_device_id == ent->device_id) &&
 409
 410                     ((pci_subvendor_id == ent->subvendor_id) ||
 411                      (ent->subvendor_id == 0)) &&
 412
 413                     ((pci_subdevice_id == ent->subdevice_id) ||
 414                      (ent->subdevice_id == 0))) {
 415                         sprintf(adapter_name, "%s, Version - %s",
 416                                 ixgbe_strings[ent->index],
 417                                 ixgbe_driver_version);
 418                         device_set_desc_copy(dev, adapter_name);
 419                         ++ixgbe_total_ports;
 420                         return (BUS_PROBE_DEFAULT);
 421                 }
 422                 ent++;
 423         }
 424         return (ENXIO);
 425 }
 426
 427 /*********************************************************************
 428  *  Device initialization routine
 429  *
 430  *  The attach entry point is called when the driver is being loaded.
 431  *  This routine identifies the type of hardware, allocates all resources
 432  *  and initializes the hardware.
 433  *
 434  *  return 0 on success, positive on failure
 435  *********************************************************************/
 436
 437 static int
 438 ixgbe_attach(device_t dev)
 439 {
 440         struct adapter *adapter;
 441         struct ixgbe_hw *hw;
 442         int             error = 0;
 443         u16             csum;
 444         u32             ctrl_ext;
 445
 446         INIT_DEBUGOUT("ixgbe_attach: begin");
 447
 448         /* Allocate, clear, and link in our adapter structure */
 449         adapter = device_get_softc(dev);
 450         adapter->dev = adapter->osdep.dev = dev;
 451         hw = &adapter->hw;
 452
 453         /* Core Lock Init*/
 454         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 455
 456         /* SYSCTL APIs */
 457
 458         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 459                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 460                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 461                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 462
 463         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 464                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 465                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 466                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 467
 468         /*
 469         ** Allow a kind of speed control by forcing the autoneg
 470         ** advertised speed list to only a certain value, this
 471         ** supports 1G on 82599 devices, and 100Mb on x540.
 472         */
 473         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 474                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 475                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 476                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 477
 478         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 479                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 480                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 481                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 482
 483         /* Set up the timer callout */
 484         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 485
 486         /* Determine hardware revision */
 487         ixgbe_identify_hardware(adapter);
 488
 489         /* Do base PCI setup - map BAR0 */
 490         if (ixgbe_allocate_pci_resources(adapter)) {
 491                 device_printf(dev, "Allocation of PCI resources failed\n");
 492                 error = ENXIO;
 493                 goto err_out;
 494         }
 495
 496         /* Do descriptor calc and sanity checks */
 497         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 498             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 499                 device_printf(dev, "TXD config issue, using default!\n");
 500                 adapter->num_tx_desc = DEFAULT_TXD;
 501         } else
 502                 adapter->num_tx_desc = ixgbe_txd;
 503
 504         /*
 505         ** With many RX rings it is easy to exceed the
 506         ** system mbuf allocation. Tuning nmbclusters
 507         ** can alleviate this.
 508         */
 509         if (nmbclusters > 0 ) {
 510                 int s;
 511                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 512                 if (s > nmbclusters) {
 513                         device_printf(dev, "RX Descriptors exceed "
 514                             "system mbuf max, using default instead!\n");
 515                         ixgbe_rxd = DEFAULT_RXD;
 516                 }
 517         }
 518
 519         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 520             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 521                 device_printf(dev, "RXD config issue, using default!\n");
 522                 adapter->num_rx_desc = DEFAULT_RXD;
 523         } else
 524                 adapter->num_rx_desc = ixgbe_rxd;
 525
 526         /* Allocate our TX/RX Queues */
 527         if (ixgbe_allocate_queues(adapter)) {
 528                 error = ENOMEM;
 529                 goto err_out;
 530         }
 531
 532         /* Allocate multicast array memory. */
 533         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 534             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 535         if (adapter->mta == NULL) {
 536                 device_printf(dev, "Can not allocate multicast setup array\n");
 537                 error = ENOMEM;
 538                 goto err_late;
 539         }
 540
 541         /* Initialize the shared code */
 542         hw->allow_unsupported_sfp = allow_unsupported_sfp;
 543         error = ixgbe_init_shared_code(hw);
 544         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 545                 /*
 546                 ** No optics in this port, set up
 547                 ** so the timer routine will probe
 548                 ** for later insertion.
 549                 */
 550                 adapter->sfp_probe = TRUE;
 551                 error = 0;
 552         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 553                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 554                 error = EIO;
 555                 goto err_late;
 556         } else if (error) {
 557                 device_printf(dev,"Unable to initialize the shared code\n");
 558                 error = EIO;
 559                 goto err_late;
 560         }
 561
 562         /* Make sure we have a good EEPROM before we read from it */
 563         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 564                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 565                 error = EIO;
 566                 goto err_late;
 567         }
 568
 569         error = ixgbe_init_hw(hw);
 570         switch (error) {
 571         case IXGBE_ERR_EEPROM_VERSION:
 572                 device_printf(dev, "This device is a pre-production adapter/"
 573                     "LOM.  Please be aware there may be issues associated "
 574                     "with your hardware.\n If you are experiencing problems "
 575                     "please contact your Intel or hardware representative "
 576                     "who provided you with this hardware.\n");
 577                 break;
 578         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 579                 device_printf(dev,"Unsupported SFP+ Module\n");
 580                 error = EIO;
 581                 goto err_late;
 582         case IXGBE_ERR_SFP_NOT_PRESENT:
 583                 device_printf(dev,"No SFP+ Module found\n");
 584                 /* falls thru */
 585         default:
 586                 break;
 587         }
 588
 589         /* Detect and set physical type */
 590         ixgbe_setup_optics(adapter);
 591
 592         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 593                 error = ixgbe_allocate_msix(adapter);
 594         else
 595                 error = ixgbe_allocate_legacy(adapter);
 596         if (error)
 597                 goto err_late;
 598
 599         /* Setup OS specific network interface */
 600         if (ixgbe_setup_interface(dev, adapter) != 0)
 601                 goto err_late;
 602
 603         /* Initialize statistics */
 604         ixgbe_update_stats_counters(adapter);
 605
 606         /* Register for VLAN events */
 607         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 608             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 609         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 610             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 611
 612         /*
 613         ** Check PCIE slot type/speed/width
 614         */
 615         ixgbe_get_slot_info(hw);
 616
 617         /* Set an initial default flow control value */
 618         adapter->fc =  ixgbe_fc_full;
 619
 620         /* let hardware know driver is loaded */
 621         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 622         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 623         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 624
 625         ixgbe_add_hw_stats(adapter);
 626
 627 #ifdef DEV_NETMAP
 628         ixgbe_netmap_attach(adapter);
 629 #endif /* DEV_NETMAP */
 630         INIT_DEBUGOUT("ixgbe_attach: end");
 631         return (0);
 632 err_late:
 633         ixgbe_free_transmit_structures(adapter);
 634         ixgbe_free_receive_structures(adapter);
 635 err_out:
 636         if (adapter->ifp != NULL)
 637                 if_free(adapter->ifp);
 638         ixgbe_free_pci_resources(adapter);
 639         free(adapter->mta, M_DEVBUF);
 640         return (error);
 641
 642 }
 643
 644 /*********************************************************************
 645  *  Device removal routine
 646  *
 647  *  The detach entry point is called when the driver is being removed.
 648  *  This routine stops the adapter and deallocates all the resources
 649  *  that were allocated for driver operation.
 650  *
 651  *  return 0 on success, positive on failure
 652  *********************************************************************/
 653
 654 static int
 655 ixgbe_detach(device_t dev)
 656 {
 657         struct adapter *adapter = device_get_softc(dev);
 658         struct ix_queue *que = adapter->queues;
 659         struct tx_ring *txr = adapter->tx_rings;
 660         u32     ctrl_ext;
 661
 662         INIT_DEBUGOUT("ixgbe_detach: begin");
 663
 664         /* Make sure VLANS are not using driver */
 665         if (adapter->ifp->if_vlantrunk != NULL) {
 666                 device_printf(dev,"Vlan in use, detach first\n");
 667                 return (EBUSY);
 668         }
 669
 670         IXGBE_CORE_LOCK(adapter);
 671         ixgbe_stop(adapter);
 672         IXGBE_CORE_UNLOCK(adapter);
 673
 674         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 675                 if (que->tq) {
 676 #ifndef IXGBE_LEGACY_TX
 677                         taskqueue_drain(que->tq, &txr->txq_task);
 678 #endif
 679                         taskqueue_drain(que->tq, &que->que_task);
 680                         taskqueue_free(que->tq);
 681                 }
 682         }
 683
 684         /* Drain the Link queue */
 685         if (adapter->tq) {
 686                 taskqueue_drain(adapter->tq, &adapter->link_task);
 687                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 688                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 689 #ifdef IXGBE_FDIR
 690                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 691 #endif
 692                 taskqueue_free(adapter->tq);
 693         }
 694
 695         /* let hardware know driver is unloading */
 696         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 697         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 698         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 699
 700         /* Unregister VLAN events */
 701         if (adapter->vlan_attach != NULL)
 702                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 703         if (adapter->vlan_detach != NULL)
 704                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 705
 706         ether_ifdetach(adapter->ifp);
 707         callout_drain(&adapter->timer);
 708 #ifdef DEV_NETMAP
 709         netmap_detach(adapter->ifp);
 710 #endif /* DEV_NETMAP */
 711         ixgbe_free_pci_resources(adapter);
 712         bus_generic_detach(dev);
 713         if_free(adapter->ifp);
 714
 715         ixgbe_free_transmit_structures(adapter);
 716         ixgbe_free_receive_structures(adapter);
 717         free(adapter->mta, M_DEVBUF);
 718
 719         IXGBE_CORE_LOCK_DESTROY(adapter);
 720         return (0);
 721 }
 722
 723 /*********************************************************************
 724  *
 725  *  Shutdown entry point
 726  *
 727  **********************************************************************/
 728
 729 static int
 730 ixgbe_shutdown(device_t dev)
 731 {
 732         struct adapter *adapter = device_get_softc(dev);
 733         IXGBE_CORE_LOCK(adapter);
 734         ixgbe_stop(adapter);
 735         IXGBE_CORE_UNLOCK(adapter);
 736         return (0);
 737 }
 738
 739
 740 #ifdef IXGBE_LEGACY_TX
 741 /*********************************************************************
 742  *  Transmit entry point
 743  *
 744  *  ixgbe_start is called by the stack to initiate a transmit.
 745  *  The driver will remain in this routine as long as there are
 746  *  packets to transmit and transmit resources are available.
 747  *  In case resources are not available stack is notified and
 748  *  the packet is requeued.
 749  **********************************************************************/
 750
 751 static void
 752 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 753 {
 754         struct mbuf    *m_head;
 755         struct adapter *adapter = txr->adapter;
 756
 757         IXGBE_TX_LOCK_ASSERT(txr);
 758
 759         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 760                 return;
 761         if (!adapter->link_active)
 762                 return;
 763
 764         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 765                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 766                         break;
 767
 768                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 769                 if (m_head == NULL)
 770                         break;
 771
 772                 if (ixgbe_xmit(txr, &m_head)) {
 773                         if (m_head != NULL)
 774                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 775                         break;
 776                 }
 777                 /* Send a copy of the frame to the BPF listener */
 778                 ETHER_BPF_MTAP(ifp, m_head);
 779
 780                 /* Set watchdog on */
 781                 txr->watchdog_time = ticks;
 782                 txr->queue_status = IXGBE_QUEUE_WORKING;
 783
 784         }
 785         return;
 786 }
 787
 788 /*
 789  * Legacy TX start - called by the stack, this
 790  * always uses the first tx ring, and should
 791  * not be used with multiqueue tx enabled.
 792  */
 793 static void
 794 ixgbe_start(struct ifnet *ifp)
 795 {
 796         struct adapter *adapter = ifp->if_softc;
 797         struct tx_ring  *txr = adapter->tx_rings;
 798
 799         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 800                 IXGBE_TX_LOCK(txr);
 801                 ixgbe_start_locked(txr, ifp);
 802                 IXGBE_TX_UNLOCK(txr);
 803         }
 804         return;
 805 }
 806
 807 #else /* ! IXGBE_LEGACY_TX */
 808
 809 /*
 810 ** Multiqueue Transmit driver
 811 **
 812 */
 813 static int
 814 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 815 {
 816         struct adapter  *adapter = ifp->if_softc;
 817         struct ix_queue *que;
 818         struct tx_ring  *txr;
 819         int             i, err = 0;
 820
 821         /* Which queue to use */
 822         if ((m->m_flags & M_FLOWID) != 0)
 823                 i = m->m_pkthdr.flowid % adapter->num_queues;
 824         else
 825                 i = curcpu % adapter->num_queues;
 826
 827         txr = &adapter->tx_rings[i];
 828         que = &adapter->queues[i];
 829
 830         err = drbr_enqueue(ifp, txr->br, m);
 831         if (err)
 832                 return (err);
 833         if (IXGBE_TX_TRYLOCK(txr)) {
 834                 err = ixgbe_mq_start_locked(ifp, txr);
 835                 IXGBE_TX_UNLOCK(txr);
 836         } else
 837                 taskqueue_enqueue(que->tq, &txr->txq_task);
 838
 839         return (err);
 840 }
 841
 842 static int
 843 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 844 {
 845         struct adapter  *adapter = txr->adapter;
 846         struct mbuf     *next;
 847         int             enqueued = 0, err = 0;
 848
 849         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 850             adapter->link_active == 0)
 851                 return (ENETDOWN);
 852
 853         /* Process the queue */
 854 #if __FreeBSD_version < 901504
 855         next = drbr_dequeue(ifp, txr->br);
 856         while (next != NULL) {
 857                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 858                         if (next != NULL)
 859                                 err = drbr_enqueue(ifp, txr->br, next);
 860 #else
 861         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 862                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 863                         if (next == NULL) {
 864                                 drbr_advance(ifp, txr->br);
 865                         } else {
 866                                 drbr_putback(ifp, txr->br, next);
 867                         }
 868 #endif
 869                         break;
 870                 }
 871 #if __FreeBSD_version >= 901504
 872                 drbr_advance(ifp, txr->br);
 873 #endif
 874                 enqueued++;
 875                 /* Send a copy of the frame to the BPF listener */
 876                 ETHER_BPF_MTAP(ifp, next);
 877                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 878                         break;
 879 #if __FreeBSD_version < 901504
 880                 next = drbr_dequeue(ifp, txr->br);
 881 #endif
 882         }
 883
 884         if (enqueued > 0) {
 885                 /* Set watchdog on */
 886                 txr->queue_status = IXGBE_QUEUE_WORKING;
 887                 txr->watchdog_time = ticks;
 888         }
 889
 890         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 891                 ixgbe_txeof(txr);
 892
 893         return (err);
 894 }
 895
 896 /*
 897  * Called from a taskqueue to drain queued transmit packets.
 898  */
 899 static void
 900 ixgbe_deferred_mq_start(void *arg, int pending)
 901 {
 902         struct tx_ring *txr = arg;
 903         struct adapter *adapter = txr->adapter;
 904         struct ifnet *ifp = adapter->ifp;
 905
 906         IXGBE_TX_LOCK(txr);
 907         if (!drbr_empty(ifp, txr->br))
 908                 ixgbe_mq_start_locked(ifp, txr);
 909         IXGBE_TX_UNLOCK(txr);
 910 }
 911
 912 /*
 913 ** Flush all ring buffers
 914 */
 915 static void
 916 ixgbe_qflush(struct ifnet *ifp)
 917 {
 918         struct adapter  *adapter = ifp->if_softc;
 919         struct tx_ring  *txr = adapter->tx_rings;
 920         struct mbuf     *m;
 921
 922         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 923                 IXGBE_TX_LOCK(txr);
 924                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 925                         m_freem(m);
 926                 IXGBE_TX_UNLOCK(txr);
 927         }
 928         if_qflush(ifp);
 929 }
 930 #endif /* IXGBE_LEGACY_TX */
 931
 932 /*********************************************************************
 933  *  Ioctl entry point
 934  *
 935  *  ixgbe_ioctl is called when the user wants to configure the
 936  *  interface.
 937  *
 938  *  return 0 on success, positive on failure
 939  **********************************************************************/
 940
 941 static int
 942 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 943 {
 944         struct adapter  *adapter = ifp->if_softc;
 945         struct ixgbe_hw *hw = &adapter->hw;
 946         struct ifreq    *ifr = (struct ifreq *) data;
 947 #if defined(INET) || defined(INET6)
 948         struct ifaddr *ifa = (struct ifaddr *)data;
 949         bool            avoid_reset = FALSE;
 950 #endif
 951         int             error = 0;
 952
 953         switch (command) {
 954
 955         case SIOCSIFADDR:
 956 #ifdef INET
 957                 if (ifa->ifa_addr->sa_family == AF_INET)
 958                         avoid_reset = TRUE;
 959 #endif
 960 #ifdef INET6
 961                 if (ifa->ifa_addr->sa_family == AF_INET6)
 962                         avoid_reset = TRUE;
 963 #endif
 964 #if defined(INET) || defined(INET6)
 965                 /*
 966                 ** Calling init results in link renegotiation,
 967                 ** so we avoid doing it when possible.
 968                 */
 969                 if (avoid_reset) {
 970                         ifp->if_flags |= IFF_UP;
 971                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 972                                 ixgbe_init(adapter);
 973                         if (!(ifp->if_flags & IFF_NOARP))
 974                                 arp_ifinit(ifp, ifa);
 975                 } else
 976                         error = ether_ioctl(ifp, command, data);
 977 #endif
 978                 break;
 979         case SIOCSIFMTU:
 980                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 981                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 982                         error = EINVAL;
 983                 } else {
 984                         IXGBE_CORE_LOCK(adapter);
 985                         ifp->if_mtu = ifr->ifr_mtu;
 986                         adapter->max_frame_size =
 987                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 988                         ixgbe_init_locked(adapter);
 989                         IXGBE_CORE_UNLOCK(adapter);
 990                 }
 991                 break;
 992         case SIOCSIFFLAGS:
 993                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 994                 IXGBE_CORE_LOCK(adapter);
 995                 if (ifp->if_flags & IFF_UP) {
 996                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 997                                 if ((ifp->if_flags ^ adapter->if_flags) &
 998                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 999                                         ixgbe_set_promisc(adapter);
1000                                 }
1001                         } else
1002                                 ixgbe_init_locked(adapter);
1003                 } else
1004                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1005                                 ixgbe_stop(adapter);
1006                 adapter->if_flags = ifp->if_flags;
1007                 IXGBE_CORE_UNLOCK(adapter);
1008                 break;
1009         case SIOCADDMULTI:
1010         case SIOCDELMULTI:
1011                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1012                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1013                         IXGBE_CORE_LOCK(adapter);
1014                         ixgbe_disable_intr(adapter);
1015                         ixgbe_set_multi(adapter);
1016                         ixgbe_enable_intr(adapter);
1017                         IXGBE_CORE_UNLOCK(adapter);
1018                 }
1019                 break;
1020         case SIOCSIFMEDIA:
1021         case SIOCGIFMEDIA:
1022                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1023                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1024                 break;
1025         case SIOCSIFCAP:
1026         {
1027                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1028                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1029                 if (mask & IFCAP_HWCSUM)
1030                         ifp->if_capenable ^= IFCAP_HWCSUM;
1031                 if (mask & IFCAP_TSO4)
1032                         ifp->if_capenable ^= IFCAP_TSO4;
1033                 if (mask & IFCAP_TSO6)
1034                         ifp->if_capenable ^= IFCAP_TSO6;
1035                 if (mask & IFCAP_LRO)
1036                         ifp->if_capenable ^= IFCAP_LRO;
1037                 if (mask & IFCAP_VLAN_HWTAGGING)
1038                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1039                 if (mask & IFCAP_VLAN_HWFILTER)
1040                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1041                 if (mask & IFCAP_VLAN_HWTSO)
1042                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1043                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1044                         IXGBE_CORE_LOCK(adapter);
1045                         ixgbe_init_locked(adapter);
1046                         IXGBE_CORE_UNLOCK(adapter);
1047                 }
1048                 VLAN_CAPABILITIES(ifp);
1049                 break;
1050         }
1051         case SIOCGI2C:
1052         {
1053                 struct ixgbe_i2c_req    i2c;
1054                 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1055                 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1056                 if (error)
1057                         break;
1058                 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1059                         error = EINVAL;
1060                         break;
1061                 }
1062                 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1063                     i2c.dev_addr, i2c.data);
1064                 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1065                 break;
1066         }
1067         default:
1068                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1069                 error = ether_ioctl(ifp, command, data);
1070                 break;
1071         }
1072
1073         return (error);
1074 }
1075
1076 /*********************************************************************
1077  *  Init entry point
1078  *
1079  *  This routine is used in two ways. It is used by the stack as
1080  *  init entry point in network interface structure. It is also used
1081  *  by the driver as a hw/sw initialization routine to get to a
1082  *  consistent state.
1083  *
1084  *  return 0 on success, positive on failure
1085  **********************************************************************/
1086 #define IXGBE_MHADD_MFS_SHIFT 16
1087
1088 static void
1089 ixgbe_init_locked(struct adapter *adapter)
1090 {
1091         struct ifnet   *ifp = adapter->ifp;
1092         device_t        dev = adapter->dev;
1093         struct ixgbe_hw *hw = &adapter->hw;
1094         u32             k, txdctl, mhadd, gpie;
1095         u32             rxdctl, rxctrl;
1096
1097         mtx_assert(&adapter->core_mtx, MA_OWNED);
1098         INIT_DEBUGOUT("ixgbe_init_locked: begin");
1099         hw->adapter_stopped = FALSE;
1100         ixgbe_stop_adapter(hw);
1101         callout_stop(&adapter->timer);
1102
1103         /* reprogram the RAR[0] in case user changed it. */
1104         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1105
1106         /* Get the latest mac address, User can use a LAA */
1107         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1108               IXGBE_ETH_LENGTH_OF_ADDRESS);
1109         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1110         hw->addr_ctrl.rar_used_count = 1;
1111
1112         /* Set the various hardware offload abilities */
1113         ifp->if_hwassist = 0;
1114         if (ifp->if_capenable & IFCAP_TSO)
1115                 ifp->if_hwassist |= CSUM_TSO;
1116         if (ifp->if_capenable & IFCAP_TXCSUM) {
1117                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1118 #if __FreeBSD_version >= 800000
1119                 if (hw->mac.type != ixgbe_mac_82598EB)
1120                         ifp->if_hwassist |= CSUM_SCTP;
1121 #endif
1122         }
1123
1124         /* Prepare transmit descriptors and buffers */
1125         if (ixgbe_setup_transmit_structures(adapter)) {
1126                 device_printf(dev,"Could not setup transmit structures\n");
1127                 ixgbe_stop(adapter);
1128                 return;
1129         }
1130
1131         ixgbe_init_hw(hw);
1132         ixgbe_initialize_transmit_units(adapter);
1133
1134         /* Setup Multicast table */
1135         ixgbe_set_multi(adapter);
1136
1137         /*
1138         ** Determine the correct mbuf pool
1139         ** for doing jumbo frames
1140         */
1141         if (adapter->max_frame_size <= 2048)
1142                 adapter->rx_mbuf_sz = MCLBYTES;
1143         else if (adapter->max_frame_size <= 4096)
1144                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1145         else if (adapter->max_frame_size <= 9216)
1146                 adapter->rx_mbuf_sz = MJUM9BYTES;
1147         else
1148                 adapter->rx_mbuf_sz = MJUM16BYTES;
1149
1150         /* Prepare receive descriptors and buffers */
1151         if (ixgbe_setup_receive_structures(adapter)) {
1152                 device_printf(dev,"Could not setup receive structures\n");
1153                 ixgbe_stop(adapter);
1154                 return;
1155         }
1156
1157         /* Configure RX settings */
1158         ixgbe_initialize_receive_units(adapter);
1159
1160         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1161
1162         /* Enable Fan Failure Interrupt */
1163         gpie |= IXGBE_SDP1_GPIEN;
1164
1165         /* Add for Module detection */
1166         if (hw->mac.type == ixgbe_mac_82599EB)
1167                 gpie |= IXGBE_SDP2_GPIEN;
1168
1169         /* Thermal Failure Detection */
1170         if (hw->mac.type == ixgbe_mac_X540)
1171                 gpie |= IXGBE_SDP0_GPIEN;
1172
1173         if (adapter->msix > 1) {
1174                 /* Enable Enhanced MSIX mode */
1175                 gpie |= IXGBE_GPIE_MSIX_MODE;
1176                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1177                     IXGBE_GPIE_OCD;
1178         }
1179         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1180
1181         /* Set MTU size */
1182         if (ifp->if_mtu > ETHERMTU) {
1183                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1184                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1185                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1186                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1187         }
1188
1189         /* Now enable all the queues */
1190
1191         for (int i = 0; i < adapter->num_queues; i++) {
1192                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1193                 txdctl |= IXGBE_TXDCTL_ENABLE;
1194                 /* Set WTHRESH to 8, burst writeback */
1195                 txdctl |= (8 << 16);
1196                 /*
1197                  * When the internal queue falls below PTHRESH (32),
1198                  * start prefetching as long as there are at least
1199                  * HTHRESH (1) buffers ready. The values are taken
1200                  * from the Intel linux driver 3.8.21.
1201                  * Prefetching enables tx line rate even with 1 queue.
1202                  */
1203                 txdctl |= (32 << 0) | (1 << 8);
1204                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1205         }
1206
1207         for (int i = 0; i < adapter->num_queues; i++) {
1208                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1209                 if (hw->mac.type == ixgbe_mac_82598EB) {
1210                         /*
1211                         ** PTHRESH = 21
1212                         ** HTHRESH = 4
1213                         ** WTHRESH = 8
1214                         */
1215                         rxdctl &= ~0x3FFFFF;
1216                         rxdctl |= 0x080420;
1217                 }
1218                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1219                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1220                 for (k = 0; k < 10; k++) {
1221                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1222                             IXGBE_RXDCTL_ENABLE)
1223                                 break;
1224                         else
1225                                 msec_delay(1);
1226                 }
1227                 wmb();
1228 #ifdef DEV_NETMAP
1229                 /*
1230                  * In netmap mode, we must preserve the buffers made
1231                  * available to userspace before the if_init()
1232                  * (this is true by default on the TX side, because
1233                  * init makes all buffers available to userspace).
1234                  *
1235                  * netmap_reset() and the device specific routines
1236                  * (e.g. ixgbe_setup_receive_rings()) map these
1237                  * buffers at the end of the NIC ring, so here we
1238                  * must set the RDT (tail) register to make sure
1239                  * they are not overwritten.
1240                  *
1241                  * In this driver the NIC ring starts at RDH = 0,
1242                  * RDT points to the last slot available for reception (?),
1243                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1244                  */
1245                 if (ifp->if_capenable & IFCAP_NETMAP) {
1246                         struct netmap_adapter *na = NA(adapter->ifp);
1247                         struct netmap_kring *kring = &na->rx_rings[i];
1248                         int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1249
1250                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1251                 } else
1252 #endif /* DEV_NETMAP */
1253                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1254         }
1255
1256         /* Enable Receive engine */
1257         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1258         if (hw->mac.type == ixgbe_mac_82598EB)
1259                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1260         rxctrl |= IXGBE_RXCTRL_RXEN;
1261         ixgbe_enable_rx_dma(hw, rxctrl);
1262
1263         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1264
1265         /* Set up MSI/X routing */
1266         if (ixgbe_enable_msix)  {
1267                 ixgbe_configure_ivars(adapter);
1268                 /* Set up auto-mask */
1269                 if (hw->mac.type == ixgbe_mac_82598EB)
1270                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1271                 else {
1272                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1273                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1274                 }
1275         } else {  /* Simple settings for Legacy/MSI */
1276                 ixgbe_set_ivar(adapter, 0, 0, 0);
1277                 ixgbe_set_ivar(adapter, 0, 0, 1);
1278                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1279         }
1280
1281 #ifdef IXGBE_FDIR
1282         /* Init Flow director */
1283         if (hw->mac.type != ixgbe_mac_82598EB) {
1284                 u32 hdrm = 32 << fdir_pballoc;
1285
1286                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1287                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1288         }
1289 #endif
1290
1291         /*
1292         ** Check on any SFP devices that
1293         ** need to be kick-started
1294         */
1295         if (hw->phy.type == ixgbe_phy_none) {
1296                 int err = hw->phy.ops.identify(hw);
1297                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1298                         device_printf(dev,
1299                             "Unsupported SFP+ module type was detected.\n");
1300                         return;
1301                 }
1302         }
1303
1304         /* Set moderation on the Link interrupt */
1305         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1306
1307         /* Config/Enable Link */
1308         ixgbe_config_link(adapter);
1309
1310         /* Hardware Packet Buffer & Flow Control setup */
1311         {
1312                 u32 rxpb, frame, size, tmp;
1313
1314                 frame = adapter->max_frame_size;
1315
1316                 /* Calculate High Water */
1317                 if (hw->mac.type == ixgbe_mac_X540)
1318                         tmp = IXGBE_DV_X540(frame, frame);
1319                 else
1320                         tmp = IXGBE_DV(frame, frame);
1321                 size = IXGBE_BT2KB(tmp);
1322                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1323                 hw->fc.high_water[0] = rxpb - size;
1324
1325                 /* Now calculate Low Water */
1326                 if (hw->mac.type == ixgbe_mac_X540)
1327                         tmp = IXGBE_LOW_DV_X540(frame);
1328                 else
1329                         tmp = IXGBE_LOW_DV(frame);
1330                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1331
1332                 hw->fc.requested_mode = adapter->fc;
1333                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1334                 hw->fc.send_xon = TRUE;
1335         }
1336         /* Initialize the FC settings */
1337         ixgbe_start_hw(hw);
1338
1339         /* Set up VLAN support and filter */
1340         ixgbe_setup_vlan_hw_support(adapter);
1341
1342         /* And now turn on interrupts */
1343         ixgbe_enable_intr(adapter);
1344
1345         /* Now inform the stack we're ready */
1346         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1347
1348         return;
1349 }
1350
1351 static void
1352 ixgbe_init(void *arg)
1353 {
1354         struct adapter *adapter = arg;
1355
1356         IXGBE_CORE_LOCK(adapter);
1357         ixgbe_init_locked(adapter);
1358         IXGBE_CORE_UNLOCK(adapter);
1359         return;
1360 }
1361
1362
1363 /*
1364 **
1365 ** MSIX Interrupt Handlers and Tasklets
1366 **
1367 */
1368
1369 static inline void
1370 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1371 {
1372         struct ixgbe_hw *hw = &adapter->hw;
1373         u64     queue = (u64)(1 << vector);
1374         u32     mask;
1375
1376         if (hw->mac.type == ixgbe_mac_82598EB) {
1377                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1379         } else {
1380                 mask = (queue & 0xFFFFFFFF);
1381                 if (mask)
1382                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1383                 mask = (queue >> 32);
1384                 if (mask)
1385                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1386         }
1387 }
1388
1389 static inline void
1390 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1391 {
1392         struct ixgbe_hw *hw = &adapter->hw;
1393         u64     queue = (u64)(1 << vector);
1394         u32     mask;
1395
1396         if (hw->mac.type == ixgbe_mac_82598EB) {
1397                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1398                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1399         } else {
1400                 mask = (queue & 0xFFFFFFFF);
1401                 if (mask)
1402                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1403                 mask = (queue >> 32);
1404                 if (mask)
1405                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1406         }
1407 }
1408
1409 static void
1410 ixgbe_handle_que(void *context, int pending)
1411 {
1412         struct ix_queue *que = context;
1413         struct adapter  *adapter = que->adapter;
1414         struct tx_ring  *txr = que->txr;
1415         struct ifnet    *ifp = adapter->ifp;
1416         bool            more;
1417
1418         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1419                 more = ixgbe_rxeof(que);
1420                 IXGBE_TX_LOCK(txr);
1421                 ixgbe_txeof(txr);
1422 #ifndef IXGBE_LEGACY_TX
1423                 if (!drbr_empty(ifp, txr->br))
1424                         ixgbe_mq_start_locked(ifp, txr);
1425 #else
1426                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1427                         ixgbe_start_locked(txr, ifp);
1428 #endif
1429                 IXGBE_TX_UNLOCK(txr);
1430         }
1431
1432         /* Reenable this interrupt */
1433         if (que->res != NULL)
1434                 ixgbe_enable_queue(adapter, que->msix);
1435         else
1436                 ixgbe_enable_intr(adapter);
1437         return;
1438 }
1439
1440
1441 /*********************************************************************
1442  *
1443  *  Legacy Interrupt Service routine
1444  *
1445  **********************************************************************/
1446
1447 static void
1448 ixgbe_legacy_irq(void *arg)
1449 {
1450         struct ix_queue *que = arg;
1451         struct adapter  *adapter = que->adapter;
1452         struct ixgbe_hw *hw = &adapter->hw;
1453         struct ifnet    *ifp = adapter->ifp;
1454         struct          tx_ring *txr = adapter->tx_rings;
1455         bool            more;
1456         u32             reg_eicr;
1457
1458
1459         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1460
1461         ++que->irqs;
1462         if (reg_eicr == 0) {
1463                 ixgbe_enable_intr(adapter);
1464                 return;
1465         }
1466
1467         more = ixgbe_rxeof(que);
1468
1469         IXGBE_TX_LOCK(txr);
1470         ixgbe_txeof(txr);
1471 #ifdef IXGBE_LEGACY_TX
1472         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1473                 ixgbe_start_locked(txr, ifp);
1474 #else
1475         if (!drbr_empty(ifp, txr->br))
1476                 ixgbe_mq_start_locked(ifp, txr);
1477 #endif
1478         IXGBE_TX_UNLOCK(txr);
1479
1480         /* Check for fan failure */
1481         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1482             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1483                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1484                     "REPLACE IMMEDIATELY!!\n");
1485                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1486         }
1487
1488         /* Link status change */
1489         if (reg_eicr & IXGBE_EICR_LSC)
1490                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1491
1492         if (more)
1493                 taskqueue_enqueue(que->tq, &que->que_task);
1494         else
1495                 ixgbe_enable_intr(adapter);
1496         return;
1497 }
1498
1499
1500 /*********************************************************************
1501  *
1502  *  MSIX Queue Interrupt Service routine
1503  *
1504  **********************************************************************/
1505 void
1506 ixgbe_msix_que(void *arg)
1507 {
1508         struct ix_queue *que = arg;
1509         struct adapter  *adapter = que->adapter;
1510         struct ifnet    *ifp = adapter->ifp;
1511         struct tx_ring  *txr = que->txr;
1512         struct rx_ring  *rxr = que->rxr;
1513         bool            more;
1514         u32             newitr = 0;
1515
1516         /* Protect against spurious interrupts */
1517         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1518                 return;
1519
1520         ixgbe_disable_queue(adapter, que->msix);
1521         ++que->irqs;
1522
1523         more = ixgbe_rxeof(que);
1524
1525         IXGBE_TX_LOCK(txr);
1526         ixgbe_txeof(txr);
1527 #ifdef IXGBE_LEGACY_TX
1528         if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1529                 ixgbe_start_locked(txr, ifp);
1530 #else
1531         if (!drbr_empty(ifp, txr->br))
1532                 ixgbe_mq_start_locked(ifp, txr);
1533 #endif
1534         IXGBE_TX_UNLOCK(txr);
1535
1536         /* Do AIM now? */
1537
1538         if (ixgbe_enable_aim == FALSE)
1539                 goto no_calc;
1540         /*
1541         ** Do Adaptive Interrupt Moderation:
1542         **  - Write out last calculated setting
1543         **  - Calculate based on average size over
1544         **    the last interval.
1545         */
1546         if (que->eitr_setting)
1547                 IXGBE_WRITE_REG(&adapter->hw,
1548                     IXGBE_EITR(que->msix), que->eitr_setting);
1549
1550         que->eitr_setting = 0;
1551
1552         /* Idle, do nothing */
1553         if ((txr->bytes == 0) && (rxr->bytes == 0))
1554                 goto no_calc;
1555
1556         if ((txr->bytes) && (txr->packets))
1557                 newitr = txr->bytes/txr->packets;
1558         if ((rxr->bytes) && (rxr->packets))
1559                 newitr = max(newitr,
1560                     (rxr->bytes / rxr->packets));
1561         newitr += 24; /* account for hardware frame, crc */
1562
1563         /* set an upper boundary */
1564         newitr = min(newitr, 3000);
1565
1566         /* Be nice to the mid range */
1567         if ((newitr > 300) && (newitr < 1200))
1568                 newitr = (newitr / 3);
1569         else
1570                 newitr = (newitr / 2);
1571
1572         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1573                 newitr |= newitr << 16;
1574         else
1575                 newitr |= IXGBE_EITR_CNT_WDIS;
1576
1577         /* save for next interrupt */
1578         que->eitr_setting = newitr;
1579
1580         /* Reset state */
1581         txr->bytes = 0;
1582         txr->packets = 0;
1583         rxr->bytes = 0;
1584         rxr->packets = 0;
1585
1586 no_calc:
1587         if (more)
1588                 taskqueue_enqueue(que->tq, &que->que_task);
1589         else
1590                 ixgbe_enable_queue(adapter, que->msix);
1591         return;
1592 }
1593
1594
1595 static void
1596 ixgbe_msix_link(void *arg)
1597 {
1598         struct adapter  *adapter = arg;
1599         struct ixgbe_hw *hw = &adapter->hw;
1600         u32             reg_eicr;
1601
1602         ++adapter->link_irq;
1603
1604         /* First get the cause */
1605         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1606         /* Be sure the queue bits are not cleared */
1607         reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1608         /* Clear interrupt with write */
1609         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1610
1611         /* Link status change */
1612         if (reg_eicr & IXGBE_EICR_LSC)
1613                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1614
1615         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1616 #ifdef IXGBE_FDIR
1617                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1618                         /* This is probably overkill :) */
1619                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1620                                 return;
1621                         /* Disable the interrupt */
1622                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1623                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1624                 } else
1625 #endif
1626                 if (reg_eicr & IXGBE_EICR_ECC) {
1627                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1628                             "Please Reboot!!\n");
1629                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1630                 } else
1631
1632                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1633                         /* Clear the interrupt */
1634                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1635                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1636                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1637                         /* Clear the interrupt */
1638                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1639                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1640                 }
1641         }
1642
1643         /* Check for fan failure */
1644         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1645             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1646                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1647                     "REPLACE IMMEDIATELY!!\n");
1648                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1649         }
1650
1651         /* Check for over temp condition */
1652         if ((hw->mac.type == ixgbe_mac_X540) &&
1653             (reg_eicr & IXGBE_EICR_TS)) {
1654                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1655                     "PHY IS SHUT DOWN!!\n");
1656                 device_printf(adapter->dev, "System shutdown required\n");
1657                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1658         }
1659
1660         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1661         return;
1662 }
1663
1664 /*********************************************************************
1665  *
1666  *  Media Ioctl callback
1667  *
1668  *  This routine is called whenever the user queries the status of
1669  *  the interface using ifconfig.
1670  *
1671  **********************************************************************/
1672 static void
1673 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1674 {
1675         struct adapter *adapter = ifp->if_softc;
1676
1677         INIT_DEBUGOUT("ixgbe_media_status: begin");
1678         IXGBE_CORE_LOCK(adapter);
1679         ixgbe_update_link_status(adapter);
1680
1681         ifmr->ifm_status = IFM_AVALID;
1682         ifmr->ifm_active = IFM_ETHER;
1683
1684         if (!adapter->link_active) {
1685                 IXGBE_CORE_UNLOCK(adapter);
1686                 return;
1687         }
1688
1689         ifmr->ifm_status |= IFM_ACTIVE;
1690
1691         switch (adapter->link_speed) {
1692                 case IXGBE_LINK_SPEED_100_FULL:
1693                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1694                         break;
1695                 case IXGBE_LINK_SPEED_1GB_FULL:
1696                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1697                         break;
1698                 case IXGBE_LINK_SPEED_10GB_FULL:
1699                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1700                         break;
1701         }
1702
1703         IXGBE_CORE_UNLOCK(adapter);
1704
1705         return;
1706 }
1707
1708 /*********************************************************************
1709  *
1710  *  Media Ioctl callback
1711  *
1712  *  This routine is called when the user changes speed/duplex using
1713  *  media/mediopt option with ifconfig.
1714  *
1715  **********************************************************************/
1716 static int
1717 ixgbe_media_change(struct ifnet * ifp)
1718 {
1719         struct adapter *adapter = ifp->if_softc;
1720         struct ifmedia *ifm = &adapter->media;
1721
1722         INIT_DEBUGOUT("ixgbe_media_change: begin");
1723
1724         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1725                 return (EINVAL);
1726
1727         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1728         case IFM_AUTO:
1729                 adapter->hw.phy.autoneg_advertised =
1730                     IXGBE_LINK_SPEED_100_FULL |
1731                     IXGBE_LINK_SPEED_1GB_FULL |
1732                     IXGBE_LINK_SPEED_10GB_FULL;
1733                 break;
1734         default:
1735                 device_printf(adapter->dev, "Only auto media type\n");
1736                 return (EINVAL);
1737         }
1738
1739         return (0);
1740 }
1741
1742 /*********************************************************************
1743  *
1744  *  This routine maps the mbufs to tx descriptors, allowing the
1745  *  TX engine to transmit the packets.
1746  *      - return 0 on success, positive on failure
1747  *
1748  **********************************************************************/
1749
1750 static int
1751 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1752 {
1753         struct adapter  *adapter = txr->adapter;
1754         u32             olinfo_status = 0, cmd_type_len;
1755         int             i, j, error, nsegs;
1756         int             first;
1757         bool            remap = TRUE;
1758         struct mbuf     *m_head;
1759         bus_dma_segment_t segs[adapter->num_segs];
1760         bus_dmamap_t    map;
1761         struct ixgbe_tx_buf *txbuf;
1762         union ixgbe_adv_tx_desc *txd = NULL;
1763
1764         m_head = *m_headp;
1765
1766         /* Basic descriptor defines */
1767         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1768             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1769
1770         if (m_head->m_flags & M_VLANTAG)
1771                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1772
1773         /*
1774          * Important to capture the first descriptor
1775          * used because it will contain the index of
1776          * the one we tell the hardware to report back
1777          */
1778         first = txr->next_avail_desc;
1779         txbuf = &txr->tx_buffers[first];
1780         map = txbuf->map;
1781
1782         /*
1783          * Map the packet for DMA.
1784          */
1785 retry:
1786         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1787             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1788
1789         if (__predict_false(error)) {
1790                 struct mbuf *m;
1791
1792                 switch (error) {
1793                 case EFBIG:
1794                         /* Try it again? - one try */
1795                         if (remap == TRUE) {
1796                                 remap = FALSE;
1797                                 m = m_defrag(*m_headp, M_NOWAIT);
1798                                 if (m == NULL) {
1799                                         adapter->mbuf_defrag_failed++;
1800                                         m_freem(*m_headp);
1801                                         *m_headp = NULL;
1802                                         return (ENOBUFS);
1803                                 }
1804                                 *m_headp = m;
1805                                 goto retry;
1806                         } else
1807                                 return (error);
1808                 case ENOMEM:
1809                         txr->no_tx_dma_setup++;
1810                         return (error);
1811                 default:
1812                         txr->no_tx_dma_setup++;
1813                         m_freem(*m_headp);
1814                         *m_headp = NULL;
1815                         return (error);
1816                 }
1817         }
1818
1819         /* Make certain there are enough descriptors */
1820         if (nsegs > txr->tx_avail - 2) {
1821                 txr->no_desc_avail++;
1822                 bus_dmamap_unload(txr->txtag, map);
1823                 return (ENOBUFS);
1824         }
1825         m_head = *m_headp;
1826
1827         /*
1828         ** Set up the appropriate offload context
1829         ** this will consume the first descriptor
1830         */
1831         error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1832         if (__predict_false(error)) {
1833                 if (error == ENOBUFS)
1834                         *m_headp = NULL;
1835                 return (error);
1836         }
1837
1838 #ifdef IXGBE_FDIR
1839         /* Do the flow director magic */
1840         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1841                 ++txr->atr_count;
1842                 if (txr->atr_count >= atr_sample_rate) {
1843                         ixgbe_atr(txr, m_head);
1844                         txr->atr_count = 0;
1845                 }
1846         }
1847 #endif
1848
1849         i = txr->next_avail_desc;
1850         for (j = 0; j < nsegs; j++) {
1851                 bus_size_t seglen;
1852                 bus_addr_t segaddr;
1853
1854                 txbuf = &txr->tx_buffers[i];
1855                 txd = &txr->tx_base[i];
1856                 seglen = segs[j].ds_len;
1857                 segaddr = htole64(segs[j].ds_addr);
1858
1859                 txd->read.buffer_addr = segaddr;
1860                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1861                     cmd_type_len |seglen);
1862                 txd->read.olinfo_status = htole32(olinfo_status);
1863
1864                 if (++i == txr->num_desc)
1865                         i = 0;
1866         }
1867
1868         txd->read.cmd_type_len |=
1869             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1870         txr->tx_avail -= nsegs;
1871         txr->next_avail_desc = i;
1872
1873         txbuf->m_head = m_head;
1874         /*
1875         ** Here we swap the map so the last descriptor,
1876         ** which gets the completion interrupt has the
1877         ** real map, and the first descriptor gets the
1878         ** unused map from this descriptor.
1879         */
1880         txr->tx_buffers[first].map = txbuf->map;
1881         txbuf->map = map;
1882         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1883
1884         /* Set the EOP descriptor that will be marked done */
1885         txbuf = &txr->tx_buffers[first];
1886         txbuf->eop = txd;
1887
1888         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1889             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1890         /*
1891          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1892          * hardware that this frame is available to transmit.
1893          */
1894         ++txr->total_packets;
1895         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1896
1897         return (0);
1898
1899 }
1900
1901 static void
1902 ixgbe_set_promisc(struct adapter *adapter)
1903 {
1904         u_int32_t       reg_rctl;
1905         struct ifnet   *ifp = adapter->ifp;
1906         int             mcnt = 0;
1907
1908         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1909         reg_rctl &= (~IXGBE_FCTRL_UPE);
1910         if (ifp->if_flags & IFF_ALLMULTI)
1911                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1912         else {
1913                 struct  ifmultiaddr *ifma;
1914 #if __FreeBSD_version < 800000
1915                 IF_ADDR_LOCK(ifp);
1916 #else
1917                 if_maddr_rlock(ifp);
1918 #endif
1919                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1920                         if (ifma->ifma_addr->sa_family != AF_LINK)
1921                                 continue;
1922                         if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1923                                 break;
1924                         mcnt++;
1925                 }
1926 #if __FreeBSD_version < 800000
1927                 IF_ADDR_UNLOCK(ifp);
1928 #else
1929                 if_maddr_runlock(ifp);
1930 #endif
1931         }
1932         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1933                 reg_rctl &= (~IXGBE_FCTRL_MPE);
1934         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1935
1936         if (ifp->if_flags & IFF_PROMISC) {
1937                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1938                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1939         } else if (ifp->if_flags & IFF_ALLMULTI) {
1940                 reg_rctl |= IXGBE_FCTRL_MPE;
1941                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1942                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1943         }
1944         return;
1945 }
1946
1947
1948 /*********************************************************************
1949  *  Multicast Update
1950  *
1951  *  This routine is called whenever multicast address list is updated.
1952  *
1953  **********************************************************************/
1954 #define IXGBE_RAR_ENTRIES 16
1955
1956 static void
1957 ixgbe_set_multi(struct adapter *adapter)
1958 {
1959         u32     fctrl;
1960         u8      *mta;
1961         u8      *update_ptr;
1962         struct  ifmultiaddr *ifma;
1963         int     mcnt = 0;
1964         struct ifnet   *ifp = adapter->ifp;
1965
1966         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1967
1968         mta = adapter->mta;
1969         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1970             MAX_NUM_MULTICAST_ADDRESSES);
1971
1972 #if __FreeBSD_version < 800000
1973         IF_ADDR_LOCK(ifp);
1974 #else
1975         if_maddr_rlock(ifp);
1976 #endif
1977         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1978                 if (ifma->ifma_addr->sa_family != AF_LINK)
1979                         continue;
1980                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1981                         break;
1982                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1983                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1984                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1985                 mcnt++;
1986         }
1987 #if __FreeBSD_version < 800000
1988         IF_ADDR_UNLOCK(ifp);
1989 #else
1990         if_maddr_runlock(ifp);
1991 #endif
1992
1993         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1994         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1995         if (ifp->if_flags & IFF_PROMISC)
1996                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1997         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1998             ifp->if_flags & IFF_ALLMULTI) {
1999                 fctrl |= IXGBE_FCTRL_MPE;
2000                 fctrl &= ~IXGBE_FCTRL_UPE;
2001         } else
2002                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2003
2004         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
2005
2006         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
2007                 update_ptr = mta;
2008                 ixgbe_update_mc_addr_list(&adapter->hw,
2009                     update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
2010         }
2011
2012         return;
2013 }
2014
2015 /*
2016  * This is an iterator function now needed by the multicast
2017  * shared code. It simply feeds the shared code routine the
2018  * addresses in the array of ixgbe_set_multi() one by one.
2019  */
2020 static u8 *
2021 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2022 {
2023         u8 *addr = *update_ptr;
2024         u8 *newptr;
2025         *vmdq = 0;
2026
2027         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2028         *update_ptr = newptr;
2029         return addr;
2030 }
2031
2032
2033 /*********************************************************************
2034  *  Timer routine
2035  *
2036  *  This routine checks for link status,updates statistics,
2037  *  and runs the watchdog check.
2038  *
2039  **********************************************************************/
2040
2041 static void
2042 ixgbe_local_timer(void *arg)
2043 {
2044         struct adapter  *adapter = arg;
2045         device_t        dev = adapter->dev;
2046         struct ix_queue *que = adapter->queues;
2047         struct tx_ring  *txr = adapter->tx_rings;
2048         int             hung = 0, paused = 0;
2049
2050         mtx_assert(&adapter->core_mtx, MA_OWNED);
2051
2052         /* Check for pluggable optics */
2053         if (adapter->sfp_probe)
2054                 if (!ixgbe_sfp_probe(adapter))
2055                         goto out; /* Nothing to do */
2056
2057         ixgbe_update_link_status(adapter);
2058         ixgbe_update_stats_counters(adapter);
2059
2060         /*
2061          * If the interface has been paused
2062          * then don't do the watchdog check
2063          */
2064         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2065                 paused = 1;
2066
2067         /*
2068         ** Check the TX queues status
2069         **      - watchdog only if all queues show hung
2070         */
2071         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2072                 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2073                     (paused == 0))
2074                         ++hung;
2075                 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2076                         taskqueue_enqueue(que->tq, &txr->txq_task);
2077         }
2078         /* Only truely watchdog if all queues show hung */
2079         if (hung == adapter->num_queues)
2080                 goto watchdog;
2081
2082 out:
2083         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2084         return;
2085
2086 watchdog:
2087         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2088         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2089             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2090             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2091         device_printf(dev,"TX(%d) desc avail = %d,"
2092             "Next TX to Clean = %d\n",
2093             txr->me, txr->tx_avail, txr->next_to_clean);
2094         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2095         adapter->watchdog_events++;
2096         ixgbe_init_locked(adapter);
2097 }
2098
2099 /*
2100 ** Note: this routine updates the OS on the link state
2101 **      the real check of the hardware only happens with
2102 **      a link interrupt.
2103 */
2104 static void
2105 ixgbe_update_link_status(struct adapter *adapter)
2106 {
2107         struct ifnet    *ifp = adapter->ifp;
2108         device_t dev = adapter->dev;
2109
2110
2111         if (adapter->link_up){
2112                 if (adapter->link_active == FALSE) {
2113                         if (bootverbose)
2114                                 device_printf(dev,"Link is up %d Gbps %s \n",
2115                                     ((adapter->link_speed == 128)? 10:1),
2116                                     "Full Duplex");
2117                         adapter->link_active = TRUE;
2118                         /* Update any Flow Control changes */
2119                         ixgbe_fc_enable(&adapter->hw);
2120                         if_link_state_change(ifp, LINK_STATE_UP);
2121                 }
2122         } else { /* Link down */
2123                 if (adapter->link_active == TRUE) {
2124                         if (bootverbose)
2125                                 device_printf(dev,"Link is Down\n");
2126                         if_link_state_change(ifp, LINK_STATE_DOWN);
2127                         adapter->link_active = FALSE;
2128                 }
2129         }
2130
2131         return;
2132 }
2133
2134
2135 /*********************************************************************
2136  *
2137  *  This routine disables all traffic on the adapter by issuing a
2138  *  global reset on the MAC and deallocates TX/RX buffers.
2139  *
2140  **********************************************************************/
2141
2142 static void
2143 ixgbe_stop(void *arg)
2144 {
2145         struct ifnet   *ifp;
2146         struct adapter *adapter = arg;
2147         struct ixgbe_hw *hw = &adapter->hw;
2148         ifp = adapter->ifp;
2149
2150         mtx_assert(&adapter->core_mtx, MA_OWNED);
2151
2152         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2153         ixgbe_disable_intr(adapter);
2154         callout_stop(&adapter->timer);
2155
2156         /* Let the stack know...*/
2157         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2158
2159         ixgbe_reset_hw(hw);
2160         hw->adapter_stopped = FALSE;
2161         ixgbe_stop_adapter(hw);
2162         if (hw->mac.type == ixgbe_mac_82599EB)
2163                 ixgbe_stop_mac_link_on_d3_82599(hw);
2164         /* Turn off the laser - noop with no optics */
2165         ixgbe_disable_tx_laser(hw);
2166
2167         /* Update the stack */
2168         adapter->link_up = FALSE;
2169         ixgbe_update_link_status(adapter);
2170
2171         /* reprogram the RAR[0] in case user changed it. */
2172         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2173
2174         return;
2175 }
2176
2177
2178 /*********************************************************************
2179  *
2180  *  Determine hardware revision.
2181  *
2182  **********************************************************************/
2183 static void
2184 ixgbe_identify_hardware(struct adapter *adapter)
2185 {
2186         device_t        dev = adapter->dev;
2187         struct ixgbe_hw *hw = &adapter->hw;
2188
2189         /* Save off the information about this board */
2190         hw->vendor_id = pci_get_vendor(dev);
2191         hw->device_id = pci_get_device(dev);
2192         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2193         hw->subsystem_vendor_id =
2194             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2195         hw->subsystem_device_id =
2196             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2197
2198         /* We need this here to set the num_segs below */
2199         ixgbe_set_mac_type(hw);
2200
2201         /* Pick up the 82599 and VF settings */
2202         if (hw->mac.type != ixgbe_mac_82598EB) {
2203                 hw->phy.smart_speed = ixgbe_smart_speed;
2204                 adapter->num_segs = IXGBE_82599_SCATTER;
2205         } else
2206                 adapter->num_segs = IXGBE_82598_SCATTER;
2207
2208         return;
2209 }
2210
2211 /*********************************************************************
2212  *
2213  *  Determine optic type
2214  *
2215  **********************************************************************/
2216 static void
2217 ixgbe_setup_optics(struct adapter *adapter)
2218 {
2219         struct ixgbe_hw *hw = &adapter->hw;
2220         int             layer;
2221
2222         layer = ixgbe_get_supported_physical_layer(hw);
2223
2224         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2225                 adapter->optics = IFM_10G_T;
2226                 return;
2227         }
2228
2229         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2230                 adapter->optics = IFM_1000_T;
2231                 return;
2232         }
2233
2234         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2235                 adapter->optics = IFM_1000_SX;
2236                 return;
2237         }
2238
2239         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2240             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2241                 adapter->optics = IFM_10G_LR;
2242                 return;
2243         }
2244
2245         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2246                 adapter->optics = IFM_10G_SR;
2247                 return;
2248         }
2249
2250         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2251                 adapter->optics = IFM_10G_TWINAX;
2252                 return;
2253         }
2254
2255         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2256             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2257                 adapter->optics = IFM_10G_CX4;
2258                 return;
2259         }
2260
2261         /* If we get here just set the default */
2262         adapter->optics = IFM_ETHER | IFM_AUTO;
2263         return;
2264 }
2265
2266 /*********************************************************************
2267  *
2268  *  Setup the Legacy or MSI Interrupt handler
2269  *
2270  **********************************************************************/
2271 static int
2272 ixgbe_allocate_legacy(struct adapter *adapter)
2273 {
2274         device_t        dev = adapter->dev;
2275         struct          ix_queue *que = adapter->queues;
2276 #ifndef IXGBE_LEGACY_TX
2277         struct tx_ring          *txr = adapter->tx_rings;
2278 #endif
2279         int             error, rid = 0;
2280
2281         /* MSI RID at 1 */
2282         if (adapter->msix == 1)
2283                 rid = 1;
2284
2285         /* We allocate a single interrupt resource */
2286         adapter->res = bus_alloc_resource_any(dev,
2287             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2288         if (adapter->res == NULL) {
2289                 device_printf(dev, "Unable to allocate bus resource: "
2290                     "interrupt\n");
2291                 return (ENXIO);
2292         }
2293
2294         /*
2295          * Try allocating a fast interrupt and the associated deferred
2296          * processing contexts.
2297          */
2298 #ifndef IXGBE_LEGACY_TX
2299         TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2300 #endif
2301         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2302         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2303             taskqueue_thread_enqueue, &que->tq);
2304         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2305             device_get_nameunit(adapter->dev));
2306
2307         /* Tasklets for Link, SFP and Multispeed Fiber */
2308         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2309         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2310         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2311 #ifdef IXGBE_FDIR
2312         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2313 #endif
2314         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2315             taskqueue_thread_enqueue, &adapter->tq);
2316         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2317             device_get_nameunit(adapter->dev));
2318
2319         if ((error = bus_setup_intr(dev, adapter->res,
2320             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2321             que, &adapter->tag)) != 0) {
2322                 device_printf(dev, "Failed to register fast interrupt "
2323                     "handler: %d\n", error);
2324                 taskqueue_free(que->tq);
2325                 taskqueue_free(adapter->tq);
2326                 que->tq = NULL;
2327                 adapter->tq = NULL;
2328                 return (error);
2329         }
2330         /* For simplicity in the handlers */
2331         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2332
2333         return (0);
2334 }
2335
2336
2337 /*********************************************************************
2338  *
2339  *  Setup MSIX Interrupt resources and handlers
2340  *
2341  **********************************************************************/
2342 static int
2343 ixgbe_allocate_msix(struct adapter *adapter)
2344 {
2345         device_t        dev = adapter->dev;
2346         struct          ix_queue *que = adapter->queues;
2347         struct          tx_ring *txr = adapter->tx_rings;
2348         int             error, rid, vector = 0;
2349
2350         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2351                 rid = vector + 1;
2352                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2353                     RF_SHAREABLE | RF_ACTIVE);
2354                 if (que->res == NULL) {
2355                         device_printf(dev,"Unable to allocate"
2356                             " bus resource: que interrupt [%d]\n", vector);
2357                         return (ENXIO);
2358                 }
2359                 /* Set the handler function */
2360                 error = bus_setup_intr(dev, que->res,
2361                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2362                     ixgbe_msix_que, que, &que->tag);
2363                 if (error) {
2364                         que->res = NULL;
2365                         device_printf(dev, "Failed to register QUE handler");
2366                         return (error);
2367                 }
2368 #if __FreeBSD_version >= 800504
2369                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2370 #endif
2371                 que->msix = vector;
2372                 adapter->que_mask |= (u64)(1 << que->msix);
2373                 /*
2374                 ** Bind the msix vector, and thus the
2375                 ** ring to the corresponding cpu.
2376                 */
2377                 if (adapter->num_queues > 1)
2378                         bus_bind_intr(dev, que->res, i);
2379
2380 #ifndef IXGBE_LEGACY_TX
2381                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2382 #endif
2383                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2384                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2385                     taskqueue_thread_enqueue, &que->tq);
2386                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2387                     device_get_nameunit(adapter->dev));
2388         }
2389
2390         /* and Link */
2391         rid = vector + 1;
2392         adapter->res = bus_alloc_resource_any(dev,
2393             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2394         if (!adapter->res) {
2395                 device_printf(dev,"Unable to allocate"
2396             " bus resource: Link interrupt [%d]\n", rid);
2397                 return (ENXIO);
2398         }
2399         /* Set the link handler function */
2400         error = bus_setup_intr(dev, adapter->res,
2401             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2402             ixgbe_msix_link, adapter, &adapter->tag);
2403         if (error) {
2404                 adapter->res = NULL;
2405                 device_printf(dev, "Failed to register LINK handler");
2406                 return (error);
2407         }
2408 #if __FreeBSD_version >= 800504
2409         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2410 #endif
2411         adapter->linkvec = vector;
2412         /* Tasklets for Link, SFP and Multispeed Fiber */
2413         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2414         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2415         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2416 #ifdef IXGBE_FDIR
2417         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2418 #endif
2419         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2420             taskqueue_thread_enqueue, &adapter->tq);
2421         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2422             device_get_nameunit(adapter->dev));
2423
2424         return (0);
2425 }
2426
2427 /*
2428  * Setup Either MSI/X or MSI
2429  */
2430 static int
2431 ixgbe_setup_msix(struct adapter *adapter)
2432 {
2433         device_t dev = adapter->dev;
2434         int rid, want, queues, msgs;
2435
2436         /* Override by tuneable */
2437         if (ixgbe_enable_msix == 0)
2438                 goto msi;
2439
2440         /* First try MSI/X */
2441         msgs = pci_msix_count(dev);
2442         if (msgs == 0)
2443                 goto msi;
2444         rid = PCIR_BAR(MSIX_82598_BAR);
2445         adapter->msix_mem = bus_alloc_resource_any(dev,
2446             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2447         if (adapter->msix_mem == NULL) {
2448                 rid += 4;       /* 82599 maps in higher BAR */
2449                 adapter->msix_mem = bus_alloc_resource_any(dev,
2450                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2451         }
2452         if (adapter->msix_mem == NULL) {
2453                 /* May not be enabled */
2454                 device_printf(adapter->dev,
2455                     "Unable to map MSIX table \n");
2456                 goto msi;
2457         }
2458
2459         /* Figure out a reasonable auto config value */
2460         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2461
2462         if (ixgbe_num_queues != 0)
2463                 queues = ixgbe_num_queues;
2464         /* Set max queues to 8 when autoconfiguring */
2465         else if ((ixgbe_num_queues == 0) && (queues > 8))
2466                 queues = 8;
2467
2468         /* reflect correct sysctl value */
2469         ixgbe_num_queues = queues;
2470
2471         /*
2472         ** Want one vector (RX/TX pair) per queue
2473         ** plus an additional for Link.
2474         */
2475         want = queues + 1;
2476         if (msgs >= want)
2477                 msgs = want;
2478         else {
2479                 device_printf(adapter->dev,
2480                     "MSIX Configuration Problem, "
2481                     "%d vectors but %d queues wanted!\n",
2482                     msgs, want);
2483                 goto msi;
2484         }
2485         if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
2486                 device_printf(adapter->dev,
2487                     "Using MSIX interrupts with %d vectors\n", msgs);
2488                 adapter->num_queues = queues;
2489                 return (msgs);
2490         }
2491         /*
2492         ** If MSIX alloc failed or provided us with
2493         ** less than needed, free and fall through to MSI
2494         */
2495         pci_release_msi(dev);
2496
2497 msi:
2498         if (adapter->msix_mem != NULL) {
2499                 bus_release_resource(dev, SYS_RES_MEMORY,
2500                     rid, adapter->msix_mem);
2501                 adapter->msix_mem = NULL;
2502         }
2503         msgs = 1;
2504         if (pci_alloc_msi(dev, &msgs) == 0) {
2505                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2506                 return (msgs);
2507         }
2508         device_printf(adapter->dev,"Using a Legacy interrupt\n");
2509         return (0);
2510 }
2511
2512
2513 static int
2514 ixgbe_allocate_pci_resources(struct adapter *adapter)
2515 {
2516         int             rid;
2517         device_t        dev = adapter->dev;
2518
2519         rid = PCIR_BAR(0);
2520         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2521             &rid, RF_ACTIVE);
2522
2523         if (!(adapter->pci_mem)) {
2524                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2525                 return (ENXIO);
2526         }
2527
2528         adapter->osdep.mem_bus_space_tag =
2529                 rman_get_bustag(adapter->pci_mem);
2530         adapter->osdep.mem_bus_space_handle =
2531                 rman_get_bushandle(adapter->pci_mem);
2532         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2533
2534         /* Legacy defaults */
2535         adapter->num_queues = 1;
2536         adapter->hw.back = &adapter->osdep;
2537
2538         /*
2539         ** Now setup MSI or MSI/X, should
2540         ** return us the number of supported
2541         ** vectors. (Will be 1 for MSI)
2542         */
2543         adapter->msix = ixgbe_setup_msix(adapter);
2544         return (0);
2545 }
2546
2547 static void
2548 ixgbe_free_pci_resources(struct adapter * adapter)
2549 {
2550         struct          ix_queue *que = adapter->queues;
2551         device_t        dev = adapter->dev;
2552         int             rid, memrid;
2553
2554         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2555                 memrid = PCIR_BAR(MSIX_82598_BAR);
2556         else
2557                 memrid = PCIR_BAR(MSIX_82599_BAR);
2558
2559         /*
2560         ** There is a slight possibility of a failure mode
2561         ** in attach that will result in entering this function
2562         ** before interrupt resources have been initialized, and
2563         ** in that case we do not want to execute the loops below
2564         ** We can detect this reliably by the state of the adapter
2565         ** res pointer.
2566         */
2567         if (adapter->res == NULL)
2568                 goto mem;
2569
2570         /*
2571         **  Release all msix queue resources:
2572         */
2573         for (int i = 0; i < adapter->num_queues; i++, que++) {
2574                 rid = que->msix + 1;
2575                 if (que->tag != NULL) {
2576                         bus_teardown_intr(dev, que->res, que->tag);
2577                         que->tag = NULL;
2578                 }
2579                 if (que->res != NULL)
2580                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2581         }
2582
2583
2584         /* Clean the Legacy or Link interrupt last */
2585         if (adapter->linkvec) /* we are doing MSIX */
2586                 rid = adapter->linkvec + 1;
2587         else
2588                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2589
2590         if (adapter->tag != NULL) {
2591                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2592                 adapter->tag = NULL;
2593         }
2594         if (adapter->res != NULL)
2595                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2596
2597 mem:
2598         if (adapter->msix)
2599                 pci_release_msi(dev);
2600
2601         if (adapter->msix_mem != NULL)
2602                 bus_release_resource(dev, SYS_RES_MEMORY,
2603                     memrid, adapter->msix_mem);
2604
2605         if (adapter->pci_mem != NULL)
2606                 bus_release_resource(dev, SYS_RES_MEMORY,
2607                     PCIR_BAR(0), adapter->pci_mem);
2608
2609         return;
2610 }
2611
2612 /*********************************************************************
2613  *
2614  *  Setup networking device structure and register an interface.
2615  *
2616  **********************************************************************/
2617 static int
2618 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2619 {
2620         struct ixgbe_hw *hw = &adapter->hw;
2621         struct ifnet   *ifp;
2622
2623         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2624
2625         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2626         if (ifp == NULL) {
2627                 device_printf(dev, "can not allocate ifnet structure\n");
2628                 return (-1);
2629         }
2630         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2631         ifp->if_baudrate = IF_Gbps(10);
2632         ifp->if_init = ixgbe_init;
2633         ifp->if_softc = adapter;
2634         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2635         ifp->if_ioctl = ixgbe_ioctl;
2636 #ifndef IXGBE_LEGACY_TX
2637         ifp->if_transmit = ixgbe_mq_start;
2638         ifp->if_qflush = ixgbe_qflush;
2639 #else
2640         ifp->if_start = ixgbe_start;
2641         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2642         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2643         IFQ_SET_READY(&ifp->if_snd);
2644 #endif
2645
2646         ether_ifattach(ifp, adapter->hw.mac.addr);
2647
2648         adapter->max_frame_size =
2649             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2650
2651         /*
2652          * Tell the upper layer(s) we support long frames.
2653          */
2654         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2655
2656         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2657         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2658         ifp->if_capabilities |= IFCAP_LRO;
2659         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2660                              |  IFCAP_VLAN_HWTSO
2661                              |  IFCAP_VLAN_MTU
2662                              |  IFCAP_HWSTATS;
2663         ifp->if_capenable = ifp->if_capabilities;
2664
2665         /*
2666         ** Don't turn this on by default, if vlans are
2667         ** created on another pseudo device (eg. lagg)
2668         ** then vlan events are not passed thru, breaking
2669         ** operation, but with HW FILTER off it works. If
2670         ** using vlans directly on the ixgbe driver you can
2671         ** enable this and get full hardware tag filtering.
2672         */
2673         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2674
2675         /*
2676          * Specify the media types supported by this adapter and register
2677          * callbacks to update media and link information
2678          */
2679         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2680                      ixgbe_media_status);
2681         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2682         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2683         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2684                 ifmedia_add(&adapter->media,
2685                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2686                 ifmedia_add(&adapter->media,
2687                     IFM_ETHER | IFM_1000_T, 0, NULL);
2688         }
2689         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2690         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2691
2692         return (0);
2693 }
2694
2695 static void
2696 ixgbe_config_link(struct adapter *adapter)
2697 {
2698         struct ixgbe_hw *hw = &adapter->hw;
2699         u32     autoneg, err = 0;
2700         bool    sfp, negotiate;
2701
2702         sfp = ixgbe_is_sfp(hw);
2703
2704         if (sfp) {
2705                 if (hw->phy.multispeed_fiber) {
2706                         hw->mac.ops.setup_sfp(hw);
2707                         ixgbe_enable_tx_laser(hw);
2708                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2709                 } else
2710                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2711         } else {
2712                 if (hw->mac.ops.check_link)
2713                         err = ixgbe_check_link(hw, &adapter->link_speed,
2714                             &adapter->link_up, FALSE);
2715                 if (err)
2716                         goto out;
2717                 autoneg = hw->phy.autoneg_advertised;
2718                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2719                         err  = hw->mac.ops.get_link_capabilities(hw,
2720                             &autoneg, &negotiate);
2721                 if (err)
2722                         goto out;
2723                 if (hw->mac.ops.setup_link)
2724                         err = hw->mac.ops.setup_link(hw,
2725                             autoneg, adapter->link_up);
2726         }
2727 out:
2728         return;
2729 }
2730
2731 /********************************************************************
2732  * Manage DMA'able memory.
2733  *******************************************************************/
2734 static void
2735 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2736 {
2737         if (error)
2738                 return;
2739         *(bus_addr_t *) arg = segs->ds_addr;
2740         return;
2741 }
2742
2743 static int
2744 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2745                 struct ixgbe_dma_alloc *dma, int mapflags)
2746 {
2747         device_t dev = adapter->dev;
2748         int             r;
2749
2750         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2751                                DBA_ALIGN, 0,    /* alignment, bounds */
2752                                BUS_SPACE_MAXADDR,       /* lowaddr */
2753                                BUS_SPACE_MAXADDR,       /* highaddr */
2754                                NULL, NULL,      /* filter, filterarg */
2755                                size,    /* maxsize */
2756                                1,       /* nsegments */
2757                                size,    /* maxsegsize */
2758                                BUS_DMA_ALLOCNOW,        /* flags */
2759                                NULL,    /* lockfunc */
2760                                NULL,    /* lockfuncarg */
2761                                &dma->dma_tag);
2762         if (r != 0) {
2763                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2764                        "error %u\n", r);
2765                 goto fail_0;
2766         }
2767         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2768                              BUS_DMA_NOWAIT, &dma->dma_map);
2769         if (r != 0) {
2770                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2771                        "error %u\n", r);
2772                 goto fail_1;
2773         }
2774         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2775                             size,
2776                             ixgbe_dmamap_cb,
2777                             &dma->dma_paddr,
2778                             mapflags | BUS_DMA_NOWAIT);
2779         if (r != 0) {
2780                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2781                        "error %u\n", r);
2782                 goto fail_2;
2783         }
2784         dma->dma_size = size;
2785         return (0);
2786 fail_2:
2787         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2788 fail_1:
2789         bus_dma_tag_destroy(dma->dma_tag);
2790 fail_0:
2791         dma->dma_map = NULL;
2792         dma->dma_tag = NULL;
2793         return (r);
2794 }
2795
2796 static void
2797 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2798 {
2799         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2800             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2801         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2802         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2803         bus_dma_tag_destroy(dma->dma_tag);
2804 }
2805
2806
2807 /*********************************************************************
2808  *
2809  *  Allocate memory for the transmit and receive rings, and then
2810  *  the descriptors associated with each, called only once at attach.
2811  *
2812  **********************************************************************/
2813 static int
2814 ixgbe_allocate_queues(struct adapter *adapter)
2815 {
2816         device_t        dev = adapter->dev;
2817         struct ix_queue *que;
2818         struct tx_ring  *txr;
2819         struct rx_ring  *rxr;
2820         int rsize, tsize, error = IXGBE_SUCCESS;
2821         int txconf = 0, rxconf = 0;
2822
2823         /* First allocate the top level queue structs */
2824         if (!(adapter->queues =
2825             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2826             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2827                 device_printf(dev, "Unable to allocate queue memory\n");
2828                 error = ENOMEM;
2829                 goto fail;
2830         }
2831
2832         /* First allocate the TX ring struct memory */
2833         if (!(adapter->tx_rings =
2834             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2835             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2836                 device_printf(dev, "Unable to allocate TX ring memory\n");
2837                 error = ENOMEM;
2838                 goto tx_fail;
2839         }
2840
2841         /* Next allocate the RX */
2842         if (!(adapter->rx_rings =
2843             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2844             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2845                 device_printf(dev, "Unable to allocate RX ring memory\n");
2846                 error = ENOMEM;
2847                 goto rx_fail;
2848         }
2849
2850         /* For the ring itself */
2851         tsize = roundup2(adapter->num_tx_desc *
2852             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2853
2854         /*
2855          * Now set up the TX queues, txconf is needed to handle the
2856          * possibility that things fail midcourse and we need to
2857          * undo memory gracefully
2858          */
2859         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2860                 /* Set up some basics */
2861                 txr = &adapter->tx_rings[i];
2862                 txr->adapter = adapter;
2863                 txr->me = i;
2864                 txr->num_desc = adapter->num_tx_desc;
2865
2866                 /* Initialize the TX side lock */
2867                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2868                     device_get_nameunit(dev), txr->me);
2869                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2870
2871                 if (ixgbe_dma_malloc(adapter, tsize,
2872                         &txr->txdma, BUS_DMA_NOWAIT)) {
2873                         device_printf(dev,
2874                             "Unable to allocate TX Descriptor memory\n");
2875                         error = ENOMEM;
2876                         goto err_tx_desc;
2877                 }
2878                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2879                 bzero((void *)txr->tx_base, tsize);
2880
2881                 /* Now allocate transmit buffers for the ring */
2882                 if (ixgbe_allocate_transmit_buffers(txr)) {
2883                         device_printf(dev,
2884                             "Critical Failure setting up transmit buffers\n");
2885                         error = ENOMEM;
2886                         goto err_tx_desc;
2887                 }
2888 #ifndef IXGBE_LEGACY_TX
2889                 /* Allocate a buf ring */
2890                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2891                     M_WAITOK, &txr->tx_mtx);
2892                 if (txr->br == NULL) {
2893                         device_printf(dev,
2894                             "Critical Failure setting up buf ring\n");
2895                         error = ENOMEM;
2896                         goto err_tx_desc;
2897                 }
2898 #endif
2899         }
2900
2901         /*
2902          * Next the RX queues...
2903          */
2904         rsize = roundup2(adapter->num_rx_desc *
2905             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2906         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2907                 rxr = &adapter->rx_rings[i];
2908                 /* Set up some basics */
2909                 rxr->adapter = adapter;
2910                 rxr->me = i;
2911                 rxr->num_desc = adapter->num_rx_desc;
2912
2913                 /* Initialize the RX side lock */
2914                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2915                     device_get_nameunit(dev), rxr->me);
2916                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2917
2918                 if (ixgbe_dma_malloc(adapter, rsize,
2919                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2920                         device_printf(dev,
2921                             "Unable to allocate RxDescriptor memory\n");
2922                         error = ENOMEM;
2923                         goto err_rx_desc;
2924                 }
2925                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2926                 bzero((void *)rxr->rx_base, rsize);
2927
2928                 /* Allocate receive buffers for the ring*/
2929                 if (ixgbe_allocate_receive_buffers(rxr)) {
2930                         device_printf(dev,
2931                             "Critical Failure setting up receive buffers\n");
2932                         error = ENOMEM;
2933                         goto err_rx_desc;
2934                 }
2935         }
2936
2937         /*
2938         ** Finally set up the queue holding structs
2939         */
2940         for (int i = 0; i < adapter->num_queues; i++) {
2941                 que = &adapter->queues[i];
2942                 que->adapter = adapter;
2943                 que->txr = &adapter->tx_rings[i];
2944                 que->rxr = &adapter->rx_rings[i];
2945         }
2946
2947         return (0);
2948
2949 err_rx_desc:
2950         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2951                 ixgbe_dma_free(adapter, &rxr->rxdma);
2952 err_tx_desc:
2953         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2954                 ixgbe_dma_free(adapter, &txr->txdma);
2955         free(adapter->rx_rings, M_DEVBUF);
2956 rx_fail:
2957         free(adapter->tx_rings, M_DEVBUF);
2958 tx_fail:
2959         free(adapter->queues, M_DEVBUF);
2960 fail:
2961         return (error);
2962 }
2963
2964 /*********************************************************************
2965  *
2966  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2967  *  the information needed to transmit a packet on the wire. This is
2968  *  called only once at attach, setup is done every reset.
2969  *
2970  **********************************************************************/
2971 static int
2972 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2973 {
2974         struct adapter *adapter = txr->adapter;
2975         device_t dev = adapter->dev;
2976         struct ixgbe_tx_buf *txbuf;
2977         int error, i;
2978
2979         /*
2980          * Setup DMA descriptor areas.
2981          */
2982         if ((error = bus_dma_tag_create(
2983                                bus_get_dma_tag(adapter->dev),   /* parent */
2984                                1, 0,            /* alignment, bounds */
2985                                BUS_SPACE_MAXADDR,       /* lowaddr */
2986                                BUS_SPACE_MAXADDR,       /* highaddr */
2987                                NULL, NULL,              /* filter, filterarg */
2988                                IXGBE_TSO_SIZE,          /* maxsize */
2989                                adapter->num_segs,       /* nsegments */
2990                                PAGE_SIZE,               /* maxsegsize */
2991                                0,                       /* flags */
2992                                NULL,                    /* lockfunc */
2993                                NULL,                    /* lockfuncarg */
2994                                &txr->txtag))) {
2995                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2996                 goto fail;
2997         }
2998
2999         if (!(txr->tx_buffers =
3000             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
3001             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3002                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3003                 error = ENOMEM;
3004                 goto fail;
3005         }
3006
3007         /* Create the descriptor buffer dma maps */
3008         txbuf = txr->tx_buffers;
3009         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3010                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3011                 if (error != 0) {
3012                         device_printf(dev, "Unable to create TX DMA map\n");
3013                         goto fail;
3014                 }
3015         }
3016
3017         return 0;
3018 fail:
3019         /* We free all, it handles case where we are in the middle */
3020         ixgbe_free_transmit_structures(adapter);
3021         return (error);
3022 }
3023
3024 /*********************************************************************
3025  *
3026  *  Initialize a transmit ring.
3027  *
3028  **********************************************************************/
3029 static void
3030 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3031 {
3032         struct adapter *adapter = txr->adapter;
3033         struct ixgbe_tx_buf *txbuf;
3034         int i;
3035 #ifdef DEV_NETMAP
3036         struct netmap_adapter *na = NA(adapter->ifp);
3037         struct netmap_slot *slot;
3038 #endif /* DEV_NETMAP */
3039
3040         /* Clear the old ring contents */
3041         IXGBE_TX_LOCK(txr);
3042 #ifdef DEV_NETMAP
3043         /*
3044          * (under lock): if in netmap mode, do some consistency
3045          * checks and set slot to entry 0 of the netmap ring.
3046          */
3047         slot = netmap_reset(na, NR_TX, txr->me, 0);
3048 #endif /* DEV_NETMAP */
3049         bzero((void *)txr->tx_base,
3050               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3051         /* Reset indices */
3052         txr->next_avail_desc = 0;
3053         txr->next_to_clean = 0;
3054
3055         /* Free any existing tx buffers. */
3056         txbuf = txr->tx_buffers;
3057         for (i = 0; i < txr->num_desc; i++, txbuf++) {
3058                 if (txbuf->m_head != NULL) {
3059                         bus_dmamap_sync(txr->txtag, txbuf->map,
3060                             BUS_DMASYNC_POSTWRITE);
3061                         bus_dmamap_unload(txr->txtag, txbuf->map);
3062                         m_freem(txbuf->m_head);
3063                         txbuf->m_head = NULL;
3064                 }
3065 #ifdef DEV_NETMAP
3066                 /*
3067                  * In netmap mode, set the map for the packet buffer.
3068                  * NOTE: Some drivers (not this one) also need to set
3069                  * the physical buffer address in the NIC ring.
3070                  * Slots in the netmap ring (indexed by "si") are
3071                  * kring->nkr_hwofs positions "ahead" wrt the
3072                  * corresponding slot in the NIC ring. In some drivers
3073                  * (not here) nkr_hwofs can be negative. Function
3074                  * netmap_idx_n2k() handles wraparounds properly.
3075                  */
3076                 if (slot) {
3077                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3078                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3079                 }
3080 #endif /* DEV_NETMAP */
3081                 /* Clear the EOP descriptor pointer */
3082                 txbuf->eop = NULL;
3083         }
3084
3085 #ifdef IXGBE_FDIR
3086         /* Set the rate at which we sample packets */
3087         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3088                 txr->atr_sample = atr_sample_rate;
3089 #endif
3090
3091         /* Set number of descriptors available */
3092         txr->tx_avail = adapter->num_tx_desc;
3093
3094         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3095             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3096         IXGBE_TX_UNLOCK(txr);
3097 }
3098
3099 /*********************************************************************
3100  *
3101  *  Initialize all transmit rings.
3102  *
3103  **********************************************************************/
3104 static int
3105 ixgbe_setup_transmit_structures(struct adapter *adapter)
3106 {
3107         struct tx_ring *txr = adapter->tx_rings;
3108
3109         for (int i = 0; i < adapter->num_queues; i++, txr++)
3110                 ixgbe_setup_transmit_ring(txr);
3111
3112         return (0);
3113 }
3114
3115 /*********************************************************************
3116  *
3117  *  Enable transmit unit.
3118  *
3119  **********************************************************************/
3120 static void
3121 ixgbe_initialize_transmit_units(struct adapter *adapter)
3122 {
3123         struct tx_ring  *txr = adapter->tx_rings;
3124         struct ixgbe_hw *hw = &adapter->hw;
3125
3126         /* Setup the Base and Length of the Tx Descriptor Ring */
3127
3128         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3129                 u64     tdba = txr->txdma.dma_paddr;
3130                 u32     txctrl;
3131
3132                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3133                        (tdba & 0x00000000ffffffffULL));
3134                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3135                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3136                     adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3137
3138                 /* Setup the HW Tx Head and Tail descriptor pointers */
3139                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3140                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3141
3142                 /* Setup Transmit Descriptor Cmd Settings */
3143                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3144                 txr->queue_status = IXGBE_QUEUE_IDLE;
3145
3146                 /* Set the processing limit */
3147                 txr->process_limit = ixgbe_tx_process_limit;
3148
3149                 /* Disable Head Writeback */
3150                 switch (hw->mac.type) {
3151                 case ixgbe_mac_82598EB:
3152                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3153                         break;
3154                 case ixgbe_mac_82599EB:
3155                 case ixgbe_mac_X540:
3156                 default:
3157                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3158                         break;
3159                 }
3160                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3161                 switch (hw->mac.type) {
3162                 case ixgbe_mac_82598EB:
3163                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3164                         break;
3165                 case ixgbe_mac_82599EB:
3166                 case ixgbe_mac_X540:
3167                 default:
3168                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3169                         break;
3170                 }
3171
3172         }
3173
3174         if (hw->mac.type != ixgbe_mac_82598EB) {
3175                 u32 dmatxctl, rttdcs;
3176                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3177                 dmatxctl |= IXGBE_DMATXCTL_TE;
3178                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3179                 /* Disable arbiter to set MTQC */
3180                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3181                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3182                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3183                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3184                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3185                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3186         }
3187
3188         return;
3189 }
3190
3191 /*********************************************************************
3192  *
3193  *  Free all transmit rings.
3194  *
3195  **********************************************************************/
3196 static void
3197 ixgbe_free_transmit_structures(struct adapter *adapter)
3198 {
3199         struct tx_ring *txr = adapter->tx_rings;
3200
3201         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3202                 IXGBE_TX_LOCK(txr);
3203                 ixgbe_free_transmit_buffers(txr);
3204                 ixgbe_dma_free(adapter, &txr->txdma);
3205                 IXGBE_TX_UNLOCK(txr);
3206                 IXGBE_TX_LOCK_DESTROY(txr);
3207         }
3208         free(adapter->tx_rings, M_DEVBUF);
3209 }
3210
3211 /*********************************************************************
3212  *
3213  *  Free transmit ring related data structures.
3214  *
3215  **********************************************************************/
3216 static void
3217 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3218 {
3219         struct adapter *adapter = txr->adapter;
3220         struct ixgbe_tx_buf *tx_buffer;
3221         int             i;
3222
3223         INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin");
3224
3225         if (txr->tx_buffers == NULL)
3226                 return;
3227
3228         tx_buffer = txr->tx_buffers;
3229         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3230                 if (tx_buffer->m_head != NULL) {
3231                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3232                             BUS_DMASYNC_POSTWRITE);
3233                         bus_dmamap_unload(txr->txtag,
3234                             tx_buffer->map);
3235                         m_freem(tx_buffer->m_head);
3236                         tx_buffer->m_head = NULL;
3237                         if (tx_buffer->map != NULL) {
3238                                 bus_dmamap_destroy(txr->txtag,
3239                                     tx_buffer->map);
3240                                 tx_buffer->map = NULL;
3241                         }
3242                 } else if (tx_buffer->map != NULL) {
3243                         bus_dmamap_unload(txr->txtag,
3244                             tx_buffer->map);
3245                         bus_dmamap_destroy(txr->txtag,
3246                             tx_buffer->map);
3247                         tx_buffer->map = NULL;
3248                 }
3249         }
3250 #ifdef IXGBE_LEGACY_TX
3251         if (txr->br != NULL)
3252                 buf_ring_free(txr->br, M_DEVBUF);
3253 #endif
3254         if (txr->tx_buffers != NULL) {
3255                 free(txr->tx_buffers, M_DEVBUF);
3256                 txr->tx_buffers = NULL;
3257         }
3258         if (txr->txtag != NULL) {
3259                 bus_dma_tag_destroy(txr->txtag);
3260                 txr->txtag = NULL;
3261         }
3262         return;
3263 }
3264
3265 /*********************************************************************
3266  *
3267  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
3268  *
3269  **********************************************************************/
3270
3271 static int
3272 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3273     u32 *cmd_type_len, u32 *olinfo_status)
3274 {
3275         struct ixgbe_adv_tx_context_desc *TXD;
3276         struct ether_vlan_header *eh;
3277         struct ip *ip;
3278         struct ip6_hdr *ip6;
3279         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3280         int     ehdrlen, ip_hlen = 0;
3281         u16     etype;
3282         u8      ipproto = 0;
3283         int     offload = TRUE;
3284         int     ctxd = txr->next_avail_desc;
3285         u16     vtag = 0;
3286
3287         /* First check if TSO is to be used */
3288         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3289                 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3290
3291         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3292                 offload = FALSE;
3293
3294         /* Indicate the whole packet as payload when not doing TSO */
3295         *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3296
3297         /* Now ready a context descriptor */
3298         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3299
3300         /*
3301         ** In advanced descriptors the vlan tag must
3302         ** be placed into the context descriptor. Hence
3303         ** we need to make one even if not doing offloads.
3304         */
3305         if (mp->m_flags & M_VLANTAG) {
3306                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3307                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3308         } else if (offload == FALSE) /* ... no offload to do */
3309                 return (0);
3310
3311         /*
3312          * Determine where frame payload starts.
3313          * Jump over vlan headers if already present,
3314          * helpful for QinQ too.
3315          */
3316         eh = mtod(mp, struct ether_vlan_header *);
3317         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3318                 etype = ntohs(eh->evl_proto);
3319                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3320         } else {
3321                 etype = ntohs(eh->evl_encap_proto);
3322                 ehdrlen = ETHER_HDR_LEN;
3323         }
3324
3325         /* Set the ether header length */
3326         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3327
3328         switch (etype) {
3329                 case ETHERTYPE_IP:
3330                         ip = (struct ip *)(mp->m_data + ehdrlen);
3331                         ip_hlen = ip->ip_hl << 2;
3332                         ipproto = ip->ip_p;
3333                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3334                         break;
3335                 case ETHERTYPE_IPV6:
3336                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3337                         ip_hlen = sizeof(struct ip6_hdr);
3338                         /* XXX-BZ this will go badly in case of ext hdrs. */
3339                         ipproto = ip6->ip6_nxt;
3340                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3341                         break;
3342                 default:
3343                         offload = FALSE;
3344                         break;
3345         }
3346
3347         vlan_macip_lens |= ip_hlen;
3348         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3349
3350         switch (ipproto) {
3351                 case IPPROTO_TCP:
3352                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3353                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3354                         break;
3355
3356                 case IPPROTO_UDP:
3357                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3358                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3359                         break;
3360
3361 #if __FreeBSD_version >= 800000
3362                 case IPPROTO_SCTP:
3363                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3364                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3365                         break;
3366 #endif
3367                 default:
3368                         offload = FALSE;
3369                         break;
3370         }
3371
3372         if (offload) /* For the TX descriptor setup */
3373                 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3374
3375         /* Now copy bits into descriptor */
3376         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3377         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3378         TXD->seqnum_seed = htole32(0);
3379         TXD->mss_l4len_idx = htole32(0);
3380
3381         /* We've consumed the first desc, adjust counters */
3382         if (++ctxd == txr->num_desc)
3383                 ctxd = 0;
3384         txr->next_avail_desc = ctxd;
3385         --txr->tx_avail;
3386
3387         return (0);
3388 }
3389
3390 /**********************************************************************
3391  *
3392  *  Setup work for hardware segmentation offload (TSO) on
3393  *  adapters using advanced tx descriptors
3394  *
3395  **********************************************************************/
3396 static int
3397 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3398     u32 *cmd_type_len, u32 *olinfo_status)
3399 {
3400         struct ixgbe_adv_tx_context_desc *TXD;
3401         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3402         u32 mss_l4len_idx = 0, paylen;
3403         u16 vtag = 0, eh_type;
3404         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3405         struct ether_vlan_header *eh;
3406 #ifdef INET6
3407         struct ip6_hdr *ip6;
3408 #endif
3409 #ifdef INET
3410         struct ip *ip;
3411 #endif
3412         struct tcphdr *th;
3413
3414
3415         /*
3416          * Determine where frame payload starts.
3417          * Jump over vlan headers if already present
3418          */
3419         eh = mtod(mp, struct ether_vlan_header *);
3420         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3421                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3422                 eh_type = eh->evl_proto;
3423         } else {
3424                 ehdrlen = ETHER_HDR_LEN;
3425                 eh_type = eh->evl_encap_proto;
3426         }
3427
3428         switch (ntohs(eh_type)) {
3429 #ifdef INET6
3430         case ETHERTYPE_IPV6:
3431                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3432                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3433                 if (ip6->ip6_nxt != IPPROTO_TCP)
3434                         return (ENXIO);
3435                 ip_hlen = sizeof(struct ip6_hdr);
3436                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3437                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3438                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3439                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3440                 break;
3441 #endif
3442 #ifdef INET
3443         case ETHERTYPE_IP:
3444                 ip = (struct ip *)(mp->m_data + ehdrlen);
3445                 if (ip->ip_p != IPPROTO_TCP)
3446                         return (ENXIO);
3447                 ip->ip_sum = 0;
3448                 ip_hlen = ip->ip_hl << 2;
3449                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3450                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3451                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3452                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3453                 /* Tell transmit desc to also do IPv4 checksum. */
3454                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3455                 break;
3456 #endif
3457         default:
3458                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3459                     __func__, ntohs(eh_type));
3460                 break;
3461         }
3462
3463         ctxd = txr->next_avail_desc;
3464         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3465
3466         tcp_hlen = th->th_off << 2;
3467
3468         /* This is used in the transmit desc in encap */
3469         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3470
3471         /* VLAN MACLEN IPLEN */
3472         if (mp->m_flags & M_VLANTAG) {
3473                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3474                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3475         }
3476
3477         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3478         vlan_macip_lens |= ip_hlen;
3479         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3480
3481         /* ADV DTYPE TUCMD */
3482         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3483         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3484         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3485
3486         /* MSS L4LEN IDX */
3487         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3488         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3489         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3490
3491         TXD->seqnum_seed = htole32(0);
3492
3493         if (++ctxd == txr->num_desc)
3494                 ctxd = 0;
3495
3496         txr->tx_avail--;
3497         txr->next_avail_desc = ctxd;
3498         *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3499         *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3500         *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3501         ++txr->tso_tx;
3502         return (0);
3503 }
3504
3505 #ifdef IXGBE_FDIR
3506 /*
3507 ** This routine parses packet headers so that Flow
3508 ** Director can make a hashed filter table entry
3509 ** allowing traffic flows to be identified and kept
3510 ** on the same cpu.  This would be a performance
3511 ** hit, but we only do it at IXGBE_FDIR_RATE of
3512 ** packets.
3513 */
3514 static void
3515 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3516 {
3517         struct adapter                  *adapter = txr->adapter;
3518         struct ix_queue                 *que;
3519         struct ip                       *ip;
3520         struct tcphdr                   *th;
3521         struct udphdr                   *uh;
3522         struct ether_vlan_header        *eh;
3523         union ixgbe_atr_hash_dword      input = {.dword = 0};
3524         union ixgbe_atr_hash_dword      common = {.dword = 0};
3525         int                             ehdrlen, ip_hlen;
3526         u16                             etype;
3527
3528         eh = mtod(mp, struct ether_vlan_header *);
3529         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3530                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3531                 etype = eh->evl_proto;
3532         } else {
3533                 ehdrlen = ETHER_HDR_LEN;
3534                 etype = eh->evl_encap_proto;
3535         }
3536
3537         /* Only handling IPv4 */
3538         if (etype != htons(ETHERTYPE_IP))
3539                 return;
3540
3541         ip = (struct ip *)(mp->m_data + ehdrlen);
3542         ip_hlen = ip->ip_hl << 2;
3543
3544         /* check if we're UDP or TCP */
3545         switch (ip->ip_p) {
3546         case IPPROTO_TCP:
3547                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3548                 /* src and dst are inverted */
3549                 common.port.dst ^= th->th_sport;
3550                 common.port.src ^= th->th_dport;
3551                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3552                 break;
3553         case IPPROTO_UDP:
3554                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3555                 /* src and dst are inverted */
3556                 common.port.dst ^= uh->uh_sport;
3557                 common.port.src ^= uh->uh_dport;
3558                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3559                 break;
3560         default:
3561                 return;
3562         }
3563
3564         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3565         if (mp->m_pkthdr.ether_vtag)
3566                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3567         else
3568                 common.flex_bytes ^= etype;
3569         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3570
3571         que = &adapter->queues[txr->me];
3572         /*
3573         ** This assumes the Rx queue and Tx
3574         ** queue are bound to the same CPU
3575         */
3576         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3577             input, common, que->msix);
3578 }
3579 #endif /* IXGBE_FDIR */
3580
3581 /**********************************************************************
3582  *
3583  *  Examine each tx_buffer in the used queue. If the hardware is done
3584  *  processing the packet then free associated resources. The
3585  *  tx_buffer is put back on the free queue.
3586  *
3587  **********************************************************************/
3588 static void
3589 ixgbe_txeof(struct tx_ring *txr)
3590 {
3591 #ifdef DEV_NETMAP
3592         struct adapter          *adapter = txr->adapter;
3593         struct ifnet            *ifp = adapter->ifp;
3594 #endif
3595         u32                     work, processed = 0;
3596         u16                     limit = txr->process_limit;
3597         struct ixgbe_tx_buf     *buf;
3598         union ixgbe_adv_tx_desc *txd;
3599
3600         mtx_assert(&txr->tx_mtx, MA_OWNED);
3601
3602 #ifdef DEV_NETMAP
3603         if (ifp->if_capenable & IFCAP_NETMAP) {
3604                 struct netmap_adapter *na = NA(ifp);
3605                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3606                 txd = txr->tx_base;
3607                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3608                     BUS_DMASYNC_POSTREAD);
3609                 /*
3610                  * In netmap mode, all the work is done in the context
3611                  * of the client thread. Interrupt handlers only wake up
3612                  * clients, which may be sleeping on individual rings
3613                  * or on a global resource for all rings.
3614                  * To implement tx interrupt mitigation, we wake up the client
3615                  * thread roughly every half ring, even if the NIC interrupts
3616                  * more frequently. This is implemented as follows:
3617                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3618                  *   the slot that should wake up the thread (nkr_num_slots
3619                  *   means the user thread should not be woken up);
3620                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3621                  *   or the slot has the DD bit set.
3622                  */
3623                 if (!netmap_mitigate ||
3624                     (kring->nr_kflags < kring->nkr_num_slots &&
3625                     txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3626                         netmap_tx_irq(ifp, txr->me);
3627                 }
3628                 return;
3629         }
3630 #endif /* DEV_NETMAP */
3631
3632         if (txr->tx_avail == txr->num_desc) {
3633                 txr->queue_status = IXGBE_QUEUE_IDLE;
3634                 return;
3635         }
3636
3637         /* Get work starting point */
3638         work = txr->next_to_clean;
3639         buf = &txr->tx_buffers[work];
3640         txd = &txr->tx_base[work];
3641         work -= txr->num_desc; /* The distance to ring end */
3642         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3643             BUS_DMASYNC_POSTREAD);
3644
3645         do {
3646                 union ixgbe_adv_tx_desc *eop= buf->eop;
3647                 if (eop == NULL) /* No work */
3648                         break;
3649
3650                 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3651                         break;  /* I/O not complete */
3652
3653                 if (buf->m_head) {
3654                         txr->bytes +=
3655                             buf->m_head->m_pkthdr.len;
3656                         bus_dmamap_sync(txr->txtag,
3657                             buf->map,
3658                             BUS_DMASYNC_POSTWRITE);
3659                         bus_dmamap_unload(txr->txtag,
3660                             buf->map);
3661                         m_freem(buf->m_head);
3662                         buf->m_head = NULL;
3663                         buf->map = NULL;
3664                 }
3665                 buf->eop = NULL;
3666                 ++txr->tx_avail;
3667
3668                 /* We clean the range if multi segment */
3669                 while (txd != eop) {
3670                         ++txd;
3671                         ++buf;
3672                         ++work;
3673                         /* wrap the ring? */
3674                         if (__predict_false(!work)) {
3675                                 work -= txr->num_desc;
3676                                 buf = txr->tx_buffers;
3677                                 txd = txr->tx_base;
3678                         }
3679                         if (buf->m_head) {
3680                                 txr->bytes +=
3681                                     buf->m_head->m_pkthdr.len;
3682                                 bus_dmamap_sync(txr->txtag,
3683                                     buf->map,
3684                                     BUS_DMASYNC_POSTWRITE);
3685                                 bus_dmamap_unload(txr->txtag,
3686                                     buf->map);
3687                                 m_freem(buf->m_head);
3688                                 buf->m_head = NULL;
3689                                 buf->map = NULL;
3690                         }
3691                         ++txr->tx_avail;
3692                         buf->eop = NULL;
3693
3694                 }
3695                 ++txr->packets;
3696                 ++processed;
3697                 txr->watchdog_time = ticks;
3698
3699                 /* Try the next packet */
3700                 ++txd;
3701                 ++buf;
3702                 ++work;
3703                 /* reset with a wrap */
3704                 if (__predict_false(!work)) {
3705                         work -= txr->num_desc;
3706                         buf = txr->tx_buffers;
3707                         txd = txr->tx_base;
3708                 }
3709                 prefetch(txd);
3710         } while (__predict_true(--limit));
3711
3712         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3713             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3714
3715         work += txr->num_desc;
3716         txr->next_to_clean = work;
3717
3718         /*
3719         ** Watchdog calculation, we know there's
3720         ** work outstanding or the first return
3721         ** would have been taken, so none processed
3722         ** for too long indicates a hang.
3723         */
3724         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3725                 txr->queue_status = IXGBE_QUEUE_HUNG;
3726
3727         if (txr->tx_avail == txr->num_desc)
3728                 txr->queue_status = IXGBE_QUEUE_IDLE;
3729
3730         return;
3731 }
3732
3733 /*********************************************************************
3734  *
3735  *  Refresh mbuf buffers for RX descriptor rings
3736  *   - now keeps its own state so discards due to resource
3737  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3738  *     it just returns, keeping its placeholder, thus it can simply
3739  *     be recalled to try again.
3740  *
3741  **********************************************************************/
3742 static void
3743 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3744 {
3745         struct adapter          *adapter = rxr->adapter;
3746         bus_dma_segment_t       seg[1];
3747         struct ixgbe_rx_buf     *rxbuf;
3748         struct mbuf             *mp;
3749         int                     i, j, nsegs, error;
3750         bool                    refreshed = FALSE;
3751
3752         i = j = rxr->next_to_refresh;
3753         /* Control the loop with one beyond */
3754         if (++j == rxr->num_desc)
3755                 j = 0;
3756
3757         while (j != limit) {
3758                 rxbuf = &rxr->rx_buffers[i];
3759                 if (rxbuf->buf == NULL) {
3760                         mp = m_getjcl(M_NOWAIT, MT_DATA,
3761                             M_PKTHDR, rxr->mbuf_sz);
3762                         if (mp == NULL)
3763                                 goto update;
3764                         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3765                                 m_adj(mp, ETHER_ALIGN);
3766                 } else
3767                         mp = rxbuf->buf;
3768
3769                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3770
3771                 /* If we're dealing with an mbuf that was copied rather
3772                  * than replaced, there's no need to go through busdma.
3773                  */
3774                 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3775                         /* Get the memory mapping */
3776                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3777                             rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3778                         if (error != 0) {
3779                                 printf("Refresh mbufs: payload dmamap load"
3780                                     " failure - %d\n", error);
3781                                 m_free(mp);
3782                                 rxbuf->buf = NULL;
3783                                 goto update;
3784                         }
3785                         rxbuf->buf = mp;
3786                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3787                             BUS_DMASYNC_PREREAD);
3788                         rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3789                             htole64(seg[0].ds_addr);
3790                 } else {
3791                         rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3792                         rxbuf->flags &= ~IXGBE_RX_COPY;
3793                 }
3794
3795                 refreshed = TRUE;
3796                 /* Next is precalculated */
3797                 i = j;
3798                 rxr->next_to_refresh = i;
3799                 if (++j == rxr->num_desc)
3800                         j = 0;
3801         }
3802 update:
3803         if (refreshed) /* Update hardware tail index */
3804                 IXGBE_WRITE_REG(&adapter->hw,
3805                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3806         return;
3807 }
3808
3809 /*********************************************************************
3810  *
3811  *  Allocate memory for rx_buffer structures. Since we use one
3812  *  rx_buffer per received packet, the maximum number of rx_buffer's
3813  *  that we'll need is equal to the number of receive descriptors
3814  *  that we've allocated.
3815  *
3816  **********************************************************************/
3817 static int
3818 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3819 {
3820         struct  adapter         *adapter = rxr->adapter;
3821         device_t                dev = adapter->dev;
3822         struct ixgbe_rx_buf     *rxbuf;
3823         int                     i, bsize, error;
3824
3825         bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3826         if (!(rxr->rx_buffers =
3827             (struct ixgbe_rx_buf *) malloc(bsize,
3828             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3829                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3830                 error = ENOMEM;
3831                 goto fail;
3832         }
3833
3834         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3835                                    1, 0,        /* alignment, bounds */
3836                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3837                                    BUS_SPACE_MAXADDR,   /* highaddr */
3838                                    NULL, NULL,          /* filter, filterarg */
3839                                    MJUM16BYTES,         /* maxsize */
3840                                    1,                   /* nsegments */
3841                                    MJUM16BYTES,         /* maxsegsize */
3842                                    0,                   /* flags */
3843                                    NULL,                /* lockfunc */
3844                                    NULL,                /* lockfuncarg */
3845                                    &rxr->ptag))) {
3846                 device_printf(dev, "Unable to create RX DMA tag\n");
3847                 goto fail;
3848         }
3849
3850         for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3851                 rxbuf = &rxr->rx_buffers[i];
3852                 error = bus_dmamap_create(rxr->ptag,
3853                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3854                 if (error) {
3855                         device_printf(dev, "Unable to create RX dma map\n");
3856                         goto fail;
3857                 }
3858         }
3859
3860         return (0);
3861
3862 fail:
3863         /* Frees all, but can handle partial completion */
3864         ixgbe_free_receive_structures(adapter);
3865         return (error);
3866 }
3867
3868 /*
3869 ** Used to detect a descriptor that has
3870 ** been merged by Hardware RSC.
3871 */
3872 static inline u32
3873 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3874 {
3875         return (le32toh(rx->wb.lower.lo_dword.data) &
3876             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3877 }
3878
3879 /*********************************************************************
3880  *
3881  *  Initialize Hardware RSC (LRO) feature on 82599
3882  *  for an RX ring, this is toggled by the LRO capability
3883  *  even though it is transparent to the stack.
3884  *
3885  *  NOTE: since this HW feature only works with IPV4 and
3886  *        our testing has shown soft LRO to be as effective
3887  *        I have decided to disable this by default.
3888  *
3889  **********************************************************************/
3890 static void
3891 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3892 {
3893         struct  adapter         *adapter = rxr->adapter;
3894         struct  ixgbe_hw        *hw = &adapter->hw;
3895         u32                     rscctrl, rdrxctl;
3896
3897         /* If turning LRO/RSC off we need to disable it */
3898         if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3899                 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3900                 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3901                 return;
3902         }
3903
3904         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3905         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3906 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3907         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3908 #endif /* DEV_NETMAP */
3909         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3910         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3911         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3912
3913         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3914         rscctrl |= IXGBE_RSCCTL_RSCEN;
3915         /*
3916         ** Limit the total number of descriptors that
3917         ** can be combined, so it does not exceed 64K
3918         */
3919         if (rxr->mbuf_sz == MCLBYTES)
3920                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3921         else if (rxr->mbuf_sz == MJUMPAGESIZE)
3922                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3923         else if (rxr->mbuf_sz == MJUM9BYTES)
3924                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3925         else  /* Using 16K cluster */
3926                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3927
3928         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3929
3930         /* Enable TCP header recognition */
3931         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3932             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3933             IXGBE_PSRTYPE_TCPHDR));
3934
3935         /* Disable RSC for ACK packets */
3936         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3937             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3938
3939         rxr->hw_rsc = TRUE;
3940 }
3941
3942
3943 static void
3944 ixgbe_free_receive_ring(struct rx_ring *rxr)
3945 {
3946         struct ixgbe_rx_buf       *rxbuf;
3947         int i;
3948
3949         for (i = 0; i < rxr->num_desc; i++) {
3950                 rxbuf = &rxr->rx_buffers[i];
3951                 if (rxbuf->buf != NULL) {
3952                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3953                             BUS_DMASYNC_POSTREAD);
3954                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3955                         rxbuf->buf->m_flags |= M_PKTHDR;
3956                         m_freem(rxbuf->buf);
3957                         rxbuf->buf = NULL;
3958                         rxbuf->flags = 0;
3959                 }
3960         }
3961 }
3962
3963
3964 /*********************************************************************
3965  *
3966  *  Initialize a receive ring and its buffers.
3967  *
3968  **********************************************************************/
3969 static int
3970 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3971 {
3972         struct  adapter         *adapter;
3973         struct ifnet            *ifp;
3974         device_t                dev;
3975         struct ixgbe_rx_buf     *rxbuf;
3976         bus_dma_segment_t       seg[1];
3977         struct lro_ctrl         *lro = &rxr->lro;
3978         int                     rsize, nsegs, error = 0;
3979 #ifdef DEV_NETMAP
3980         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3981         struct netmap_slot *slot;
3982 #endif /* DEV_NETMAP */
3983
3984         adapter = rxr->adapter;
3985         ifp = adapter->ifp;
3986         dev = adapter->dev;
3987
3988         /* Clear the ring contents */
3989         IXGBE_RX_LOCK(rxr);
3990 #ifdef DEV_NETMAP
3991         /* same as in ixgbe_setup_transmit_ring() */
3992         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3993 #endif /* DEV_NETMAP */
3994         rsize = roundup2(adapter->num_rx_desc *
3995             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3996         bzero((void *)rxr->rx_base, rsize);
3997         /* Cache the size */
3998         rxr->mbuf_sz = adapter->rx_mbuf_sz;
3999
4000         /* Free current RX buffer structs and their mbufs */
4001         ixgbe_free_receive_ring(rxr);
4002
4003         /* Now replenish the mbufs */
4004         for (int j = 0; j != rxr->num_desc; ++j) {
4005                 struct mbuf     *mp;
4006
4007                 rxbuf = &rxr->rx_buffers[j];
4008 #ifdef DEV_NETMAP
4009                 /*
4010                  * In netmap mode, fill the map and set the buffer
4011                  * address in the NIC ring, considering the offset
4012                  * between the netmap and NIC rings (see comment in
4013                  * ixgbe_setup_transmit_ring() ). No need to allocate
4014                  * an mbuf, so end the block with a continue;
4015                  */
4016                 if (slot) {
4017                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4018                         uint64_t paddr;
4019                         void *addr;
4020
4021                         addr = PNMB(slot + sj, &paddr);
4022                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4023                         /* Update descriptor and the cached value */
4024                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4025                         rxbuf->addr = htole64(paddr);
4026                         continue;
4027                 }
4028 #endif /* DEV_NETMAP */
4029                 rxbuf->flags = 0;
4030                 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
4031                     M_PKTHDR, adapter->rx_mbuf_sz);
4032                 if (rxbuf->buf == NULL) {
4033                         error = ENOBUFS;
4034                         goto fail;
4035                 }
4036                 mp = rxbuf->buf;
4037                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4038                 /* Get the memory mapping */
4039                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4040                     rxbuf->pmap, mp, seg,
4041                     &nsegs, BUS_DMA_NOWAIT);
4042                 if (error != 0)
4043                         goto fail;
4044                 bus_dmamap_sync(rxr->ptag,
4045                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4046                 /* Update the descriptor and the cached value */
4047                 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4048                 rxbuf->addr = htole64(seg[0].ds_addr);
4049         }
4050
4051
4052         /* Setup our descriptor indices */
4053         rxr->next_to_check = 0;
4054         rxr->next_to_refresh = 0;
4055         rxr->lro_enabled = FALSE;
4056         rxr->rx_copies = 0;
4057         rxr->rx_bytes = 0;
4058         rxr->discard = FALSE;
4059         rxr->vtag_strip = FALSE;
4060
4061         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4062             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4063
4064         /*
4065         ** Now set up the LRO interface:
4066         */
4067         if (ixgbe_rsc_enable)
4068                 ixgbe_setup_hw_rsc(rxr);
4069         else if (ifp->if_capenable & IFCAP_LRO) {
4070                 int err = tcp_lro_init(lro);
4071                 if (err) {
4072                         device_printf(dev, "LRO Initialization failed!\n");
4073                         goto fail;
4074                 }
4075                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4076                 rxr->lro_enabled = TRUE;
4077                 lro->ifp = adapter->ifp;
4078         }
4079
4080         IXGBE_RX_UNLOCK(rxr);
4081         return (0);
4082
4083 fail:
4084         ixgbe_free_receive_ring(rxr);
4085         IXGBE_RX_UNLOCK(rxr);
4086         return (error);
4087 }
4088
4089 /*********************************************************************
4090  *
4091  *  Initialize all receive rings.
4092  *
4093  **********************************************************************/
4094 static int
4095 ixgbe_setup_receive_structures(struct adapter *adapter)
4096 {
4097         struct rx_ring *rxr = adapter->rx_rings;
4098         int j;
4099
4100         for (j = 0; j < adapter->num_queues; j++, rxr++)
4101                 if (ixgbe_setup_receive_ring(rxr))
4102                         goto fail;
4103
4104         return (0);
4105 fail:
4106         /*
4107          * Free RX buffers allocated so far, we will only handle
4108          * the rings that completed, the failing case will have
4109          * cleaned up for itself. 'j' failed, so its the terminus.
4110          */
4111         for (int i = 0; i < j; ++i) {
4112                 rxr = &adapter->rx_rings[i];
4113                 ixgbe_free_receive_ring(rxr);
4114         }
4115
4116         return (ENOBUFS);
4117 }
4118
4119 /*********************************************************************
4120  *
4121  *  Setup receive registers and features.
4122  *
4123  **********************************************************************/
4124 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4125
4126 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4127
4128 static void
4129 ixgbe_initialize_receive_units(struct adapter *adapter)
4130 {
4131         struct  rx_ring *rxr = adapter->rx_rings;
4132         struct ixgbe_hw *hw = &adapter->hw;
4133         struct ifnet   *ifp = adapter->ifp;
4134         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4135         u32             reta, mrqc = 0, hlreg, random[10];
4136
4137
4138         /*
4139          * Make sure receives are disabled while
4140          * setting up the descriptor ring
4141          */
4142         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4143         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4144             rxctrl & ~IXGBE_RXCTRL_RXEN);
4145
4146         /* Enable broadcasts */
4147         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4148         fctrl |= IXGBE_FCTRL_BAM;
4149         fctrl |= IXGBE_FCTRL_DPF;
4150         fctrl |= IXGBE_FCTRL_PMCF;
4151         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4152
4153         /* Set for Jumbo Frames? */
4154         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4155         if (ifp->if_mtu > ETHERMTU)
4156                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4157         else
4158                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4159 #ifdef DEV_NETMAP
4160         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4161         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4162                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4163         else
4164                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4165 #endif /* DEV_NETMAP */
4166         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4167
4168         bufsz = (adapter->rx_mbuf_sz +
4169             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4170
4171         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4172                 u64 rdba = rxr->rxdma.dma_paddr;
4173
4174                 /* Setup the Base and Length of the Rx Descriptor Ring */
4175                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4176                                (rdba & 0x00000000ffffffffULL));
4177                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4178                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4179                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4180
4181                 /* Set up the SRRCTL register */
4182                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4183                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4184                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4185                 srrctl |= bufsz;
4186                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4187                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4188
4189                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4190                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4191                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4192
4193                 /* Set the processing limit */
4194                 rxr->process_limit = ixgbe_rx_process_limit;
4195         }
4196
4197         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4198                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4199                               IXGBE_PSRTYPE_UDPHDR |
4200                               IXGBE_PSRTYPE_IPV4HDR |
4201                               IXGBE_PSRTYPE_IPV6HDR;
4202                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4203         }
4204
4205         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4206
4207         /* Setup RSS */
4208         if (adapter->num_queues > 1) {
4209                 int i, j;
4210                 reta = 0;
4211
4212                 /* set up random bits */
4213                 arc4rand(&random, sizeof(random), 0);
4214
4215                 /* Set up the redirection table */
4216                 for (i = 0, j = 0; i < 128; i++, j++) {
4217                         if (j == adapter->num_queues) j = 0;
4218                         reta = (reta << 8) | (j * 0x11);
4219                         if ((i & 3) == 3)
4220                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4221                 }
4222
4223                 /* Now fill our hash function seeds */
4224                 for (int i = 0; i < 10; i++)
4225                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4226
4227                 /* Perform hash on these packet types */
4228                 mrqc = IXGBE_MRQC_RSSEN
4229                      | IXGBE_MRQC_RSS_FIELD_IPV4
4230                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4231                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4232                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4233                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4234                      | IXGBE_MRQC_RSS_FIELD_IPV6
4235                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4236                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4237                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4238                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4239
4240                 /* RSS and RX IPP Checksum are mutually exclusive */
4241                 rxcsum |= IXGBE_RXCSUM_PCSD;
4242         }
4243
4244         if (ifp->if_capenable & IFCAP_RXCSUM)
4245                 rxcsum |= IXGBE_RXCSUM_PCSD;
4246
4247         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4248                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4249
4250         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4251
4252         return;
4253 }
4254
4255 /*********************************************************************
4256  *
4257  *  Free all receive rings.
4258  *
4259  **********************************************************************/
4260 static void
4261 ixgbe_free_receive_structures(struct adapter *adapter)
4262 {
4263         struct rx_ring *rxr = adapter->rx_rings;
4264
4265         INIT_DEBUGOUT("ixgbe_free_receive_structures: begin");
4266
4267         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4268                 struct lro_ctrl         *lro = &rxr->lro;
4269                 ixgbe_free_receive_buffers(rxr);
4270                 /* Free LRO memory */
4271                 tcp_lro_free(lro);
4272                 /* Free the ring memory as well */
4273                 ixgbe_dma_free(adapter, &rxr->rxdma);
4274         }
4275
4276         free(adapter->rx_rings, M_DEVBUF);
4277 }
4278
4279
4280 /*********************************************************************
4281  *
4282  *  Free receive ring data structures
4283  *
4284  **********************************************************************/
4285 static void
4286 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4287 {
4288         struct adapter          *adapter = rxr->adapter;
4289         struct ixgbe_rx_buf     *rxbuf;
4290
4291         INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin");
4292
4293         /* Cleanup any existing buffers */
4294         if (rxr->rx_buffers != NULL) {
4295                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4296                         rxbuf = &rxr->rx_buffers[i];
4297                         if (rxbuf->buf != NULL) {
4298                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4299                                     BUS_DMASYNC_POSTREAD);
4300                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4301                                 rxbuf->buf->m_flags |= M_PKTHDR;
4302                                 m_freem(rxbuf->buf);
4303                         }
4304                         rxbuf->buf = NULL;
4305                         if (rxbuf->pmap != NULL) {
4306                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4307                                 rxbuf->pmap = NULL;
4308                         }
4309                 }
4310                 if (rxr->rx_buffers != NULL) {
4311                         free(rxr->rx_buffers, M_DEVBUF);
4312                         rxr->rx_buffers = NULL;
4313                 }
4314         }
4315
4316         if (rxr->ptag != NULL) {
4317                 bus_dma_tag_destroy(rxr->ptag);
4318                 rxr->ptag = NULL;
4319         }
4320
4321         return;
4322 }
4323
4324 static __inline void
4325 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4326 {
4327
4328         /*
4329          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4330          * should be computed by hardware. Also it should not have VLAN tag in
4331          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4332          */
4333         if (rxr->lro_enabled &&
4334             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4335             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4336             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4337             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4338             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4339             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4340             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4341             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4342                 /*
4343                  * Send to the stack if:
4344                  **  - LRO not enabled, or
4345                  **  - no LRO resources, or
4346                  **  - lro enqueue fails
4347                  */
4348                 if (rxr->lro.lro_cnt != 0)
4349                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4350                                 return;
4351         }
4352         IXGBE_RX_UNLOCK(rxr);
4353         (*ifp->if_input)(ifp, m);
4354         IXGBE_RX_LOCK(rxr);
4355 }
4356
4357 static __inline void
4358 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4359 {
4360         struct ixgbe_rx_buf     *rbuf;
4361
4362         rbuf = &rxr->rx_buffers[i];
4363
4364         if (rbuf->fmp != NULL) {/* Partial chain ? */
4365                 rbuf->fmp->m_flags |= M_PKTHDR;
4366                 m_freem(rbuf->fmp);
4367                 rbuf->fmp = NULL;
4368         }
4369
4370         /*
4371         ** With advanced descriptors the writeback
4372         ** clobbers the buffer addrs, so its easier
4373         ** to just free the existing mbufs and take
4374         ** the normal refresh path to get new buffers
4375         ** and mapping.
4376         */
4377         if (rbuf->buf) {
4378                 m_free(rbuf->buf);
4379                 rbuf->buf = NULL;
4380         }
4381
4382         rbuf->flags = 0;
4383
4384         return;
4385 }
4386
4387
4388 /*********************************************************************
4389  *
4390  *  This routine executes in interrupt context. It replenishes
4391  *  the mbufs in the descriptor and sends data which has been
4392  *  dma'ed into host memory to upper layer.
4393  *
4394  *  We loop at most count times if count is > 0, or until done if
4395  *  count < 0.
4396  *
4397  *  Return TRUE for more work, FALSE for all clean.
4398  *********************************************************************/
4399 static bool
4400 ixgbe_rxeof(struct ix_queue *que)
4401 {
4402         struct adapter          *adapter = que->adapter;
4403         struct rx_ring          *rxr = que->rxr;
4404         struct ifnet            *ifp = adapter->ifp;
4405         struct lro_ctrl         *lro = &rxr->lro;
4406         struct lro_entry        *queued;
4407         int                     i, nextp, processed = 0;
4408         u32                     staterr = 0;
4409         u16                     count = rxr->process_limit;
4410         union ixgbe_adv_rx_desc *cur;
4411         struct ixgbe_rx_buf     *rbuf, *nbuf;
4412
4413         IXGBE_RX_LOCK(rxr);
4414
4415 #ifdef DEV_NETMAP
4416         /* Same as the txeof routine: wakeup clients on intr. */
4417         if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4418                 IXGBE_RX_UNLOCK(rxr);
4419                 return (FALSE);
4420         }
4421 #endif /* DEV_NETMAP */
4422
4423         for (i = rxr->next_to_check; count != 0;) {
4424                 struct mbuf     *sendmp, *mp;
4425                 u32             rsc, ptype;
4426                 u16             len;
4427                 u16             vtag = 0;
4428                 bool            eop;
4429
4430                 /* Sync the ring. */
4431                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4432                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4433
4434                 cur = &rxr->rx_base[i];
4435                 staterr = le32toh(cur->wb.upper.status_error);
4436
4437                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4438                         break;
4439                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4440                         break;
4441
4442                 count--;
4443                 sendmp = NULL;
4444                 nbuf = NULL;
4445                 rsc = 0;
4446                 cur->wb.upper.status_error = 0;
4447                 rbuf = &rxr->rx_buffers[i];
4448                 mp = rbuf->buf;
4449
4450                 len = le16toh(cur->wb.upper.length);
4451                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4452                     IXGBE_RXDADV_PKTTYPE_MASK;
4453                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4454
4455                 /* Make sure bad packets are discarded */
4456                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4457                     (rxr->discard)) {
4458                         rxr->rx_discarded++;
4459                         if (eop)
4460                                 rxr->discard = FALSE;
4461                         else
4462                                 rxr->discard = TRUE;
4463                         ixgbe_rx_discard(rxr, i);
4464                         goto next_desc;
4465                 }
4466
4467                 /*
4468                 ** On 82599 which supports a hardware
4469                 ** LRO (called HW RSC), packets need
4470                 ** not be fragmented across sequential
4471                 ** descriptors, rather the next descriptor
4472                 ** is indicated in bits of the descriptor.
4473                 ** This also means that we might proceses
4474                 ** more than one packet at a time, something
4475                 ** that has never been true before, it
4476                 ** required eliminating global chain pointers
4477                 ** in favor of what we are doing here.  -jfv
4478                 */
4479                 if (!eop) {
4480                         /*
4481                         ** Figure out the next descriptor
4482                         ** of this frame.
4483                         */
4484                         if (rxr->hw_rsc == TRUE) {
4485                                 rsc = ixgbe_rsc_count(cur);
4486                                 rxr->rsc_num += (rsc - 1);
4487                         }
4488                         if (rsc) { /* Get hardware index */
4489                                 nextp = ((staterr &
4490                                     IXGBE_RXDADV_NEXTP_MASK) >>
4491                                     IXGBE_RXDADV_NEXTP_SHIFT);
4492                         } else { /* Just sequential */
4493                                 nextp = i + 1;
4494                                 if (nextp == adapter->num_rx_desc)
4495                                         nextp = 0;
4496                         }
4497                         nbuf = &rxr->rx_buffers[nextp];
4498                         prefetch(nbuf);
4499                 }
4500                 /*
4501                 ** Rather than using the fmp/lmp global pointers
4502                 ** we now keep the head of a packet chain in the
4503                 ** buffer struct and pass this along from one
4504                 ** descriptor to the next, until we get EOP.
4505                 */
4506                 mp->m_len = len;
4507                 /*
4508                 ** See if there is a stored head
4509                 ** that determines what we are
4510                 */
4511                 sendmp = rbuf->fmp;
4512                 if (sendmp != NULL) {  /* secondary frag */
4513                         rbuf->buf = rbuf->fmp = NULL;
4514                         mp->m_flags &= ~M_PKTHDR;
4515                         sendmp->m_pkthdr.len += mp->m_len;
4516                 } else {
4517                         /*
4518                          * Optimize.  This might be a small packet,
4519                          * maybe just a TCP ACK.  Do a fast copy that
4520                          * is cache aligned into a new mbuf, and
4521                          * leave the old mbuf+cluster for re-use.
4522                          */
4523                         if (eop && len <= IXGBE_RX_COPY_LEN) {
4524                                 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4525                                 if (sendmp != NULL) {
4526                                         sendmp->m_data +=
4527                                             IXGBE_RX_COPY_ALIGN;
4528                                         ixgbe_bcopy(mp->m_data,
4529                                             sendmp->m_data, len);
4530                                         sendmp->m_len = len;
4531                                         rxr->rx_copies++;
4532                                         rbuf->flags |= IXGBE_RX_COPY;
4533                                 }
4534                         }
4535                         if (sendmp == NULL) {
4536                                 rbuf->buf = rbuf->fmp = NULL;
4537                                 sendmp = mp;
4538                         }
4539
4540                         /* first desc of a non-ps chain */
4541                         sendmp->m_flags |= M_PKTHDR;
4542                         sendmp->m_pkthdr.len = mp->m_len;
4543                 }
4544                 ++processed;
4545
4546                 /* Pass the head pointer on */
4547                 if (eop == 0) {
4548                         nbuf->fmp = sendmp;
4549                         sendmp = NULL;
4550                         mp->m_next = nbuf->buf;
4551                 } else { /* Sending this frame */
4552                         sendmp->m_pkthdr.rcvif = ifp;
4553                         rxr->rx_packets++;
4554                         /* capture data for AIM */
4555                         rxr->bytes += sendmp->m_pkthdr.len;
4556                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4557                         /* Process vlan info */
4558                         if ((rxr->vtag_strip) &&
4559                             (staterr & IXGBE_RXD_STAT_VP))
4560                                 vtag = le16toh(cur->wb.upper.vlan);
4561                         if (vtag) {
4562                                 sendmp->m_pkthdr.ether_vtag = vtag;
4563                                 sendmp->m_flags |= M_VLANTAG;
4564                         }
4565                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4566                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4567 #if __FreeBSD_version >= 800000
4568                         sendmp->m_pkthdr.flowid = que->msix;
4569                         sendmp->m_flags |= M_FLOWID;
4570 #endif
4571                 }
4572 next_desc:
4573                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4574                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4575
4576                 /* Advance our pointers to the next descriptor. */
4577                 if (++i == rxr->num_desc)
4578                         i = 0;
4579
4580                 /* Now send to the stack or do LRO */
4581                 if (sendmp != NULL) {
4582                         rxr->next_to_check = i;
4583                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4584                         i = rxr->next_to_check;
4585                 }
4586
4587                /* Every 8 descriptors we go to refresh mbufs */
4588                 if (processed == 8) {
4589                         ixgbe_refresh_mbufs(rxr, i);
4590                         processed = 0;
4591                 }
4592         }
4593
4594         /* Refresh any remaining buf structs */
4595         if (ixgbe_rx_unrefreshed(rxr))
4596                 ixgbe_refresh_mbufs(rxr, i);
4597
4598         rxr->next_to_check = i;
4599
4600         /*
4601          * Flush any outstanding LRO work
4602          */
4603         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4604                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4605                 tcp_lro_flush(lro, queued);
4606         }
4607
4608         IXGBE_RX_UNLOCK(rxr);
4609
4610         /*
4611         ** Still have cleaning to do?
4612         */
4613         if ((staterr & IXGBE_RXD_STAT_DD) != 0)
4614                 return (TRUE);
4615         else
4616                 return (FALSE);
4617 }
4618
4619
4620 /*********************************************************************
4621  *
4622  *  Verify that the hardware indicated that the checksum is valid.
4623  *  Inform the stack about the status of checksum so that stack
4624  *  doesn't spend time verifying the checksum.
4625  *
4626  *********************************************************************/
4627 static void
4628 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4629 {
4630         u16     status = (u16) staterr;
4631         u8      errors = (u8) (staterr >> 24);
4632         bool    sctp = FALSE;
4633
4634         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4635             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4636                 sctp = TRUE;
4637
4638         if (status & IXGBE_RXD_STAT_IPCS) {
4639                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4640                         /* IP Checksum Good */
4641                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4642                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4643
4644                 } else
4645                         mp->m_pkthdr.csum_flags = 0;
4646         }
4647         if (status & IXGBE_RXD_STAT_L4CS) {
4648                 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4649 #if __FreeBSD_version >= 800000
4650                 if (sctp)
4651                         type = CSUM_SCTP_VALID;
4652 #endif
4653                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4654                         mp->m_pkthdr.csum_flags |= type;
4655                         if (!sctp)
4656                                 mp->m_pkthdr.csum_data = htons(0xffff);
4657                 }
4658         }
4659         return;
4660 }
4661
4662
4663 /*
4664 ** This routine is run via an vlan config EVENT,
4665 ** it enables us to use the HW Filter table since
4666 ** we can get the vlan id. This just creates the
4667 ** entry in the soft version of the VFTA, init will
4668 ** repopulate the real table.
4669 */
4670 static void
4671 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4672 {
4673         struct adapter  *adapter = ifp->if_softc;
4674         u16             index, bit;
4675
4676         if (ifp->if_softc !=  arg)   /* Not our event */
4677                 return;
4678
4679         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4680                 return;
4681
4682         IXGBE_CORE_LOCK(adapter);
4683         index = (vtag >> 5) & 0x7F;
4684         bit = vtag & 0x1F;
4685         adapter->shadow_vfta[index] |= (1 << bit);
4686         ++adapter->num_vlans;
4687         ixgbe_setup_vlan_hw_support(adapter);
4688         IXGBE_CORE_UNLOCK(adapter);
4689 }
4690
4691 /*
4692 ** This routine is run via an vlan
4693 ** unconfig EVENT, remove our entry
4694 ** in the soft vfta.
4695 */
4696 static void
4697 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4698 {
4699         struct adapter  *adapter = ifp->if_softc;
4700         u16             index, bit;
4701
4702         if (ifp->if_softc !=  arg)
4703                 return;
4704
4705         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4706                 return;
4707
4708         IXGBE_CORE_LOCK(adapter);
4709         index = (vtag >> 5) & 0x7F;
4710         bit = vtag & 0x1F;
4711         adapter->shadow_vfta[index] &= ~(1 << bit);
4712         --adapter->num_vlans;
4713         /* Re-init to load the changes */
4714         ixgbe_setup_vlan_hw_support(adapter);
4715         IXGBE_CORE_UNLOCK(adapter);
4716 }
4717
4718 static void
4719 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4720 {
4721         struct ifnet    *ifp = adapter->ifp;
4722         struct ixgbe_hw *hw = &adapter->hw;
4723         struct rx_ring  *rxr;
4724         u32             ctrl;
4725
4726
4727         /*
4728         ** We get here thru init_locked, meaning
4729         ** a soft reset, this has already cleared
4730         ** the VFTA and other state, so if there
4731         ** have been no vlan's registered do nothing.
4732         */
4733         if (adapter->num_vlans == 0)
4734                 return;
4735
4736         /* Setup the queues for vlans */
4737         for (int i = 0; i < adapter->num_queues; i++) {
4738                 rxr = &adapter->rx_rings[i];
4739                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4740                 if (hw->mac.type != ixgbe_mac_82598EB) {
4741                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4742                         ctrl |= IXGBE_RXDCTL_VME;
4743                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4744                 }
4745                 rxr->vtag_strip = TRUE;
4746         }
4747
4748         if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
4749                 return;
4750         /*
4751         ** A soft reset zero's out the VFTA, so
4752         ** we need to repopulate it now.
4753         */
4754         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4755                 if (adapter->shadow_vfta[i] != 0)
4756                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4757                             adapter->shadow_vfta[i]);
4758
4759         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4760         /* Enable the Filter Table if enabled */
4761         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4762                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4763                 ctrl |= IXGBE_VLNCTRL_VFE;
4764         }
4765         if (hw->mac.type == ixgbe_mac_82598EB)
4766                 ctrl |= IXGBE_VLNCTRL_VME;
4767         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4768 }
4769
4770 static void
4771 ixgbe_enable_intr(struct adapter *adapter)
4772 {
4773         struct ixgbe_hw *hw = &adapter->hw;
4774         struct ix_queue *que = adapter->queues;
4775         u32             mask, fwsm;
4776
4777         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4778         /* Enable Fan Failure detection */
4779         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4780                     mask |= IXGBE_EIMS_GPI_SDP1;
4781
4782         switch (adapter->hw.mac.type) {
4783                 case ixgbe_mac_82599EB:
4784                         mask |= IXGBE_EIMS_ECC;
4785                         mask |= IXGBE_EIMS_GPI_SDP0;
4786                         mask |= IXGBE_EIMS_GPI_SDP1;
4787                         mask |= IXGBE_EIMS_GPI_SDP2;
4788 #ifdef IXGBE_FDIR
4789                         mask |= IXGBE_EIMS_FLOW_DIR;
4790 #endif
4791                         break;
4792                 case ixgbe_mac_X540:
4793                         mask |= IXGBE_EIMS_ECC;
4794                         /* Detect if Thermal Sensor is enabled */
4795                         fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
4796                         if (fwsm & IXGBE_FWSM_TS_ENABLED)
4797                                 mask |= IXGBE_EIMS_TS;
4798 #ifdef IXGBE_FDIR
4799                         mask |= IXGBE_EIMS_FLOW_DIR;
4800 #endif
4801                 /* falls through */
4802                 default:
4803                         break;
4804         }
4805
4806         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4807
4808         /* With RSS we use auto clear */
4809         if (adapter->msix_mem) {
4810                 mask = IXGBE_EIMS_ENABLE_MASK;
4811                 /* Don't autoclear Link */
4812                 mask &= ~IXGBE_EIMS_OTHER;
4813                 mask &= ~IXGBE_EIMS_LSC;
4814                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4815         }
4816
4817         /*
4818         ** Now enable all queues, this is done separately to
4819         ** allow for handling the extended (beyond 32) MSIX
4820         ** vectors that can be used by 82599
4821         */
4822         for (int i = 0; i < adapter->num_queues; i++, que++)
4823                 ixgbe_enable_queue(adapter, que->msix);
4824
4825         IXGBE_WRITE_FLUSH(hw);
4826
4827         return;
4828 }
4829
4830 static void
4831 ixgbe_disable_intr(struct adapter *adapter)
4832 {
4833         if (adapter->msix_mem)
4834                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4835         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4836                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4837         } else {
4838                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4839                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4840                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4841         }
4842         IXGBE_WRITE_FLUSH(&adapter->hw);
4843         return;
4844 }
4845
4846 u16
4847 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4848 {
4849         u16 value;
4850
4851         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4852             reg, 2);
4853
4854         return (value);
4855 }
4856
4857 void
4858 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4859 {
4860         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4861             reg, value, 2);
4862
4863         return;
4864 }
4865
4866 /*
4867 ** Get the width and transaction speed of
4868 ** the slot this adapter is plugged into.
4869 */
4870 static void
4871 ixgbe_get_slot_info(struct ixgbe_hw *hw)
4872 {
4873         device_t                dev = ((struct ixgbe_osdep *)hw->back)->dev;
4874         struct ixgbe_mac_info   *mac = &hw->mac;
4875         u16                     link;
4876         u32                     offset;
4877
4878         /* For most devices simply call the shared code routine */
4879         if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
4880                 ixgbe_get_bus_info(hw);
4881                 goto display;
4882         }
4883
4884         /*
4885         ** For the Quad port adapter we need to parse back
4886         ** up the PCI tree to find the speed of the expansion
4887         ** slot into which this adapter is plugged. A bit more work.
4888         */
4889         dev = device_get_parent(device_get_parent(dev));
4890 #ifdef IXGBE_DEBUG
4891         device_printf(dev, "parent pcib = %x,%x,%x\n",
4892             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4893 #endif
4894         dev = device_get_parent(device_get_parent(dev));
4895 #ifdef IXGBE_DEBUG
4896         device_printf(dev, "slot pcib = %x,%x,%x\n",
4897             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4898 #endif
4899         /* Now get the PCI Express Capabilities offset */
4900         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4901         /* ...and read the Link Status Register */
4902         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4903         switch (link & IXGBE_PCI_LINK_WIDTH) {
4904         case IXGBE_PCI_LINK_WIDTH_1:
4905                 hw->bus.width = ixgbe_bus_width_pcie_x1;
4906                 break;
4907         case IXGBE_PCI_LINK_WIDTH_2:
4908                 hw->bus.width = ixgbe_bus_width_pcie_x2;
4909                 break;
4910         case IXGBE_PCI_LINK_WIDTH_4:
4911                 hw->bus.width = ixgbe_bus_width_pcie_x4;
4912                 break;
4913         case IXGBE_PCI_LINK_WIDTH_8:
4914                 hw->bus.width = ixgbe_bus_width_pcie_x8;
4915                 break;
4916         default:
4917                 hw->bus.width = ixgbe_bus_width_unknown;
4918                 break;
4919         }
4920
4921         switch (link & IXGBE_PCI_LINK_SPEED) {
4922         case IXGBE_PCI_LINK_SPEED_2500:
4923                 hw->bus.speed = ixgbe_bus_speed_2500;
4924                 break;
4925         case IXGBE_PCI_LINK_SPEED_5000:
4926                 hw->bus.speed = ixgbe_bus_speed_5000;
4927                 break;
4928         case IXGBE_PCI_LINK_SPEED_8000:
4929                 hw->bus.speed = ixgbe_bus_speed_8000;
4930                 break;
4931         default:
4932                 hw->bus.speed = ixgbe_bus_speed_unknown;
4933                 break;
4934         }
4935
4936         mac->ops.set_lan_id(hw);
4937
4938 display:
4939         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4940             ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
4941             (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
4942             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4943             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
4944             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
4945             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
4946             ("Unknown"));
4947
4948         if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4949             ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
4950             (hw->bus.speed == ixgbe_bus_speed_2500))) {
4951                 device_printf(dev, "PCI-Express bandwidth available"
4952                     " for this card\n     is not sufficient for"
4953                     " optimal performance.\n");
4954                 device_printf(dev, "For optimal performance a x8 "
4955                     "PCIE, or x4 PCIE Gen2 slot is required.\n");
4956         }
4957         if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4958             ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
4959             (hw->bus.speed < ixgbe_bus_speed_8000))) {
4960                 device_printf(dev, "PCI-Express bandwidth available"
4961                     " for this card\n     is not sufficient for"
4962                     " optimal performance.\n");
4963                 device_printf(dev, "For optimal performance a x8 "
4964                     "PCIE Gen3 slot is required.\n");
4965         }
4966
4967         return;
4968 }
4969
4970
4971 /*
4972 ** Setup the correct IVAR register for a particular MSIX interrupt
4973 **   (yes this is all very magic and confusing :)
4974 **  - entry is the register array entry
4975 **  - vector is the MSIX vector for this queue
4976 **  - type is RX/TX/MISC
4977 */
4978 static void
4979 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4980 {
4981         struct ixgbe_hw *hw = &adapter->hw;
4982         u32 ivar, index;
4983
4984         vector |= IXGBE_IVAR_ALLOC_VAL;
4985
4986         switch (hw->mac.type) {
4987
4988         case ixgbe_mac_82598EB:
4989                 if (type == -1)
4990                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4991                 else
4992                         entry += (type * 64);
4993                 index = (entry >> 2) & 0x1F;
4994                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4995                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4996                 ivar |= (vector << (8 * (entry & 0x3)));
4997                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4998                 break;
4999
5000         case ixgbe_mac_82599EB:
5001         case ixgbe_mac_X540:
5002                 if (type == -1) { /* MISC IVAR */
5003                         index = (entry & 1) * 8;
5004                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5005                         ivar &= ~(0xFF << index);
5006                         ivar |= (vector << index);
5007                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5008                 } else {        /* RX/TX IVARS */
5009                         index = (16 * (entry & 1)) + (8 * type);
5010                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5011                         ivar &= ~(0xFF << index);
5012                         ivar |= (vector << index);
5013                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5014                 }
5015
5016         default:
5017                 break;
5018         }
5019 }
5020
5021 static void
5022 ixgbe_configure_ivars(struct adapter *adapter)
5023 {
5024         struct  ix_queue *que = adapter->queues;
5025         u32 newitr;
5026
5027         if (ixgbe_max_interrupt_rate > 0)
5028                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5029         else
5030                 newitr = 0;
5031
5032         for (int i = 0; i < adapter->num_queues; i++, que++) {
5033                 /* First the RX queue entry */
5034                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5035                 /* ... and the TX */
5036                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5037                 /* Set an Initial EITR value */
5038                 IXGBE_WRITE_REG(&adapter->hw,
5039                     IXGBE_EITR(que->msix), newitr);
5040         }
5041
5042         /* For the Link interrupt */
5043         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5044 }
5045
5046 /*
5047 ** ixgbe_sfp_probe - called in the local timer to
5048 ** determine if a port had optics inserted.
5049 */
5050 static bool ixgbe_sfp_probe(struct adapter *adapter)
5051 {
5052         struct ixgbe_hw *hw = &adapter->hw;
5053         device_t        dev = adapter->dev;
5054         bool            result = FALSE;
5055
5056         if ((hw->phy.type == ixgbe_phy_nl) &&
5057             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5058                 s32 ret = hw->phy.ops.identify_sfp(hw);
5059                 if (ret)
5060                         goto out;
5061                 ret = hw->phy.ops.reset(hw);
5062                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5063                         device_printf(dev,"Unsupported SFP+ module detected!");
5064                         printf(" Reload driver with supported module.\n");
5065                         adapter->sfp_probe = FALSE;
5066                         goto out;
5067                 } else
5068                         device_printf(dev,"SFP+ module detected!\n");
5069                 /* We now have supported optics */
5070                 adapter->sfp_probe = FALSE;
5071                 /* Set the optics type so system reports correctly */
5072                 ixgbe_setup_optics(adapter);
5073                 result = TRUE;
5074         }
5075 out:
5076         return (result);
5077 }
5078
5079 /*
5080 ** Tasklet handler for MSIX Link interrupts
5081 **  - do outside interrupt since it might sleep
5082 */
5083 static void
5084 ixgbe_handle_link(void *context, int pending)
5085 {
5086         struct adapter  *adapter = context;
5087
5088         ixgbe_check_link(&adapter->hw,
5089             &adapter->link_speed, &adapter->link_up, 0);
5090         ixgbe_update_link_status(adapter);
5091 }
5092
5093 /*
5094 ** Tasklet for handling SFP module interrupts
5095 */
5096 static void
5097 ixgbe_handle_mod(void *context, int pending)
5098 {
5099         struct adapter  *adapter = context;
5100         struct ixgbe_hw *hw = &adapter->hw;
5101         device_t        dev = adapter->dev;
5102         u32 err;
5103
5104         err = hw->phy.ops.identify_sfp(hw);
5105         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5106                 device_printf(dev,
5107                     "Unsupported SFP+ module type was detected.\n");
5108                 return;
5109         }
5110         err = hw->mac.ops.setup_sfp(hw);
5111         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5112                 device_printf(dev,
5113                     "Setup failure - unsupported SFP+ module type.\n");
5114                 return;
5115         }
5116         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5117         return;
5118 }
5119
5120
5121 /*
5122 ** Tasklet for handling MSF (multispeed fiber) interrupts
5123 */
5124 static void
5125 ixgbe_handle_msf(void *context, int pending)
5126 {
5127         struct adapter  *adapter = context;
5128         struct ixgbe_hw *hw = &adapter->hw;
5129         u32 autoneg;
5130         bool negotiate;
5131
5132         autoneg = hw->phy.autoneg_advertised;
5133         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5134                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5135         if (hw->mac.ops.setup_link)
5136                 hw->mac.ops.setup_link(hw, autoneg, TRUE);
5137         return;
5138 }
5139
5140 #ifdef IXGBE_FDIR
5141 /*
5142 ** Tasklet for reinitializing the Flow Director filter table
5143 */
5144 static void
5145 ixgbe_reinit_fdir(void *context, int pending)
5146 {
5147         struct adapter  *adapter = context;
5148         struct ifnet   *ifp = adapter->ifp;
5149
5150         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5151                 return;
5152         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5153         adapter->fdir_reinit = 0;
5154         /* re-enable flow director interrupts */
5155         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5156         /* Restart the interface */
5157         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5158         return;
5159 }
5160 #endif
5161
5162 /**********************************************************************
5163  *
5164  *  Update the board statistics counters.
5165  *
5166  **********************************************************************/
5167 static void
5168 ixgbe_update_stats_counters(struct adapter *adapter)
5169 {
5170         struct ifnet   *ifp = adapter->ifp;
5171         struct ixgbe_hw *hw = &adapter->hw;
5172         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5173         u64  total_missed_rx = 0;
5174
5175         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5176         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5177         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5178         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5179
5180         /*
5181         ** Note: these are for the 8 possible traffic classes,
5182         **       which in current implementation is unused,
5183         **       therefore only 0 should read real data.
5184         */
5185         for (int i = 0; i < 8; i++) {
5186                 u32 mp;
5187                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5188                 /* missed_rx tallies misses for the gprc workaround */
5189                 missed_rx += mp;
5190                 /* global total per queue */
5191                 adapter->stats.mpc[i] += mp;
5192                 /* Running comprehensive total for stats display */
5193                 total_missed_rx += adapter->stats.mpc[i];
5194                 if (hw->mac.type == ixgbe_mac_82598EB) {
5195                         adapter->stats.rnbc[i] +=
5196                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5197                         adapter->stats.qbtc[i] +=
5198                             IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5199                         adapter->stats.qbrc[i] +=
5200                             IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5201                         adapter->stats.pxonrxc[i] +=
5202                             IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5203                 } else
5204                         adapter->stats.pxonrxc[i] +=
5205                             IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5206                 adapter->stats.pxontxc[i] +=
5207                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5208                 adapter->stats.pxofftxc[i] +=
5209                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5210                 adapter->stats.pxoffrxc[i] +=
5211                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5212                 adapter->stats.pxon2offc[i] +=
5213                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5214         }
5215         for (int i = 0; i < 16; i++) {
5216                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5217                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5218                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5219         }
5220         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5221         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5222         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5223
5224         /* Hardware workaround, gprc counts missed packets */
5225         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5226         adapter->stats.gprc -= missed_rx;
5227
5228         if (hw->mac.type != ixgbe_mac_82598EB) {
5229                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5230                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5231                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5232                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5233                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5234                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5235                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5236                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5237         } else {
5238                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5239                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5240                 /* 82598 only has a counter in the high register */
5241                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5242                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5243                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5244         }
5245
5246         /*
5247          * Workaround: mprc hardware is incorrectly counting
5248          * broadcasts, so for now we subtract those.
5249          */
5250         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5251         adapter->stats.bprc += bprc;
5252         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5253         if (hw->mac.type == ixgbe_mac_82598EB)
5254                 adapter->stats.mprc -= bprc;
5255
5256         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5257         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5258         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5259         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5260         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5261         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5262
5263         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5264         adapter->stats.lxontxc += lxon;
5265         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5266         adapter->stats.lxofftxc += lxoff;
5267         total = lxon + lxoff;
5268
5269         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5270         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5271         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5272         adapter->stats.gptc -= total;
5273         adapter->stats.mptc -= total;
5274         adapter->stats.ptc64 -= total;
5275         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5276
5277         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5278         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5279         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5280         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5281         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5282         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5283         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5284         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5285         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5286         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5287         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5288         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5289         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5290         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5291         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5292         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5293         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5294         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5295         /* Only read FCOE on 82599 */
5296         if (hw->mac.type != ixgbe_mac_82598EB) {
5297                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5298                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5299                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5300                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5301                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5302         }
5303
5304         /* Fill out the OS statistics structure */
5305         ifp->if_ipackets = adapter->stats.gprc;
5306         ifp->if_opackets = adapter->stats.gptc;
5307         ifp->if_ibytes = adapter->stats.gorc;
5308         ifp->if_obytes = adapter->stats.gotc;
5309         ifp->if_imcasts = adapter->stats.mprc;
5310         ifp->if_omcasts = adapter->stats.mptc;
5311         ifp->if_collisions = 0;
5312
5313         /* Rx Errors */
5314         ifp->if_iqdrops = total_missed_rx;
5315         ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec;
5316 }
5317
5318 /** ixgbe_sysctl_tdh_handler - Handler function
5319  *  Retrieves the TDH value from the hardware
5320  */
5321 static int
5322 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5323 {
5324         int error;
5325
5326         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5327         if (!txr) return 0;
5328
5329         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5330         error = sysctl_handle_int(oidp, &val, 0, req);
5331         if (error || !req->newptr)
5332                 return error;
5333         return 0;
5334 }
5335
5336 /** ixgbe_sysctl_tdt_handler - Handler function
5337  *  Retrieves the TDT value from the hardware
5338  */
5339 static int
5340 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5341 {
5342         int error;
5343
5344         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5345         if (!txr) return 0;
5346
5347         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5348         error = sysctl_handle_int(oidp, &val, 0, req);
5349         if (error || !req->newptr)
5350                 return error;
5351         return 0;
5352 }
5353
5354 /** ixgbe_sysctl_rdh_handler - Handler function
5355  *  Retrieves the RDH value from the hardware
5356  */
5357 static int
5358 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5359 {
5360         int error;
5361
5362         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5363         if (!rxr) return 0;
5364
5365         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5366         error = sysctl_handle_int(oidp, &val, 0, req);
5367         if (error || !req->newptr)
5368                 return error;
5369         return 0;
5370 }
5371
5372 /** ixgbe_sysctl_rdt_handler - Handler function
5373  *  Retrieves the RDT value from the hardware
5374  */
5375 static int
5376 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5377 {
5378         int error;
5379
5380         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5381         if (!rxr) return 0;
5382
5383         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5384         error = sysctl_handle_int(oidp, &val, 0, req);
5385         if (error || !req->newptr)
5386                 return error;
5387         return 0;
5388 }
5389
5390 static int
5391 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5392 {
5393         int error;
5394         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5395         unsigned int reg, usec, rate;
5396
5397         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5398         usec = ((reg & 0x0FF8) >> 3);
5399         if (usec > 0)
5400                 rate = 500000 / usec;
5401         else
5402                 rate = 0;
5403         error = sysctl_handle_int(oidp, &rate, 0, req);
5404         if (error || !req->newptr)
5405                 return error;
5406         reg &= ~0xfff; /* default, no limitation */
5407         ixgbe_max_interrupt_rate = 0;
5408         if (rate > 0 && rate < 500000) {
5409                 if (rate < 1000)
5410                         rate = 1000;
5411                 ixgbe_max_interrupt_rate = rate;
5412                 reg |= ((4000000/rate) & 0xff8 );
5413         }
5414         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5415         return 0;
5416 }
5417
5418 /*
5419  * Add sysctl variables, one per statistic, to the system.
5420  */
5421 static void
5422 ixgbe_add_hw_stats(struct adapter *adapter)
5423 {
5424
5425         device_t dev = adapter->dev;
5426
5427         struct tx_ring *txr = adapter->tx_rings;
5428         struct rx_ring *rxr = adapter->rx_rings;
5429
5430         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5431         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5432         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5433         struct ixgbe_hw_stats *stats = &adapter->stats;
5434
5435         struct sysctl_oid *stat_node, *queue_node;
5436         struct sysctl_oid_list *stat_list, *queue_list;
5437
5438 #define QUEUE_NAME_LEN 32
5439         char namebuf[QUEUE_NAME_LEN];
5440
5441         /* Driver Statistics */
5442         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5443                         CTLFLAG_RD, &adapter->dropped_pkts,
5444                         "Driver dropped packets");
5445         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5446                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5447                         "m_defrag() failed");
5448         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5449                         CTLFLAG_RD, &adapter->watchdog_events,
5450                         "Watchdog timeouts");
5451         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5452                         CTLFLAG_RD, &adapter->link_irq,
5453                         "Link MSIX IRQ Handled");
5454
5455         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5456                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5457                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5458                                             CTLFLAG_RD, NULL, "Queue Name");
5459                 queue_list = SYSCTL_CHILDREN(queue_node);
5460
5461                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5462                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5463                                 sizeof(&adapter->queues[i]),
5464                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5465                                 "Interrupt Rate");
5466                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5467                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
5468                                 "irqs on this queue");
5469                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5470                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5471                                 ixgbe_sysctl_tdh_handler, "IU",
5472                                 "Transmit Descriptor Head");
5473                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5474                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5475                                 ixgbe_sysctl_tdt_handler, "IU",
5476                                 "Transmit Descriptor Tail");
5477                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5478                                 CTLFLAG_RD, &txr->tso_tx,
5479                                 "TSO");
5480                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5481                                 CTLFLAG_RD, &txr->no_tx_dma_setup,
5482                                 "Driver tx dma failure in xmit");
5483                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5484                                 CTLFLAG_RD, &txr->no_desc_avail,
5485                                 "Queue No Descriptor Available");
5486                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5487                                 CTLFLAG_RD, &txr->total_packets,
5488                                 "Queue Packets Transmitted");
5489         }
5490
5491         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5492                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5493                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5494                                             CTLFLAG_RD, NULL, "Queue Name");
5495                 queue_list = SYSCTL_CHILDREN(queue_node);
5496
5497                 struct lro_ctrl *lro = &rxr->lro;
5498
5499                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5500                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5501                                             CTLFLAG_RD, NULL, "Queue Name");
5502                 queue_list = SYSCTL_CHILDREN(queue_node);
5503
5504                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5505                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5506                                 ixgbe_sysctl_rdh_handler, "IU",
5507                                 "Receive Descriptor Head");
5508                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5509                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5510                                 ixgbe_sysctl_rdt_handler, "IU",
5511                                 "Receive Descriptor Tail");
5512                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5513                                 CTLFLAG_RD, &rxr->rx_packets,
5514                                 "Queue Packets Received");
5515                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5516                                 CTLFLAG_RD, &rxr->rx_bytes,
5517                                 "Queue Bytes Received");
5518                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5519                                 CTLFLAG_RD, &rxr->rx_copies,
5520                                 "Copied RX Frames");
5521                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5522                                 CTLFLAG_RD, &lro->lro_queued, 0,
5523                                 "LRO Queued");
5524                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5525                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5526                                 "LRO Flushed");
5527         }
5528
5529         /* MAC stats get the own sub node */
5530
5531         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5532                                     CTLFLAG_RD, NULL, "MAC Statistics");
5533         stat_list = SYSCTL_CHILDREN(stat_node);
5534
5535         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5536                         CTLFLAG_RD, &stats->crcerrs,
5537                         "CRC Errors");
5538         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5539                         CTLFLAG_RD, &stats->illerrc,
5540                         "Illegal Byte Errors");
5541         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5542                         CTLFLAG_RD, &stats->errbc,
5543                         "Byte Errors");
5544         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5545                         CTLFLAG_RD, &stats->mspdc,
5546                         "MAC Short Packets Discarded");
5547         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5548                         CTLFLAG_RD, &stats->mlfc,
5549                         "MAC Local Faults");
5550         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5551                         CTLFLAG_RD, &stats->mrfc,
5552                         "MAC Remote Faults");
5553         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5554                         CTLFLAG_RD, &stats->rlec,
5555                         "Receive Length Errors");
5556
5557         /* Flow Control stats */
5558         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5559                         CTLFLAG_RD, &stats->lxontxc,
5560                         "Link XON Transmitted");
5561         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5562                         CTLFLAG_RD, &stats->lxonrxc,
5563                         "Link XON Received");
5564         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5565                         CTLFLAG_RD, &stats->lxofftxc,
5566                         "Link XOFF Transmitted");
5567         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5568                         CTLFLAG_RD, &stats->lxoffrxc,
5569                         "Link XOFF Received");
5570
5571         /* Packet Reception Stats */
5572         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5573                         CTLFLAG_RD, &stats->tor,
5574                         "Total Octets Received");
5575         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5576                         CTLFLAG_RD, &stats->gorc,
5577                         "Good Octets Received");
5578         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5579                         CTLFLAG_RD, &stats->tpr,
5580                         "Total Packets Received");
5581         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5582                         CTLFLAG_RD, &stats->gprc,
5583                         "Good Packets Received");
5584         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5585                         CTLFLAG_RD, &stats->mprc,
5586                         "Multicast Packets Received");
5587         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5588                         CTLFLAG_RD, &stats->bprc,
5589                         "Broadcast Packets Received");
5590         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5591                         CTLFLAG_RD, &stats->prc64,
5592                         "64 byte frames received ");
5593         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5594                         CTLFLAG_RD, &stats->prc127,
5595                         "65-127 byte frames received");
5596         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5597                         CTLFLAG_RD, &stats->prc255,
5598                         "128-255 byte frames received");
5599         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5600                         CTLFLAG_RD, &stats->prc511,
5601                         "256-511 byte frames received");
5602         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5603                         CTLFLAG_RD, &stats->prc1023,
5604                         "512-1023 byte frames received");
5605         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5606                         CTLFLAG_RD, &stats->prc1522,
5607                         "1023-1522 byte frames received");
5608         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5609                         CTLFLAG_RD, &stats->ruc,
5610                         "Receive Undersized");
5611         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5612                         CTLFLAG_RD, &stats->rfc,
5613                         "Fragmented Packets Received ");
5614         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5615                         CTLFLAG_RD, &stats->roc,
5616                         "Oversized Packets Received");
5617         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5618                         CTLFLAG_RD, &stats->rjc,
5619                         "Received Jabber");
5620         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5621                         CTLFLAG_RD, &stats->mngprc,
5622                         "Management Packets Received");
5623         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5624                         CTLFLAG_RD, &stats->mngptc,
5625                         "Management Packets Dropped");
5626         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5627                         CTLFLAG_RD, &stats->xec,
5628                         "Checksum Errors");
5629
5630         /* Packet Transmission Stats */
5631         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5632                         CTLFLAG_RD, &stats->gotc,
5633                         "Good Octets Transmitted");
5634         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5635                         CTLFLAG_RD, &stats->tpt,
5636                         "Total Packets Transmitted");
5637         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5638                         CTLFLAG_RD, &stats->gptc,
5639                         "Good Packets Transmitted");
5640         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5641                         CTLFLAG_RD, &stats->bptc,
5642                         "Broadcast Packets Transmitted");
5643         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5644                         CTLFLAG_RD, &stats->mptc,
5645                         "Multicast Packets Transmitted");
5646         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5647                         CTLFLAG_RD, &stats->mngptc,
5648                         "Management Packets Transmitted");
5649         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5650                         CTLFLAG_RD, &stats->ptc64,
5651                         "64 byte frames transmitted ");
5652         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5653                         CTLFLAG_RD, &stats->ptc127,
5654                         "65-127 byte frames transmitted");
5655         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5656                         CTLFLAG_RD, &stats->ptc255,
5657                         "128-255 byte frames transmitted");
5658         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5659                         CTLFLAG_RD, &stats->ptc511,
5660                         "256-511 byte frames transmitted");
5661         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5662                         CTLFLAG_RD, &stats->ptc1023,
5663                         "512-1023 byte frames transmitted");
5664         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5665                         CTLFLAG_RD, &stats->ptc1522,
5666                         "1024-1522 byte frames transmitted");
5667 }
5668
5669 /*
5670 ** Set flow control using sysctl:
5671 ** Flow control values:
5672 **      0 - off
5673 **      1 - rx pause
5674 **      2 - tx pause
5675 **      3 - full
5676 */
5677 static int
5678 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5679 {
5680         int error, last;
5681         struct adapter *adapter = (struct adapter *) arg1;
5682
5683         last = adapter->fc;
5684         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5685         if ((error) || (req->newptr == NULL))
5686                 return (error);
5687
5688         /* Don't bother if it's not changed */
5689         if (adapter->fc == last)
5690                 return (0);
5691
5692         switch (adapter->fc) {
5693                 case ixgbe_fc_rx_pause:
5694                 case ixgbe_fc_tx_pause:
5695                 case ixgbe_fc_full:
5696                         adapter->hw.fc.requested_mode = adapter->fc;
5697                         if (adapter->num_queues > 1)
5698                                 ixgbe_disable_rx_drop(adapter);
5699                         break;
5700                 case ixgbe_fc_none:
5701                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5702                         if (adapter->num_queues > 1)
5703                                 ixgbe_enable_rx_drop(adapter);
5704                         break;
5705                 default:
5706                         adapter->fc = last;
5707                         return (EINVAL);
5708         }
5709         /* Don't autoneg if forcing a value */
5710         adapter->hw.fc.disable_fc_autoneg = TRUE;
5711         ixgbe_fc_enable(&adapter->hw);
5712         return error;
5713 }
5714
5715 /*
5716 ** Control link advertise speed:
5717 **      1 - advertise only 1G
5718 **      2 - advertise 100Mb
5719 **      3 - advertise normal
5720 */
5721 static int
5722 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5723 {
5724         int                     error = 0;
5725         struct adapter          *adapter;
5726         device_t                dev;
5727         struct ixgbe_hw         *hw;
5728         ixgbe_link_speed        speed, last;
5729
5730         adapter = (struct adapter *) arg1;
5731         dev = adapter->dev;
5732         hw = &adapter->hw;
5733         last = adapter->advertise;
5734
5735         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5736         if ((error) || (req->newptr == NULL))
5737                 return (error);
5738
5739         if (adapter->advertise == last) /* no change */
5740                 return (0);
5741
5742         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5743             (hw->phy.multispeed_fiber)))
5744                 return (EINVAL);
5745
5746         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5747                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5748                 return (EINVAL);
5749         }
5750
5751         if (adapter->advertise == 1)
5752                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5753         else if (adapter->advertise == 2)
5754                 speed = IXGBE_LINK_SPEED_100_FULL;
5755         else if (adapter->advertise == 3)
5756                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5757                         IXGBE_LINK_SPEED_10GB_FULL;
5758         else {  /* bogus value */
5759                 adapter->advertise = last;
5760                 return (EINVAL);
5761         }
5762
5763         hw->mac.autotry_restart = TRUE;
5764         hw->mac.ops.setup_link(hw, speed, TRUE);
5765
5766         return (error);
5767 }
5768
5769 /*
5770 ** Thermal Shutdown Trigger
5771 **   - cause a Thermal Overtemp IRQ
5772 **   - this now requires firmware enabling
5773 */
5774 static int
5775 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5776 {
5777         int             error, fire = 0;
5778         struct adapter  *adapter = (struct adapter *) arg1;
5779         struct ixgbe_hw *hw = &adapter->hw;
5780
5781
5782         if (hw->mac.type != ixgbe_mac_X540)
5783                 return (0);
5784
5785         error = sysctl_handle_int(oidp, &fire, 0, req);
5786         if ((error) || (req->newptr == NULL))
5787                 return (error);
5788
5789         if (fire) {
5790                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5791                 reg |= IXGBE_EICR_TS;
5792                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5793         }
5794
5795         return (0);
5796 }
5797
5798 /*
5799 ** Enable the hardware to drop packets when the buffer is
5800 ** full. This is useful when multiqueue,so that no single
5801 ** queue being full stalls the entire RX engine. We only
5802 ** enable this when Multiqueue AND when Flow Control is
5803 ** disabled.
5804 */
5805 static void
5806 ixgbe_enable_rx_drop(struct adapter *adapter)
5807 {
5808         struct ixgbe_hw *hw = &adapter->hw;
5809
5810         for (int i = 0; i < adapter->num_queues; i++) {
5811                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5812                 srrctl |= IXGBE_SRRCTL_DROP_EN;
5813                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5814         }
5815 }
5816
5817 static void
5818 ixgbe_disable_rx_drop(struct adapter *adapter)
5819 {
5820         struct ixgbe_hw *hw = &adapter->hw;
5821
5822         for (int i = 0; i < adapter->num_queues; i++) {
5823                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5824                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5825                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5826         }
5827 }