sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2013, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.5.13";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 #ifdef IXGBE_LEGACY_TX
 106 static void     ixgbe_start(struct ifnet *);
 107 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 108 #else /* ! IXGBE_LEGACY_TX */
 109 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 110 static int      ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 static void     ixgbe_deferred_mq_start(void *, int);
 113 #endif /* IXGBE_LEGACY_TX */
 114 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 115 static void     ixgbe_init(void *);
 116 static void     ixgbe_init_locked(struct adapter *);
 117 static void     ixgbe_stop(void *);
 118 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 119 static int      ixgbe_media_change(struct ifnet *);
 120 static void     ixgbe_identify_hardware(struct adapter *);
 121 static int      ixgbe_allocate_pci_resources(struct adapter *);
 122 static void     ixgbe_get_slot_info(struct ixgbe_hw *);
 123 static int      ixgbe_allocate_msix(struct adapter *);
 124 static int      ixgbe_allocate_legacy(struct adapter *);
 125 static int      ixgbe_allocate_queues(struct adapter *);
 126 static int      ixgbe_setup_msix(struct adapter *);
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static void     ixgbe_txeof(struct tx_ring *);
 151 static bool     ixgbe_rxeof(struct ix_queue *);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static int      ixgbe_tx_ctx_setup(struct tx_ring *,
 165                     struct mbuf *, u32 *, u32 *);
 166 static int      ixgbe_tso_setup(struct tx_ring *,
 167                     struct mbuf *, u32 *, u32 *);
 168 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 169 static void     ixgbe_configure_ivars(struct adapter *);
 170 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 171
 172 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 173 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 174 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 175
 176 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 177
 178 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 179 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 180                     struct mbuf *, u32);
 181
 182 static void     ixgbe_enable_rx_drop(struct adapter *);
 183 static void     ixgbe_disable_rx_drop(struct adapter *);
 184
 185 /* Support for pluggable optic modules */
 186 static bool     ixgbe_sfp_probe(struct adapter *);
 187 static void     ixgbe_setup_optics(struct adapter *);
 188
 189 /* Legacy (single vector interrupt handler */
 190 static void     ixgbe_legacy_irq(void *);
 191
 192 /* The MSI/X Interrupt handlers */
 193 static void     ixgbe_msix_que(void *);
 194 static void     ixgbe_msix_link(void *);
 195
 196 /* Deferred interrupt tasklets */
 197 static void     ixgbe_handle_que(void *, int);
 198 static void     ixgbe_handle_link(void *, int);
 199 static void     ixgbe_handle_msf(void *, int);
 200 static void     ixgbe_handle_mod(void *, int);
 201
 202 #ifdef IXGBE_FDIR
 203 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 204 static void     ixgbe_reinit_fdir(void *, int);
 205 #endif
 206
 207 /* Missing shared code prototype */
 208 extern void ixgbe_stop_mac_link_on_d3_82599(struct ixgbe_hw *hw);
 209
 210 /*********************************************************************
 211  *  FreeBSD Device Interface Entry Points
 212  *********************************************************************/
 213
 214 static device_method_t ixgbe_methods[] = {
 215         /* Device interface */
 216         DEVMETHOD(device_probe, ixgbe_probe),
 217         DEVMETHOD(device_attach, ixgbe_attach),
 218         DEVMETHOD(device_detach, ixgbe_detach),
 219         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 220         DEVMETHOD_END
 221 };
 222
 223 static driver_t ixgbe_driver = {
 224         "ix", ixgbe_methods, sizeof(struct adapter),
 225 };
 226
 227 devclass_t ixgbe_devclass;
 228 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 229
 230 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 231 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 232
 233 /*
 234 ** TUNEABLE PARAMETERS:
 235 */
 236
 237 /*
 238 ** AIM: Adaptive Interrupt Moderation
 239 ** which means that the interrupt rate
 240 ** is varied over time based on the
 241 ** traffic for that interrupt vector
 242 */
 243 static int ixgbe_enable_aim = TRUE;
 244 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 245
 246 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 247 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 248
 249 /* How many packets rxeof tries to clean at a time */
 250 static int ixgbe_rx_process_limit = 256;
 251 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 252
 253 /* How many packets txeof tries to clean at a time */
 254 static int ixgbe_tx_process_limit = 256;
 255 TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit);
 256
 257 /*
 258 ** Smart speed setting, default to on
 259 ** this only works as a compile option
 260 ** right now as its during attach, set
 261 ** this to 'ixgbe_smart_speed_off' to
 262 ** disable.
 263 */
 264 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 265
 266 /*
 267  * MSIX should be the default for best performance,
 268  * but this allows it to be forced off for testing.
 269  */
 270 static int ixgbe_enable_msix = 1;
 271 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 272
 273 /*
 274  * Number of Queues, can be set to 0,
 275  * it then autoconfigures based on the
 276  * number of cpus with a max of 8. This
 277  * can be overriden manually here.
 278  */
 279 static int ixgbe_num_queues = 0;
 280 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 281
 282 /*
 283 ** Number of TX descriptors per ring,
 284 ** setting higher than RX as this seems
 285 ** the better performing choice.
 286 */
 287 static int ixgbe_txd = PERFORM_TXD;
 288 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 289
 290 /* Number of RX descriptors per ring */
 291 static int ixgbe_rxd = PERFORM_RXD;
 292 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 293
 294 /*
 295 ** Defining this on will allow the use
 296 ** of unsupported SFP+ modules, note that
 297 ** doing so you are on your own :)
 298 */
 299 static int allow_unsupported_sfp = FALSE;
 300
 301 /*
 302 ** HW RSC control:
 303 **  this feature only works with
 304 **  IPv4, and only on 82599 and later.
 305 **  Also this will cause IP forwarding to
 306 **  fail and that can't be controlled by
 307 **  the stack as LRO can. For all these
 308 **  reasons I've deemed it best to leave
 309 **  this off and not bother with a tuneable
 310 **  interface, this would need to be compiled
 311 **  to enable.
 312 */
 313 static bool ixgbe_rsc_enable = FALSE;
 314
 315 /* Keep running tab on them for sanity check */
 316 static int ixgbe_total_ports;
 317
 318 #ifdef IXGBE_FDIR
 319 /*
 320 ** For Flow Director: this is the
 321 ** number of TX packets we sample
 322 ** for the filter pool, this means
 323 ** every 20th packet will be probed.
 324 **
 325 ** This feature can be disabled by
 326 ** setting this to 0.
 327 */
 328 static int atr_sample_rate = 20;
 329 /*
 330 ** Flow Director actually 'steals'
 331 ** part of the packet buffer as its
 332 ** filter pool, this variable controls
 333 ** how much it uses:
 334 **  0 = 64K, 1 = 128K, 2 = 256K
 335 */
 336 static int fdir_pballoc = 1;
 337 #endif
 338
 339 #ifdef DEV_NETMAP
 340 /*
 341  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 342  * be a reference on how to implement netmap support in a driver.
 343  * Additional comments are in ixgbe_netmap.h .
 344  *
 345  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 346  * that extend the standard driver.
 347  */
 348 #include <dev/netmap/ixgbe_netmap.h>
 349 #endif /* DEV_NETMAP */
 350
 351 /*********************************************************************
 352  *  Device identification routine
 353  *
 354  *  ixgbe_probe determines if the driver should be loaded on
 355  *  adapter based on PCI vendor/device id of the adapter.
 356  *
 357  *  return BUS_PROBE_DEFAULT on success, positive on failure
 358  *********************************************************************/
 359
 360 static int
 361 ixgbe_probe(device_t dev)
 362 {
 363         ixgbe_vendor_info_t *ent;
 364
 365         u16     pci_vendor_id = 0;
 366         u16     pci_device_id = 0;
 367         u16     pci_subvendor_id = 0;
 368         u16     pci_subdevice_id = 0;
 369         char    adapter_name[256];
 370
 371         INIT_DEBUGOUT("ixgbe_probe: begin");
 372
 373         pci_vendor_id = pci_get_vendor(dev);
 374         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 375                 return (ENXIO);
 376
 377         pci_device_id = pci_get_device(dev);
 378         pci_subvendor_id = pci_get_subvendor(dev);
 379         pci_subdevice_id = pci_get_subdevice(dev);
 380
 381         ent = ixgbe_vendor_info_array;
 382         while (ent->vendor_id != 0) {
 383                 if ((pci_vendor_id == ent->vendor_id) &&
 384                     (pci_device_id == ent->device_id) &&
 385
 386                     ((pci_subvendor_id == ent->subvendor_id) ||
 387                      (ent->subvendor_id == 0)) &&
 388
 389                     ((pci_subdevice_id == ent->subdevice_id) ||
 390                      (ent->subdevice_id == 0))) {
 391                         sprintf(adapter_name, "%s, Version - %s",
 392                                 ixgbe_strings[ent->index],
 393                                 ixgbe_driver_version);
 394                         device_set_desc_copy(dev, adapter_name);
 395                         ++ixgbe_total_ports;
 396                         return (BUS_PROBE_DEFAULT);
 397                 }
 398                 ent++;
 399         }
 400         return (ENXIO);
 401 }
 402
 403 /*********************************************************************
 404  *  Device initialization routine
 405  *
 406  *  The attach entry point is called when the driver is being loaded.
 407  *  This routine identifies the type of hardware, allocates all resources
 408  *  and initializes the hardware.
 409  *
 410  *  return 0 on success, positive on failure
 411  *********************************************************************/
 412
 413 static int
 414 ixgbe_attach(device_t dev)
 415 {
 416         struct adapter *adapter;
 417         struct ixgbe_hw *hw;
 418         int             error = 0;
 419         u16             csum;
 420         u32             ctrl_ext;
 421
 422         INIT_DEBUGOUT("ixgbe_attach: begin");
 423
 424         /* Allocate, clear, and link in our adapter structure */
 425         adapter = device_get_softc(dev);
 426         adapter->dev = adapter->osdep.dev = dev;
 427         hw = &adapter->hw;
 428
 429         /* Core Lock Init*/
 430         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 431
 432         /* SYSCTL APIs */
 433
 434         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 435                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 436                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 437                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 438
 439         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 440                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 441                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 442                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 443
 444         /*
 445         ** Allow a kind of speed control by forcing the autoneg
 446         ** advertised speed list to only a certain value, this
 447         ** supports 1G on 82599 devices, and 100Mb on x540.
 448         */
 449         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 450                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 451                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 452                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 453
 454         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 455                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 456                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 457                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 458
 459         /* Set up the timer callout */
 460         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 461
 462         /* Determine hardware revision */
 463         ixgbe_identify_hardware(adapter);
 464
 465         /* Do base PCI setup - map BAR0 */
 466         if (ixgbe_allocate_pci_resources(adapter)) {
 467                 device_printf(dev, "Allocation of PCI resources failed\n");
 468                 error = ENXIO;
 469                 goto err_out;
 470         }
 471
 472         /* Do descriptor calc and sanity checks */
 473         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 474             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 475                 device_printf(dev, "TXD config issue, using default!\n");
 476                 adapter->num_tx_desc = DEFAULT_TXD;
 477         } else
 478                 adapter->num_tx_desc = ixgbe_txd;
 479
 480         /*
 481         ** With many RX rings it is easy to exceed the
 482         ** system mbuf allocation. Tuning nmbclusters
 483         ** can alleviate this.
 484         */
 485         if (nmbclusters > 0 ) {
 486                 int s;
 487                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 488                 if (s > nmbclusters) {
 489                         device_printf(dev, "RX Descriptors exceed "
 490                             "system mbuf max, using default instead!\n");
 491                         ixgbe_rxd = DEFAULT_RXD;
 492                 }
 493         }
 494
 495         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 496             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 497                 device_printf(dev, "RXD config issue, using default!\n");
 498                 adapter->num_rx_desc = DEFAULT_RXD;
 499         } else
 500                 adapter->num_rx_desc = ixgbe_rxd;
 501
 502         /* Allocate our TX/RX Queues */
 503         if (ixgbe_allocate_queues(adapter)) {
 504                 error = ENOMEM;
 505                 goto err_out;
 506         }
 507
 508         /* Allocate multicast array memory. */
 509         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 510             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 511         if (adapter->mta == NULL) {
 512                 device_printf(dev, "Can not allocate multicast setup array\n");
 513                 error = ENOMEM;
 514                 goto err_late;
 515         }
 516
 517         /* Initialize the shared code */
 518         hw->allow_unsupported_sfp = allow_unsupported_sfp;
 519         error = ixgbe_init_shared_code(hw);
 520         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 521                 /*
 522                 ** No optics in this port, set up
 523                 ** so the timer routine will probe
 524                 ** for later insertion.
 525                 */
 526                 adapter->sfp_probe = TRUE;
 527                 error = 0;
 528         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 529                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 530                 error = EIO;
 531                 goto err_late;
 532         } else if (error) {
 533                 device_printf(dev,"Unable to initialize the shared code\n");
 534                 error = EIO;
 535                 goto err_late;
 536         }
 537
 538         /* Make sure we have a good EEPROM before we read from it */
 539         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 540                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 541                 error = EIO;
 542                 goto err_late;
 543         }
 544
 545         error = ixgbe_init_hw(hw);
 546         switch (error) {
 547         case IXGBE_ERR_EEPROM_VERSION:
 548                 device_printf(dev, "This device is a pre-production adapter/"
 549                     "LOM.  Please be aware there may be issues associated "
 550                     "with your hardware.\n If you are experiencing problems "
 551                     "please contact your Intel or hardware representative "
 552                     "who provided you with this hardware.\n");
 553                 break;
 554         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 555                 device_printf(dev,"Unsupported SFP+ Module\n");
 556                 error = EIO;
 557                 goto err_late;
 558         case IXGBE_ERR_SFP_NOT_PRESENT:
 559                 device_printf(dev,"No SFP+ Module found\n");
 560                 /* falls thru */
 561         default:
 562                 break;
 563         }
 564
 565         /* Detect and set physical type */
 566         ixgbe_setup_optics(adapter);
 567
 568         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 569                 error = ixgbe_allocate_msix(adapter);
 570         else
 571                 error = ixgbe_allocate_legacy(adapter);
 572         if (error)
 573                 goto err_late;
 574
 575         /* Setup OS specific network interface */
 576         if (ixgbe_setup_interface(dev, adapter) != 0)
 577                 goto err_late;
 578
 579         /* Initialize statistics */
 580         ixgbe_update_stats_counters(adapter);
 581
 582         /* Register for VLAN events */
 583         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 584             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 585         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 586             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 587
 588         /*
 589         ** Check PCIE slot type/speed/width
 590         */
 591         ixgbe_get_slot_info(hw);
 592
 593         /* Set an initial default flow control value */
 594         adapter->fc =  ixgbe_fc_full;
 595
 596         /* let hardware know driver is loaded */
 597         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 598         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 599         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 600
 601         ixgbe_add_hw_stats(adapter);
 602
 603 #ifdef DEV_NETMAP
 604         ixgbe_netmap_attach(adapter);
 605 #endif /* DEV_NETMAP */
 606         INIT_DEBUGOUT("ixgbe_attach: end");
 607         return (0);
 608 err_late:
 609         ixgbe_free_transmit_structures(adapter);
 610         ixgbe_free_receive_structures(adapter);
 611 err_out:
 612         if (adapter->ifp != NULL)
 613                 if_free(adapter->ifp);
 614         ixgbe_free_pci_resources(adapter);
 615         free(adapter->mta, M_DEVBUF);
 616         return (error);
 617
 618 }
 619
 620 /*********************************************************************
 621  *  Device removal routine
 622  *
 623  *  The detach entry point is called when the driver is being removed.
 624  *  This routine stops the adapter and deallocates all the resources
 625  *  that were allocated for driver operation.
 626  *
 627  *  return 0 on success, positive on failure
 628  *********************************************************************/
 629
 630 static int
 631 ixgbe_detach(device_t dev)
 632 {
 633         struct adapter *adapter = device_get_softc(dev);
 634         struct ix_queue *que = adapter->queues;
 635         struct tx_ring *txr = adapter->tx_rings;
 636         u32     ctrl_ext;
 637
 638         INIT_DEBUGOUT("ixgbe_detach: begin");
 639
 640         /* Make sure VLANS are not using driver */
 641         if (adapter->ifp->if_vlantrunk != NULL) {
 642                 device_printf(dev,"Vlan in use, detach first\n");
 643                 return (EBUSY);
 644         }
 645
 646         IXGBE_CORE_LOCK(adapter);
 647         ixgbe_stop(adapter);
 648         IXGBE_CORE_UNLOCK(adapter);
 649
 650         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 651                 if (que->tq) {
 652 #ifndef IXGBE_LEGACY_TX
 653                         taskqueue_drain(que->tq, &txr->txq_task);
 654 #endif
 655                         taskqueue_drain(que->tq, &que->que_task);
 656                         taskqueue_free(que->tq);
 657                 }
 658         }
 659
 660         /* Drain the Link queue */
 661         if (adapter->tq) {
 662                 taskqueue_drain(adapter->tq, &adapter->link_task);
 663                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 664                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 665 #ifdef IXGBE_FDIR
 666                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 667 #endif
 668                 taskqueue_free(adapter->tq);
 669         }
 670
 671         /* let hardware know driver is unloading */
 672         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 673         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 674         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 675
 676         /* Unregister VLAN events */
 677         if (adapter->vlan_attach != NULL)
 678                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 679         if (adapter->vlan_detach != NULL)
 680                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 681
 682         ether_ifdetach(adapter->ifp);
 683         callout_drain(&adapter->timer);
 684 #ifdef DEV_NETMAP
 685         netmap_detach(adapter->ifp);
 686 #endif /* DEV_NETMAP */
 687         ixgbe_free_pci_resources(adapter);
 688         bus_generic_detach(dev);
 689         if_free(adapter->ifp);
 690
 691         ixgbe_free_transmit_structures(adapter);
 692         ixgbe_free_receive_structures(adapter);
 693         free(adapter->mta, M_DEVBUF);
 694
 695         IXGBE_CORE_LOCK_DESTROY(adapter);
 696         return (0);
 697 }
 698
 699 /*********************************************************************
 700  *
 701  *  Shutdown entry point
 702  *
 703  **********************************************************************/
 704
 705 static int
 706 ixgbe_shutdown(device_t dev)
 707 {
 708         struct adapter *adapter = device_get_softc(dev);
 709         IXGBE_CORE_LOCK(adapter);
 710         ixgbe_stop(adapter);
 711         IXGBE_CORE_UNLOCK(adapter);
 712         return (0);
 713 }
 714
 715
 716 #ifdef IXGBE_LEGACY_TX
 717 /*********************************************************************
 718  *  Transmit entry point
 719  *
 720  *  ixgbe_start is called by the stack to initiate a transmit.
 721  *  The driver will remain in this routine as long as there are
 722  *  packets to transmit and transmit resources are available.
 723  *  In case resources are not available stack is notified and
 724  *  the packet is requeued.
 725  **********************************************************************/
 726
 727 static void
 728 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 729 {
 730         struct mbuf    *m_head;
 731         struct adapter *adapter = txr->adapter;
 732
 733         IXGBE_TX_LOCK_ASSERT(txr);
 734
 735         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 736                 return;
 737         if (!adapter->link_active)
 738                 return;
 739
 740         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 741                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 742                         break;
 743
 744                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 745                 if (m_head == NULL)
 746                         break;
 747
 748                 if (ixgbe_xmit(txr, &m_head)) {
 749                         if (m_head != NULL)
 750                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 751                         break;
 752                 }
 753                 /* Send a copy of the frame to the BPF listener */
 754                 ETHER_BPF_MTAP(ifp, m_head);
 755
 756                 /* Set watchdog on */
 757                 txr->watchdog_time = ticks;
 758                 txr->queue_status = IXGBE_QUEUE_WORKING;
 759
 760         }
 761         return;
 762 }
 763
 764 /*
 765  * Legacy TX start - called by the stack, this
 766  * always uses the first tx ring, and should
 767  * not be used with multiqueue tx enabled.
 768  */
 769 static void
 770 ixgbe_start(struct ifnet *ifp)
 771 {
 772         struct adapter *adapter = ifp->if_softc;
 773         struct tx_ring  *txr = adapter->tx_rings;
 774
 775         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 776                 IXGBE_TX_LOCK(txr);
 777                 ixgbe_start_locked(txr, ifp);
 778                 IXGBE_TX_UNLOCK(txr);
 779         }
 780         return;
 781 }
 782
 783 #else /* ! IXGBE_LEGACY_TX */
 784
 785 /*
 786 ** Multiqueue Transmit driver
 787 **
 788 */
 789 static int
 790 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 791 {
 792         struct adapter  *adapter = ifp->if_softc;
 793         struct ix_queue *que;
 794         struct tx_ring  *txr;
 795         int             i, err = 0;
 796
 797         /* Which queue to use */
 798         if ((m->m_flags & M_FLOWID) != 0)
 799                 i = m->m_pkthdr.flowid % adapter->num_queues;
 800         else
 801                 i = curcpu % adapter->num_queues;
 802
 803         txr = &adapter->tx_rings[i];
 804         que = &adapter->queues[i];
 805
 806         err = drbr_enqueue(ifp, txr->br, m);
 807         if (err)
 808                 return (err);
 809         if (IXGBE_TX_TRYLOCK(txr)) {
 810                 err = ixgbe_mq_start_locked(ifp, txr);
 811                 IXGBE_TX_UNLOCK(txr);
 812         } else
 813                 taskqueue_enqueue(que->tq, &txr->txq_task);
 814
 815         return (err);
 816 }
 817
 818 static int
 819 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 820 {
 821         struct adapter  *adapter = txr->adapter;
 822         struct mbuf     *next;
 823         int             enqueued = 0, err = 0;
 824
 825         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 826             adapter->link_active == 0)
 827                 return (ENETDOWN);
 828
 829         /* Process the queue */
 830 #if __FreeBSD_version < 901504
 831         next = drbr_dequeue(ifp, txr->br);
 832         while (next != NULL) {
 833                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 834                         if (next != NULL)
 835                                 err = drbr_enqueue(ifp, txr->br, next);
 836 #else
 837         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 838                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 839                         if (next == NULL) {
 840                                 drbr_advance(ifp, txr->br);
 841                         } else {
 842                                 drbr_putback(ifp, txr->br, next);
 843                         }
 844 #endif
 845                         break;
 846                 }
 847 #if __FreeBSD_version >= 901504
 848                 drbr_advance(ifp, txr->br);
 849 #endif
 850                 enqueued++;
 851                 /* Send a copy of the frame to the BPF listener */
 852                 ETHER_BPF_MTAP(ifp, next);
 853                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 854                         break;
 855 #if __FreeBSD_version < 901504
 856                 next = drbr_dequeue(ifp, txr->br);
 857 #endif
 858         }
 859
 860         if (enqueued > 0) {
 861                 /* Set watchdog on */
 862                 txr->queue_status = IXGBE_QUEUE_WORKING;
 863                 txr->watchdog_time = ticks;
 864         }
 865
 866         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 867                 ixgbe_txeof(txr);
 868
 869         return (err);
 870 }
 871
 872 /*
 873  * Called from a taskqueue to drain queued transmit packets.
 874  */
 875 static void
 876 ixgbe_deferred_mq_start(void *arg, int pending)
 877 {
 878         struct tx_ring *txr = arg;
 879         struct adapter *adapter = txr->adapter;
 880         struct ifnet *ifp = adapter->ifp;
 881
 882         IXGBE_TX_LOCK(txr);
 883         if (!drbr_empty(ifp, txr->br))
 884                 ixgbe_mq_start_locked(ifp, txr);
 885         IXGBE_TX_UNLOCK(txr);
 886 }
 887
 888 /*
 889 ** Flush all ring buffers
 890 */
 891 static void
 892 ixgbe_qflush(struct ifnet *ifp)
 893 {
 894         struct adapter  *adapter = ifp->if_softc;
 895         struct tx_ring  *txr = adapter->tx_rings;
 896         struct mbuf     *m;
 897
 898         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 899                 IXGBE_TX_LOCK(txr);
 900                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 901                         m_freem(m);
 902                 IXGBE_TX_UNLOCK(txr);
 903         }
 904         if_qflush(ifp);
 905 }
 906 #endif /* IXGBE_LEGACY_TX */
 907
 908 /*********************************************************************
 909  *  Ioctl entry point
 910  *
 911  *  ixgbe_ioctl is called when the user wants to configure the
 912  *  interface.
 913  *
 914  *  return 0 on success, positive on failure
 915  **********************************************************************/
 916
 917 static int
 918 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 919 {
 920         struct adapter  *adapter = ifp->if_softc;
 921         struct ixgbe_hw *hw = &adapter->hw;
 922         struct ifreq    *ifr = (struct ifreq *) data;
 923 #if defined(INET) || defined(INET6)
 924         struct ifaddr *ifa = (struct ifaddr *)data;
 925         bool            avoid_reset = FALSE;
 926 #endif
 927         int             error = 0;
 928
 929         switch (command) {
 930
 931         case SIOCSIFADDR:
 932 #ifdef INET
 933                 if (ifa->ifa_addr->sa_family == AF_INET)
 934                         avoid_reset = TRUE;
 935 #endif
 936 #ifdef INET6
 937                 if (ifa->ifa_addr->sa_family == AF_INET6)
 938                         avoid_reset = TRUE;
 939 #endif
 940 #if defined(INET) || defined(INET6)
 941                 /*
 942                 ** Calling init results in link renegotiation,
 943                 ** so we avoid doing it when possible.
 944                 */
 945                 if (avoid_reset) {
 946                         ifp->if_flags |= IFF_UP;
 947                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 948                                 ixgbe_init(adapter);
 949                         if (!(ifp->if_flags & IFF_NOARP))
 950                                 arp_ifinit(ifp, ifa);
 951                 } else
 952                         error = ether_ioctl(ifp, command, data);
 953 #endif
 954                 break;
 955         case SIOCSIFMTU:
 956                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 957                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 958                         error = EINVAL;
 959                 } else {
 960                         IXGBE_CORE_LOCK(adapter);
 961                         ifp->if_mtu = ifr->ifr_mtu;
 962                         adapter->max_frame_size =
 963                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 964                         ixgbe_init_locked(adapter);
 965                         IXGBE_CORE_UNLOCK(adapter);
 966                 }
 967                 break;
 968         case SIOCSIFFLAGS:
 969                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 970                 IXGBE_CORE_LOCK(adapter);
 971                 if (ifp->if_flags & IFF_UP) {
 972                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 973                                 if ((ifp->if_flags ^ adapter->if_flags) &
 974                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 975                                         ixgbe_set_promisc(adapter);
 976                                 }
 977                         } else
 978                                 ixgbe_init_locked(adapter);
 979                 } else
 980                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 981                                 ixgbe_stop(adapter);
 982                 adapter->if_flags = ifp->if_flags;
 983                 IXGBE_CORE_UNLOCK(adapter);
 984                 break;
 985         case SIOCADDMULTI:
 986         case SIOCDELMULTI:
 987                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 988                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 989                         IXGBE_CORE_LOCK(adapter);
 990                         ixgbe_disable_intr(adapter);
 991                         ixgbe_set_multi(adapter);
 992                         ixgbe_enable_intr(adapter);
 993                         IXGBE_CORE_UNLOCK(adapter);
 994                 }
 995                 break;
 996         case SIOCSIFMEDIA:
 997         case SIOCGIFMEDIA:
 998                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 999                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1000                 break;
1001         case SIOCSIFCAP:
1002         {
1003                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1004                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1005                 if (mask & IFCAP_HWCSUM)
1006                         ifp->if_capenable ^= IFCAP_HWCSUM;
1007                 if (mask & IFCAP_TSO4)
1008                         ifp->if_capenable ^= IFCAP_TSO4;
1009                 if (mask & IFCAP_TSO6)
1010                         ifp->if_capenable ^= IFCAP_TSO6;
1011                 if (mask & IFCAP_LRO)
1012                         ifp->if_capenable ^= IFCAP_LRO;
1013                 if (mask & IFCAP_VLAN_HWTAGGING)
1014                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1015                 if (mask & IFCAP_VLAN_HWFILTER)
1016                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1017                 if (mask & IFCAP_VLAN_HWTSO)
1018                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1019                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1020                         IXGBE_CORE_LOCK(adapter);
1021                         ixgbe_init_locked(adapter);
1022                         IXGBE_CORE_UNLOCK(adapter);
1023                 }
1024                 VLAN_CAPABILITIES(ifp);
1025                 break;
1026         }
1027         case SIOCGI2C:
1028         {
1029                 struct ixgbe_i2c_req    i2c;
1030                 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
1031                 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1032                 if (error)
1033                         break;
1034                 if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){
1035                         error = EINVAL;
1036                         break;
1037                 }
1038                 hw->phy.ops.read_i2c_byte(hw, i2c.offset,
1039                     i2c.dev_addr, i2c.data);
1040                 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1041                 break;
1042         }
1043         default:
1044                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1045                 error = ether_ioctl(ifp, command, data);
1046                 break;
1047         }
1048
1049         return (error);
1050 }
1051
1052 /*********************************************************************
1053  *  Init entry point
1054  *
1055  *  This routine is used in two ways. It is used by the stack as
1056  *  init entry point in network interface structure. It is also used
1057  *  by the driver as a hw/sw initialization routine to get to a
1058  *  consistent state.
1059  *
1060  *  return 0 on success, positive on failure
1061  **********************************************************************/
1062 #define IXGBE_MHADD_MFS_SHIFT 16
1063
1064 static void
1065 ixgbe_init_locked(struct adapter *adapter)
1066 {
1067         struct ifnet   *ifp = adapter->ifp;
1068         device_t        dev = adapter->dev;
1069         struct ixgbe_hw *hw = &adapter->hw;
1070         u32             k, txdctl, mhadd, gpie;
1071         u32             rxdctl, rxctrl;
1072
1073         mtx_assert(&adapter->core_mtx, MA_OWNED);
1074         INIT_DEBUGOUT("ixgbe_init: begin");
1075         hw->adapter_stopped = FALSE;
1076         ixgbe_stop_adapter(hw);
1077         callout_stop(&adapter->timer);
1078
1079         /* reprogram the RAR[0] in case user changed it. */
1080         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1081
1082         /* Get the latest mac address, User can use a LAA */
1083         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1084               IXGBE_ETH_LENGTH_OF_ADDRESS);
1085         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1086         hw->addr_ctrl.rar_used_count = 1;
1087
1088         /* Set the various hardware offload abilities */
1089         ifp->if_hwassist = 0;
1090         if (ifp->if_capenable & IFCAP_TSO)
1091                 ifp->if_hwassist |= CSUM_TSO;
1092         if (ifp->if_capenable & IFCAP_TXCSUM) {
1093                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1094 #if __FreeBSD_version >= 800000
1095                 if (hw->mac.type != ixgbe_mac_82598EB)
1096                         ifp->if_hwassist |= CSUM_SCTP;
1097 #endif
1098         }
1099
1100         /* Prepare transmit descriptors and buffers */
1101         if (ixgbe_setup_transmit_structures(adapter)) {
1102                 device_printf(dev,"Could not setup transmit structures\n");
1103                 ixgbe_stop(adapter);
1104                 return;
1105         }
1106
1107         ixgbe_init_hw(hw);
1108         ixgbe_initialize_transmit_units(adapter);
1109
1110         /* Setup Multicast table */
1111         ixgbe_set_multi(adapter);
1112
1113         /*
1114         ** Determine the correct mbuf pool
1115         ** for doing jumbo frames
1116         */
1117         if (adapter->max_frame_size <= 2048)
1118                 adapter->rx_mbuf_sz = MCLBYTES;
1119         else if (adapter->max_frame_size <= 4096)
1120                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1121         else if (adapter->max_frame_size <= 9216)
1122                 adapter->rx_mbuf_sz = MJUM9BYTES;
1123         else
1124                 adapter->rx_mbuf_sz = MJUM16BYTES;
1125
1126         /* Prepare receive descriptors and buffers */
1127         if (ixgbe_setup_receive_structures(adapter)) {
1128                 device_printf(dev,"Could not setup receive structures\n");
1129                 ixgbe_stop(adapter);
1130                 return;
1131         }
1132
1133         /* Configure RX settings */
1134         ixgbe_initialize_receive_units(adapter);
1135
1136         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1137
1138         /* Enable Fan Failure Interrupt */
1139         gpie |= IXGBE_SDP1_GPIEN;
1140
1141         /* Add for Module detection */
1142         if (hw->mac.type == ixgbe_mac_82599EB)
1143                 gpie |= IXGBE_SDP2_GPIEN;
1144
1145         /* Thermal Failure Detection */
1146         if (hw->mac.type == ixgbe_mac_X540)
1147                 gpie |= IXGBE_SDP0_GPIEN;
1148
1149         if (adapter->msix > 1) {
1150                 /* Enable Enhanced MSIX mode */
1151                 gpie |= IXGBE_GPIE_MSIX_MODE;
1152                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1153                     IXGBE_GPIE_OCD;
1154         }
1155         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1156
1157         /* Set MTU size */
1158         if (ifp->if_mtu > ETHERMTU) {
1159                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1160                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1161                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1162                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1163         }
1164
1165         /* Now enable all the queues */
1166
1167         for (int i = 0; i < adapter->num_queues; i++) {
1168                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1169                 txdctl |= IXGBE_TXDCTL_ENABLE;
1170                 /* Set WTHRESH to 8, burst writeback */
1171                 txdctl |= (8 << 16);
1172                 /*
1173                  * When the internal queue falls below PTHRESH (32),
1174                  * start prefetching as long as there are at least
1175                  * HTHRESH (1) buffers ready. The values are taken
1176                  * from the Intel linux driver 3.8.21.
1177                  * Prefetching enables tx line rate even with 1 queue.
1178                  */
1179                 txdctl |= (32 << 0) | (1 << 8);
1180                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1181         }
1182
1183         for (int i = 0; i < adapter->num_queues; i++) {
1184                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1185                 if (hw->mac.type == ixgbe_mac_82598EB) {
1186                         /*
1187                         ** PTHRESH = 21
1188                         ** HTHRESH = 4
1189                         ** WTHRESH = 8
1190                         */
1191                         rxdctl &= ~0x3FFFFF;
1192                         rxdctl |= 0x080420;
1193                 }
1194                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1195                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1196                 for (k = 0; k < 10; k++) {
1197                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1198                             IXGBE_RXDCTL_ENABLE)
1199                                 break;
1200                         else
1201                                 msec_delay(1);
1202                 }
1203                 wmb();
1204 #ifdef DEV_NETMAP
1205                 /*
1206                  * In netmap mode, we must preserve the buffers made
1207                  * available to userspace before the if_init()
1208                  * (this is true by default on the TX side, because
1209                  * init makes all buffers available to userspace).
1210                  *
1211                  * netmap_reset() and the device specific routines
1212                  * (e.g. ixgbe_setup_receive_rings()) map these
1213                  * buffers at the end of the NIC ring, so here we
1214                  * must set the RDT (tail) register to make sure
1215                  * they are not overwritten.
1216                  *
1217                  * In this driver the NIC ring starts at RDH = 0,
1218                  * RDT points to the last slot available for reception (?),
1219                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1220                  */
1221                 if (ifp->if_capenable & IFCAP_NETMAP) {
1222                         struct netmap_adapter *na = NA(adapter->ifp);
1223                         struct netmap_kring *kring = &na->rx_rings[i];
1224                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1225
1226                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1227                 } else
1228 #endif /* DEV_NETMAP */
1229                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1230         }
1231
1232         /* Set up VLAN support and filter */
1233         ixgbe_setup_vlan_hw_support(adapter);
1234
1235         /* Enable Receive engine */
1236         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1237         if (hw->mac.type == ixgbe_mac_82598EB)
1238                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1239         rxctrl |= IXGBE_RXCTRL_RXEN;
1240         ixgbe_enable_rx_dma(hw, rxctrl);
1241
1242         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1243
1244         /* Set up MSI/X routing */
1245         if (ixgbe_enable_msix)  {
1246                 ixgbe_configure_ivars(adapter);
1247                 /* Set up auto-mask */
1248                 if (hw->mac.type == ixgbe_mac_82598EB)
1249                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1250                 else {
1251                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1252                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1253                 }
1254         } else {  /* Simple settings for Legacy/MSI */
1255                 ixgbe_set_ivar(adapter, 0, 0, 0);
1256                 ixgbe_set_ivar(adapter, 0, 0, 1);
1257                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1258         }
1259
1260 #ifdef IXGBE_FDIR
1261         /* Init Flow director */
1262         if (hw->mac.type != ixgbe_mac_82598EB) {
1263                 u32 hdrm = 32 << fdir_pballoc;
1264
1265                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1266                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1267         }
1268 #endif
1269
1270         /*
1271         ** Check on any SFP devices that
1272         ** need to be kick-started
1273         */
1274         if (hw->phy.type == ixgbe_phy_none) {
1275                 int err = hw->phy.ops.identify(hw);
1276                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1277                         device_printf(dev,
1278                             "Unsupported SFP+ module type was detected.\n");
1279                         return;
1280                 }
1281         }
1282
1283         /* Set moderation on the Link interrupt */
1284         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1285
1286         /* Config/Enable Link */
1287         ixgbe_config_link(adapter);
1288
1289         /* Hardware Packet Buffer & Flow Control setup */
1290         {
1291                 u32 rxpb, frame, size, tmp;
1292
1293                 frame = adapter->max_frame_size;
1294
1295                 /* Calculate High Water */
1296                 if (hw->mac.type == ixgbe_mac_X540)
1297                         tmp = IXGBE_DV_X540(frame, frame);
1298                 else
1299                         tmp = IXGBE_DV(frame, frame);
1300                 size = IXGBE_BT2KB(tmp);
1301                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1302                 hw->fc.high_water[0] = rxpb - size;
1303
1304                 /* Now calculate Low Water */
1305                 if (hw->mac.type == ixgbe_mac_X540)
1306                         tmp = IXGBE_LOW_DV_X540(frame);
1307                 else
1308                         tmp = IXGBE_LOW_DV(frame);
1309                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1310
1311                 hw->fc.requested_mode = adapter->fc;
1312                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1313                 hw->fc.send_xon = TRUE;
1314         }
1315         /* Initialize the FC settings */
1316         ixgbe_start_hw(hw);
1317
1318         /* And now turn on interrupts */
1319         ixgbe_enable_intr(adapter);
1320
1321         /* Now inform the stack we're ready */
1322         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1323
1324         return;
1325 }
1326
1327 static void
1328 ixgbe_init(void *arg)
1329 {
1330         struct adapter *adapter = arg;
1331
1332         IXGBE_CORE_LOCK(adapter);
1333         ixgbe_init_locked(adapter);
1334         IXGBE_CORE_UNLOCK(adapter);
1335         return;
1336 }
1337
1338
1339 /*
1340 **
1341 ** MSIX Interrupt Handlers and Tasklets
1342 **
1343 */
1344
1345 static inline void
1346 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1347 {
1348         struct ixgbe_hw *hw = &adapter->hw;
1349         u64     queue = (u64)(1 << vector);
1350         u32     mask;
1351
1352         if (hw->mac.type == ixgbe_mac_82598EB) {
1353                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1354                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1355         } else {
1356                 mask = (queue & 0xFFFFFFFF);
1357                 if (mask)
1358                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1359                 mask = (queue >> 32);
1360                 if (mask)
1361                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1362         }
1363 }
1364
1365 static inline void
1366 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1367 {
1368         struct ixgbe_hw *hw = &adapter->hw;
1369         u64     queue = (u64)(1 << vector);
1370         u32     mask;
1371
1372         if (hw->mac.type == ixgbe_mac_82598EB) {
1373                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1374                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1375         } else {
1376                 mask = (queue & 0xFFFFFFFF);
1377                 if (mask)
1378                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1379                 mask = (queue >> 32);
1380                 if (mask)
1381                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1382         }
1383 }
1384
1385 static inline void
1386 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1387 {
1388         u32 mask;
1389
1390         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1391                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1392                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1393         } else {
1394                 mask = (queues & 0xFFFFFFFF);
1395                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1396                 mask = (queues >> 32);
1397                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1398         }
1399 }
1400
1401
1402 static void
1403 ixgbe_handle_que(void *context, int pending)
1404 {
1405         struct ix_queue *que = context;
1406         struct adapter  *adapter = que->adapter;
1407         struct tx_ring  *txr = que->txr;
1408         struct ifnet    *ifp = adapter->ifp;
1409         bool            more;
1410
1411         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1412                 more = ixgbe_rxeof(que);
1413                 IXGBE_TX_LOCK(txr);
1414                 ixgbe_txeof(txr);
1415 #ifndef IXGBE_LEGACY_TX
1416                 if (!drbr_empty(ifp, txr->br))
1417                         ixgbe_mq_start_locked(ifp, txr);
1418 #else
1419                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1420                         ixgbe_start_locked(txr, ifp);
1421 #endif
1422                 IXGBE_TX_UNLOCK(txr);
1423         }
1424
1425         /* Reenable this interrupt */
1426         if (que->res != NULL)
1427                 ixgbe_enable_queue(adapter, que->msix);
1428         else
1429                 ixgbe_enable_intr(adapter);
1430         return;
1431 }
1432
1433
1434 /*********************************************************************
1435  *
1436  *  Legacy Interrupt Service routine
1437  *
1438  **********************************************************************/
1439
1440 static void
1441 ixgbe_legacy_irq(void *arg)
1442 {
1443         struct ix_queue *que = arg;
1444         struct adapter  *adapter = que->adapter;
1445         struct ixgbe_hw *hw = &adapter->hw;
1446         struct ifnet    *ifp = adapter->ifp;
1447         struct          tx_ring *txr = adapter->tx_rings;
1448         bool            more;
1449         u32             reg_eicr;
1450
1451
1452         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1453
1454         ++que->irqs;
1455         if (reg_eicr == 0) {
1456                 ixgbe_enable_intr(adapter);
1457                 return;
1458         }
1459
1460         more = ixgbe_rxeof(que);
1461
1462         IXGBE_TX_LOCK(txr);
1463         ixgbe_txeof(txr);
1464 #ifdef IXGBE_LEGACY_TX
1465         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1466                 ixgbe_start_locked(txr, ifp);
1467 #else
1468         if (!drbr_empty(ifp, txr->br))
1469                 ixgbe_mq_start_locked(ifp, txr);
1470 #endif
1471         IXGBE_TX_UNLOCK(txr);
1472
1473         /* Check for fan failure */
1474         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1475             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1476                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1477                     "REPLACE IMMEDIATELY!!\n");
1478                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1479         }
1480
1481         /* Link status change */
1482         if (reg_eicr & IXGBE_EICR_LSC)
1483                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1484
1485         if (more)
1486                 taskqueue_enqueue(que->tq, &que->que_task);
1487         else
1488                 ixgbe_enable_intr(adapter);
1489         return;
1490 }
1491
1492
1493 /*********************************************************************
1494  *
1495  *  MSIX Queue Interrupt Service routine
1496  *
1497  **********************************************************************/
1498 void
1499 ixgbe_msix_que(void *arg)
1500 {
1501         struct ix_queue *que = arg;
1502         struct adapter  *adapter = que->adapter;
1503         struct ifnet    *ifp = adapter->ifp;
1504         struct tx_ring  *txr = que->txr;
1505         struct rx_ring  *rxr = que->rxr;
1506         bool            more;
1507         u32             newitr = 0;
1508
1509         ixgbe_disable_queue(adapter, que->msix);
1510         ++que->irqs;
1511
1512         more = ixgbe_rxeof(que);
1513
1514         IXGBE_TX_LOCK(txr);
1515         ixgbe_txeof(txr);
1516 #ifdef IXGBE_LEGACY_TX
1517         if (!IFQ_DRV_IS_EMPTY(ifp->if_snd))
1518                 ixgbe_start_locked(txr, ifp);
1519 #else
1520         if (!drbr_empty(ifp, txr->br))
1521                 ixgbe_mq_start_locked(ifp, txr);
1522 #endif
1523         IXGBE_TX_UNLOCK(txr);
1524
1525         /* Do AIM now? */
1526
1527         if (ixgbe_enable_aim == FALSE)
1528                 goto no_calc;
1529         /*
1530         ** Do Adaptive Interrupt Moderation:
1531         **  - Write out last calculated setting
1532         **  - Calculate based on average size over
1533         **    the last interval.
1534         */
1535         if (que->eitr_setting)
1536                 IXGBE_WRITE_REG(&adapter->hw,
1537                     IXGBE_EITR(que->msix), que->eitr_setting);
1538
1539         que->eitr_setting = 0;
1540
1541         /* Idle, do nothing */
1542         if ((txr->bytes == 0) && (rxr->bytes == 0))
1543                 goto no_calc;
1544
1545         if ((txr->bytes) && (txr->packets))
1546                 newitr = txr->bytes/txr->packets;
1547         if ((rxr->bytes) && (rxr->packets))
1548                 newitr = max(newitr,
1549                     (rxr->bytes / rxr->packets));
1550         newitr += 24; /* account for hardware frame, crc */
1551
1552         /* set an upper boundary */
1553         newitr = min(newitr, 3000);
1554
1555         /* Be nice to the mid range */
1556         if ((newitr > 300) && (newitr < 1200))
1557                 newitr = (newitr / 3);
1558         else
1559                 newitr = (newitr / 2);
1560
1561         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1562                 newitr |= newitr << 16;
1563         else
1564                 newitr |= IXGBE_EITR_CNT_WDIS;
1565
1566         /* save for next interrupt */
1567         que->eitr_setting = newitr;
1568
1569         /* Reset state */
1570         txr->bytes = 0;
1571         txr->packets = 0;
1572         rxr->bytes = 0;
1573         rxr->packets = 0;
1574
1575 no_calc:
1576         if (more)
1577                 taskqueue_enqueue(que->tq, &que->que_task);
1578         else
1579                 ixgbe_enable_queue(adapter, que->msix);
1580         return;
1581 }
1582
1583
1584 static void
1585 ixgbe_msix_link(void *arg)
1586 {
1587         struct adapter  *adapter = arg;
1588         struct ixgbe_hw *hw = &adapter->hw;
1589         u32             reg_eicr;
1590
1591         ++adapter->link_irq;
1592
1593         /* First get the cause */
1594         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1595         /* Clear interrupt with write */
1596         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1597
1598         /* Link status change */
1599         if (reg_eicr & IXGBE_EICR_LSC)
1600                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1601
1602         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1603 #ifdef IXGBE_FDIR
1604                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1605                         /* This is probably overkill :) */
1606                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1607                                 return;
1608                         /* Disable the interrupt */
1609                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1610                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1611                 } else
1612 #endif
1613                 if (reg_eicr & IXGBE_EICR_ECC) {
1614                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1615                             "Please Reboot!!\n");
1616                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1617                 } else
1618
1619                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1620                         /* Clear the interrupt */
1621                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1622                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1623                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1624                         /* Clear the interrupt */
1625                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1626                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1627                 }
1628         }
1629
1630         /* Check for fan failure */
1631         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1632             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1633                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1634                     "REPLACE IMMEDIATELY!!\n");
1635                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1636         }
1637
1638         /* Check for over temp condition */
1639         if ((hw->mac.type == ixgbe_mac_X540) &&
1640             (reg_eicr & IXGBE_EICR_TS)) {
1641                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1642                     "PHY IS SHUT DOWN!!\n");
1643                 device_printf(adapter->dev, "System shutdown required\n");
1644                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1645         }
1646
1647         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1648         return;
1649 }
1650
1651 /*********************************************************************
1652  *
1653  *  Media Ioctl callback
1654  *
1655  *  This routine is called whenever the user queries the status of
1656  *  the interface using ifconfig.
1657  *
1658  **********************************************************************/
1659 static void
1660 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1661 {
1662         struct adapter *adapter = ifp->if_softc;
1663
1664         INIT_DEBUGOUT("ixgbe_media_status: begin");
1665         IXGBE_CORE_LOCK(adapter);
1666         ixgbe_update_link_status(adapter);
1667
1668         ifmr->ifm_status = IFM_AVALID;
1669         ifmr->ifm_active = IFM_ETHER;
1670
1671         if (!adapter->link_active) {
1672                 IXGBE_CORE_UNLOCK(adapter);
1673                 return;
1674         }
1675
1676         ifmr->ifm_status |= IFM_ACTIVE;
1677
1678         switch (adapter->link_speed) {
1679                 case IXGBE_LINK_SPEED_100_FULL:
1680                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1681                         break;
1682                 case IXGBE_LINK_SPEED_1GB_FULL:
1683                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1684                         break;
1685                 case IXGBE_LINK_SPEED_10GB_FULL:
1686                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1687                         break;
1688         }
1689
1690         IXGBE_CORE_UNLOCK(adapter);
1691
1692         return;
1693 }
1694
1695 /*********************************************************************
1696  *
1697  *  Media Ioctl callback
1698  *
1699  *  This routine is called when the user changes speed/duplex using
1700  *  media/mediopt option with ifconfig.
1701  *
1702  **********************************************************************/
1703 static int
1704 ixgbe_media_change(struct ifnet * ifp)
1705 {
1706         struct adapter *adapter = ifp->if_softc;
1707         struct ifmedia *ifm = &adapter->media;
1708
1709         INIT_DEBUGOUT("ixgbe_media_change: begin");
1710
1711         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1712                 return (EINVAL);
1713
1714         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1715         case IFM_AUTO:
1716                 adapter->hw.phy.autoneg_advertised =
1717                     IXGBE_LINK_SPEED_100_FULL |
1718                     IXGBE_LINK_SPEED_1GB_FULL |
1719                     IXGBE_LINK_SPEED_10GB_FULL;
1720                 break;
1721         default:
1722                 device_printf(adapter->dev, "Only auto media type\n");
1723                 return (EINVAL);
1724         }
1725
1726         return (0);
1727 }
1728
1729 /*********************************************************************
1730  *
1731  *  This routine maps the mbufs to tx descriptors, allowing the
1732  *  TX engine to transmit the packets.
1733  *      - return 0 on success, positive on failure
1734  *
1735  **********************************************************************/
1736
1737 static int
1738 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1739 {
1740         struct adapter  *adapter = txr->adapter;
1741         u32             olinfo_status = 0, cmd_type_len;
1742         int             i, j, error, nsegs;
1743         int             first;
1744         bool            remap = TRUE;
1745         struct mbuf     *m_head;
1746         bus_dma_segment_t segs[adapter->num_segs];
1747         bus_dmamap_t    map;
1748         struct ixgbe_tx_buf *txbuf;
1749         union ixgbe_adv_tx_desc *txd = NULL;
1750
1751         m_head = *m_headp;
1752
1753         /* Basic descriptor defines */
1754         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1755             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1756
1757         if (m_head->m_flags & M_VLANTAG)
1758                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1759
1760         /*
1761          * Important to capture the first descriptor
1762          * used because it will contain the index of
1763          * the one we tell the hardware to report back
1764          */
1765         first = txr->next_avail_desc;
1766         txbuf = &txr->tx_buffers[first];
1767         map = txbuf->map;
1768
1769         /*
1770          * Map the packet for DMA.
1771          */
1772 retry:
1773         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1774             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1775
1776         if (__predict_false(error)) {
1777                 struct mbuf *m;
1778
1779                 switch (error) {
1780                 case EFBIG:
1781                         /* Try it again? - one try */
1782                         if (remap == TRUE) {
1783                                 remap = FALSE;
1784                                 m = m_defrag(*m_headp, M_NOWAIT);
1785                                 if (m == NULL) {
1786                                         adapter->mbuf_defrag_failed++;
1787                                         m_freem(*m_headp);
1788                                         *m_headp = NULL;
1789                                         return (ENOBUFS);
1790                                 }
1791                                 *m_headp = m;
1792                                 goto retry;
1793                         } else
1794                                 return (error);
1795                 case ENOMEM:
1796                         txr->no_tx_dma_setup++;
1797                         return (error);
1798                 default:
1799                         txr->no_tx_dma_setup++;
1800                         m_freem(*m_headp);
1801                         *m_headp = NULL;
1802                         return (error);
1803                 }
1804         }
1805
1806         /* Make certain there are enough descriptors */
1807         if (nsegs > txr->tx_avail - 2) {
1808                 txr->no_desc_avail++;
1809                 bus_dmamap_unload(txr->txtag, map);
1810                 return (ENOBUFS);
1811         }
1812         m_head = *m_headp;
1813
1814         /*
1815         ** Set up the appropriate offload context
1816         ** this will consume the first descriptor
1817         */
1818         error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1819         if (__predict_false(error)) {
1820                 if (error == ENOBUFS)
1821                         *m_headp = NULL;
1822                 return (error);
1823         }
1824
1825 #ifdef IXGBE_FDIR
1826         /* Do the flow director magic */
1827         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1828                 ++txr->atr_count;
1829                 if (txr->atr_count >= atr_sample_rate) {
1830                         ixgbe_atr(txr, m_head);
1831                         txr->atr_count = 0;
1832                 }
1833         }
1834 #endif
1835
1836         i = txr->next_avail_desc;
1837         for (j = 0; j < nsegs; j++) {
1838                 bus_size_t seglen;
1839                 bus_addr_t segaddr;
1840
1841                 txbuf = &txr->tx_buffers[i];
1842                 txd = &txr->tx_base[i];
1843                 seglen = segs[j].ds_len;
1844                 segaddr = htole64(segs[j].ds_addr);
1845
1846                 txd->read.buffer_addr = segaddr;
1847                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1848                     cmd_type_len |seglen);
1849                 txd->read.olinfo_status = htole32(olinfo_status);
1850
1851                 if (++i == txr->num_desc)
1852                         i = 0;
1853         }
1854
1855         txd->read.cmd_type_len |=
1856             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1857         txr->tx_avail -= nsegs;
1858         txr->next_avail_desc = i;
1859
1860         txbuf->m_head = m_head;
1861         /*
1862         ** Here we swap the map so the last descriptor,
1863         ** which gets the completion interrupt has the
1864         ** real map, and the first descriptor gets the
1865         ** unused map from this descriptor.
1866         */
1867         txr->tx_buffers[first].map = txbuf->map;
1868         txbuf->map = map;
1869         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1870
1871         /* Set the EOP descriptor that will be marked done */
1872         txbuf = &txr->tx_buffers[first];
1873         txbuf->eop = txd;
1874
1875         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1876             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1877         /*
1878          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1879          * hardware that this frame is available to transmit.
1880          */
1881         ++txr->total_packets;
1882         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1883
1884         return (0);
1885
1886 }
1887
1888 static void
1889 ixgbe_set_promisc(struct adapter *adapter)
1890 {
1891         u_int32_t       reg_rctl;
1892         struct ifnet   *ifp = adapter->ifp;
1893         int             mcnt = 0;
1894
1895         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1896         reg_rctl &= (~IXGBE_FCTRL_UPE);
1897         if (ifp->if_flags & IFF_ALLMULTI)
1898                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1899         else {
1900                 struct  ifmultiaddr *ifma;
1901 #if __FreeBSD_version < 800000
1902                 IF_ADDR_LOCK(ifp);
1903 #else
1904                 if_maddr_rlock(ifp);
1905 #endif
1906                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1907                         if (ifma->ifma_addr->sa_family != AF_LINK)
1908                                 continue;
1909                         if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1910                                 break;
1911                         mcnt++;
1912                 }
1913 #if __FreeBSD_version < 800000
1914                 IF_ADDR_UNLOCK(ifp);
1915 #else
1916                 if_maddr_runlock(ifp);
1917 #endif
1918         }
1919         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1920                 reg_rctl &= (~IXGBE_FCTRL_MPE);
1921         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1922
1923         if (ifp->if_flags & IFF_PROMISC) {
1924                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1925                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1926         } else if (ifp->if_flags & IFF_ALLMULTI) {
1927                 reg_rctl |= IXGBE_FCTRL_MPE;
1928                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1929                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1930         }
1931         return;
1932 }
1933
1934
1935 /*********************************************************************
1936  *  Multicast Update
1937  *
1938  *  This routine is called whenever multicast address list is updated.
1939  *
1940  **********************************************************************/
1941 #define IXGBE_RAR_ENTRIES 16
1942
1943 static void
1944 ixgbe_set_multi(struct adapter *adapter)
1945 {
1946         u32     fctrl;
1947         u8      *mta;
1948         u8      *update_ptr;
1949         struct  ifmultiaddr *ifma;
1950         int     mcnt = 0;
1951         struct ifnet   *ifp = adapter->ifp;
1952
1953         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1954
1955         mta = adapter->mta;
1956         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1957             MAX_NUM_MULTICAST_ADDRESSES);
1958
1959 #if __FreeBSD_version < 800000
1960         IF_ADDR_LOCK(ifp);
1961 #else
1962         if_maddr_rlock(ifp);
1963 #endif
1964         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1965                 if (ifma->ifma_addr->sa_family != AF_LINK)
1966                         continue;
1967                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1968                         break;
1969                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1970                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1971                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1972                 mcnt++;
1973         }
1974 #if __FreeBSD_version < 800000
1975         IF_ADDR_UNLOCK(ifp);
1976 #else
1977         if_maddr_runlock(ifp);
1978 #endif
1979
1980         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1981         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1982         if (ifp->if_flags & IFF_PROMISC)
1983                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1984         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1985             ifp->if_flags & IFF_ALLMULTI) {
1986                 fctrl |= IXGBE_FCTRL_MPE;
1987                 fctrl &= ~IXGBE_FCTRL_UPE;
1988         } else
1989                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1990
1991         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1992
1993         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
1994                 update_ptr = mta;
1995                 ixgbe_update_mc_addr_list(&adapter->hw,
1996                     update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1997         }
1998
1999         return;
2000 }
2001
2002 /*
2003  * This is an iterator function now needed by the multicast
2004  * shared code. It simply feeds the shared code routine the
2005  * addresses in the array of ixgbe_set_multi() one by one.
2006  */
2007 static u8 *
2008 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2009 {
2010         u8 *addr = *update_ptr;
2011         u8 *newptr;
2012         *vmdq = 0;
2013
2014         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2015         *update_ptr = newptr;
2016         return addr;
2017 }
2018
2019
2020 /*********************************************************************
2021  *  Timer routine
2022  *
2023  *  This routine checks for link status,updates statistics,
2024  *  and runs the watchdog check.
2025  *
2026  **********************************************************************/
2027
2028 static void
2029 ixgbe_local_timer(void *arg)
2030 {
2031         struct adapter  *adapter = arg;
2032         device_t        dev = adapter->dev;
2033         struct ix_queue *que = adapter->queues;
2034         struct tx_ring  *txr = adapter->tx_rings;
2035         int             hung = 0, paused = 0;
2036
2037         mtx_assert(&adapter->core_mtx, MA_OWNED);
2038
2039         /* Check for pluggable optics */
2040         if (adapter->sfp_probe)
2041                 if (!ixgbe_sfp_probe(adapter))
2042                         goto out; /* Nothing to do */
2043
2044         ixgbe_update_link_status(adapter);
2045         ixgbe_update_stats_counters(adapter);
2046
2047         /*
2048          * If the interface has been paused
2049          * then don't do the watchdog check
2050          */
2051         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2052                 paused = 1;
2053
2054         /*
2055         ** Check the TX queues status
2056         **      - watchdog only if all queues show hung
2057         */
2058         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2059                 if ((txr->queue_status == IXGBE_QUEUE_HUNG) &&
2060                     (paused == 0))
2061                         ++hung;
2062                 else if (txr->queue_status == IXGBE_QUEUE_WORKING)
2063                         taskqueue_enqueue(que->tq, &txr->txq_task);
2064         }
2065         /* Only truely watchdog if all queues show hung */
2066         if (hung == adapter->num_queues)
2067                 goto watchdog;
2068
2069 out:
2070         ixgbe_rearm_queues(adapter, adapter->que_mask);
2071         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2072         return;
2073
2074 watchdog:
2075         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2076         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2077             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2078             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2079         device_printf(dev,"TX(%d) desc avail = %d,"
2080             "Next TX to Clean = %d\n",
2081             txr->me, txr->tx_avail, txr->next_to_clean);
2082         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2083         adapter->watchdog_events++;
2084         ixgbe_init_locked(adapter);
2085 }
2086
2087 /*
2088 ** Note: this routine updates the OS on the link state
2089 **      the real check of the hardware only happens with
2090 **      a link interrupt.
2091 */
2092 static void
2093 ixgbe_update_link_status(struct adapter *adapter)
2094 {
2095         struct ifnet    *ifp = adapter->ifp;
2096         device_t dev = adapter->dev;
2097
2098
2099         if (adapter->link_up){
2100                 if (adapter->link_active == FALSE) {
2101                         if (bootverbose)
2102                                 device_printf(dev,"Link is up %d Gbps %s \n",
2103                                     ((adapter->link_speed == 128)? 10:1),
2104                                     "Full Duplex");
2105                         adapter->link_active = TRUE;
2106                         /* Update any Flow Control changes */
2107                         ixgbe_fc_enable(&adapter->hw);
2108                         if_link_state_change(ifp, LINK_STATE_UP);
2109                 }
2110         } else { /* Link down */
2111                 if (adapter->link_active == TRUE) {
2112                         if (bootverbose)
2113                                 device_printf(dev,"Link is Down\n");
2114                         if_link_state_change(ifp, LINK_STATE_DOWN);
2115                         adapter->link_active = FALSE;
2116                 }
2117         }
2118
2119         return;
2120 }
2121
2122
2123 /*********************************************************************
2124  *
2125  *  This routine disables all traffic on the adapter by issuing a
2126  *  global reset on the MAC and deallocates TX/RX buffers.
2127  *
2128  **********************************************************************/
2129
2130 static void
2131 ixgbe_stop(void *arg)
2132 {
2133         struct ifnet   *ifp;
2134         struct adapter *adapter = arg;
2135         struct ixgbe_hw *hw = &adapter->hw;
2136         ifp = adapter->ifp;
2137
2138         mtx_assert(&adapter->core_mtx, MA_OWNED);
2139
2140         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2141         ixgbe_disable_intr(adapter);
2142         callout_stop(&adapter->timer);
2143
2144         /* Let the stack know...*/
2145         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2146
2147         ixgbe_reset_hw(hw);
2148         hw->adapter_stopped = FALSE;
2149         ixgbe_stop_adapter(hw);
2150         if (hw->mac.type == ixgbe_mac_82599EB)
2151                 ixgbe_stop_mac_link_on_d3_82599(hw);
2152         /* Turn off the laser - noop with no optics */
2153         ixgbe_disable_tx_laser(hw);
2154
2155         /* Update the stack */
2156         adapter->link_up = FALSE;
2157         ixgbe_update_link_status(adapter);
2158
2159         /* reprogram the RAR[0] in case user changed it. */
2160         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2161
2162         return;
2163 }
2164
2165
2166 /*********************************************************************
2167  *
2168  *  Determine hardware revision.
2169  *
2170  **********************************************************************/
2171 static void
2172 ixgbe_identify_hardware(struct adapter *adapter)
2173 {
2174         device_t        dev = adapter->dev;
2175         struct ixgbe_hw *hw = &adapter->hw;
2176
2177         /* Save off the information about this board */
2178         hw->vendor_id = pci_get_vendor(dev);
2179         hw->device_id = pci_get_device(dev);
2180         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2181         hw->subsystem_vendor_id =
2182             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2183         hw->subsystem_device_id =
2184             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2185
2186         /* We need this here to set the num_segs below */
2187         ixgbe_set_mac_type(hw);
2188
2189         /* Pick up the 82599 and VF settings */
2190         if (hw->mac.type != ixgbe_mac_82598EB) {
2191                 hw->phy.smart_speed = ixgbe_smart_speed;
2192                 adapter->num_segs = IXGBE_82599_SCATTER;
2193         } else
2194                 adapter->num_segs = IXGBE_82598_SCATTER;
2195
2196         return;
2197 }
2198
2199 /*********************************************************************
2200  *
2201  *  Determine optic type
2202  *
2203  **********************************************************************/
2204 static void
2205 ixgbe_setup_optics(struct adapter *adapter)
2206 {
2207         struct ixgbe_hw *hw = &adapter->hw;
2208         int             layer;
2209
2210         layer = ixgbe_get_supported_physical_layer(hw);
2211
2212         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2213                 adapter->optics = IFM_10G_T;
2214                 return;
2215         }
2216
2217         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2218                 adapter->optics = IFM_1000_T;
2219                 return;
2220         }
2221
2222         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
2223                 adapter->optics = IFM_1000_SX;
2224                 return;
2225         }
2226
2227         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2228             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2229                 adapter->optics = IFM_10G_LR;
2230                 return;
2231         }
2232
2233         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2234                 adapter->optics = IFM_10G_SR;
2235                 return;
2236         }
2237
2238         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2239                 adapter->optics = IFM_10G_TWINAX;
2240                 return;
2241         }
2242
2243         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2244             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2245                 adapter->optics = IFM_10G_CX4;
2246                 return;
2247         }
2248
2249         /* If we get here just set the default */
2250         adapter->optics = IFM_ETHER | IFM_AUTO;
2251         return;
2252 }
2253
2254 /*********************************************************************
2255  *
2256  *  Setup the Legacy or MSI Interrupt handler
2257  *
2258  **********************************************************************/
2259 static int
2260 ixgbe_allocate_legacy(struct adapter *adapter)
2261 {
2262         device_t        dev = adapter->dev;
2263         struct          ix_queue *que = adapter->queues;
2264 #ifndef IXGBE_LEGACY_TX
2265         struct tx_ring          *txr = adapter->tx_rings;
2266 #endif
2267         int             error, rid = 0;
2268
2269         /* MSI RID at 1 */
2270         if (adapter->msix == 1)
2271                 rid = 1;
2272
2273         /* We allocate a single interrupt resource */
2274         adapter->res = bus_alloc_resource_any(dev,
2275             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2276         if (adapter->res == NULL) {
2277                 device_printf(dev, "Unable to allocate bus resource: "
2278                     "interrupt\n");
2279                 return (ENXIO);
2280         }
2281
2282         /*
2283          * Try allocating a fast interrupt and the associated deferred
2284          * processing contexts.
2285          */
2286 #ifndef IXGBE_LEGACY_TX
2287         TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2288 #endif
2289         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2290         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2291             taskqueue_thread_enqueue, &que->tq);
2292         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2293             device_get_nameunit(adapter->dev));
2294
2295         /* Tasklets for Link, SFP and Multispeed Fiber */
2296         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2297         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2298         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2299 #ifdef IXGBE_FDIR
2300         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2301 #endif
2302         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2303             taskqueue_thread_enqueue, &adapter->tq);
2304         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2305             device_get_nameunit(adapter->dev));
2306
2307         if ((error = bus_setup_intr(dev, adapter->res,
2308             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2309             que, &adapter->tag)) != 0) {
2310                 device_printf(dev, "Failed to register fast interrupt "
2311                     "handler: %d\n", error);
2312                 taskqueue_free(que->tq);
2313                 taskqueue_free(adapter->tq);
2314                 que->tq = NULL;
2315                 adapter->tq = NULL;
2316                 return (error);
2317         }
2318         /* For simplicity in the handlers */
2319         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2320
2321         return (0);
2322 }
2323
2324
2325 /*********************************************************************
2326  *
2327  *  Setup MSIX Interrupt resources and handlers
2328  *
2329  **********************************************************************/
2330 static int
2331 ixgbe_allocate_msix(struct adapter *adapter)
2332 {
2333         device_t        dev = adapter->dev;
2334         struct          ix_queue *que = adapter->queues;
2335         struct          tx_ring *txr = adapter->tx_rings;
2336         int             error, rid, vector = 0;
2337
2338         for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) {
2339                 rid = vector + 1;
2340                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2341                     RF_SHAREABLE | RF_ACTIVE);
2342                 if (que->res == NULL) {
2343                         device_printf(dev,"Unable to allocate"
2344                             " bus resource: que interrupt [%d]\n", vector);
2345                         return (ENXIO);
2346                 }
2347                 /* Set the handler function */
2348                 error = bus_setup_intr(dev, que->res,
2349                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2350                     ixgbe_msix_que, que, &que->tag);
2351                 if (error) {
2352                         que->res = NULL;
2353                         device_printf(dev, "Failed to register QUE handler");
2354                         return (error);
2355                 }
2356 #if __FreeBSD_version >= 800504
2357                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2358 #endif
2359                 que->msix = vector;
2360                 adapter->que_mask |= (u64)(1 << que->msix);
2361                 /*
2362                 ** Bind the msix vector, and thus the
2363                 ** ring to the corresponding cpu.
2364                 */
2365                 if (adapter->num_queues > 1)
2366                         bus_bind_intr(dev, que->res, i);
2367
2368 #ifndef IXGBE_LEGACY_TX
2369                 TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr);
2370 #endif
2371                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2372                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2373                     taskqueue_thread_enqueue, &que->tq);
2374                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2375                     device_get_nameunit(adapter->dev));
2376         }
2377
2378         /* and Link */
2379         rid = vector + 1;
2380         adapter->res = bus_alloc_resource_any(dev,
2381             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2382         if (!adapter->res) {
2383                 device_printf(dev,"Unable to allocate"
2384             " bus resource: Link interrupt [%d]\n", rid);
2385                 return (ENXIO);
2386         }
2387         /* Set the link handler function */
2388         error = bus_setup_intr(dev, adapter->res,
2389             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2390             ixgbe_msix_link, adapter, &adapter->tag);
2391         if (error) {
2392                 adapter->res = NULL;
2393                 device_printf(dev, "Failed to register LINK handler");
2394                 return (error);
2395         }
2396 #if __FreeBSD_version >= 800504
2397         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2398 #endif
2399         adapter->linkvec = vector;
2400         /* Tasklets for Link, SFP and Multispeed Fiber */
2401         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2402         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2403         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2404 #ifdef IXGBE_FDIR
2405         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2406 #endif
2407         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2408             taskqueue_thread_enqueue, &adapter->tq);
2409         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2410             device_get_nameunit(adapter->dev));
2411
2412         return (0);
2413 }
2414
2415 /*
2416  * Setup Either MSI/X or MSI
2417  */
2418 static int
2419 ixgbe_setup_msix(struct adapter *adapter)
2420 {
2421         device_t dev = adapter->dev;
2422         int rid, want, queues, msgs;
2423
2424         /* Override by tuneable */
2425         if (ixgbe_enable_msix == 0)
2426                 goto msi;
2427
2428         /* First try MSI/X */
2429         rid = PCIR_BAR(MSIX_82598_BAR);
2430         adapter->msix_mem = bus_alloc_resource_any(dev,
2431             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2432         if (!adapter->msix_mem) {
2433                 rid += 4;       /* 82599 maps in higher BAR */
2434                 adapter->msix_mem = bus_alloc_resource_any(dev,
2435                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2436         }
2437         if (!adapter->msix_mem) {
2438                 /* May not be enabled */
2439                 device_printf(adapter->dev,
2440                     "Unable to map MSIX table \n");
2441                 goto msi;
2442         }
2443
2444         msgs = pci_msix_count(dev);
2445         if (msgs == 0) { /* system has msix disabled */
2446                 bus_release_resource(dev, SYS_RES_MEMORY,
2447                     rid, adapter->msix_mem);
2448                 adapter->msix_mem = NULL;
2449                 goto msi;
2450         }
2451
2452         /* Figure out a reasonable auto config value */
2453         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2454
2455         if (ixgbe_num_queues != 0)
2456                 queues = ixgbe_num_queues;
2457         /* Set max queues to 8 when autoconfiguring */
2458         else if ((ixgbe_num_queues == 0) && (queues > 8))
2459                 queues = 8;
2460
2461         /*
2462         ** Want one vector (RX/TX pair) per queue
2463         ** plus an additional for Link.
2464         */
2465         want = queues + 1;
2466         if (msgs >= want)
2467                 msgs = want;
2468         else {
2469                 device_printf(adapter->dev,
2470                     "MSIX Configuration Problem, "
2471                     "%d vectors but %d queues wanted!\n",
2472                     msgs, want);
2473                 return (0); /* Will go to Legacy setup */
2474         }
2475         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2476                 device_printf(adapter->dev,
2477                     "Using MSIX interrupts with %d vectors\n", msgs);
2478                 adapter->num_queues = queues;
2479                 return (msgs);
2480         }
2481 msi:
2482         msgs = pci_msi_count(dev);
2483         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2484                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2485         else
2486                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2487         return (msgs);
2488 }
2489
2490
2491 static int
2492 ixgbe_allocate_pci_resources(struct adapter *adapter)
2493 {
2494         int             rid;
2495         device_t        dev = adapter->dev;
2496
2497         rid = PCIR_BAR(0);
2498         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2499             &rid, RF_ACTIVE);
2500
2501         if (!(adapter->pci_mem)) {
2502                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2503                 return (ENXIO);
2504         }
2505
2506         adapter->osdep.mem_bus_space_tag =
2507                 rman_get_bustag(adapter->pci_mem);
2508         adapter->osdep.mem_bus_space_handle =
2509                 rman_get_bushandle(adapter->pci_mem);
2510         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2511
2512         /* Legacy defaults */
2513         adapter->num_queues = 1;
2514         adapter->hw.back = &adapter->osdep;
2515
2516         /*
2517         ** Now setup MSI or MSI/X, should
2518         ** return us the number of supported
2519         ** vectors. (Will be 1 for MSI)
2520         */
2521         adapter->msix = ixgbe_setup_msix(adapter);
2522         return (0);
2523 }
2524
2525 static void
2526 ixgbe_free_pci_resources(struct adapter * adapter)
2527 {
2528         struct          ix_queue *que = adapter->queues;
2529         device_t        dev = adapter->dev;
2530         int             rid, memrid;
2531
2532         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2533                 memrid = PCIR_BAR(MSIX_82598_BAR);
2534         else
2535                 memrid = PCIR_BAR(MSIX_82599_BAR);
2536
2537         /*
2538         ** There is a slight possibility of a failure mode
2539         ** in attach that will result in entering this function
2540         ** before interrupt resources have been initialized, and
2541         ** in that case we do not want to execute the loops below
2542         ** We can detect this reliably by the state of the adapter
2543         ** res pointer.
2544         */
2545         if (adapter->res == NULL)
2546                 goto mem;
2547
2548         /*
2549         **  Release all msix queue resources:
2550         */
2551         for (int i = 0; i < adapter->num_queues; i++, que++) {
2552                 rid = que->msix + 1;
2553                 if (que->tag != NULL) {
2554                         bus_teardown_intr(dev, que->res, que->tag);
2555                         que->tag = NULL;
2556                 }
2557                 if (que->res != NULL)
2558                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2559         }
2560
2561
2562         /* Clean the Legacy or Link interrupt last */
2563         if (adapter->linkvec) /* we are doing MSIX */
2564                 rid = adapter->linkvec + 1;
2565         else
2566                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2567
2568         if (adapter->tag != NULL) {
2569                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2570                 adapter->tag = NULL;
2571         }
2572         if (adapter->res != NULL)
2573                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2574
2575 mem:
2576         if (adapter->msix)
2577                 pci_release_msi(dev);
2578
2579         if (adapter->msix_mem != NULL)
2580                 bus_release_resource(dev, SYS_RES_MEMORY,
2581                     memrid, adapter->msix_mem);
2582
2583         if (adapter->pci_mem != NULL)
2584                 bus_release_resource(dev, SYS_RES_MEMORY,
2585                     PCIR_BAR(0), adapter->pci_mem);
2586
2587         return;
2588 }
2589
2590 /*********************************************************************
2591  *
2592  *  Setup networking device structure and register an interface.
2593  *
2594  **********************************************************************/
2595 static int
2596 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2597 {
2598         struct ixgbe_hw *hw = &adapter->hw;
2599         struct ifnet   *ifp;
2600
2601         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2602
2603         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2604         if (ifp == NULL) {
2605                 device_printf(dev, "can not allocate ifnet structure\n");
2606                 return (-1);
2607         }
2608         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2609 #if __FreeBSD_version < 1000025
2610         ifp->if_baudrate = 1000000000;
2611 #else
2612         if_initbaudrate(ifp, IF_Gbps(10));
2613 #endif
2614         ifp->if_init = ixgbe_init;
2615         ifp->if_softc = adapter;
2616         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2617         ifp->if_ioctl = ixgbe_ioctl;
2618 #ifndef IXGBE_LEGACY_TX
2619         ifp->if_transmit = ixgbe_mq_start;
2620         ifp->if_qflush = ixgbe_qflush;
2621 #else
2622         ifp->if_start = ixgbe_start;
2623         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
2624         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
2625         IFQ_SET_READY(&ifp->if_snd);
2626 #endif
2627
2628         ether_ifattach(ifp, adapter->hw.mac.addr);
2629
2630         adapter->max_frame_size =
2631             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2632
2633         /*
2634          * Tell the upper layer(s) we support long frames.
2635          */
2636         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2637
2638         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2639         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2640         ifp->if_capabilities |= IFCAP_LRO;
2641         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2642                              |  IFCAP_VLAN_HWTSO
2643                              |  IFCAP_VLAN_MTU;
2644         ifp->if_capenable = ifp->if_capabilities;
2645
2646         /*
2647         ** Don't turn this on by default, if vlans are
2648         ** created on another pseudo device (eg. lagg)
2649         ** then vlan events are not passed thru, breaking
2650         ** operation, but with HW FILTER off it works. If
2651         ** using vlans directly on the ixgbe driver you can
2652         ** enable this and get full hardware tag filtering.
2653         */
2654         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2655
2656         /*
2657          * Specify the media types supported by this adapter and register
2658          * callbacks to update media and link information
2659          */
2660         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2661                      ixgbe_media_status);
2662         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2663         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2664         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2665                 ifmedia_add(&adapter->media,
2666                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2667                 ifmedia_add(&adapter->media,
2668                     IFM_ETHER | IFM_1000_T, 0, NULL);
2669         }
2670         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2671         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2672
2673         return (0);
2674 }
2675
2676 static void
2677 ixgbe_config_link(struct adapter *adapter)
2678 {
2679         struct ixgbe_hw *hw = &adapter->hw;
2680         u32     autoneg, err = 0;
2681         bool    sfp, negotiate;
2682
2683         sfp = ixgbe_is_sfp(hw);
2684
2685         if (sfp) {
2686                 if (hw->phy.multispeed_fiber) {
2687                         hw->mac.ops.setup_sfp(hw);
2688                         ixgbe_enable_tx_laser(hw);
2689                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2690                 } else
2691                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2692         } else {
2693                 if (hw->mac.ops.check_link)
2694                         err = ixgbe_check_link(hw, &adapter->link_speed,
2695                             &adapter->link_up, FALSE);
2696                 if (err)
2697                         goto out;
2698                 autoneg = hw->phy.autoneg_advertised;
2699                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2700                         err  = hw->mac.ops.get_link_capabilities(hw,
2701                             &autoneg, &negotiate);
2702                 if (err)
2703                         goto out;
2704                 if (hw->mac.ops.setup_link)
2705                         err = hw->mac.ops.setup_link(hw,
2706                             autoneg, adapter->link_up);
2707         }
2708 out:
2709         return;
2710 }
2711
2712 /********************************************************************
2713  * Manage DMA'able memory.
2714  *******************************************************************/
2715 static void
2716 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2717 {
2718         if (error)
2719                 return;
2720         *(bus_addr_t *) arg = segs->ds_addr;
2721         return;
2722 }
2723
2724 static int
2725 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2726                 struct ixgbe_dma_alloc *dma, int mapflags)
2727 {
2728         device_t dev = adapter->dev;
2729         int             r;
2730
2731         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2732                                DBA_ALIGN, 0,    /* alignment, bounds */
2733                                BUS_SPACE_MAXADDR,       /* lowaddr */
2734                                BUS_SPACE_MAXADDR,       /* highaddr */
2735                                NULL, NULL,      /* filter, filterarg */
2736                                size,    /* maxsize */
2737                                1,       /* nsegments */
2738                                size,    /* maxsegsize */
2739                                BUS_DMA_ALLOCNOW,        /* flags */
2740                                NULL,    /* lockfunc */
2741                                NULL,    /* lockfuncarg */
2742                                &dma->dma_tag);
2743         if (r != 0) {
2744                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2745                        "error %u\n", r);
2746                 goto fail_0;
2747         }
2748         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2749                              BUS_DMA_NOWAIT, &dma->dma_map);
2750         if (r != 0) {
2751                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2752                        "error %u\n", r);
2753                 goto fail_1;
2754         }
2755         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2756                             size,
2757                             ixgbe_dmamap_cb,
2758                             &dma->dma_paddr,
2759                             mapflags | BUS_DMA_NOWAIT);
2760         if (r != 0) {
2761                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2762                        "error %u\n", r);
2763                 goto fail_2;
2764         }
2765         dma->dma_size = size;
2766         return (0);
2767 fail_2:
2768         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2769 fail_1:
2770         bus_dma_tag_destroy(dma->dma_tag);
2771 fail_0:
2772         dma->dma_map = NULL;
2773         dma->dma_tag = NULL;
2774         return (r);
2775 }
2776
2777 static void
2778 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2779 {
2780         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2781             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2782         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2783         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2784         bus_dma_tag_destroy(dma->dma_tag);
2785 }
2786
2787
2788 /*********************************************************************
2789  *
2790  *  Allocate memory for the transmit and receive rings, and then
2791  *  the descriptors associated with each, called only once at attach.
2792  *
2793  **********************************************************************/
2794 static int
2795 ixgbe_allocate_queues(struct adapter *adapter)
2796 {
2797         device_t        dev = adapter->dev;
2798         struct ix_queue *que;
2799         struct tx_ring  *txr;
2800         struct rx_ring  *rxr;
2801         int rsize, tsize, error = IXGBE_SUCCESS;
2802         int txconf = 0, rxconf = 0;
2803
2804         /* First allocate the top level queue structs */
2805         if (!(adapter->queues =
2806             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2807             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2808                 device_printf(dev, "Unable to allocate queue memory\n");
2809                 error = ENOMEM;
2810                 goto fail;
2811         }
2812
2813         /* First allocate the TX ring struct memory */
2814         if (!(adapter->tx_rings =
2815             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2816             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2817                 device_printf(dev, "Unable to allocate TX ring memory\n");
2818                 error = ENOMEM;
2819                 goto tx_fail;
2820         }
2821
2822         /* Next allocate the RX */
2823         if (!(adapter->rx_rings =
2824             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2825             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2826                 device_printf(dev, "Unable to allocate RX ring memory\n");
2827                 error = ENOMEM;
2828                 goto rx_fail;
2829         }
2830
2831         /* For the ring itself */
2832         tsize = roundup2(adapter->num_tx_desc *
2833             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2834
2835         /*
2836          * Now set up the TX queues, txconf is needed to handle the
2837          * possibility that things fail midcourse and we need to
2838          * undo memory gracefully
2839          */
2840         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2841                 /* Set up some basics */
2842                 txr = &adapter->tx_rings[i];
2843                 txr->adapter = adapter;
2844                 txr->me = i;
2845                 txr->num_desc = adapter->num_tx_desc;
2846
2847                 /* Initialize the TX side lock */
2848                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2849                     device_get_nameunit(dev), txr->me);
2850                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2851
2852                 if (ixgbe_dma_malloc(adapter, tsize,
2853                         &txr->txdma, BUS_DMA_NOWAIT)) {
2854                         device_printf(dev,
2855                             "Unable to allocate TX Descriptor memory\n");
2856                         error = ENOMEM;
2857                         goto err_tx_desc;
2858                 }
2859                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2860                 bzero((void *)txr->tx_base, tsize);
2861
2862                 /* Now allocate transmit buffers for the ring */
2863                 if (ixgbe_allocate_transmit_buffers(txr)) {
2864                         device_printf(dev,
2865                             "Critical Failure setting up transmit buffers\n");
2866                         error = ENOMEM;
2867                         goto err_tx_desc;
2868                 }
2869 #ifndef IXGBE_LEGACY_TX
2870                 /* Allocate a buf ring */
2871                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2872                     M_WAITOK, &txr->tx_mtx);
2873                 if (txr->br == NULL) {
2874                         device_printf(dev,
2875                             "Critical Failure setting up buf ring\n");
2876                         error = ENOMEM;
2877                         goto err_tx_desc;
2878                 }
2879 #endif
2880         }
2881
2882         /*
2883          * Next the RX queues...
2884          */
2885         rsize = roundup2(adapter->num_rx_desc *
2886             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2887         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2888                 rxr = &adapter->rx_rings[i];
2889                 /* Set up some basics */
2890                 rxr->adapter = adapter;
2891                 rxr->me = i;
2892                 rxr->num_desc = adapter->num_rx_desc;
2893
2894                 /* Initialize the RX side lock */
2895                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2896                     device_get_nameunit(dev), rxr->me);
2897                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2898
2899                 if (ixgbe_dma_malloc(adapter, rsize,
2900                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2901                         device_printf(dev,
2902                             "Unable to allocate RxDescriptor memory\n");
2903                         error = ENOMEM;
2904                         goto err_rx_desc;
2905                 }
2906                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2907                 bzero((void *)rxr->rx_base, rsize);
2908
2909                 /* Allocate receive buffers for the ring*/
2910                 if (ixgbe_allocate_receive_buffers(rxr)) {
2911                         device_printf(dev,
2912                             "Critical Failure setting up receive buffers\n");
2913                         error = ENOMEM;
2914                         goto err_rx_desc;
2915                 }
2916         }
2917
2918         /*
2919         ** Finally set up the queue holding structs
2920         */
2921         for (int i = 0; i < adapter->num_queues; i++) {
2922                 que = &adapter->queues[i];
2923                 que->adapter = adapter;
2924                 que->txr = &adapter->tx_rings[i];
2925                 que->rxr = &adapter->rx_rings[i];
2926         }
2927
2928         return (0);
2929
2930 err_rx_desc:
2931         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2932                 ixgbe_dma_free(adapter, &rxr->rxdma);
2933 err_tx_desc:
2934         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2935                 ixgbe_dma_free(adapter, &txr->txdma);
2936         free(adapter->rx_rings, M_DEVBUF);
2937 rx_fail:
2938         free(adapter->tx_rings, M_DEVBUF);
2939 tx_fail:
2940         free(adapter->queues, M_DEVBUF);
2941 fail:
2942         return (error);
2943 }
2944
2945 /*********************************************************************
2946  *
2947  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2948  *  the information needed to transmit a packet on the wire. This is
2949  *  called only once at attach, setup is done every reset.
2950  *
2951  **********************************************************************/
2952 static int
2953 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2954 {
2955         struct adapter *adapter = txr->adapter;
2956         device_t dev = adapter->dev;
2957         struct ixgbe_tx_buf *txbuf;
2958         int error, i;
2959
2960         /*
2961          * Setup DMA descriptor areas.
2962          */
2963         if ((error = bus_dma_tag_create(
2964                                bus_get_dma_tag(adapter->dev),   /* parent */
2965                                1, 0,            /* alignment, bounds */
2966                                BUS_SPACE_MAXADDR,       /* lowaddr */
2967                                BUS_SPACE_MAXADDR,       /* highaddr */
2968                                NULL, NULL,              /* filter, filterarg */
2969                                IXGBE_TSO_SIZE,          /* maxsize */
2970                                adapter->num_segs,       /* nsegments */
2971                                PAGE_SIZE,               /* maxsegsize */
2972                                0,                       /* flags */
2973                                NULL,                    /* lockfunc */
2974                                NULL,                    /* lockfuncarg */
2975                                &txr->txtag))) {
2976                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2977                 goto fail;
2978         }
2979
2980         if (!(txr->tx_buffers =
2981             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2982             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2983                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2984                 error = ENOMEM;
2985                 goto fail;
2986         }
2987
2988         /* Create the descriptor buffer dma maps */
2989         txbuf = txr->tx_buffers;
2990         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2991                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2992                 if (error != 0) {
2993                         device_printf(dev, "Unable to create TX DMA map\n");
2994                         goto fail;
2995                 }
2996         }
2997
2998         return 0;
2999 fail:
3000         /* We free all, it handles case where we are in the middle */
3001         ixgbe_free_transmit_structures(adapter);
3002         return (error);
3003 }
3004
3005 /*********************************************************************
3006  *
3007  *  Initialize a transmit ring.
3008  *
3009  **********************************************************************/
3010 static void
3011 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3012 {
3013         struct adapter *adapter = txr->adapter;
3014         struct ixgbe_tx_buf *txbuf;
3015         int i;
3016 #ifdef DEV_NETMAP
3017         struct netmap_adapter *na = NA(adapter->ifp);
3018         struct netmap_slot *slot;
3019 #endif /* DEV_NETMAP */
3020
3021         /* Clear the old ring contents */
3022         IXGBE_TX_LOCK(txr);
3023 #ifdef DEV_NETMAP
3024         /*
3025          * (under lock): if in netmap mode, do some consistency
3026          * checks and set slot to entry 0 of the netmap ring.
3027          */
3028         slot = netmap_reset(na, NR_TX, txr->me, 0);
3029 #endif /* DEV_NETMAP */
3030         bzero((void *)txr->tx_base,
3031               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3032         /* Reset indices */
3033         txr->next_avail_desc = 0;
3034         txr->next_to_clean = 0;
3035
3036         /* Free any existing tx buffers. */
3037         txbuf = txr->tx_buffers;
3038         for (i = 0; i < txr->num_desc; i++, txbuf++) {
3039                 if (txbuf->m_head != NULL) {
3040                         bus_dmamap_sync(txr->txtag, txbuf->map,
3041                             BUS_DMASYNC_POSTWRITE);
3042                         bus_dmamap_unload(txr->txtag, txbuf->map);
3043                         m_freem(txbuf->m_head);
3044                         txbuf->m_head = NULL;
3045                 }
3046 #ifdef DEV_NETMAP
3047                 /*
3048                  * In netmap mode, set the map for the packet buffer.
3049                  * NOTE: Some drivers (not this one) also need to set
3050                  * the physical buffer address in the NIC ring.
3051                  * Slots in the netmap ring (indexed by "si") are
3052                  * kring->nkr_hwofs positions "ahead" wrt the
3053                  * corresponding slot in the NIC ring. In some drivers
3054                  * (not here) nkr_hwofs can be negative. Function
3055                  * netmap_idx_n2k() handles wraparounds properly.
3056                  */
3057                 if (slot) {
3058                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3059                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3060                 }
3061 #endif /* DEV_NETMAP */
3062                 /* Clear the EOP descriptor pointer */
3063                 txbuf->eop = NULL;
3064         }
3065
3066 #ifdef IXGBE_FDIR
3067         /* Set the rate at which we sample packets */
3068         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3069                 txr->atr_sample = atr_sample_rate;
3070 #endif
3071
3072         /* Set number of descriptors available */
3073         txr->tx_avail = adapter->num_tx_desc;
3074
3075         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3076             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3077         IXGBE_TX_UNLOCK(txr);
3078 }
3079
3080 /*********************************************************************
3081  *
3082  *  Initialize all transmit rings.
3083  *
3084  **********************************************************************/
3085 static int
3086 ixgbe_setup_transmit_structures(struct adapter *adapter)
3087 {
3088         struct tx_ring *txr = adapter->tx_rings;
3089
3090         for (int i = 0; i < adapter->num_queues; i++, txr++)
3091                 ixgbe_setup_transmit_ring(txr);
3092
3093         return (0);
3094 }
3095
3096 /*********************************************************************
3097  *
3098  *  Enable transmit unit.
3099  *
3100  **********************************************************************/
3101 static void
3102 ixgbe_initialize_transmit_units(struct adapter *adapter)
3103 {
3104         struct tx_ring  *txr = adapter->tx_rings;
3105         struct ixgbe_hw *hw = &adapter->hw;
3106
3107         /* Setup the Base and Length of the Tx Descriptor Ring */
3108
3109         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3110                 u64     tdba = txr->txdma.dma_paddr;
3111                 u32     txctrl;
3112
3113                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3114                        (tdba & 0x00000000ffffffffULL));
3115                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3116                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3117                     adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3118
3119                 /* Setup the HW Tx Head and Tail descriptor pointers */
3120                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3121                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3122
3123                 /* Setup Transmit Descriptor Cmd Settings */
3124                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3125                 txr->queue_status = IXGBE_QUEUE_IDLE;
3126
3127                 /* Set the processing limit */
3128                 txr->process_limit = ixgbe_tx_process_limit;
3129
3130                 /* Disable Head Writeback */
3131                 switch (hw->mac.type) {
3132                 case ixgbe_mac_82598EB:
3133                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3134                         break;
3135                 case ixgbe_mac_82599EB:
3136                 case ixgbe_mac_X540:
3137                 default:
3138                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3139                         break;
3140                 }
3141                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3142                 switch (hw->mac.type) {
3143                 case ixgbe_mac_82598EB:
3144                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3145                         break;
3146                 case ixgbe_mac_82599EB:
3147                 case ixgbe_mac_X540:
3148                 default:
3149                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3150                         break;
3151                 }
3152
3153         }
3154
3155         if (hw->mac.type != ixgbe_mac_82598EB) {
3156                 u32 dmatxctl, rttdcs;
3157                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3158                 dmatxctl |= IXGBE_DMATXCTL_TE;
3159                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3160                 /* Disable arbiter to set MTQC */
3161                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3162                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3163                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3164                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3165                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3166                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3167         }
3168
3169         return;
3170 }
3171
3172 /*********************************************************************
3173  *
3174  *  Free all transmit rings.
3175  *
3176  **********************************************************************/
3177 static void
3178 ixgbe_free_transmit_structures(struct adapter *adapter)
3179 {
3180         struct tx_ring *txr = adapter->tx_rings;
3181
3182         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3183                 IXGBE_TX_LOCK(txr);
3184                 ixgbe_free_transmit_buffers(txr);
3185                 ixgbe_dma_free(adapter, &txr->txdma);
3186                 IXGBE_TX_UNLOCK(txr);
3187                 IXGBE_TX_LOCK_DESTROY(txr);
3188         }
3189         free(adapter->tx_rings, M_DEVBUF);
3190 }
3191
3192 /*********************************************************************
3193  *
3194  *  Free transmit ring related data structures.
3195  *
3196  **********************************************************************/
3197 static void
3198 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3199 {
3200         struct adapter *adapter = txr->adapter;
3201         struct ixgbe_tx_buf *tx_buffer;
3202         int             i;
3203
3204         INIT_DEBUGOUT("free_transmit_ring: begin");
3205
3206         if (txr->tx_buffers == NULL)
3207                 return;
3208
3209         tx_buffer = txr->tx_buffers;
3210         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3211                 if (tx_buffer->m_head != NULL) {
3212                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3213                             BUS_DMASYNC_POSTWRITE);
3214                         bus_dmamap_unload(txr->txtag,
3215                             tx_buffer->map);
3216                         m_freem(tx_buffer->m_head);
3217                         tx_buffer->m_head = NULL;
3218                         if (tx_buffer->map != NULL) {
3219                                 bus_dmamap_destroy(txr->txtag,
3220                                     tx_buffer->map);
3221                                 tx_buffer->map = NULL;
3222                         }
3223                 } else if (tx_buffer->map != NULL) {
3224                         bus_dmamap_unload(txr->txtag,
3225                             tx_buffer->map);
3226                         bus_dmamap_destroy(txr->txtag,
3227                             tx_buffer->map);
3228                         tx_buffer->map = NULL;
3229                 }
3230         }
3231 #ifdef IXGBE_LEGACY_TX
3232         if (txr->br != NULL)
3233                 buf_ring_free(txr->br, M_DEVBUF);
3234 #endif
3235         if (txr->tx_buffers != NULL) {
3236                 free(txr->tx_buffers, M_DEVBUF);
3237                 txr->tx_buffers = NULL;
3238         }
3239         if (txr->txtag != NULL) {
3240                 bus_dma_tag_destroy(txr->txtag);
3241                 txr->txtag = NULL;
3242         }
3243         return;
3244 }
3245
3246 /*********************************************************************
3247  *
3248  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
3249  *
3250  **********************************************************************/
3251
3252 static int
3253 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
3254     u32 *cmd_type_len, u32 *olinfo_status)
3255 {
3256         struct ixgbe_adv_tx_context_desc *TXD;
3257         struct ether_vlan_header *eh;
3258         struct ip *ip;
3259         struct ip6_hdr *ip6;
3260         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3261         int     ehdrlen, ip_hlen = 0;
3262         u16     etype;
3263         u8      ipproto = 0;
3264         int     offload = TRUE;
3265         int     ctxd = txr->next_avail_desc;
3266         u16     vtag = 0;
3267
3268         /* First check if TSO is to be used */
3269         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
3270                 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
3271
3272         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3273                 offload = FALSE;
3274
3275         /* Indicate the whole packet as payload when not doing TSO */
3276         *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
3277
3278         /* Now ready a context descriptor */
3279         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3280
3281         /*
3282         ** In advanced descriptors the vlan tag must
3283         ** be placed into the context descriptor. Hence
3284         ** we need to make one even if not doing offloads.
3285         */
3286         if (mp->m_flags & M_VLANTAG) {
3287                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3288                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3289         } else if (offload == FALSE) /* ... no offload to do */
3290                 return (0);
3291
3292         /*
3293          * Determine where frame payload starts.
3294          * Jump over vlan headers if already present,
3295          * helpful for QinQ too.
3296          */
3297         eh = mtod(mp, struct ether_vlan_header *);
3298         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3299                 etype = ntohs(eh->evl_proto);
3300                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3301         } else {
3302                 etype = ntohs(eh->evl_encap_proto);
3303                 ehdrlen = ETHER_HDR_LEN;
3304         }
3305
3306         /* Set the ether header length */
3307         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3308
3309         switch (etype) {
3310                 case ETHERTYPE_IP:
3311                         ip = (struct ip *)(mp->m_data + ehdrlen);
3312                         ip_hlen = ip->ip_hl << 2;
3313                         ipproto = ip->ip_p;
3314                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3315                         break;
3316                 case ETHERTYPE_IPV6:
3317                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3318                         ip_hlen = sizeof(struct ip6_hdr);
3319                         /* XXX-BZ this will go badly in case of ext hdrs. */
3320                         ipproto = ip6->ip6_nxt;
3321                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3322                         break;
3323                 default:
3324                         offload = FALSE;
3325                         break;
3326         }
3327
3328         vlan_macip_lens |= ip_hlen;
3329         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3330
3331         switch (ipproto) {
3332                 case IPPROTO_TCP:
3333                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3334                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3335                         break;
3336
3337                 case IPPROTO_UDP:
3338                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3339                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3340                         break;
3341
3342 #if __FreeBSD_version >= 800000
3343                 case IPPROTO_SCTP:
3344                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3345                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3346                         break;
3347 #endif
3348                 default:
3349                         offload = FALSE;
3350                         break;
3351         }
3352
3353         if (offload) /* For the TX descriptor setup */
3354                 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3355
3356         /* Now copy bits into descriptor */
3357         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3358         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3359         TXD->seqnum_seed = htole32(0);
3360         TXD->mss_l4len_idx = htole32(0);
3361
3362         /* We've consumed the first desc, adjust counters */
3363         if (++ctxd == txr->num_desc)
3364                 ctxd = 0;
3365         txr->next_avail_desc = ctxd;
3366         --txr->tx_avail;
3367
3368         return (0);
3369 }
3370
3371 /**********************************************************************
3372  *
3373  *  Setup work for hardware segmentation offload (TSO) on
3374  *  adapters using advanced tx descriptors
3375  *
3376  **********************************************************************/
3377 static int
3378 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
3379     u32 *cmd_type_len, u32 *olinfo_status)
3380 {
3381         struct ixgbe_adv_tx_context_desc *TXD;
3382         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3383         u32 mss_l4len_idx = 0, paylen;
3384         u16 vtag = 0, eh_type;
3385         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3386         struct ether_vlan_header *eh;
3387 #ifdef INET6
3388         struct ip6_hdr *ip6;
3389 #endif
3390 #ifdef INET
3391         struct ip *ip;
3392 #endif
3393         struct tcphdr *th;
3394
3395
3396         /*
3397          * Determine where frame payload starts.
3398          * Jump over vlan headers if already present
3399          */
3400         eh = mtod(mp, struct ether_vlan_header *);
3401         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3402                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3403                 eh_type = eh->evl_proto;
3404         } else {
3405                 ehdrlen = ETHER_HDR_LEN;
3406                 eh_type = eh->evl_encap_proto;
3407         }
3408
3409         switch (ntohs(eh_type)) {
3410 #ifdef INET6
3411         case ETHERTYPE_IPV6:
3412                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3413                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3414                 if (ip6->ip6_nxt != IPPROTO_TCP)
3415                         return (ENXIO);
3416                 ip_hlen = sizeof(struct ip6_hdr);
3417                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3418                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3419                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3420                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3421                 break;
3422 #endif
3423 #ifdef INET
3424         case ETHERTYPE_IP:
3425                 ip = (struct ip *)(mp->m_data + ehdrlen);
3426                 if (ip->ip_p != IPPROTO_TCP)
3427                         return (ENXIO);
3428                 ip->ip_sum = 0;
3429                 ip_hlen = ip->ip_hl << 2;
3430                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3431                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3432                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3433                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3434                 /* Tell transmit desc to also do IPv4 checksum. */
3435                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3436                 break;
3437 #endif
3438         default:
3439                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3440                     __func__, ntohs(eh_type));
3441                 break;
3442         }
3443
3444         ctxd = txr->next_avail_desc;
3445         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3446
3447         tcp_hlen = th->th_off << 2;
3448
3449         /* This is used in the transmit desc in encap */
3450         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3451
3452         /* VLAN MACLEN IPLEN */
3453         if (mp->m_flags & M_VLANTAG) {
3454                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3455                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3456         }
3457
3458         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3459         vlan_macip_lens |= ip_hlen;
3460         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
3461
3462         /* ADV DTYPE TUCMD */
3463         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3464         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3465         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
3466
3467         /* MSS L4LEN IDX */
3468         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3469         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3470         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3471
3472         TXD->seqnum_seed = htole32(0);
3473
3474         if (++ctxd == txr->num_desc)
3475                 ctxd = 0;
3476
3477         txr->tx_avail--;
3478         txr->next_avail_desc = ctxd;
3479         *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3480         *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
3481         *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
3482         ++txr->tso_tx;
3483         return (0);
3484 }
3485
3486 #ifdef IXGBE_FDIR
3487 /*
3488 ** This routine parses packet headers so that Flow
3489 ** Director can make a hashed filter table entry
3490 ** allowing traffic flows to be identified and kept
3491 ** on the same cpu.  This would be a performance
3492 ** hit, but we only do it at IXGBE_FDIR_RATE of
3493 ** packets.
3494 */
3495 static void
3496 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3497 {
3498         struct adapter                  *adapter = txr->adapter;
3499         struct ix_queue                 *que;
3500         struct ip                       *ip;
3501         struct tcphdr                   *th;
3502         struct udphdr                   *uh;
3503         struct ether_vlan_header        *eh;
3504         union ixgbe_atr_hash_dword      input = {.dword = 0};
3505         union ixgbe_atr_hash_dword      common = {.dword = 0};
3506         int                             ehdrlen, ip_hlen;
3507         u16                             etype;
3508
3509         eh = mtod(mp, struct ether_vlan_header *);
3510         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3511                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3512                 etype = eh->evl_proto;
3513         } else {
3514                 ehdrlen = ETHER_HDR_LEN;
3515                 etype = eh->evl_encap_proto;
3516         }
3517
3518         /* Only handling IPv4 */
3519         if (etype != htons(ETHERTYPE_IP))
3520                 return;
3521
3522         ip = (struct ip *)(mp->m_data + ehdrlen);
3523         ip_hlen = ip->ip_hl << 2;
3524
3525         /* check if we're UDP or TCP */
3526         switch (ip->ip_p) {
3527         case IPPROTO_TCP:
3528                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3529                 /* src and dst are inverted */
3530                 common.port.dst ^= th->th_sport;
3531                 common.port.src ^= th->th_dport;
3532                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3533                 break;
3534         case IPPROTO_UDP:
3535                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3536                 /* src and dst are inverted */
3537                 common.port.dst ^= uh->uh_sport;
3538                 common.port.src ^= uh->uh_dport;
3539                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3540                 break;
3541         default:
3542                 return;
3543         }
3544
3545         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3546         if (mp->m_pkthdr.ether_vtag)
3547                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3548         else
3549                 common.flex_bytes ^= etype;
3550         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3551
3552         que = &adapter->queues[txr->me];
3553         /*
3554         ** This assumes the Rx queue and Tx
3555         ** queue are bound to the same CPU
3556         */
3557         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3558             input, common, que->msix);
3559 }
3560 #endif /* IXGBE_FDIR */
3561
3562 /**********************************************************************
3563  *
3564  *  Examine each tx_buffer in the used queue. If the hardware is done
3565  *  processing the packet then free associated resources. The
3566  *  tx_buffer is put back on the free queue.
3567  *
3568  **********************************************************************/
3569 static void
3570 ixgbe_txeof(struct tx_ring *txr)
3571 {
3572         struct adapter          *adapter = txr->adapter;
3573         struct ifnet            *ifp = adapter->ifp;
3574         u32                     work, processed = 0;
3575         u16                     limit = txr->process_limit;
3576         struct ixgbe_tx_buf     *buf;
3577         union ixgbe_adv_tx_desc *txd;
3578
3579         mtx_assert(&txr->tx_mtx, MA_OWNED);
3580
3581 #ifdef DEV_NETMAP
3582         if (ifp->if_capenable & IFCAP_NETMAP) {
3583                 struct netmap_adapter *na = NA(ifp);
3584                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3585                 txd = txr->tx_base;
3586                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3587                     BUS_DMASYNC_POSTREAD);
3588                 /*
3589                  * In netmap mode, all the work is done in the context
3590                  * of the client thread. Interrupt handlers only wake up
3591                  * clients, which may be sleeping on individual rings
3592                  * or on a global resource for all rings.
3593                  * To implement tx interrupt mitigation, we wake up the client
3594                  * thread roughly every half ring, even if the NIC interrupts
3595                  * more frequently. This is implemented as follows:
3596                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3597                  *   the slot that should wake up the thread (nkr_num_slots
3598                  *   means the user thread should not be woken up);
3599                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3600                  *   or the slot has the DD bit set.
3601                  *
3602                  * When the driver has separate locks, we need to
3603                  * release and re-acquire txlock to avoid deadlocks.
3604                  * XXX see if we can find a better way.
3605                  */
3606                 if (!netmap_mitigate ||
3607                     (kring->nr_kflags < kring->nkr_num_slots &&
3608                     txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
3609                         netmap_tx_irq(ifp, txr->me |
3610                             (NETMAP_LOCKED_ENTER|NETMAP_LOCKED_EXIT));
3611                 }
3612                 return;
3613         }
3614 #endif /* DEV_NETMAP */
3615
3616         if (txr->tx_avail == txr->num_desc) {
3617                 txr->queue_status = IXGBE_QUEUE_IDLE;
3618                 return;
3619         }
3620
3621         /* Get work starting point */
3622         work = txr->next_to_clean;
3623         buf = &txr->tx_buffers[work];
3624         txd = &txr->tx_base[work];
3625         work -= txr->num_desc; /* The distance to ring end */
3626         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3627             BUS_DMASYNC_POSTREAD);
3628
3629         do {
3630                 union ixgbe_adv_tx_desc *eop= buf->eop;
3631                 if (eop == NULL) /* No work */
3632                         break;
3633
3634                 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
3635                         break;  /* I/O not complete */
3636
3637                 if (buf->m_head) {
3638                         txr->bytes +=
3639                             buf->m_head->m_pkthdr.len;
3640                         bus_dmamap_sync(txr->txtag,
3641                             buf->map,
3642                             BUS_DMASYNC_POSTWRITE);
3643                         bus_dmamap_unload(txr->txtag,
3644                             buf->map);
3645                         m_freem(buf->m_head);
3646                         buf->m_head = NULL;
3647                         buf->map = NULL;
3648                 }
3649                 buf->eop = NULL;
3650                 ++txr->tx_avail;
3651
3652                 /* We clean the range if multi segment */
3653                 while (txd != eop) {
3654                         ++txd;
3655                         ++buf;
3656                         ++work;
3657                         /* wrap the ring? */
3658                         if (__predict_false(!work)) {
3659                                 work -= txr->num_desc;
3660                                 buf = txr->tx_buffers;
3661                                 txd = txr->tx_base;
3662                         }
3663                         if (buf->m_head) {
3664                                 txr->bytes +=
3665                                     buf->m_head->m_pkthdr.len;
3666                                 bus_dmamap_sync(txr->txtag,
3667                                     buf->map,
3668                                     BUS_DMASYNC_POSTWRITE);
3669                                 bus_dmamap_unload(txr->txtag,
3670                                     buf->map);
3671                                 m_freem(buf->m_head);
3672                                 buf->m_head = NULL;
3673                                 buf->map = NULL;
3674                         }
3675                         ++txr->tx_avail;
3676                         buf->eop = NULL;
3677
3678                 }
3679                 ++txr->packets;
3680                 ++processed;
3681                 ++ifp->if_opackets;
3682                 txr->watchdog_time = ticks;
3683
3684                 /* Try the next packet */
3685                 ++txd;
3686                 ++buf;
3687                 ++work;
3688                 /* reset with a wrap */
3689                 if (__predict_false(!work)) {
3690                         work -= txr->num_desc;
3691                         buf = txr->tx_buffers;
3692                         txd = txr->tx_base;
3693                 }
3694                 prefetch(txd);
3695         } while (__predict_true(--limit));
3696
3697         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3698             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3699
3700         work += txr->num_desc;
3701         txr->next_to_clean = work;
3702
3703         /*
3704         ** Watchdog calculation, we know there's
3705         ** work outstanding or the first return
3706         ** would have been taken, so none processed
3707         ** for too long indicates a hang.
3708         */
3709         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3710                 txr->queue_status = IXGBE_QUEUE_HUNG;
3711
3712         if (txr->tx_avail == txr->num_desc)
3713                 txr->queue_status = IXGBE_QUEUE_IDLE;
3714
3715         return;
3716 }
3717
3718 /*********************************************************************
3719  *
3720  *  Refresh mbuf buffers for RX descriptor rings
3721  *   - now keeps its own state so discards due to resource
3722  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3723  *     it just returns, keeping its placeholder, thus it can simply
3724  *     be recalled to try again.
3725  *
3726  **********************************************************************/
3727 static void
3728 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3729 {
3730         struct adapter          *adapter = rxr->adapter;
3731         bus_dma_segment_t       seg[1];
3732         struct ixgbe_rx_buf     *rxbuf;
3733         struct mbuf             *mp;
3734         int                     i, j, nsegs, error;
3735         bool                    refreshed = FALSE;
3736
3737         i = j = rxr->next_to_refresh;
3738         /* Control the loop with one beyond */
3739         if (++j == rxr->num_desc)
3740                 j = 0;
3741
3742         while (j != limit) {
3743                 rxbuf = &rxr->rx_buffers[i];
3744                 if (rxbuf->buf == NULL) {
3745                         mp = m_getjcl(M_NOWAIT, MT_DATA,
3746                             M_PKTHDR, rxr->mbuf_sz);
3747                         if (mp == NULL)
3748                                 goto update;
3749                         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3750                                 m_adj(mp, ETHER_ALIGN);
3751                 } else
3752                         mp = rxbuf->buf;
3753
3754                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
3755
3756                 /* If we're dealing with an mbuf that was copied rather
3757                  * than replaced, there's no need to go through busdma.
3758                  */
3759                 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3760                         /* Get the memory mapping */
3761                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3762                             rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
3763                         if (error != 0) {
3764                                 printf("Refresh mbufs: payload dmamap load"
3765                                     " failure - %d\n", error);
3766                                 m_free(mp);
3767                                 rxbuf->buf = NULL;
3768                                 goto update;
3769                         }
3770                         rxbuf->buf = mp;
3771                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3772                             BUS_DMASYNC_PREREAD);
3773                         rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
3774                             htole64(seg[0].ds_addr);
3775                 } else {
3776                         rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
3777                         rxbuf->flags &= ~IXGBE_RX_COPY;
3778                 }
3779
3780                 refreshed = TRUE;
3781                 /* Next is precalculated */
3782                 i = j;
3783                 rxr->next_to_refresh = i;
3784                 if (++j == rxr->num_desc)
3785                         j = 0;
3786         }
3787 update:
3788         if (refreshed) /* Update hardware tail index */
3789                 IXGBE_WRITE_REG(&adapter->hw,
3790                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3791         return;
3792 }
3793
3794 /*********************************************************************
3795  *
3796  *  Allocate memory for rx_buffer structures. Since we use one
3797  *  rx_buffer per received packet, the maximum number of rx_buffer's
3798  *  that we'll need is equal to the number of receive descriptors
3799  *  that we've allocated.
3800  *
3801  **********************************************************************/
3802 static int
3803 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3804 {
3805         struct  adapter         *adapter = rxr->adapter;
3806         device_t                dev = adapter->dev;
3807         struct ixgbe_rx_buf     *rxbuf;
3808         int                     i, bsize, error;
3809
3810         bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
3811         if (!(rxr->rx_buffers =
3812             (struct ixgbe_rx_buf *) malloc(bsize,
3813             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3814                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3815                 error = ENOMEM;
3816                 goto fail;
3817         }
3818
3819         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3820                                    1, 0,        /* alignment, bounds */
3821                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3822                                    BUS_SPACE_MAXADDR,   /* highaddr */
3823                                    NULL, NULL,          /* filter, filterarg */
3824                                    MJUM16BYTES,         /* maxsize */
3825                                    1,                   /* nsegments */
3826                                    MJUM16BYTES,         /* maxsegsize */
3827                                    0,                   /* flags */
3828                                    NULL,                /* lockfunc */
3829                                    NULL,                /* lockfuncarg */
3830                                    &rxr->ptag))) {
3831                 device_printf(dev, "Unable to create RX DMA tag\n");
3832                 goto fail;
3833         }
3834
3835         for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
3836                 rxbuf = &rxr->rx_buffers[i];
3837                 error = bus_dmamap_create(rxr->ptag,
3838                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3839                 if (error) {
3840                         device_printf(dev, "Unable to create RX dma map\n");
3841                         goto fail;
3842                 }
3843         }
3844
3845         return (0);
3846
3847 fail:
3848         /* Frees all, but can handle partial completion */
3849         ixgbe_free_receive_structures(adapter);
3850         return (error);
3851 }
3852
3853 /*
3854 ** Used to detect a descriptor that has
3855 ** been merged by Hardware RSC.
3856 */
3857 static inline u32
3858 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3859 {
3860         return (le32toh(rx->wb.lower.lo_dword.data) &
3861             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3862 }
3863
3864 /*********************************************************************
3865  *
3866  *  Initialize Hardware RSC (LRO) feature on 82599
3867  *  for an RX ring, this is toggled by the LRO capability
3868  *  even though it is transparent to the stack.
3869  *
3870  *  NOTE: since this HW feature only works with IPV4 and
3871  *        our testing has shown soft LRO to be as effective
3872  *        I have decided to disable this by default.
3873  *
3874  **********************************************************************/
3875 static void
3876 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3877 {
3878         struct  adapter         *adapter = rxr->adapter;
3879         struct  ixgbe_hw        *hw = &adapter->hw;
3880         u32                     rscctrl, rdrxctl;
3881
3882         /* If turning LRO/RSC off we need to disable it */
3883         if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
3884                 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3885                 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
3886                 return;
3887         }
3888
3889         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3890         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3891 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3892         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3893 #endif /* DEV_NETMAP */
3894         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3895         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3896         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3897
3898         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3899         rscctrl |= IXGBE_RSCCTL_RSCEN;
3900         /*
3901         ** Limit the total number of descriptors that
3902         ** can be combined, so it does not exceed 64K
3903         */
3904         if (rxr->mbuf_sz == MCLBYTES)
3905                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3906         else if (rxr->mbuf_sz == MJUMPAGESIZE)
3907                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3908         else if (rxr->mbuf_sz == MJUM9BYTES)
3909                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3910         else  /* Using 16K cluster */
3911                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3912
3913         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3914
3915         /* Enable TCP header recognition */
3916         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3917             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3918             IXGBE_PSRTYPE_TCPHDR));
3919
3920         /* Disable RSC for ACK packets */
3921         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3922             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3923
3924         rxr->hw_rsc = TRUE;
3925 }
3926
3927
3928 static void
3929 ixgbe_free_receive_ring(struct rx_ring *rxr)
3930 {
3931         struct ixgbe_rx_buf       *rxbuf;
3932         int i;
3933
3934         for (i = 0; i < rxr->num_desc; i++) {
3935                 rxbuf = &rxr->rx_buffers[i];
3936                 if (rxbuf->buf != NULL) {
3937                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3938                             BUS_DMASYNC_POSTREAD);
3939                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3940                         rxbuf->buf->m_flags |= M_PKTHDR;
3941                         m_freem(rxbuf->buf);
3942                         rxbuf->buf = NULL;
3943                 }
3944         }
3945 }
3946
3947
3948 /*********************************************************************
3949  *
3950  *  Initialize a receive ring and its buffers.
3951  *
3952  **********************************************************************/
3953 static int
3954 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3955 {
3956         struct  adapter         *adapter;
3957         struct ifnet            *ifp;
3958         device_t                dev;
3959         struct ixgbe_rx_buf     *rxbuf;
3960         bus_dma_segment_t       seg[1];
3961         struct lro_ctrl         *lro = &rxr->lro;
3962         int                     rsize, nsegs, error = 0;
3963 #ifdef DEV_NETMAP
3964         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3965         struct netmap_slot *slot;
3966 #endif /* DEV_NETMAP */
3967
3968         adapter = rxr->adapter;
3969         ifp = adapter->ifp;
3970         dev = adapter->dev;
3971
3972         /* Clear the ring contents */
3973         IXGBE_RX_LOCK(rxr);
3974 #ifdef DEV_NETMAP
3975         /* same as in ixgbe_setup_transmit_ring() */
3976         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3977 #endif /* DEV_NETMAP */
3978         rsize = roundup2(adapter->num_rx_desc *
3979             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3980         bzero((void *)rxr->rx_base, rsize);
3981         /* Cache the size */
3982         rxr->mbuf_sz = adapter->rx_mbuf_sz;
3983
3984         /* Free current RX buffer structs and their mbufs */
3985         ixgbe_free_receive_ring(rxr);
3986
3987         /* Now replenish the mbufs */
3988         for (int j = 0; j != rxr->num_desc; ++j) {
3989                 struct mbuf     *mp;
3990
3991                 rxbuf = &rxr->rx_buffers[j];
3992 #ifdef DEV_NETMAP
3993                 /*
3994                  * In netmap mode, fill the map and set the buffer
3995                  * address in the NIC ring, considering the offset
3996                  * between the netmap and NIC rings (see comment in
3997                  * ixgbe_setup_transmit_ring() ). No need to allocate
3998                  * an mbuf, so end the block with a continue;
3999                  */
4000                 if (slot) {
4001                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4002                         uint64_t paddr;
4003                         void *addr;
4004
4005                         addr = PNMB(slot + sj, &paddr);
4006                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4007                         /* Update descriptor */
4008                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4009                         continue;
4010                 }
4011 #endif /* DEV_NETMAP */
4012                 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
4013                     M_PKTHDR, adapter->rx_mbuf_sz);
4014                 if (rxbuf->buf == NULL) {
4015                         error = ENOBUFS;
4016                         goto fail;
4017                 }
4018                 mp = rxbuf->buf;
4019                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
4020                 /* Get the memory mapping */
4021                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4022                     rxbuf->pmap, mp, seg,
4023                     &nsegs, BUS_DMA_NOWAIT);
4024                 if (error != 0)
4025                         goto fail;
4026                 bus_dmamap_sync(rxr->ptag,
4027                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4028                 /* Update descriptor */
4029                 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
4030         }
4031
4032
4033         /* Setup our descriptor indices */
4034         rxr->next_to_check = 0;
4035         rxr->next_to_refresh = 0;
4036         rxr->lro_enabled = FALSE;
4037         rxr->rx_copies = 0;
4038         rxr->rx_bytes = 0;
4039         rxr->discard = FALSE;
4040         rxr->vtag_strip = FALSE;
4041
4042         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4043             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4044
4045         /*
4046         ** Now set up the LRO interface:
4047         */
4048         if (ixgbe_rsc_enable)
4049                 ixgbe_setup_hw_rsc(rxr);
4050         else if (ifp->if_capenable & IFCAP_LRO) {
4051                 int err = tcp_lro_init(lro);
4052                 if (err) {
4053                         device_printf(dev, "LRO Initialization failed!\n");
4054                         goto fail;
4055                 }
4056                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4057                 rxr->lro_enabled = TRUE;
4058                 lro->ifp = adapter->ifp;
4059         }
4060
4061         IXGBE_RX_UNLOCK(rxr);
4062         return (0);
4063
4064 fail:
4065         ixgbe_free_receive_ring(rxr);
4066         IXGBE_RX_UNLOCK(rxr);
4067         return (error);
4068 }
4069
4070 /*********************************************************************
4071  *
4072  *  Initialize all receive rings.
4073  *
4074  **********************************************************************/
4075 static int
4076 ixgbe_setup_receive_structures(struct adapter *adapter)
4077 {
4078         struct rx_ring *rxr = adapter->rx_rings;
4079         int j;
4080
4081         for (j = 0; j < adapter->num_queues; j++, rxr++)
4082                 if (ixgbe_setup_receive_ring(rxr))
4083                         goto fail;
4084
4085         return (0);
4086 fail:
4087         /*
4088          * Free RX buffers allocated so far, we will only handle
4089          * the rings that completed, the failing case will have
4090          * cleaned up for itself. 'j' failed, so its the terminus.
4091          */
4092         for (int i = 0; i < j; ++i) {
4093                 rxr = &adapter->rx_rings[i];
4094                 ixgbe_free_receive_ring(rxr);
4095         }
4096
4097         return (ENOBUFS);
4098 }
4099
4100 /*********************************************************************
4101  *
4102  *  Setup receive registers and features.
4103  *
4104  **********************************************************************/
4105 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4106
4107 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4108
4109 static void
4110 ixgbe_initialize_receive_units(struct adapter *adapter)
4111 {
4112         struct  rx_ring *rxr = adapter->rx_rings;
4113         struct ixgbe_hw *hw = &adapter->hw;
4114         struct ifnet   *ifp = adapter->ifp;
4115         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4116         u32             reta, mrqc = 0, hlreg, random[10];
4117
4118
4119         /*
4120          * Make sure receives are disabled while
4121          * setting up the descriptor ring
4122          */
4123         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4124         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4125             rxctrl & ~IXGBE_RXCTRL_RXEN);
4126
4127         /* Enable broadcasts */
4128         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4129         fctrl |= IXGBE_FCTRL_BAM;
4130         fctrl |= IXGBE_FCTRL_DPF;
4131         fctrl |= IXGBE_FCTRL_PMCF;
4132         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4133
4134         /* Set for Jumbo Frames? */
4135         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4136         if (ifp->if_mtu > ETHERMTU)
4137                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4138         else
4139                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4140 #ifdef DEV_NETMAP
4141         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4142         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4143                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4144         else
4145                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4146 #endif /* DEV_NETMAP */
4147         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4148
4149         bufsz = (adapter->rx_mbuf_sz +
4150             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4151
4152         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4153                 u64 rdba = rxr->rxdma.dma_paddr;
4154
4155                 /* Setup the Base and Length of the Rx Descriptor Ring */
4156                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4157                                (rdba & 0x00000000ffffffffULL));
4158                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4159                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4160                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4161
4162                 /* Set up the SRRCTL register */
4163                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4164                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4165                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4166                 srrctl |= bufsz;
4167                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4168                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4169
4170                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4171                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4172                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4173
4174                 /* Set the processing limit */
4175                 rxr->process_limit = ixgbe_rx_process_limit;
4176         }
4177
4178         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4179                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4180                               IXGBE_PSRTYPE_UDPHDR |
4181                               IXGBE_PSRTYPE_IPV4HDR |
4182                               IXGBE_PSRTYPE_IPV6HDR;
4183                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4184         }
4185
4186         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4187
4188         /* Setup RSS */
4189         if (adapter->num_queues > 1) {
4190                 int i, j;
4191                 reta = 0;
4192
4193                 /* set up random bits */
4194                 arc4rand(&random, sizeof(random), 0);
4195
4196                 /* Set up the redirection table */
4197                 for (i = 0, j = 0; i < 128; i++, j++) {
4198                         if (j == adapter->num_queues) j = 0;
4199                         reta = (reta << 8) | (j * 0x11);
4200                         if ((i & 3) == 3)
4201                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4202                 }
4203
4204                 /* Now fill our hash function seeds */
4205                 for (int i = 0; i < 10; i++)
4206                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4207
4208                 /* Perform hash on these packet types */
4209                 mrqc = IXGBE_MRQC_RSSEN
4210                      | IXGBE_MRQC_RSS_FIELD_IPV4
4211                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4212                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4213                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4214                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4215                      | IXGBE_MRQC_RSS_FIELD_IPV6
4216                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4217                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4218                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4219                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4220
4221                 /* RSS and RX IPP Checksum are mutually exclusive */
4222                 rxcsum |= IXGBE_RXCSUM_PCSD;
4223         }
4224
4225         if (ifp->if_capenable & IFCAP_RXCSUM)
4226                 rxcsum |= IXGBE_RXCSUM_PCSD;
4227
4228         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4229                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4230
4231         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4232
4233         return;
4234 }
4235
4236 /*********************************************************************
4237  *
4238  *  Free all receive rings.
4239  *
4240  **********************************************************************/
4241 static void
4242 ixgbe_free_receive_structures(struct adapter *adapter)
4243 {
4244         struct rx_ring *rxr = adapter->rx_rings;
4245
4246         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4247                 struct lro_ctrl         *lro = &rxr->lro;
4248                 ixgbe_free_receive_buffers(rxr);
4249                 /* Free LRO memory */
4250                 tcp_lro_free(lro);
4251                 /* Free the ring memory as well */
4252                 ixgbe_dma_free(adapter, &rxr->rxdma);
4253         }
4254
4255         free(adapter->rx_rings, M_DEVBUF);
4256 }
4257
4258
4259 /*********************************************************************
4260  *
4261  *  Free receive ring data structures
4262  *
4263  **********************************************************************/
4264 static void
4265 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4266 {
4267         struct adapter          *adapter = rxr->adapter;
4268         struct ixgbe_rx_buf     *rxbuf;
4269
4270         INIT_DEBUGOUT("free_receive_structures: begin");
4271
4272         /* Cleanup any existing buffers */
4273         if (rxr->rx_buffers != NULL) {
4274                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4275                         rxbuf = &rxr->rx_buffers[i];
4276                         if (rxbuf->buf != NULL) {
4277                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4278                                     BUS_DMASYNC_POSTREAD);
4279                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4280                                 rxbuf->buf->m_flags |= M_PKTHDR;
4281                                 m_freem(rxbuf->buf);
4282                         }
4283                         rxbuf->buf = NULL;
4284                         if (rxbuf->pmap != NULL) {
4285                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4286                                 rxbuf->pmap = NULL;
4287                         }
4288                 }
4289                 if (rxr->rx_buffers != NULL) {
4290                         free(rxr->rx_buffers, M_DEVBUF);
4291                         rxr->rx_buffers = NULL;
4292                 }
4293         }
4294
4295         if (rxr->ptag != NULL) {
4296                 bus_dma_tag_destroy(rxr->ptag);
4297                 rxr->ptag = NULL;
4298         }
4299
4300         return;
4301 }
4302
4303 static __inline void
4304 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4305 {
4306
4307         /*
4308          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4309          * should be computed by hardware. Also it should not have VLAN tag in
4310          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4311          */
4312         if (rxr->lro_enabled &&
4313             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4314             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4315             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4316             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4317             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4318             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4319             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4320             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4321                 /*
4322                  * Send to the stack if:
4323                  **  - LRO not enabled, or
4324                  **  - no LRO resources, or
4325                  **  - lro enqueue fails
4326                  */
4327                 if (rxr->lro.lro_cnt != 0)
4328                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4329                                 return;
4330         }
4331         IXGBE_RX_UNLOCK(rxr);
4332         (*ifp->if_input)(ifp, m);
4333         IXGBE_RX_LOCK(rxr);
4334 }
4335
4336 static __inline void
4337 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4338 {
4339         struct ixgbe_rx_buf     *rbuf;
4340
4341         rbuf = &rxr->rx_buffers[i];
4342
4343         if (rbuf->fmp != NULL) {/* Partial chain ? */
4344                 rbuf->fmp->m_flags |= M_PKTHDR;
4345                 m_freem(rbuf->fmp);
4346                 rbuf->fmp = NULL;
4347         }
4348
4349         /*
4350         ** With advanced descriptors the writeback
4351         ** clobbers the buffer addrs, so its easier
4352         ** to just free the existing mbufs and take
4353         ** the normal refresh path to get new buffers
4354         ** and mapping.
4355         */
4356         if (rbuf->buf) {
4357                 m_free(rbuf->buf);
4358                 rbuf->buf = NULL;
4359         }
4360
4361         return;
4362 }
4363
4364
4365 /*********************************************************************
4366  *
4367  *  This routine executes in interrupt context. It replenishes
4368  *  the mbufs in the descriptor and sends data which has been
4369  *  dma'ed into host memory to upper layer.
4370  *
4371  *  We loop at most count times if count is > 0, or until done if
4372  *  count < 0.
4373  *
4374  *  Return TRUE for more work, FALSE for all clean.
4375  *********************************************************************/
4376 static bool
4377 ixgbe_rxeof(struct ix_queue *que)
4378 {
4379         struct adapter          *adapter = que->adapter;
4380         struct rx_ring          *rxr = que->rxr;
4381         struct ifnet            *ifp = adapter->ifp;
4382         struct lro_ctrl         *lro = &rxr->lro;
4383         struct lro_entry        *queued;
4384         int                     i, nextp, processed = 0;
4385         u32                     staterr = 0;
4386         u16                     count = rxr->process_limit;
4387         union ixgbe_adv_rx_desc *cur;
4388         struct ixgbe_rx_buf     *rbuf, *nbuf;
4389
4390         IXGBE_RX_LOCK(rxr);
4391
4392 #ifdef DEV_NETMAP
4393         /* Same as the txeof routine: wakeup clients on intr. */
4394         if (netmap_rx_irq(ifp, rxr->me | NETMAP_LOCKED_ENTER, &processed))
4395                 return (FALSE);
4396 #endif /* DEV_NETMAP */
4397
4398         for (i = rxr->next_to_check; count != 0;) {
4399                 struct mbuf     *sendmp, *mp;
4400                 u32             rsc, ptype;
4401                 u16             len;
4402                 u16             vtag = 0;
4403                 bool            eop;
4404
4405                 /* Sync the ring. */
4406                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4407                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4408
4409                 cur = &rxr->rx_base[i];
4410                 staterr = le32toh(cur->wb.upper.status_error);
4411
4412                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4413                         break;
4414                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4415                         break;
4416
4417                 count--;
4418                 sendmp = NULL;
4419                 nbuf = NULL;
4420                 rsc = 0;
4421                 cur->wb.upper.status_error = 0;
4422                 rbuf = &rxr->rx_buffers[i];
4423                 mp = rbuf->buf;
4424
4425                 len = le16toh(cur->wb.upper.length);
4426                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4427                     IXGBE_RXDADV_PKTTYPE_MASK;
4428                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4429
4430                 /* Make sure bad packets are discarded */
4431                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4432                     (rxr->discard)) {
4433                         rxr->rx_discarded++;
4434                         if (eop)
4435                                 rxr->discard = FALSE;
4436                         else
4437                                 rxr->discard = TRUE;
4438                         ixgbe_rx_discard(rxr, i);
4439                         goto next_desc;
4440                 }
4441
4442                 /*
4443                 ** On 82599 which supports a hardware
4444                 ** LRO (called HW RSC), packets need
4445                 ** not be fragmented across sequential
4446                 ** descriptors, rather the next descriptor
4447                 ** is indicated in bits of the descriptor.
4448                 ** This also means that we might proceses
4449                 ** more than one packet at a time, something
4450                 ** that has never been true before, it
4451                 ** required eliminating global chain pointers
4452                 ** in favor of what we are doing here.  -jfv
4453                 */
4454                 if (!eop) {
4455                         /*
4456                         ** Figure out the next descriptor
4457                         ** of this frame.
4458                         */
4459                         if (rxr->hw_rsc == TRUE) {
4460                                 rsc = ixgbe_rsc_count(cur);
4461                                 rxr->rsc_num += (rsc - 1);
4462                         }
4463                         if (rsc) { /* Get hardware index */
4464                                 nextp = ((staterr &
4465                                     IXGBE_RXDADV_NEXTP_MASK) >>
4466                                     IXGBE_RXDADV_NEXTP_SHIFT);
4467                         } else { /* Just sequential */
4468                                 nextp = i + 1;
4469                                 if (nextp == adapter->num_rx_desc)
4470                                         nextp = 0;
4471                         }
4472                         nbuf = &rxr->rx_buffers[nextp];
4473                         prefetch(nbuf);
4474                 }
4475                 /*
4476                 ** Rather than using the fmp/lmp global pointers
4477                 ** we now keep the head of a packet chain in the
4478                 ** buffer struct and pass this along from one
4479                 ** descriptor to the next, until we get EOP.
4480                 */
4481                 mp->m_len = len;
4482                 /*
4483                 ** See if there is a stored head
4484                 ** that determines what we are
4485                 */
4486                 sendmp = rbuf->fmp;
4487                 if (sendmp != NULL) {  /* secondary frag */
4488                         rbuf->buf = rbuf->fmp = NULL;
4489                         mp->m_flags &= ~M_PKTHDR;
4490                         sendmp->m_pkthdr.len += mp->m_len;
4491                 } else {
4492                         /*
4493                          * Optimize.  This might be a small packet,
4494                          * maybe just a TCP ACK.  Do a fast copy that
4495                          * is cache aligned into a new mbuf, and
4496                          * leave the old mbuf+cluster for re-use.
4497                          */
4498                         if (eop && len <= IXGBE_RX_COPY_LEN) {
4499                                 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
4500                                 if (sendmp != NULL) {
4501                                         sendmp->m_data +=
4502                                             IXGBE_RX_COPY_ALIGN;
4503                                         ixgbe_bcopy(mp->m_data,
4504                                             sendmp->m_data, len);
4505                                         sendmp->m_len = len;
4506                                         rxr->rx_copies++;
4507                                         rbuf->flags |= IXGBE_RX_COPY;
4508                                 }
4509                         }
4510                         if (sendmp == NULL) {
4511                                 rbuf->buf = rbuf->fmp = NULL;
4512                                 sendmp = mp;
4513                         }
4514
4515                         /* first desc of a non-ps chain */
4516                         sendmp->m_flags |= M_PKTHDR;
4517                         sendmp->m_pkthdr.len = mp->m_len;
4518                 }
4519                 ++processed;
4520
4521                 /* Pass the head pointer on */
4522                 if (eop == 0) {
4523                         nbuf->fmp = sendmp;
4524                         sendmp = NULL;
4525                         mp->m_next = nbuf->buf;
4526                 } else { /* Sending this frame */
4527                         sendmp->m_pkthdr.rcvif = ifp;
4528                         ifp->if_ipackets++;
4529                         rxr->rx_packets++;
4530                         /* capture data for AIM */
4531                         rxr->bytes += sendmp->m_pkthdr.len;
4532                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4533                         /* Process vlan info */
4534                         if ((rxr->vtag_strip) &&
4535                             (staterr & IXGBE_RXD_STAT_VP))
4536                                 vtag = le16toh(cur->wb.upper.vlan);
4537                         if (vtag) {
4538                                 sendmp->m_pkthdr.ether_vtag = vtag;
4539                                 sendmp->m_flags |= M_VLANTAG;
4540                         }
4541                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4542                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4543 #if __FreeBSD_version >= 800000
4544                         sendmp->m_pkthdr.flowid = que->msix;
4545                         sendmp->m_flags |= M_FLOWID;
4546 #endif
4547                 }
4548 next_desc:
4549                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4550                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4551
4552                 /* Advance our pointers to the next descriptor. */
4553                 if (++i == rxr->num_desc)
4554                         i = 0;
4555
4556                 /* Now send to the stack or do LRO */
4557                 if (sendmp != NULL) {
4558                         rxr->next_to_check = i;
4559                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4560                         i = rxr->next_to_check;
4561                 }
4562
4563                /* Every 8 descriptors we go to refresh mbufs */
4564                 if (processed == 8) {
4565                         ixgbe_refresh_mbufs(rxr, i);
4566                         processed = 0;
4567                 }
4568         }
4569
4570         /* Refresh any remaining buf structs */
4571         if (ixgbe_rx_unrefreshed(rxr))
4572                 ixgbe_refresh_mbufs(rxr, i);
4573
4574         rxr->next_to_check = i;
4575
4576         /*
4577          * Flush any outstanding LRO work
4578          */
4579         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4580                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4581                 tcp_lro_flush(lro, queued);
4582         }
4583
4584         IXGBE_RX_UNLOCK(rxr);
4585
4586         /*
4587         ** Still have cleaning to do?
4588         */
4589         if ((staterr & IXGBE_RXD_STAT_DD) != 0)
4590                 return (TRUE);
4591         else
4592                 return (FALSE);
4593 }
4594
4595
4596 /*********************************************************************
4597  *
4598  *  Verify that the hardware indicated that the checksum is valid.
4599  *  Inform the stack about the status of checksum so that stack
4600  *  doesn't spend time verifying the checksum.
4601  *
4602  *********************************************************************/
4603 static void
4604 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4605 {
4606         u16     status = (u16) staterr;
4607         u8      errors = (u8) (staterr >> 24);
4608         bool    sctp = FALSE;
4609
4610         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4611             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4612                 sctp = TRUE;
4613
4614         if (status & IXGBE_RXD_STAT_IPCS) {
4615                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4616                         /* IP Checksum Good */
4617                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4618                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4619
4620                 } else
4621                         mp->m_pkthdr.csum_flags = 0;
4622         }
4623         if (status & IXGBE_RXD_STAT_L4CS) {
4624                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4625 #if __FreeBSD_version >= 800000
4626                 if (sctp)
4627                         type = CSUM_SCTP_VALID;
4628 #endif
4629                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4630                         mp->m_pkthdr.csum_flags |= type;
4631                         if (!sctp)
4632                                 mp->m_pkthdr.csum_data = htons(0xffff);
4633                 }
4634         }
4635         return;
4636 }
4637
4638
4639 /*
4640 ** This routine is run via an vlan config EVENT,
4641 ** it enables us to use the HW Filter table since
4642 ** we can get the vlan id. This just creates the
4643 ** entry in the soft version of the VFTA, init will
4644 ** repopulate the real table.
4645 */
4646 static void
4647 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4648 {
4649         struct adapter  *adapter = ifp->if_softc;
4650         u16             index, bit;
4651
4652         if (ifp->if_softc !=  arg)   /* Not our event */
4653                 return;
4654
4655         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4656                 return;
4657
4658         IXGBE_CORE_LOCK(adapter);
4659         index = (vtag >> 5) & 0x7F;
4660         bit = vtag & 0x1F;
4661         adapter->shadow_vfta[index] |= (1 << bit);
4662         ++adapter->num_vlans;
4663         ixgbe_init_locked(adapter);
4664         IXGBE_CORE_UNLOCK(adapter);
4665 }
4666
4667 /*
4668 ** This routine is run via an vlan
4669 ** unconfig EVENT, remove our entry
4670 ** in the soft vfta.
4671 */
4672 static void
4673 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4674 {
4675         struct adapter  *adapter = ifp->if_softc;
4676         u16             index, bit;
4677
4678         if (ifp->if_softc !=  arg)
4679                 return;
4680
4681         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4682                 return;
4683
4684         IXGBE_CORE_LOCK(adapter);
4685         index = (vtag >> 5) & 0x7F;
4686         bit = vtag & 0x1F;
4687         adapter->shadow_vfta[index] &= ~(1 << bit);
4688         --adapter->num_vlans;
4689         /* Re-init to load the changes */
4690         ixgbe_init_locked(adapter);
4691         IXGBE_CORE_UNLOCK(adapter);
4692 }
4693
4694 static void
4695 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4696 {
4697         struct ifnet    *ifp = adapter->ifp;
4698         struct ixgbe_hw *hw = &adapter->hw;
4699         struct rx_ring  *rxr;
4700         u32             ctrl;
4701
4702
4703         /*
4704         ** We get here thru init_locked, meaning
4705         ** a soft reset, this has already cleared
4706         ** the VFTA and other state, so if there
4707         ** have been no vlan's registered do nothing.
4708         */
4709         if (adapter->num_vlans == 0)
4710                 return;
4711
4712         /*
4713         ** A soft reset zero's out the VFTA, so
4714         ** we need to repopulate it now.
4715         */
4716         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4717                 if (adapter->shadow_vfta[i] != 0)
4718                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4719                             adapter->shadow_vfta[i]);
4720
4721         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4722         /* Enable the Filter Table if enabled */
4723         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4724                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4725                 ctrl |= IXGBE_VLNCTRL_VFE;
4726         }
4727         if (hw->mac.type == ixgbe_mac_82598EB)
4728                 ctrl |= IXGBE_VLNCTRL_VME;
4729         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4730
4731         /* Setup the queues for vlans */
4732         for (int i = 0; i < adapter->num_queues; i++) {
4733                 rxr = &adapter->rx_rings[i];
4734                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4735                 if (hw->mac.type != ixgbe_mac_82598EB) {
4736                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4737                         ctrl |= IXGBE_RXDCTL_VME;
4738                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4739                 }
4740                 rxr->vtag_strip = TRUE;
4741         }
4742 }
4743
4744 static void
4745 ixgbe_enable_intr(struct adapter *adapter)
4746 {
4747         struct ixgbe_hw *hw = &adapter->hw;
4748         struct ix_queue *que = adapter->queues;
4749         u32             mask, fwsm;
4750
4751         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4752         /* Enable Fan Failure detection */
4753         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4754                     mask |= IXGBE_EIMS_GPI_SDP1;
4755
4756         switch (adapter->hw.mac.type) {
4757                 case ixgbe_mac_82599EB:
4758                         mask |= IXGBE_EIMS_ECC;
4759                         mask |= IXGBE_EIMS_GPI_SDP0;
4760                         mask |= IXGBE_EIMS_GPI_SDP1;
4761                         mask |= IXGBE_EIMS_GPI_SDP2;
4762 #ifdef IXGBE_FDIR
4763                         mask |= IXGBE_EIMS_FLOW_DIR;
4764 #endif
4765                         break;
4766                 case ixgbe_mac_X540:
4767                         mask |= IXGBE_EIMS_ECC;
4768                         /* Detect if Thermal Sensor is enabled */
4769                         fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
4770                         if (fwsm & IXGBE_FWSM_TS_ENABLED)
4771                                 mask |= IXGBE_EIMS_TS;
4772 #ifdef IXGBE_FDIR
4773                         mask |= IXGBE_EIMS_FLOW_DIR;
4774 #endif
4775                 /* falls through */
4776                 default:
4777                         break;
4778         }
4779
4780         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4781
4782         /* With RSS we use auto clear */
4783         if (adapter->msix_mem) {
4784                 mask = IXGBE_EIMS_ENABLE_MASK;
4785                 /* Don't autoclear Link */
4786                 mask &= ~IXGBE_EIMS_OTHER;
4787                 mask &= ~IXGBE_EIMS_LSC;
4788                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4789         }
4790
4791         /*
4792         ** Now enable all queues, this is done separately to
4793         ** allow for handling the extended (beyond 32) MSIX
4794         ** vectors that can be used by 82599
4795         */
4796         for (int i = 0; i < adapter->num_queues; i++, que++)
4797                 ixgbe_enable_queue(adapter, que->msix);
4798
4799         IXGBE_WRITE_FLUSH(hw);
4800
4801         return;
4802 }
4803
4804 static void
4805 ixgbe_disable_intr(struct adapter *adapter)
4806 {
4807         if (adapter->msix_mem)
4808                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4809         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4810                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4811         } else {
4812                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4813                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4814                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4815         }
4816         IXGBE_WRITE_FLUSH(&adapter->hw);
4817         return;
4818 }
4819
4820 u16
4821 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4822 {
4823         u16 value;
4824
4825         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4826             reg, 2);
4827
4828         return (value);
4829 }
4830
4831 void
4832 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4833 {
4834         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4835             reg, value, 2);
4836
4837         return;
4838 }
4839
4840 /*
4841 ** Get the width and transaction speed of
4842 ** the slot this adapter is plugged into.
4843 */
4844 static void
4845 ixgbe_get_slot_info(struct ixgbe_hw *hw)
4846 {
4847         device_t                dev = ((struct ixgbe_osdep *)hw->back)->dev;
4848         struct ixgbe_mac_info   *mac = &hw->mac;
4849         u16                     link;
4850         u32                     offset;
4851
4852         /* For most devices simply call the shared code routine */
4853         if (hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) {
4854                 ixgbe_get_bus_info(hw);
4855                 goto display;
4856         }
4857
4858         /*
4859         ** For the Quad port adapter we need to parse back
4860         ** up the PCI tree to find the speed of the expansion
4861         ** slot into which this adapter is plugged. A bit more work.
4862         */
4863         dev = device_get_parent(device_get_parent(dev));
4864 #ifdef IXGBE_DEBUG
4865         device_printf(dev, "parent pcib = %x,%x,%x\n",
4866             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4867 #endif
4868         dev = device_get_parent(device_get_parent(dev));
4869 #ifdef IXGBE_DEBUG
4870         device_printf(dev, "slot pcib = %x,%x,%x\n",
4871             pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev));
4872 #endif
4873         /* Now get the PCI Express Capabilities offset */
4874         pci_find_cap(dev, PCIY_EXPRESS, &offset);
4875         /* ...and read the Link Status Register */
4876         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4877         switch (link & IXGBE_PCI_LINK_WIDTH) {
4878         case IXGBE_PCI_LINK_WIDTH_1:
4879                 hw->bus.width = ixgbe_bus_width_pcie_x1;
4880                 break;
4881         case IXGBE_PCI_LINK_WIDTH_2:
4882                 hw->bus.width = ixgbe_bus_width_pcie_x2;
4883                 break;
4884         case IXGBE_PCI_LINK_WIDTH_4:
4885                 hw->bus.width = ixgbe_bus_width_pcie_x4;
4886                 break;
4887         case IXGBE_PCI_LINK_WIDTH_8:
4888                 hw->bus.width = ixgbe_bus_width_pcie_x8;
4889                 break;
4890         default:
4891                 hw->bus.width = ixgbe_bus_width_unknown;
4892                 break;
4893         }
4894
4895         switch (link & IXGBE_PCI_LINK_SPEED) {
4896         case IXGBE_PCI_LINK_SPEED_2500:
4897                 hw->bus.speed = ixgbe_bus_speed_2500;
4898                 break;
4899         case IXGBE_PCI_LINK_SPEED_5000:
4900                 hw->bus.speed = ixgbe_bus_speed_5000;
4901                 break;
4902         case IXGBE_PCI_LINK_SPEED_8000:
4903                 hw->bus.speed = ixgbe_bus_speed_8000;
4904                 break;
4905         default:
4906                 hw->bus.speed = ixgbe_bus_speed_unknown;
4907                 break;
4908         }
4909
4910         mac->ops.set_lan_id(hw);
4911
4912 display:
4913         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4914             ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s":
4915             (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s":
4916             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4917             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
4918             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
4919             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
4920             ("Unknown"));
4921
4922         if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4923             ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
4924             (hw->bus.speed == ixgbe_bus_speed_2500))) {
4925                 device_printf(dev, "PCI-Express bandwidth available"
4926                     " for this card\n     is not sufficient for"
4927                     " optimal performance.\n");
4928                 device_printf(dev, "For optimal performance a x8 "
4929                     "PCIE, or x4 PCIE Gen2 slot is required.\n");
4930         }
4931         if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
4932             ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
4933             (hw->bus.speed < ixgbe_bus_speed_8000))) {
4934                 device_printf(dev, "PCI-Express bandwidth available"
4935                     " for this card\n     is not sufficient for"
4936                     " optimal performance.\n");
4937                 device_printf(dev, "For optimal performance a x8 "
4938                     "PCIE Gen3 slot is required.\n");
4939         }
4940
4941         return;
4942 }
4943
4944
4945 /*
4946 ** Setup the correct IVAR register for a particular MSIX interrupt
4947 **   (yes this is all very magic and confusing :)
4948 **  - entry is the register array entry
4949 **  - vector is the MSIX vector for this queue
4950 **  - type is RX/TX/MISC
4951 */
4952 static void
4953 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4954 {
4955         struct ixgbe_hw *hw = &adapter->hw;
4956         u32 ivar, index;
4957
4958         vector |= IXGBE_IVAR_ALLOC_VAL;
4959
4960         switch (hw->mac.type) {
4961
4962         case ixgbe_mac_82598EB:
4963                 if (type == -1)
4964                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4965                 else
4966                         entry += (type * 64);
4967                 index = (entry >> 2) & 0x1F;
4968                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4969                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4970                 ivar |= (vector << (8 * (entry & 0x3)));
4971                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4972                 break;
4973
4974         case ixgbe_mac_82599EB:
4975         case ixgbe_mac_X540:
4976                 if (type == -1) { /* MISC IVAR */
4977                         index = (entry & 1) * 8;
4978                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4979                         ivar &= ~(0xFF << index);
4980                         ivar |= (vector << index);
4981                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4982                 } else {        /* RX/TX IVARS */
4983                         index = (16 * (entry & 1)) + (8 * type);
4984                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4985                         ivar &= ~(0xFF << index);
4986                         ivar |= (vector << index);
4987                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4988                 }
4989
4990         default:
4991                 break;
4992         }
4993 }
4994
4995 static void
4996 ixgbe_configure_ivars(struct adapter *adapter)
4997 {
4998         struct  ix_queue *que = adapter->queues;
4999         u32 newitr;
5000
5001         if (ixgbe_max_interrupt_rate > 0)
5002                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5003         else
5004                 newitr = 0;
5005
5006         for (int i = 0; i < adapter->num_queues; i++, que++) {
5007                 /* First the RX queue entry */
5008                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5009                 /* ... and the TX */
5010                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5011                 /* Set an Initial EITR value */
5012                 IXGBE_WRITE_REG(&adapter->hw,
5013                     IXGBE_EITR(que->msix), newitr);
5014         }
5015
5016         /* For the Link interrupt */
5017         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5018 }
5019
5020 /*
5021 ** ixgbe_sfp_probe - called in the local timer to
5022 ** determine if a port had optics inserted.
5023 */
5024 static bool ixgbe_sfp_probe(struct adapter *adapter)
5025 {
5026         struct ixgbe_hw *hw = &adapter->hw;
5027         device_t        dev = adapter->dev;
5028         bool            result = FALSE;
5029
5030         if ((hw->phy.type == ixgbe_phy_nl) &&
5031             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5032                 s32 ret = hw->phy.ops.identify_sfp(hw);
5033                 if (ret)
5034                         goto out;
5035                 ret = hw->phy.ops.reset(hw);
5036                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5037                         device_printf(dev,"Unsupported SFP+ module detected!");
5038                         printf(" Reload driver with supported module.\n");
5039                         adapter->sfp_probe = FALSE;
5040                         goto out;
5041                 } else
5042                         device_printf(dev,"SFP+ module detected!\n");
5043                 /* We now have supported optics */
5044                 adapter->sfp_probe = FALSE;
5045                 /* Set the optics type so system reports correctly */
5046                 ixgbe_setup_optics(adapter);
5047                 result = TRUE;
5048         }
5049 out:
5050         return (result);
5051 }
5052
5053 /*
5054 ** Tasklet handler for MSIX Link interrupts
5055 **  - do outside interrupt since it might sleep
5056 */
5057 static void
5058 ixgbe_handle_link(void *context, int pending)
5059 {
5060         struct adapter  *adapter = context;
5061
5062         ixgbe_check_link(&adapter->hw,
5063             &adapter->link_speed, &adapter->link_up, 0);
5064         ixgbe_update_link_status(adapter);
5065 }
5066
5067 /*
5068 ** Tasklet for handling SFP module interrupts
5069 */
5070 static void
5071 ixgbe_handle_mod(void *context, int pending)
5072 {
5073         struct adapter  *adapter = context;
5074         struct ixgbe_hw *hw = &adapter->hw;
5075         device_t        dev = adapter->dev;
5076         u32 err;
5077
5078         err = hw->phy.ops.identify_sfp(hw);
5079         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5080                 device_printf(dev,
5081                     "Unsupported SFP+ module type was detected.\n");
5082                 return;
5083         }
5084         err = hw->mac.ops.setup_sfp(hw);
5085         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5086                 device_printf(dev,
5087                     "Setup failure - unsupported SFP+ module type.\n");
5088                 return;
5089         }
5090         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5091         return;
5092 }
5093
5094
5095 /*
5096 ** Tasklet for handling MSF (multispeed fiber) interrupts
5097 */
5098 static void
5099 ixgbe_handle_msf(void *context, int pending)
5100 {
5101         struct adapter  *adapter = context;
5102         struct ixgbe_hw *hw = &adapter->hw;
5103         u32 autoneg;
5104         bool negotiate;
5105
5106         autoneg = hw->phy.autoneg_advertised;
5107         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5108                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5109         if (hw->mac.ops.setup_link)
5110                 hw->mac.ops.setup_link(hw, autoneg, TRUE);
5111         return;
5112 }
5113
5114 #ifdef IXGBE_FDIR
5115 /*
5116 ** Tasklet for reinitializing the Flow Director filter table
5117 */
5118 static void
5119 ixgbe_reinit_fdir(void *context, int pending)
5120 {
5121         struct adapter  *adapter = context;
5122         struct ifnet   *ifp = adapter->ifp;
5123
5124         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5125                 return;
5126         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5127         adapter->fdir_reinit = 0;
5128         /* re-enable flow director interrupts */
5129         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5130         /* Restart the interface */
5131         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5132         return;
5133 }
5134 #endif
5135
5136 /**********************************************************************
5137  *
5138  *  Update the board statistics counters.
5139  *
5140  **********************************************************************/
5141 static void
5142 ixgbe_update_stats_counters(struct adapter *adapter)
5143 {
5144         struct ifnet   *ifp = adapter->ifp;
5145         struct ixgbe_hw *hw = &adapter->hw;
5146         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5147         u64  total_missed_rx = 0;
5148
5149         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5150         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5151         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5152         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5153
5154         /*
5155         ** Note: these are for the 8 possible traffic classes,
5156         **       which in current implementation is unused,
5157         **       therefore only 0 should read real data.
5158         */
5159         for (int i = 0; i < 8; i++) {
5160                 u32 mp;
5161                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5162                 /* missed_rx tallies misses for the gprc workaround */
5163                 missed_rx += mp;
5164                 /* global total per queue */
5165                 adapter->stats.mpc[i] += mp;
5166                 /* Running comprehensive total for stats display */
5167                 total_missed_rx += adapter->stats.mpc[i];
5168                 if (hw->mac.type == ixgbe_mac_82598EB) {
5169                         adapter->stats.rnbc[i] +=
5170                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5171                         adapter->stats.qbtc[i] +=
5172                             IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5173                         adapter->stats.qbrc[i] +=
5174                             IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5175                         adapter->stats.pxonrxc[i] +=
5176                             IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5177                 } else
5178                         adapter->stats.pxonrxc[i] +=
5179                             IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5180                 adapter->stats.pxontxc[i] +=
5181                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5182                 adapter->stats.pxofftxc[i] +=
5183                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5184                 adapter->stats.pxoffrxc[i] +=
5185                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5186                 adapter->stats.pxon2offc[i] +=
5187                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5188         }
5189         for (int i = 0; i < 16; i++) {
5190                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5191                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5192                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5193         }
5194         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5195         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5196         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5197
5198         /* Hardware workaround, gprc counts missed packets */
5199         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5200         adapter->stats.gprc -= missed_rx;
5201
5202         if (hw->mac.type != ixgbe_mac_82598EB) {
5203                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5204                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5205                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5206                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5207                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5208                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5209                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5210                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5211         } else {
5212                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5213                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5214                 /* 82598 only has a counter in the high register */
5215                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5216                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5217                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5218         }
5219
5220         /*
5221          * Workaround: mprc hardware is incorrectly counting
5222          * broadcasts, so for now we subtract those.
5223          */
5224         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5225         adapter->stats.bprc += bprc;
5226         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5227         if (hw->mac.type == ixgbe_mac_82598EB)
5228                 adapter->stats.mprc -= bprc;
5229
5230         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5231         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5232         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5233         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5234         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5235         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5236
5237         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5238         adapter->stats.lxontxc += lxon;
5239         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5240         adapter->stats.lxofftxc += lxoff;
5241         total = lxon + lxoff;
5242
5243         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5244         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5245         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5246         adapter->stats.gptc -= total;
5247         adapter->stats.mptc -= total;
5248         adapter->stats.ptc64 -= total;
5249         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5250
5251         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5252         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5253         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5254         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5255         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5256         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5257         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5258         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5259         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5260         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5261         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5262         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5263         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5264         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5265         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5266         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5267         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5268         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5269         /* Only read FCOE on 82599 */
5270         if (hw->mac.type != ixgbe_mac_82598EB) {
5271                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5272                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5273                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5274                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5275                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5276         }
5277
5278         /* Fill out the OS statistics structure */
5279         ifp->if_ipackets = adapter->stats.gprc;
5280         ifp->if_opackets = adapter->stats.gptc;
5281         ifp->if_ibytes = adapter->stats.gorc;
5282         ifp->if_obytes = adapter->stats.gotc;
5283         ifp->if_imcasts = adapter->stats.mprc;
5284         ifp->if_omcasts = adapter->stats.mptc;
5285         ifp->if_collisions = 0;
5286
5287         /* Rx Errors */
5288         ifp->if_iqdrops = total_missed_rx;
5289         ifp->if_ierrors = adapter->stats.crcerrs + adapter->stats.rlec;
5290 }
5291
5292 /** ixgbe_sysctl_tdh_handler - Handler function
5293  *  Retrieves the TDH value from the hardware
5294  */
5295 static int
5296 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5297 {
5298         int error;
5299
5300         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5301         if (!txr) return 0;
5302
5303         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5304         error = sysctl_handle_int(oidp, &val, 0, req);
5305         if (error || !req->newptr)
5306                 return error;
5307         return 0;
5308 }
5309
5310 /** ixgbe_sysctl_tdt_handler - Handler function
5311  *  Retrieves the TDT value from the hardware
5312  */
5313 static int
5314 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5315 {
5316         int error;
5317
5318         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5319         if (!txr) return 0;
5320
5321         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5322         error = sysctl_handle_int(oidp, &val, 0, req);
5323         if (error || !req->newptr)
5324                 return error;
5325         return 0;
5326 }
5327
5328 /** ixgbe_sysctl_rdh_handler - Handler function
5329  *  Retrieves the RDH value from the hardware
5330  */
5331 static int
5332 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5333 {
5334         int error;
5335
5336         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5337         if (!rxr) return 0;
5338
5339         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5340         error = sysctl_handle_int(oidp, &val, 0, req);
5341         if (error || !req->newptr)
5342                 return error;
5343         return 0;
5344 }
5345
5346 /** ixgbe_sysctl_rdt_handler - Handler function
5347  *  Retrieves the RDT value from the hardware
5348  */
5349 static int
5350 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5351 {
5352         int error;
5353
5354         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5355         if (!rxr) return 0;
5356
5357         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5358         error = sysctl_handle_int(oidp, &val, 0, req);
5359         if (error || !req->newptr)
5360                 return error;
5361         return 0;
5362 }
5363
5364 static int
5365 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5366 {
5367         int error;
5368         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5369         unsigned int reg, usec, rate;
5370
5371         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5372         usec = ((reg & 0x0FF8) >> 3);
5373         if (usec > 0)
5374                 rate = 500000 / usec;
5375         else
5376                 rate = 0;
5377         error = sysctl_handle_int(oidp, &rate, 0, req);
5378         if (error || !req->newptr)
5379                 return error;
5380         reg &= ~0xfff; /* default, no limitation */
5381         ixgbe_max_interrupt_rate = 0;
5382         if (rate > 0 && rate < 500000) {
5383                 if (rate < 1000)
5384                         rate = 1000;
5385                 ixgbe_max_interrupt_rate = rate;
5386                 reg |= ((4000000/rate) & 0xff8 );
5387         }
5388         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5389         return 0;
5390 }
5391
5392 /*
5393  * Add sysctl variables, one per statistic, to the system.
5394  */
5395 static void
5396 ixgbe_add_hw_stats(struct adapter *adapter)
5397 {
5398
5399         device_t dev = adapter->dev;
5400
5401         struct tx_ring *txr = adapter->tx_rings;
5402         struct rx_ring *rxr = adapter->rx_rings;
5403
5404         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5405         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5406         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5407         struct ixgbe_hw_stats *stats = &adapter->stats;
5408
5409         struct sysctl_oid *stat_node, *queue_node;
5410         struct sysctl_oid_list *stat_list, *queue_list;
5411
5412 #define QUEUE_NAME_LEN 32
5413         char namebuf[QUEUE_NAME_LEN];
5414
5415         /* Driver Statistics */
5416         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5417                         CTLFLAG_RD, &adapter->dropped_pkts,
5418                         "Driver dropped packets");
5419         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5420                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5421                         "m_defrag() failed");
5422         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5423                         CTLFLAG_RD, &adapter->watchdog_events,
5424                         "Watchdog timeouts");
5425         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5426                         CTLFLAG_RD, &adapter->link_irq,
5427                         "Link MSIX IRQ Handled");
5428
5429         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5430                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5431                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5432                                             CTLFLAG_RD, NULL, "Queue Name");
5433                 queue_list = SYSCTL_CHILDREN(queue_node);
5434
5435                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5436                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5437                                 sizeof(&adapter->queues[i]),
5438                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5439                                 "Interrupt Rate");
5440                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5441                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
5442                                 "irqs on this queue");
5443                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5444                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5445                                 ixgbe_sysctl_tdh_handler, "IU",
5446                                 "Transmit Descriptor Head");
5447                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5448                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5449                                 ixgbe_sysctl_tdt_handler, "IU",
5450                                 "Transmit Descriptor Tail");
5451                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx",
5452                                 CTLFLAG_RD, &txr->tso_tx,
5453                                 "TSO");
5454                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup",
5455                                 CTLFLAG_RD, &txr->no_tx_dma_setup,
5456                                 "Driver tx dma failure in xmit");
5457                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5458                                 CTLFLAG_RD, &txr->no_desc_avail,
5459                                 "Queue No Descriptor Available");
5460                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5461                                 CTLFLAG_RD, &txr->total_packets,
5462                                 "Queue Packets Transmitted");
5463         }
5464
5465         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5466                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5467                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5468                                             CTLFLAG_RD, NULL, "Queue Name");
5469                 queue_list = SYSCTL_CHILDREN(queue_node);
5470
5471                 struct lro_ctrl *lro = &rxr->lro;
5472
5473                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5474                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5475                                             CTLFLAG_RD, NULL, "Queue Name");
5476                 queue_list = SYSCTL_CHILDREN(queue_node);
5477
5478                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5479                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5480                                 ixgbe_sysctl_rdh_handler, "IU",
5481                                 "Receive Descriptor Head");
5482                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5483                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5484                                 ixgbe_sysctl_rdt_handler, "IU",
5485                                 "Receive Descriptor Tail");
5486                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5487                                 CTLFLAG_RD, &rxr->rx_packets,
5488                                 "Queue Packets Received");
5489                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5490                                 CTLFLAG_RD, &rxr->rx_bytes,
5491                                 "Queue Bytes Received");
5492                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5493                                 CTLFLAG_RD, &rxr->rx_copies,
5494                                 "Copied RX Frames");
5495                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5496                                 CTLFLAG_RD, &lro->lro_queued, 0,
5497                                 "LRO Queued");
5498                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5499                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5500                                 "LRO Flushed");
5501         }
5502
5503         /* MAC stats get the own sub node */
5504
5505         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5506                                     CTLFLAG_RD, NULL, "MAC Statistics");
5507         stat_list = SYSCTL_CHILDREN(stat_node);
5508
5509         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5510                         CTLFLAG_RD, &stats->crcerrs,
5511                         "CRC Errors");
5512         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5513                         CTLFLAG_RD, &stats->illerrc,
5514                         "Illegal Byte Errors");
5515         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5516                         CTLFLAG_RD, &stats->errbc,
5517                         "Byte Errors");
5518         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5519                         CTLFLAG_RD, &stats->mspdc,
5520                         "MAC Short Packets Discarded");
5521         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5522                         CTLFLAG_RD, &stats->mlfc,
5523                         "MAC Local Faults");
5524         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5525                         CTLFLAG_RD, &stats->mrfc,
5526                         "MAC Remote Faults");
5527         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5528                         CTLFLAG_RD, &stats->rlec,
5529                         "Receive Length Errors");
5530
5531         /* Flow Control stats */
5532         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5533                         CTLFLAG_RD, &stats->lxontxc,
5534                         "Link XON Transmitted");
5535         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5536                         CTLFLAG_RD, &stats->lxonrxc,
5537                         "Link XON Received");
5538         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5539                         CTLFLAG_RD, &stats->lxofftxc,
5540                         "Link XOFF Transmitted");
5541         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5542                         CTLFLAG_RD, &stats->lxoffrxc,
5543                         "Link XOFF Received");
5544
5545         /* Packet Reception Stats */
5546         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5547                         CTLFLAG_RD, &stats->tor,
5548                         "Total Octets Received");
5549         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5550                         CTLFLAG_RD, &stats->gorc,
5551                         "Good Octets Received");
5552         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5553                         CTLFLAG_RD, &stats->tpr,
5554                         "Total Packets Received");
5555         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5556                         CTLFLAG_RD, &stats->gprc,
5557                         "Good Packets Received");
5558         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5559                         CTLFLAG_RD, &stats->mprc,
5560                         "Multicast Packets Received");
5561         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5562                         CTLFLAG_RD, &stats->bprc,
5563                         "Broadcast Packets Received");
5564         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5565                         CTLFLAG_RD, &stats->prc64,
5566                         "64 byte frames received ");
5567         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5568                         CTLFLAG_RD, &stats->prc127,
5569                         "65-127 byte frames received");
5570         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5571                         CTLFLAG_RD, &stats->prc255,
5572                         "128-255 byte frames received");
5573         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5574                         CTLFLAG_RD, &stats->prc511,
5575                         "256-511 byte frames received");
5576         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5577                         CTLFLAG_RD, &stats->prc1023,
5578                         "512-1023 byte frames received");
5579         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5580                         CTLFLAG_RD, &stats->prc1522,
5581                         "1023-1522 byte frames received");
5582         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5583                         CTLFLAG_RD, &stats->ruc,
5584                         "Receive Undersized");
5585         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5586                         CTLFLAG_RD, &stats->rfc,
5587                         "Fragmented Packets Received ");
5588         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5589                         CTLFLAG_RD, &stats->roc,
5590                         "Oversized Packets Received");
5591         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5592                         CTLFLAG_RD, &stats->rjc,
5593                         "Received Jabber");
5594         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5595                         CTLFLAG_RD, &stats->mngprc,
5596                         "Management Packets Received");
5597         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5598                         CTLFLAG_RD, &stats->mngptc,
5599                         "Management Packets Dropped");
5600         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5601                         CTLFLAG_RD, &stats->xec,
5602                         "Checksum Errors");
5603
5604         /* Packet Transmission Stats */
5605         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5606                         CTLFLAG_RD, &stats->gotc,
5607                         "Good Octets Transmitted");
5608         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5609                         CTLFLAG_RD, &stats->tpt,
5610                         "Total Packets Transmitted");
5611         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5612                         CTLFLAG_RD, &stats->gptc,
5613                         "Good Packets Transmitted");
5614         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5615                         CTLFLAG_RD, &stats->bptc,
5616                         "Broadcast Packets Transmitted");
5617         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5618                         CTLFLAG_RD, &stats->mptc,
5619                         "Multicast Packets Transmitted");
5620         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5621                         CTLFLAG_RD, &stats->mngptc,
5622                         "Management Packets Transmitted");
5623         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5624                         CTLFLAG_RD, &stats->ptc64,
5625                         "64 byte frames transmitted ");
5626         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5627                         CTLFLAG_RD, &stats->ptc127,
5628                         "65-127 byte frames transmitted");
5629         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5630                         CTLFLAG_RD, &stats->ptc255,
5631                         "128-255 byte frames transmitted");
5632         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5633                         CTLFLAG_RD, &stats->ptc511,
5634                         "256-511 byte frames transmitted");
5635         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5636                         CTLFLAG_RD, &stats->ptc1023,
5637                         "512-1023 byte frames transmitted");
5638         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5639                         CTLFLAG_RD, &stats->ptc1522,
5640                         "1024-1522 byte frames transmitted");
5641 }
5642
5643 /*
5644 ** Set flow control using sysctl:
5645 ** Flow control values:
5646 **      0 - off
5647 **      1 - rx pause
5648 **      2 - tx pause
5649 **      3 - full
5650 */
5651 static int
5652 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5653 {
5654         int error, last;
5655         struct adapter *adapter = (struct adapter *) arg1;
5656
5657         last = adapter->fc;
5658         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5659         if ((error) || (req->newptr == NULL))
5660                 return (error);
5661
5662         /* Don't bother if it's not changed */
5663         if (adapter->fc == last)
5664                 return (0);
5665
5666         switch (adapter->fc) {
5667                 case ixgbe_fc_rx_pause:
5668                 case ixgbe_fc_tx_pause:
5669                 case ixgbe_fc_full:
5670                         adapter->hw.fc.requested_mode = adapter->fc;
5671                         if (adapter->num_queues > 1)
5672                                 ixgbe_disable_rx_drop(adapter);
5673                         break;
5674                 case ixgbe_fc_none:
5675                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5676                         if (adapter->num_queues > 1)
5677                                 ixgbe_enable_rx_drop(adapter);
5678                         break;
5679                 default:
5680                         adapter->fc = last;
5681                         return (EINVAL);
5682         }
5683         /* Don't autoneg if forcing a value */
5684         adapter->hw.fc.disable_fc_autoneg = TRUE;
5685         ixgbe_fc_enable(&adapter->hw);
5686         return error;
5687 }
5688
5689 /*
5690 ** Control link advertise speed:
5691 **      1 - advertise only 1G
5692 **      2 - advertise 100Mb
5693 **      3 - advertise normal
5694 */
5695 static int
5696 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5697 {
5698         int                     error = 0;
5699         struct adapter          *adapter;
5700         device_t                dev;
5701         struct ixgbe_hw         *hw;
5702         ixgbe_link_speed        speed, last;
5703
5704         adapter = (struct adapter *) arg1;
5705         dev = adapter->dev;
5706         hw = &adapter->hw;
5707         last = adapter->advertise;
5708
5709         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5710         if ((error) || (req->newptr == NULL))
5711                 return (error);
5712
5713         if (adapter->advertise == last) /* no change */
5714                 return (0);
5715
5716         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5717             (hw->phy.multispeed_fiber)))
5718                 return (EINVAL);
5719
5720         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5721                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5722                 return (EINVAL);
5723         }
5724
5725         if (adapter->advertise == 1)
5726                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5727         else if (adapter->advertise == 2)
5728                 speed = IXGBE_LINK_SPEED_100_FULL;
5729         else if (adapter->advertise == 3)
5730                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5731                         IXGBE_LINK_SPEED_10GB_FULL;
5732         else {  /* bogus value */
5733                 adapter->advertise = last;
5734                 return (EINVAL);
5735         }
5736
5737         hw->mac.autotry_restart = TRUE;
5738         hw->mac.ops.setup_link(hw, speed, TRUE);
5739
5740         return (error);
5741 }
5742
5743 /*
5744 ** Thermal Shutdown Trigger
5745 **   - cause a Thermal Overtemp IRQ
5746 **   - this now requires firmware enabling
5747 */
5748 static int
5749 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5750 {
5751         int             error, fire = 0;
5752         struct adapter  *adapter = (struct adapter *) arg1;
5753         struct ixgbe_hw *hw = &adapter->hw;
5754
5755
5756         if (hw->mac.type != ixgbe_mac_X540)
5757                 return (0);
5758
5759         error = sysctl_handle_int(oidp, &fire, 0, req);
5760         if ((error) || (req->newptr == NULL))
5761                 return (error);
5762
5763         if (fire) {
5764                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5765                 reg |= IXGBE_EICR_TS;
5766                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5767         }
5768
5769         return (0);
5770 }
5771
5772 /*
5773 ** Enable the hardware to drop packets when the buffer is
5774 ** full. This is useful when multiqueue,so that no single
5775 ** queue being full stalls the entire RX engine. We only
5776 ** enable this when Multiqueue AND when Flow Control is
5777 ** disabled.
5778 */
5779 static void
5780 ixgbe_enable_rx_drop(struct adapter *adapter)
5781 {
5782         struct ixgbe_hw *hw = &adapter->hw;
5783
5784         for (int i = 0; i < adapter->num_queues; i++) {
5785                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5786                 srrctl |= IXGBE_SRRCTL_DROP_EN;
5787                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5788         }
5789 }
5790
5791 static void
5792 ixgbe_disable_rx_drop(struct adapter *adapter)
5793 {
5794         struct ixgbe_hw *hw = &adapter->hw;
5795
5796         for (int i = 0; i < adapter->num_queues; i++) {
5797                 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
5798                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
5799                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
5800         }
5801 }