sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #endif
  39
  40 #include "ixgbe.h"
  41
  42 /*********************************************************************
  43  *  Set this to one to display debug statistics
  44  *********************************************************************/
  45 int             ixgbe_display_debug_stats = 0;
  46
  47 /*********************************************************************
  48  *  Driver version
  49  *********************************************************************/
  50 char ixgbe_driver_version[] = "2.4.8";
  51
  52 /*********************************************************************
  53  *  PCI Device ID Table
  54  *
  55  *  Used by probe to select devices to load on
  56  *  Last field stores an index into ixgbe_strings
  57  *  Last entry must be all 0s
  58  *
  59  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  60  *********************************************************************/
  61
  62 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  63 {
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  86         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  87         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  88         /* required last entry */
  89         {0, 0, 0, 0, 0}
  90 };
  91
  92 /*********************************************************************
  93  *  Table of branding strings
  94  *********************************************************************/
  95
  96 static char    *ixgbe_strings[] = {
  97         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  98 };
  99
 100 /*********************************************************************
 101  *  Function prototypes
 102  *********************************************************************/
 103 static int      ixgbe_probe(device_t);
 104 static int      ixgbe_attach(device_t);
 105 static int      ixgbe_detach(device_t);
 106 static int      ixgbe_shutdown(device_t);
 107 static void     ixgbe_start(struct ifnet *);
 108 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 109 #if __FreeBSD_version >= 800000
 110 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 111 static int      ixgbe_mq_start_locked(struct ifnet *,
 112                     struct tx_ring *, struct mbuf *);
 113 static void     ixgbe_qflush(struct ifnet *);
 114 #endif
 115 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 116 static void     ixgbe_init(void *);
 117 static void     ixgbe_init_locked(struct adapter *);
 118 static void     ixgbe_stop(void *);
 119 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 120 static int      ixgbe_media_change(struct ifnet *);
 121 static void     ixgbe_identify_hardware(struct adapter *);
 122 static int      ixgbe_allocate_pci_resources(struct adapter *);
 123 static int      ixgbe_allocate_msix(struct adapter *);
 124 static int      ixgbe_allocate_legacy(struct adapter *);
 125 static int      ixgbe_allocate_queues(struct adapter *);
 126 static int      ixgbe_setup_msix(struct adapter *);
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static bool     ixgbe_txeof(struct tx_ring *);
 151 static bool     ixgbe_rxeof(struct ix_queue *, int);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 165                     const char *, int *, int);
 166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 167 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 168 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 169 static void     ixgbe_configure_ivars(struct adapter *);
 170 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 171
 172 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 173 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 174 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 175
 176 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 177
 178 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 179 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 180                     struct mbuf *, u32);
 181
 182 /* Support for pluggable optic modules */
 183 static bool     ixgbe_sfp_probe(struct adapter *);
 184 static void     ixgbe_setup_optics(struct adapter *);
 185
 186 /* Legacy (single vector interrupt handler */
 187 static void     ixgbe_legacy_irq(void *);
 188
 189 /* The MSI/X Interrupt handlers */
 190 static void     ixgbe_msix_que(void *);
 191 static void     ixgbe_msix_link(void *);
 192
 193 /* Deferred interrupt tasklets */
 194 static void     ixgbe_handle_que(void *, int);
 195 static void     ixgbe_handle_link(void *, int);
 196 static void     ixgbe_handle_msf(void *, int);
 197 static void     ixgbe_handle_mod(void *, int);
 198
 199 #ifdef IXGBE_FDIR
 200 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 201 static void     ixgbe_reinit_fdir(void *, int);
 202 #endif
 203
 204 /*********************************************************************
 205  *  FreeBSD Device Interface Entry Points
 206  *********************************************************************/
 207
 208 static device_method_t ixgbe_methods[] = {
 209         /* Device interface */
 210         DEVMETHOD(device_probe, ixgbe_probe),
 211         DEVMETHOD(device_attach, ixgbe_attach),
 212         DEVMETHOD(device_detach, ixgbe_detach),
 213         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 214         {0, 0}
 215 };
 216
 217 static driver_t ixgbe_driver = {
 218         "ix", ixgbe_methods, sizeof(struct adapter),
 219 };
 220
 221 devclass_t ixgbe_devclass;
 222 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 223
 224 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 225 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 226
 227 /*
 228 ** TUNEABLE PARAMETERS:
 229 */
 230
 231 /*
 232 ** AIM: Adaptive Interrupt Moderation
 233 ** which means that the interrupt rate
 234 ** is varied over time based on the
 235 ** traffic for that interrupt vector
 236 */
 237 static int ixgbe_enable_aim = TRUE;
 238 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 239
 240 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 241 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 242
 243 /* How many packets rxeof tries to clean at a time */
 244 static int ixgbe_rx_process_limit = 128;
 245 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 246
 247 /*
 248 ** Smart speed setting, default to on
 249 ** this only works as a compile option
 250 ** right now as its during attach, set
 251 ** this to 'ixgbe_smart_speed_off' to
 252 ** disable.
 253 */
 254 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 255
 256 /*
 257  * MSIX should be the default for best performance,
 258  * but this allows it to be forced off for testing.
 259  */
 260 static int ixgbe_enable_msix = 1;
 261 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 262
 263 /*
 264  * Header split: this causes the hardware to DMA
 265  * the header into a separate mbuf from the payload,
 266  * it can be a performance win in some workloads, but
 267  * in others it actually hurts, its off by default.
 268  */
 269 static int ixgbe_header_split = FALSE;
 270 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 271
 272 /*
 273  * Number of Queues, can be set to 0,
 274  * it then autoconfigures based on the
 275  * number of cpus with a max of 8. This
 276  * can be overriden manually here.
 277  */
 278 static int ixgbe_num_queues = 0;
 279 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 280
 281 /*
 282 ** Number of TX descriptors per ring,
 283 ** setting higher than RX as this seems
 284 ** the better performing choice.
 285 */
 286 static int ixgbe_txd = PERFORM_TXD;
 287 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 288
 289 /* Number of RX descriptors per ring */
 290 static int ixgbe_rxd = PERFORM_RXD;
 291 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 292
 293 /* Keep running tab on them for sanity check */
 294 static int ixgbe_total_ports;
 295
 296 #ifdef IXGBE_FDIR
 297 /*
 298 ** For Flow Director: this is the
 299 ** number of TX packets we sample
 300 ** for the filter pool, this means
 301 ** every 20th packet will be probed.
 302 **
 303 ** This feature can be disabled by
 304 ** setting this to 0.
 305 */
 306 static int atr_sample_rate = 20;
 307 /*
 308 ** Flow Director actually 'steals'
 309 ** part of the packet buffer as its
 310 ** filter pool, this variable controls
 311 ** how much it uses:
 312 **  0 = 64K, 1 = 128K, 2 = 256K
 313 */
 314 static int fdir_pballoc = 1;
 315 #endif
 316
 317 #ifdef DEV_NETMAP
 318 /*
 319  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 320  * be a reference on how to implement netmap support in a driver.
 321  * Additional comments are in ixgbe_netmap.h .
 322  *
 323  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 324  * that extend the standard driver.
 325  */
 326 #include <dev/netmap/ixgbe_netmap.h>
 327 #endif /* DEV_NETMAP */
 328
 329 /*********************************************************************
 330  *  Device identification routine
 331  *
 332  *  ixgbe_probe determines if the driver should be loaded on
 333  *  adapter based on PCI vendor/device id of the adapter.
 334  *
 335  *  return BUS_PROBE_DEFAULT on success, positive on failure
 336  *********************************************************************/
 337
 338 static int
 339 ixgbe_probe(device_t dev)
 340 {
 341         ixgbe_vendor_info_t *ent;
 342
 343         u16     pci_vendor_id = 0;
 344         u16     pci_device_id = 0;
 345         u16     pci_subvendor_id = 0;
 346         u16     pci_subdevice_id = 0;
 347         char    adapter_name[256];
 348
 349         INIT_DEBUGOUT("ixgbe_probe: begin");
 350
 351         pci_vendor_id = pci_get_vendor(dev);
 352         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 353                 return (ENXIO);
 354
 355         pci_device_id = pci_get_device(dev);
 356         pci_subvendor_id = pci_get_subvendor(dev);
 357         pci_subdevice_id = pci_get_subdevice(dev);
 358
 359         ent = ixgbe_vendor_info_array;
 360         while (ent->vendor_id != 0) {
 361                 if ((pci_vendor_id == ent->vendor_id) &&
 362                     (pci_device_id == ent->device_id) &&
 363
 364                     ((pci_subvendor_id == ent->subvendor_id) ||
 365                      (ent->subvendor_id == 0)) &&
 366
 367                     ((pci_subdevice_id == ent->subdevice_id) ||
 368                      (ent->subdevice_id == 0))) {
 369                         sprintf(adapter_name, "%s, Version - %s",
 370                                 ixgbe_strings[ent->index],
 371                                 ixgbe_driver_version);
 372                         device_set_desc_copy(dev, adapter_name);
 373                         ++ixgbe_total_ports;
 374                         return (BUS_PROBE_DEFAULT);
 375                 }
 376                 ent++;
 377         }
 378         return (ENXIO);
 379 }
 380
 381 /*********************************************************************
 382  *  Device initialization routine
 383  *
 384  *  The attach entry point is called when the driver is being loaded.
 385  *  This routine identifies the type of hardware, allocates all resources
 386  *  and initializes the hardware.
 387  *
 388  *  return 0 on success, positive on failure
 389  *********************************************************************/
 390
 391 static int
 392 ixgbe_attach(device_t dev)
 393 {
 394         struct adapter *adapter;
 395         struct ixgbe_hw *hw;
 396         int             error = 0;
 397         u16             csum;
 398         u32             ctrl_ext;
 399
 400         INIT_DEBUGOUT("ixgbe_attach: begin");
 401
 402         /* Allocate, clear, and link in our adapter structure */
 403         adapter = device_get_softc(dev);
 404         adapter->dev = adapter->osdep.dev = dev;
 405         hw = &adapter->hw;
 406
 407         /* Core Lock Init*/
 408         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 409
 410         /* SYSCTL APIs */
 411
 412         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 413                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 414                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 415                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 416
 417         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 418                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 419                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 420                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 421
 422         /*
 423         ** Allow a kind of speed control by forcing the autoneg
 424         ** advertised speed list to only a certain value, this
 425         ** supports 1G on 82599 devices, and 100Mb on x540.
 426         */
 427         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 428                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 429                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 430                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 431
 432         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 433                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 434                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 435                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 436
 437         /* Set up the timer callout */
 438         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 439
 440         /* Determine hardware revision */
 441         ixgbe_identify_hardware(adapter);
 442
 443         /* Do base PCI setup - map BAR0 */
 444         if (ixgbe_allocate_pci_resources(adapter)) {
 445                 device_printf(dev, "Allocation of PCI resources failed\n");
 446                 error = ENXIO;
 447                 goto err_out;
 448         }
 449
 450         /* Do descriptor calc and sanity checks */
 451         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 452             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 453                 device_printf(dev, "TXD config issue, using default!\n");
 454                 adapter->num_tx_desc = DEFAULT_TXD;
 455         } else
 456                 adapter->num_tx_desc = ixgbe_txd;
 457
 458         /*
 459         ** With many RX rings it is easy to exceed the
 460         ** system mbuf allocation. Tuning nmbclusters
 461         ** can alleviate this.
 462         */
 463         if (nmbclusters > 0 ) {
 464                 int s;
 465                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 466                 if (s > nmbclusters) {
 467                         device_printf(dev, "RX Descriptors exceed "
 468                             "system mbuf max, using default instead!\n");
 469                         ixgbe_rxd = DEFAULT_RXD;
 470                 }
 471         }
 472
 473         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 474             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 475                 device_printf(dev, "RXD config issue, using default!\n");
 476                 adapter->num_rx_desc = DEFAULT_RXD;
 477         } else
 478                 adapter->num_rx_desc = ixgbe_rxd;
 479
 480         /* Allocate our TX/RX Queues */
 481         if (ixgbe_allocate_queues(adapter)) {
 482                 error = ENOMEM;
 483                 goto err_out;
 484         }
 485
 486         /* Allocate multicast array memory. */
 487         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 488             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 489         if (adapter->mta == NULL) {
 490                 device_printf(dev, "Can not allocate multicast setup array\n");
 491                 error = ENOMEM;
 492                 goto err_late;
 493         }
 494
 495         /* Initialize the shared code */
 496         error = ixgbe_init_shared_code(hw);
 497         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 498                 /*
 499                 ** No optics in this port, set up
 500                 ** so the timer routine will probe
 501                 ** for later insertion.
 502                 */
 503                 adapter->sfp_probe = TRUE;
 504                 error = 0;
 505         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 506                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 507                 error = EIO;
 508                 goto err_late;
 509         } else if (error) {
 510                 device_printf(dev,"Unable to initialize the shared code\n");
 511                 error = EIO;
 512                 goto err_late;
 513         }
 514
 515         /* Make sure we have a good EEPROM before we read from it */
 516         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 517                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 518                 error = EIO;
 519                 goto err_late;
 520         }
 521
 522         error = ixgbe_init_hw(hw);
 523         switch (error) {
 524         case IXGBE_ERR_EEPROM_VERSION:
 525                 device_printf(dev, "This device is a pre-production adapter/"
 526                     "LOM.  Please be aware there may be issues associated "
 527                     "with your hardware.\n If you are experiencing problems "
 528                     "please contact your Intel or hardware representative "
 529                     "who provided you with this hardware.\n");
 530                 break;
 531         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 532                 device_printf(dev,"Unsupported SFP+ Module\n");
 533                 error = EIO;
 534                 device_printf(dev,"Hardware Initialization Failure\n");
 535                 goto err_late;
 536         case IXGBE_ERR_SFP_NOT_PRESENT:
 537                 device_printf(dev,"No SFP+ Module found\n");
 538                 /* falls thru */
 539         default:
 540                 break;
 541         }
 542
 543         /* Detect and set physical type */
 544         ixgbe_setup_optics(adapter);
 545
 546         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 547                 error = ixgbe_allocate_msix(adapter);
 548         else
 549                 error = ixgbe_allocate_legacy(adapter);
 550         if (error)
 551                 goto err_late;
 552
 553         /* Setup OS specific network interface */
 554         if (ixgbe_setup_interface(dev, adapter) != 0)
 555                 goto err_late;
 556
 557         /* Sysctl for limiting the amount of work done in the taskqueue */
 558         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 559             "max number of rx packets to process", &adapter->rx_process_limit,
 560             ixgbe_rx_process_limit);
 561
 562         /* Initialize statistics */
 563         ixgbe_update_stats_counters(adapter);
 564
 565         /* Register for VLAN events */
 566         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 567             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 568         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 569             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 570
 571         /* Print PCIE bus type/speed/width info */
 572         ixgbe_get_bus_info(hw);
 573         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 574             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 575             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 576             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 577             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 578             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 579             ("Unknown"));
 580
 581         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 582             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 583                 device_printf(dev, "PCI-Express bandwidth available"
 584                     " for this card\n     is not sufficient for"
 585                     " optimal performance.\n");
 586                 device_printf(dev, "For optimal performance a x8 "
 587                     "PCIE, or x4 PCIE 2 slot is required.\n");
 588         }
 589
 590         /* let hardware know driver is loaded */
 591         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 592         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 593         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 594
 595         ixgbe_add_hw_stats(adapter);
 596
 597 #ifdef DEV_NETMAP
 598         ixgbe_netmap_attach(adapter);
 599 #endif /* DEV_NETMAP */
 600         INIT_DEBUGOUT("ixgbe_attach: end");
 601         return (0);
 602 err_late:
 603         ixgbe_free_transmit_structures(adapter);
 604         ixgbe_free_receive_structures(adapter);
 605 err_out:
 606         if (adapter->ifp != NULL)
 607                 if_free(adapter->ifp);
 608         ixgbe_free_pci_resources(adapter);
 609         free(adapter->mta, M_DEVBUF);
 610         return (error);
 611
 612 }
 613
 614 /*********************************************************************
 615  *  Device removal routine
 616  *
 617  *  The detach entry point is called when the driver is being removed.
 618  *  This routine stops the adapter and deallocates all the resources
 619  *  that were allocated for driver operation.
 620  *
 621  *  return 0 on success, positive on failure
 622  *********************************************************************/
 623
 624 static int
 625 ixgbe_detach(device_t dev)
 626 {
 627         struct adapter *adapter = device_get_softc(dev);
 628         struct ix_queue *que = adapter->queues;
 629         u32     ctrl_ext;
 630
 631         INIT_DEBUGOUT("ixgbe_detach: begin");
 632
 633         /* Make sure VLANS are not using driver */
 634         if (adapter->ifp->if_vlantrunk != NULL) {
 635                 device_printf(dev,"Vlan in use, detach first\n");
 636                 return (EBUSY);
 637         }
 638
 639         IXGBE_CORE_LOCK(adapter);
 640         ixgbe_stop(adapter);
 641         IXGBE_CORE_UNLOCK(adapter);
 642
 643         for (int i = 0; i < adapter->num_queues; i++, que++) {
 644                 if (que->tq) {
 645                         taskqueue_drain(que->tq, &que->que_task);
 646                         taskqueue_free(que->tq);
 647                 }
 648         }
 649
 650         /* Drain the Link queue */
 651         if (adapter->tq) {
 652                 taskqueue_drain(adapter->tq, &adapter->link_task);
 653                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 654                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 655 #ifdef IXGBE_FDIR
 656                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 657 #endif
 658                 taskqueue_free(adapter->tq);
 659         }
 660
 661         /* let hardware know driver is unloading */
 662         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 663         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 664         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 665
 666         /* Unregister VLAN events */
 667         if (adapter->vlan_attach != NULL)
 668                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 669         if (adapter->vlan_detach != NULL)
 670                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 671
 672         ether_ifdetach(adapter->ifp);
 673         callout_drain(&adapter->timer);
 674 #ifdef DEV_NETMAP
 675         netmap_detach(adapter->ifp);
 676 #endif /* DEV_NETMAP */
 677         ixgbe_free_pci_resources(adapter);
 678         bus_generic_detach(dev);
 679         if_free(adapter->ifp);
 680
 681         ixgbe_free_transmit_structures(adapter);
 682         ixgbe_free_receive_structures(adapter);
 683         free(adapter->mta, M_DEVBUF);
 684
 685         IXGBE_CORE_LOCK_DESTROY(adapter);
 686         return (0);
 687 }
 688
 689 /*********************************************************************
 690  *
 691  *  Shutdown entry point
 692  *
 693  **********************************************************************/
 694
 695 static int
 696 ixgbe_shutdown(device_t dev)
 697 {
 698         struct adapter *adapter = device_get_softc(dev);
 699         IXGBE_CORE_LOCK(adapter);
 700         ixgbe_stop(adapter);
 701         IXGBE_CORE_UNLOCK(adapter);
 702         return (0);
 703 }
 704
 705
 706 /*********************************************************************
 707  *  Transmit entry point
 708  *
 709  *  ixgbe_start is called by the stack to initiate a transmit.
 710  *  The driver will remain in this routine as long as there are
 711  *  packets to transmit and transmit resources are available.
 712  *  In case resources are not available stack is notified and
 713  *  the packet is requeued.
 714  **********************************************************************/
 715
 716 static void
 717 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 718 {
 719         struct mbuf    *m_head;
 720         struct adapter *adapter = txr->adapter;
 721
 722         IXGBE_TX_LOCK_ASSERT(txr);
 723
 724         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 725             IFF_DRV_RUNNING)
 726                 return;
 727         if (!adapter->link_active)
 728                 return;
 729
 730         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 731                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 732                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 733                         break;
 734                 }
 735
 736                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 737                 if (m_head == NULL)
 738                         break;
 739
 740                 if (ixgbe_xmit(txr, &m_head)) {
 741                         if (m_head != NULL)
 742                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 743                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 744                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 745                         break;
 746                 }
 747                 /* Send a copy of the frame to the BPF listener */
 748                 ETHER_BPF_MTAP(ifp, m_head);
 749
 750                 /* Set watchdog on */
 751                 txr->watchdog_time = ticks;
 752                 txr->queue_status = IXGBE_QUEUE_WORKING;
 753
 754         }
 755         return;
 756 }
 757
 758 /*
 759  * Legacy TX start - called by the stack, this
 760  * always uses the first tx ring, and should
 761  * not be used with multiqueue tx enabled.
 762  */
 763 static void
 764 ixgbe_start(struct ifnet *ifp)
 765 {
 766         struct adapter *adapter = ifp->if_softc;
 767         struct tx_ring  *txr = adapter->tx_rings;
 768
 769         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 770                 IXGBE_TX_LOCK(txr);
 771                 ixgbe_start_locked(txr, ifp);
 772                 IXGBE_TX_UNLOCK(txr);
 773         }
 774         return;
 775 }
 776
 777 #if __FreeBSD_version >= 800000
 778 /*
 779 ** Multiqueue Transmit driver
 780 **
 781 */
 782 static int
 783 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 784 {
 785         struct adapter  *adapter = ifp->if_softc;
 786         struct ix_queue *que;
 787         struct tx_ring  *txr;
 788         int             i = 0, err = 0;
 789
 790         /* Which queue to use */
 791         if ((m->m_flags & M_FLOWID) != 0)
 792                 i = m->m_pkthdr.flowid % adapter->num_queues;
 793         else
 794                 i = curcpu % adapter->num_queues;
 795
 796         txr = &adapter->tx_rings[i];
 797         que = &adapter->queues[i];
 798
 799         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 800             IXGBE_TX_TRYLOCK(txr)) {
 801                 err = ixgbe_mq_start_locked(ifp, txr, m);
 802                 IXGBE_TX_UNLOCK(txr);
 803         } else {
 804                 err = drbr_enqueue(ifp, txr->br, m);
 805                 taskqueue_enqueue(que->tq, &que->que_task);
 806         }
 807
 808         return (err);
 809 }
 810
 811 static int
 812 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 813 {
 814         struct adapter  *adapter = txr->adapter;
 815         struct mbuf     *next;
 816         int             enqueued, err = 0;
 817
 818         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 819             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 820             adapter->link_active == 0) {
 821                 if (m != NULL)
 822                         err = drbr_enqueue(ifp, txr->br, m);
 823                 return (err);
 824         }
 825
 826         enqueued = 0;
 827         if (m == NULL) {
 828                 next = drbr_dequeue(ifp, txr->br);
 829         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 830                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 831                         return (err);
 832                 next = drbr_dequeue(ifp, txr->br);
 833         } else
 834                 next = m;
 835
 836         /* Process the queue */
 837         while (next != NULL) {
 838                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 839                         if (next != NULL)
 840                                 err = drbr_enqueue(ifp, txr->br, next);
 841                         break;
 842                 }
 843                 enqueued++;
 844                 /* Send a copy of the frame to the BPF listener */
 845                 ETHER_BPF_MTAP(ifp, next);
 846                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 847                         break;
 848                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 849                         ixgbe_txeof(txr);
 850                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 851                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 852                         break;
 853                 }
 854                 next = drbr_dequeue(ifp, txr->br);
 855         }
 856
 857         if (enqueued > 0) {
 858                 /* Set watchdog on */
 859                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 860                 txr->watchdog_time = ticks;
 861         }
 862
 863         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 864                 ixgbe_txeof(txr);
 865
 866         return (err);
 867 }
 868
 869 /*
 870 ** Flush all ring buffers
 871 */
 872 static void
 873 ixgbe_qflush(struct ifnet *ifp)
 874 {
 875         struct adapter  *adapter = ifp->if_softc;
 876         struct tx_ring  *txr = adapter->tx_rings;
 877         struct mbuf     *m;
 878
 879         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 880                 IXGBE_TX_LOCK(txr);
 881                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 882                         m_freem(m);
 883                 IXGBE_TX_UNLOCK(txr);
 884         }
 885         if_qflush(ifp);
 886 }
 887 #endif /* __FreeBSD_version >= 800000 */
 888
 889 /*********************************************************************
 890  *  Ioctl entry point
 891  *
 892  *  ixgbe_ioctl is called when the user wants to configure the
 893  *  interface.
 894  *
 895  *  return 0 on success, positive on failure
 896  **********************************************************************/
 897
 898 static int
 899 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 900 {
 901         struct adapter  *adapter = ifp->if_softc;
 902         struct ifreq    *ifr = (struct ifreq *) data;
 903 #if defined(INET) || defined(INET6)
 904         struct ifaddr *ifa = (struct ifaddr *)data;
 905         bool            avoid_reset = FALSE;
 906 #endif
 907         int             error = 0;
 908
 909         switch (command) {
 910
 911         case SIOCSIFADDR:
 912 #ifdef INET
 913                 if (ifa->ifa_addr->sa_family == AF_INET)
 914                         avoid_reset = TRUE;
 915 #endif
 916 #ifdef INET6
 917                 if (ifa->ifa_addr->sa_family == AF_INET6)
 918                         avoid_reset = TRUE;
 919 #endif
 920 #if defined(INET) || defined(INET6)
 921                 /*
 922                 ** Calling init results in link renegotiation,
 923                 ** so we avoid doing it when possible.
 924                 */
 925                 if (avoid_reset) {
 926                         ifp->if_flags |= IFF_UP;
 927                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 928                                 ixgbe_init(adapter);
 929                         if (!(ifp->if_flags & IFF_NOARP))
 930                                 arp_ifinit(ifp, ifa);
 931                 } else
 932                         error = ether_ioctl(ifp, command, data);
 933 #endif
 934                 break;
 935         case SIOCSIFMTU:
 936                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 937                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 938                         error = EINVAL;
 939                 } else {
 940                         IXGBE_CORE_LOCK(adapter);
 941                         ifp->if_mtu = ifr->ifr_mtu;
 942                         adapter->max_frame_size =
 943                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 944                         ixgbe_init_locked(adapter);
 945                         IXGBE_CORE_UNLOCK(adapter);
 946                 }
 947                 break;
 948         case SIOCSIFFLAGS:
 949                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 950                 IXGBE_CORE_LOCK(adapter);
 951                 if (ifp->if_flags & IFF_UP) {
 952                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 953                                 if ((ifp->if_flags ^ adapter->if_flags) &
 954                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 955                                         ixgbe_set_promisc(adapter);
 956                                 }
 957                         } else
 958                                 ixgbe_init_locked(adapter);
 959                 } else
 960                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 961                                 ixgbe_stop(adapter);
 962                 adapter->if_flags = ifp->if_flags;
 963                 IXGBE_CORE_UNLOCK(adapter);
 964                 break;
 965         case SIOCADDMULTI:
 966         case SIOCDELMULTI:
 967                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 968                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 969                         IXGBE_CORE_LOCK(adapter);
 970                         ixgbe_disable_intr(adapter);
 971                         ixgbe_set_multi(adapter);
 972                         ixgbe_enable_intr(adapter);
 973                         IXGBE_CORE_UNLOCK(adapter);
 974                 }
 975                 break;
 976         case SIOCSIFMEDIA:
 977         case SIOCGIFMEDIA:
 978                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 979                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 980                 break;
 981         case SIOCSIFCAP:
 982         {
 983                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 984                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 985                 if (mask & IFCAP_HWCSUM)
 986                         ifp->if_capenable ^= IFCAP_HWCSUM;
 987                 if (mask & IFCAP_TSO4)
 988                         ifp->if_capenable ^= IFCAP_TSO4;
 989                 if (mask & IFCAP_LRO)
 990                         ifp->if_capenable ^= IFCAP_LRO;
 991                 if (mask & IFCAP_VLAN_HWTAGGING)
 992                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 993                 if (mask & IFCAP_VLAN_HWFILTER)
 994                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 995                 if (mask & IFCAP_VLAN_HWTSO)
 996                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 997                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 998                         IXGBE_CORE_LOCK(adapter);
 999                         ixgbe_init_locked(adapter);
1000                         IXGBE_CORE_UNLOCK(adapter);
1001                 }
1002                 VLAN_CAPABILITIES(ifp);
1003                 break;
1004         }
1005
1006         default:
1007                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1008                 error = ether_ioctl(ifp, command, data);
1009                 break;
1010         }
1011
1012         return (error);
1013 }
1014
1015 /*********************************************************************
1016  *  Init entry point
1017  *
1018  *  This routine is used in two ways. It is used by the stack as
1019  *  init entry point in network interface structure. It is also used
1020  *  by the driver as a hw/sw initialization routine to get to a
1021  *  consistent state.
1022  *
1023  *  return 0 on success, positive on failure
1024  **********************************************************************/
1025 #define IXGBE_MHADD_MFS_SHIFT 16
1026
1027 static void
1028 ixgbe_init_locked(struct adapter *adapter)
1029 {
1030         struct ifnet   *ifp = adapter->ifp;
1031         device_t        dev = adapter->dev;
1032         struct ixgbe_hw *hw = &adapter->hw;
1033         u32             k, txdctl, mhadd, gpie;
1034         u32             rxdctl, rxctrl;
1035
1036         mtx_assert(&adapter->core_mtx, MA_OWNED);
1037         INIT_DEBUGOUT("ixgbe_init: begin");
1038         hw->adapter_stopped = FALSE;
1039         ixgbe_stop_adapter(hw);
1040         callout_stop(&adapter->timer);
1041
1042         /* reprogram the RAR[0] in case user changed it. */
1043         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1044
1045         /* Get the latest mac address, User can use a LAA */
1046         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1047               IXGBE_ETH_LENGTH_OF_ADDRESS);
1048         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1049         hw->addr_ctrl.rar_used_count = 1;
1050
1051         /* Set the various hardware offload abilities */
1052         ifp->if_hwassist = 0;
1053         if (ifp->if_capenable & IFCAP_TSO4)
1054                 ifp->if_hwassist |= CSUM_TSO;
1055         if (ifp->if_capenable & IFCAP_TXCSUM) {
1056                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1057 #if __FreeBSD_version >= 800000
1058                 if (hw->mac.type != ixgbe_mac_82598EB)
1059                         ifp->if_hwassist |= CSUM_SCTP;
1060 #endif
1061         }
1062
1063         /* Prepare transmit descriptors and buffers */
1064         if (ixgbe_setup_transmit_structures(adapter)) {
1065                 device_printf(dev,"Could not setup transmit structures\n");
1066                 ixgbe_stop(adapter);
1067                 return;
1068         }
1069
1070         ixgbe_init_hw(hw);
1071         ixgbe_initialize_transmit_units(adapter);
1072
1073         /* Setup Multicast table */
1074         ixgbe_set_multi(adapter);
1075
1076         /*
1077         ** Determine the correct mbuf pool
1078         ** for doing jumbo/headersplit
1079         */
1080         if (adapter->max_frame_size <= 2048)
1081                 adapter->rx_mbuf_sz = MCLBYTES;
1082         else if (adapter->max_frame_size <= 4096)
1083                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1084         else if (adapter->max_frame_size <= 9216)
1085                 adapter->rx_mbuf_sz = MJUM9BYTES;
1086         else
1087                 adapter->rx_mbuf_sz = MJUM16BYTES;
1088
1089         /* Prepare receive descriptors and buffers */
1090         if (ixgbe_setup_receive_structures(adapter)) {
1091                 device_printf(dev,"Could not setup receive structures\n");
1092                 ixgbe_stop(adapter);
1093                 return;
1094         }
1095
1096         /* Configure RX settings */
1097         ixgbe_initialize_receive_units(adapter);
1098
1099         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1100
1101         /* Enable Fan Failure Interrupt */
1102         gpie |= IXGBE_SDP1_GPIEN;
1103
1104         /* Add for Module detection */
1105         if (hw->mac.type == ixgbe_mac_82599EB)
1106                 gpie |= IXGBE_SDP2_GPIEN;
1107
1108         /* Thermal Failure Detection */
1109         if (hw->mac.type == ixgbe_mac_X540)
1110                 gpie |= IXGBE_SDP0_GPIEN;
1111
1112         if (adapter->msix > 1) {
1113                 /* Enable Enhanced MSIX mode */
1114                 gpie |= IXGBE_GPIE_MSIX_MODE;
1115                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1116                     IXGBE_GPIE_OCD;
1117         }
1118         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1119
1120         /* Set MTU size */
1121         if (ifp->if_mtu > ETHERMTU) {
1122                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1123                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1124                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1125                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1126         }
1127
1128         /* Now enable all the queues */
1129
1130         for (int i = 0; i < adapter->num_queues; i++) {
1131                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1132                 txdctl |= IXGBE_TXDCTL_ENABLE;
1133                 /* Set WTHRESH to 8, burst writeback */
1134                 txdctl |= (8 << 16);
1135                 /*
1136                  * When the internal queue falls below PTHRESH (32),
1137                  * start prefetching as long as there are at least
1138                  * HTHRESH (1) buffers ready. The values are taken
1139                  * from the Intel linux driver 3.8.21.
1140                  * Prefetching enables tx line rate even with 1 queue.
1141                  */
1142                 txdctl |= (32 << 0) | (1 << 8);
1143                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1144         }
1145
1146         for (int i = 0; i < adapter->num_queues; i++) {
1147                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1148                 if (hw->mac.type == ixgbe_mac_82598EB) {
1149                         /*
1150                         ** PTHRESH = 21
1151                         ** HTHRESH = 4
1152                         ** WTHRESH = 8
1153                         */
1154                         rxdctl &= ~0x3FFFFF;
1155                         rxdctl |= 0x080420;
1156                 }
1157                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1158                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1159                 for (k = 0; k < 10; k++) {
1160                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1161                             IXGBE_RXDCTL_ENABLE)
1162                                 break;
1163                         else
1164                                 msec_delay(1);
1165                 }
1166                 wmb();
1167 #ifdef DEV_NETMAP
1168                 /*
1169                  * In netmap mode, we must preserve the buffers made
1170                  * available to userspace before the if_init()
1171                  * (this is true by default on the TX side, because
1172                  * init makes all buffers available to userspace).
1173                  *
1174                  * netmap_reset() and the device specific routines
1175                  * (e.g. ixgbe_setup_receive_rings()) map these
1176                  * buffers at the end of the NIC ring, so here we
1177                  * must set the RDT (tail) register to make sure
1178                  * they are not overwritten.
1179                  *
1180                  * In this driver the NIC ring starts at RDH = 0,
1181                  * RDT points to the last slot available for reception (?),
1182                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1183                  */
1184                 if (ifp->if_capenable & IFCAP_NETMAP) {
1185                         struct netmap_adapter *na = NA(adapter->ifp);
1186                         struct netmap_kring *kring = &na->rx_rings[i];
1187                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1188
1189                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1190                 } else
1191 #endif /* DEV_NETMAP */
1192                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1193         }
1194
1195         /* Set up VLAN support and filter */
1196         ixgbe_setup_vlan_hw_support(adapter);
1197
1198         /* Enable Receive engine */
1199         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1200         if (hw->mac.type == ixgbe_mac_82598EB)
1201                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1202         rxctrl |= IXGBE_RXCTRL_RXEN;
1203         ixgbe_enable_rx_dma(hw, rxctrl);
1204
1205         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1206
1207         /* Set up MSI/X routing */
1208         if (ixgbe_enable_msix)  {
1209                 ixgbe_configure_ivars(adapter);
1210                 /* Set up auto-mask */
1211                 if (hw->mac.type == ixgbe_mac_82598EB)
1212                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1213                 else {
1214                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1215                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1216                 }
1217         } else {  /* Simple settings for Legacy/MSI */
1218                 ixgbe_set_ivar(adapter, 0, 0, 0);
1219                 ixgbe_set_ivar(adapter, 0, 0, 1);
1220                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1221         }
1222
1223 #ifdef IXGBE_FDIR
1224         /* Init Flow director */
1225         if (hw->mac.type != ixgbe_mac_82598EB) {
1226                 u32 hdrm = 32 << fdir_pballoc;
1227
1228                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1229                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1230         }
1231 #endif
1232
1233         /*
1234         ** Check on any SFP devices that
1235         ** need to be kick-started
1236         */
1237         if (hw->phy.type == ixgbe_phy_none) {
1238                 int err = hw->phy.ops.identify(hw);
1239                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1240                         device_printf(dev,
1241                             "Unsupported SFP+ module type was detected.\n");
1242                         return;
1243                 }
1244         }
1245
1246         /* Set moderation on the Link interrupt */
1247         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1248
1249         /* Config/Enable Link */
1250         ixgbe_config_link(adapter);
1251
1252         /* Hardware Packet Buffer & Flow Control setup */
1253         {
1254                 u32 rxpb, frame, size, tmp;
1255
1256                 frame = adapter->max_frame_size;
1257
1258                 /* Calculate High Water */
1259                 if (hw->mac.type == ixgbe_mac_X540)
1260                         tmp = IXGBE_DV_X540(frame, frame);
1261                 else
1262                         tmp = IXGBE_DV(frame, frame);
1263                 size = IXGBE_BT2KB(tmp);
1264                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1265                 hw->fc.high_water[0] = rxpb - size;
1266
1267                 /* Now calculate Low Water */
1268                 if (hw->mac.type == ixgbe_mac_X540)
1269                         tmp = IXGBE_LOW_DV_X540(frame);
1270                 else
1271                         tmp = IXGBE_LOW_DV(frame);
1272                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1273
1274                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1275                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1276                 hw->fc.send_xon = TRUE;
1277         }
1278         /* Initialize the FC settings */
1279         ixgbe_start_hw(hw);
1280
1281         /* And now turn on interrupts */
1282         ixgbe_enable_intr(adapter);
1283
1284         /* Now inform the stack we're ready */
1285         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1286         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1287
1288         return;
1289 }
1290
1291 static void
1292 ixgbe_init(void *arg)
1293 {
1294         struct adapter *adapter = arg;
1295
1296         IXGBE_CORE_LOCK(adapter);
1297         ixgbe_init_locked(adapter);
1298         IXGBE_CORE_UNLOCK(adapter);
1299         return;
1300 }
1301
1302
1303 /*
1304 **
1305 ** MSIX Interrupt Handlers and Tasklets
1306 **
1307 */
1308
1309 static inline void
1310 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1311 {
1312         struct ixgbe_hw *hw = &adapter->hw;
1313         u64     queue = (u64)(1 << vector);
1314         u32     mask;
1315
1316         if (hw->mac.type == ixgbe_mac_82598EB) {
1317                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1318                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1319         } else {
1320                 mask = (queue & 0xFFFFFFFF);
1321                 if (mask)
1322                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1323                 mask = (queue >> 32);
1324                 if (mask)
1325                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1326         }
1327 }
1328
1329 static inline void
1330 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1331 {
1332         struct ixgbe_hw *hw = &adapter->hw;
1333         u64     queue = (u64)(1 << vector);
1334         u32     mask;
1335
1336         if (hw->mac.type == ixgbe_mac_82598EB) {
1337                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1338                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1339         } else {
1340                 mask = (queue & 0xFFFFFFFF);
1341                 if (mask)
1342                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1343                 mask = (queue >> 32);
1344                 if (mask)
1345                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1346         }
1347 }
1348
1349 static inline void
1350 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1351 {
1352         u32 mask;
1353
1354         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1355                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1356                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1357         } else {
1358                 mask = (queues & 0xFFFFFFFF);
1359                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1360                 mask = (queues >> 32);
1361                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1362         }
1363 }
1364
1365
1366 static void
1367 ixgbe_handle_que(void *context, int pending)
1368 {
1369         struct ix_queue *que = context;
1370         struct adapter  *adapter = que->adapter;
1371         struct tx_ring  *txr = que->txr;
1372         struct ifnet    *ifp = adapter->ifp;
1373         bool            more;
1374
1375         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1376                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1377                 IXGBE_TX_LOCK(txr);
1378                 ixgbe_txeof(txr);
1379 #if __FreeBSD_version >= 800000
1380                 if (!drbr_empty(ifp, txr->br))
1381                         ixgbe_mq_start_locked(ifp, txr, NULL);
1382 #else
1383                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1384                         ixgbe_start_locked(txr, ifp);
1385 #endif
1386                 IXGBE_TX_UNLOCK(txr);
1387                 if (more) {
1388                         taskqueue_enqueue(que->tq, &que->que_task);
1389                         return;
1390                 }
1391         }
1392
1393         /* Reenable this interrupt */
1394         ixgbe_enable_queue(adapter, que->msix);
1395         return;
1396 }
1397
1398
1399 /*********************************************************************
1400  *
1401  *  Legacy Interrupt Service routine
1402  *
1403  **********************************************************************/
1404
1405 static void
1406 ixgbe_legacy_irq(void *arg)
1407 {
1408         struct ix_queue *que = arg;
1409         struct adapter  *adapter = que->adapter;
1410         struct ixgbe_hw *hw = &adapter->hw;
1411         struct          tx_ring *txr = adapter->tx_rings;
1412         bool            more_tx, more_rx;
1413         u32             reg_eicr, loop = MAX_LOOP;
1414
1415
1416         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1417
1418         ++que->irqs;
1419         if (reg_eicr == 0) {
1420                 ixgbe_enable_intr(adapter);
1421                 return;
1422         }
1423
1424         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1425
1426         IXGBE_TX_LOCK(txr);
1427         do {
1428                 more_tx = ixgbe_txeof(txr);
1429         } while (loop-- && more_tx);
1430         IXGBE_TX_UNLOCK(txr);
1431
1432         if (more_rx || more_tx)
1433                 taskqueue_enqueue(que->tq, &que->que_task);
1434
1435         /* Check for fan failure */
1436         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1437             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1438                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1439                     "REPLACE IMMEDIATELY!!\n");
1440                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1441         }
1442
1443         /* Link status change */
1444         if (reg_eicr & IXGBE_EICR_LSC)
1445                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1446
1447         ixgbe_enable_intr(adapter);
1448         return;
1449 }
1450
1451
1452 /*********************************************************************
1453  *
1454  *  MSIX Queue Interrupt Service routine
1455  *
1456  **********************************************************************/
1457 void
1458 ixgbe_msix_que(void *arg)
1459 {
1460         struct ix_queue *que = arg;
1461         struct adapter  *adapter = que->adapter;
1462         struct tx_ring  *txr = que->txr;
1463         struct rx_ring  *rxr = que->rxr;
1464         bool            more_tx, more_rx;
1465         u32             newitr = 0;
1466
1467         ixgbe_disable_queue(adapter, que->msix);
1468         ++que->irqs;
1469
1470         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1471
1472         IXGBE_TX_LOCK(txr);
1473         more_tx = ixgbe_txeof(txr);
1474         /*
1475         ** Make certain that if the stack
1476         ** has anything queued the task gets
1477         ** scheduled to handle it.
1478         */
1479 #if __FreeBSD_version < 800000
1480         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1481 #else
1482         if (!drbr_empty(adapter->ifp, txr->br))
1483 #endif
1484                 more_tx = 1;
1485         IXGBE_TX_UNLOCK(txr);
1486
1487         /* Do AIM now? */
1488
1489         if (ixgbe_enable_aim == FALSE)
1490                 goto no_calc;
1491         /*
1492         ** Do Adaptive Interrupt Moderation:
1493         **  - Write out last calculated setting
1494         **  - Calculate based on average size over
1495         **    the last interval.
1496         */
1497         if (que->eitr_setting)
1498                 IXGBE_WRITE_REG(&adapter->hw,
1499                     IXGBE_EITR(que->msix), que->eitr_setting);
1500
1501         que->eitr_setting = 0;
1502
1503         /* Idle, do nothing */
1504         if ((txr->bytes == 0) && (rxr->bytes == 0))
1505                 goto no_calc;
1506
1507         if ((txr->bytes) && (txr->packets))
1508                 newitr = txr->bytes/txr->packets;
1509         if ((rxr->bytes) && (rxr->packets))
1510                 newitr = max(newitr,
1511                     (rxr->bytes / rxr->packets));
1512         newitr += 24; /* account for hardware frame, crc */
1513
1514         /* set an upper boundary */
1515         newitr = min(newitr, 3000);
1516
1517         /* Be nice to the mid range */
1518         if ((newitr > 300) && (newitr < 1200))
1519                 newitr = (newitr / 3);
1520         else
1521                 newitr = (newitr / 2);
1522
1523         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1524                 newitr |= newitr << 16;
1525         else
1526                 newitr |= IXGBE_EITR_CNT_WDIS;
1527
1528         /* save for next interrupt */
1529         que->eitr_setting = newitr;
1530
1531         /* Reset state */
1532         txr->bytes = 0;
1533         txr->packets = 0;
1534         rxr->bytes = 0;
1535         rxr->packets = 0;
1536
1537 no_calc:
1538         if (more_tx || more_rx)
1539                 taskqueue_enqueue(que->tq, &que->que_task);
1540         else /* Reenable this interrupt */
1541                 ixgbe_enable_queue(adapter, que->msix);
1542         return;
1543 }
1544
1545
1546 static void
1547 ixgbe_msix_link(void *arg)
1548 {
1549         struct adapter  *adapter = arg;
1550         struct ixgbe_hw *hw = &adapter->hw;
1551         u32             reg_eicr;
1552
1553         ++adapter->link_irq;
1554
1555         /* First get the cause */
1556         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1557         /* Clear interrupt with write */
1558         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1559
1560         /* Link status change */
1561         if (reg_eicr & IXGBE_EICR_LSC)
1562                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1563
1564         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1565 #ifdef IXGBE_FDIR
1566                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1567                         /* This is probably overkill :) */
1568                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1569                                 return;
1570                         /* Disable the interrupt */
1571                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1572                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1573                 } else
1574 #endif
1575                 if (reg_eicr & IXGBE_EICR_ECC) {
1576                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1577                             "Please Reboot!!\n");
1578                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1579                 } else
1580
1581                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1582                         /* Clear the interrupt */
1583                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1584                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1585                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1586                         /* Clear the interrupt */
1587                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1588                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1589                 }
1590         }
1591
1592         /* Check for fan failure */
1593         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1594             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1595                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1596                     "REPLACE IMMEDIATELY!!\n");
1597                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1598         }
1599
1600         /* Check for over temp condition */
1601         if ((hw->mac.type == ixgbe_mac_X540) &&
1602             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1603                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1604                     "PHY IS SHUT DOWN!!\n");
1605                 device_printf(adapter->dev, "System shutdown required\n");
1606                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1607         }
1608
1609         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1610         return;
1611 }
1612
1613 /*********************************************************************
1614  *
1615  *  Media Ioctl callback
1616  *
1617  *  This routine is called whenever the user queries the status of
1618  *  the interface using ifconfig.
1619  *
1620  **********************************************************************/
1621 static void
1622 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1623 {
1624         struct adapter *adapter = ifp->if_softc;
1625
1626         INIT_DEBUGOUT("ixgbe_media_status: begin");
1627         IXGBE_CORE_LOCK(adapter);
1628         ixgbe_update_link_status(adapter);
1629
1630         ifmr->ifm_status = IFM_AVALID;
1631         ifmr->ifm_active = IFM_ETHER;
1632
1633         if (!adapter->link_active) {
1634                 IXGBE_CORE_UNLOCK(adapter);
1635                 return;
1636         }
1637
1638         ifmr->ifm_status |= IFM_ACTIVE;
1639
1640         switch (adapter->link_speed) {
1641                 case IXGBE_LINK_SPEED_100_FULL:
1642                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1643                         break;
1644                 case IXGBE_LINK_SPEED_1GB_FULL:
1645                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1646                         break;
1647                 case IXGBE_LINK_SPEED_10GB_FULL:
1648                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1649                         break;
1650         }
1651
1652         IXGBE_CORE_UNLOCK(adapter);
1653
1654         return;
1655 }
1656
1657 /*********************************************************************
1658  *
1659  *  Media Ioctl callback
1660  *
1661  *  This routine is called when the user changes speed/duplex using
1662  *  media/mediopt option with ifconfig.
1663  *
1664  **********************************************************************/
1665 static int
1666 ixgbe_media_change(struct ifnet * ifp)
1667 {
1668         struct adapter *adapter = ifp->if_softc;
1669         struct ifmedia *ifm = &adapter->media;
1670
1671         INIT_DEBUGOUT("ixgbe_media_change: begin");
1672
1673         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1674                 return (EINVAL);
1675
1676         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1677         case IFM_AUTO:
1678                 adapter->hw.phy.autoneg_advertised =
1679                     IXGBE_LINK_SPEED_100_FULL |
1680                     IXGBE_LINK_SPEED_1GB_FULL |
1681                     IXGBE_LINK_SPEED_10GB_FULL;
1682                 break;
1683         default:
1684                 device_printf(adapter->dev, "Only auto media type\n");
1685                 return (EINVAL);
1686         }
1687
1688         return (0);
1689 }
1690
1691 /*********************************************************************
1692  *
1693  *  This routine maps the mbufs to tx descriptors, allowing the
1694  *  TX engine to transmit the packets.
1695  *      - return 0 on success, positive on failure
1696  *
1697  **********************************************************************/
1698
1699 static int
1700 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1701 {
1702         struct adapter  *adapter = txr->adapter;
1703         u32             olinfo_status = 0, cmd_type_len;
1704         u32             paylen = 0;
1705         int             i, j, error, nsegs;
1706         int             first, last = 0;
1707         struct mbuf     *m_head;
1708         bus_dma_segment_t segs[adapter->num_segs];
1709         bus_dmamap_t    map;
1710         struct ixgbe_tx_buf *txbuf;
1711         union ixgbe_adv_tx_desc *txd = NULL;
1712
1713         m_head = *m_headp;
1714
1715         /* Basic descriptor defines */
1716         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1717             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1718
1719         if (m_head->m_flags & M_VLANTAG)
1720                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1721
1722         /*
1723          * Important to capture the first descriptor
1724          * used because it will contain the index of
1725          * the one we tell the hardware to report back
1726          */
1727         first = txr->next_avail_desc;
1728         txbuf = &txr->tx_buffers[first];
1729         map = txbuf->map;
1730
1731         /*
1732          * Map the packet for DMA.
1733          */
1734         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1735             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1736
1737         if (error == EFBIG) {
1738                 struct mbuf *m;
1739
1740                 m = m_defrag(*m_headp, M_DONTWAIT);
1741                 if (m == NULL) {
1742                         adapter->mbuf_defrag_failed++;
1743                         m_freem(*m_headp);
1744                         *m_headp = NULL;
1745                         return (ENOBUFS);
1746                 }
1747                 *m_headp = m;
1748
1749                 /* Try it again */
1750                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1751                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1752
1753                 if (error == ENOMEM) {
1754                         adapter->no_tx_dma_setup++;
1755                         return (error);
1756                 } else if (error != 0) {
1757                         adapter->no_tx_dma_setup++;
1758                         m_freem(*m_headp);
1759                         *m_headp = NULL;
1760                         return (error);
1761                 }
1762         } else if (error == ENOMEM) {
1763                 adapter->no_tx_dma_setup++;
1764                 return (error);
1765         } else if (error != 0) {
1766                 adapter->no_tx_dma_setup++;
1767                 m_freem(*m_headp);
1768                 *m_headp = NULL;
1769                 return (error);
1770         }
1771
1772         /* Make certain there are enough descriptors */
1773         if (nsegs > txr->tx_avail - 2) {
1774                 txr->no_desc_avail++;
1775                 error = ENOBUFS;
1776                 goto xmit_fail;
1777         }
1778         m_head = *m_headp;
1779
1780         /*
1781         ** Set up the appropriate offload context
1782         ** this becomes the first descriptor of
1783         ** a packet.
1784         */
1785         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1786                 if (ixgbe_tso_setup(txr, m_head, &paylen)) {
1787                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1788                         olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1789                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1790                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1791                         ++adapter->tso_tx;
1792                 } else
1793                         return (ENXIO);
1794         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1795                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1796
1797 #ifdef IXGBE_IEEE1588
1798         /* This is changing soon to an mtag detection */
1799         if (we detect this mbuf has a TSTAMP mtag)
1800                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1801 #endif
1802
1803 #ifdef IXGBE_FDIR
1804         /* Do the flow director magic */
1805         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1806                 ++txr->atr_count;
1807                 if (txr->atr_count >= atr_sample_rate) {
1808                         ixgbe_atr(txr, m_head);
1809                         txr->atr_count = 0;
1810                 }
1811         }
1812 #endif
1813         /* Record payload length */
1814         if (paylen == 0)
1815                 olinfo_status |= m_head->m_pkthdr.len <<
1816                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1817
1818         i = txr->next_avail_desc;
1819         for (j = 0; j < nsegs; j++) {
1820                 bus_size_t seglen;
1821                 bus_addr_t segaddr;
1822
1823                 txbuf = &txr->tx_buffers[i];
1824                 txd = &txr->tx_base[i];
1825                 seglen = segs[j].ds_len;
1826                 segaddr = htole64(segs[j].ds_addr);
1827
1828                 txd->read.buffer_addr = segaddr;
1829                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1830                     cmd_type_len |seglen);
1831                 txd->read.olinfo_status = htole32(olinfo_status);
1832                 last = i; /* descriptor that will get completion IRQ */
1833
1834                 if (++i == adapter->num_tx_desc)
1835                         i = 0;
1836
1837                 txbuf->m_head = NULL;
1838                 txbuf->eop_index = -1;
1839         }
1840
1841         txd->read.cmd_type_len |=
1842             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1843         txr->tx_avail -= nsegs;
1844         txr->next_avail_desc = i;
1845
1846         txbuf->m_head = m_head;
1847         /* Swap the dma map between the first and last descriptor */
1848         txr->tx_buffers[first].map = txbuf->map;
1849         txbuf->map = map;
1850         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1851
1852         /* Set the index of the descriptor that will be marked done */
1853         txbuf = &txr->tx_buffers[first];
1854         txbuf->eop_index = last;
1855
1856         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1857             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1858         /*
1859          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1860          * hardware that this frame is available to transmit.
1861          */
1862         ++txr->total_packets;
1863         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1864
1865         return (0);
1866
1867 xmit_fail:
1868         bus_dmamap_unload(txr->txtag, txbuf->map);
1869         return (error);
1870
1871 }
1872
1873 static void
1874 ixgbe_set_promisc(struct adapter *adapter)
1875 {
1876         u_int32_t       reg_rctl;
1877         struct ifnet   *ifp = adapter->ifp;
1878
1879         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1880         reg_rctl &= (~IXGBE_FCTRL_UPE);
1881         reg_rctl &= (~IXGBE_FCTRL_MPE);
1882         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1883
1884         if (ifp->if_flags & IFF_PROMISC) {
1885                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1886                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1887         } else if (ifp->if_flags & IFF_ALLMULTI) {
1888                 reg_rctl |= IXGBE_FCTRL_MPE;
1889                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1890                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1891         }
1892         return;
1893 }
1894
1895
1896 /*********************************************************************
1897  *  Multicast Update
1898  *
1899  *  This routine is called whenever multicast address list is updated.
1900  *
1901  **********************************************************************/
1902 #define IXGBE_RAR_ENTRIES 16
1903
1904 static void
1905 ixgbe_set_multi(struct adapter *adapter)
1906 {
1907         u32     fctrl;
1908         u8      *mta;
1909         u8      *update_ptr;
1910         struct  ifmultiaddr *ifma;
1911         int     mcnt = 0;
1912         struct ifnet   *ifp = adapter->ifp;
1913
1914         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1915
1916         mta = adapter->mta;
1917         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1918             MAX_NUM_MULTICAST_ADDRESSES);
1919
1920         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1921         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1922         if (ifp->if_flags & IFF_PROMISC)
1923                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1924         else if (ifp->if_flags & IFF_ALLMULTI) {
1925                 fctrl |= IXGBE_FCTRL_MPE;
1926                 fctrl &= ~IXGBE_FCTRL_UPE;
1927         } else
1928                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1929
1930         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1931
1932 #if __FreeBSD_version < 800000
1933         IF_ADDR_LOCK(ifp);
1934 #else
1935         if_maddr_rlock(ifp);
1936 #endif
1937         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1938                 if (ifma->ifma_addr->sa_family != AF_LINK)
1939                         continue;
1940                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1941                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1942                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1943                 mcnt++;
1944         }
1945 #if __FreeBSD_version < 800000
1946         IF_ADDR_UNLOCK(ifp);
1947 #else
1948         if_maddr_runlock(ifp);
1949 #endif
1950
1951         update_ptr = mta;
1952         ixgbe_update_mc_addr_list(&adapter->hw,
1953             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1954
1955         return;
1956 }
1957
1958 /*
1959  * This is an iterator function now needed by the multicast
1960  * shared code. It simply feeds the shared code routine the
1961  * addresses in the array of ixgbe_set_multi() one by one.
1962  */
1963 static u8 *
1964 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1965 {
1966         u8 *addr = *update_ptr;
1967         u8 *newptr;
1968         *vmdq = 0;
1969
1970         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1971         *update_ptr = newptr;
1972         return addr;
1973 }
1974
1975
1976 /*********************************************************************
1977  *  Timer routine
1978  *
1979  *  This routine checks for link status,updates statistics,
1980  *  and runs the watchdog check.
1981  *
1982  **********************************************************************/
1983
1984 static void
1985 ixgbe_local_timer(void *arg)
1986 {
1987         struct adapter  *adapter = arg;
1988         device_t        dev = adapter->dev;
1989         struct ifnet    *ifp = adapter->ifp;
1990         struct ix_queue *que = adapter->queues;
1991         struct tx_ring  *txr = adapter->tx_rings;
1992         int             hung, busy, paused;
1993
1994         mtx_assert(&adapter->core_mtx, MA_OWNED);
1995         hung = busy = paused = 0;
1996
1997         /* Check for pluggable optics */
1998         if (adapter->sfp_probe)
1999                 if (!ixgbe_sfp_probe(adapter))
2000                         goto out; /* Nothing to do */
2001
2002         ixgbe_update_link_status(adapter);
2003         ixgbe_update_stats_counters(adapter);
2004
2005         /*
2006          * If the interface has been paused
2007          * then don't do the watchdog check
2008          */
2009         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2010                 paused = 1;
2011
2012         /*
2013         ** Check the TX queues status
2014         **      - central locked handling of OACTIVE
2015         **      - watchdog only if all queues show hung
2016         */
2017         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2018                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2019                     (paused == 0))
2020                         ++hung;
2021                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2022                         ++busy;
2023                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2024                         taskqueue_enqueue(que->tq, &que->que_task);
2025         }
2026         /* Only truely watchdog if all queues show hung */
2027         if (hung == adapter->num_queues)
2028                 goto watchdog;
2029         /* Only turn off the stack flow when ALL are depleted */
2030         if (busy == adapter->num_queues)
2031                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2032         else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
2033             (busy < adapter->num_queues))
2034                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2035
2036 out:
2037         ixgbe_rearm_queues(adapter, adapter->que_mask);
2038         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2039         return;
2040
2041 watchdog:
2042         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2043         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2044             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2045             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2046         device_printf(dev,"TX(%d) desc avail = %d,"
2047             "Next TX to Clean = %d\n",
2048             txr->me, txr->tx_avail, txr->next_to_clean);
2049         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2050         adapter->watchdog_events++;
2051         ixgbe_init_locked(adapter);
2052 }
2053
2054 /*
2055 ** Note: this routine updates the OS on the link state
2056 **      the real check of the hardware only happens with
2057 **      a link interrupt.
2058 */
2059 static void
2060 ixgbe_update_link_status(struct adapter *adapter)
2061 {
2062         struct ifnet    *ifp = adapter->ifp;
2063         struct tx_ring *txr = adapter->tx_rings;
2064         device_t dev = adapter->dev;
2065
2066
2067         if (adapter->link_up){
2068                 if (adapter->link_active == FALSE) {
2069                         if (bootverbose)
2070                                 device_printf(dev,"Link is up %d Gbps %s \n",
2071                                     ((adapter->link_speed == 128)? 10:1),
2072                                     "Full Duplex");
2073                         adapter->link_active = TRUE;
2074                         /* Update any Flow Control changes */
2075                         ixgbe_fc_enable(&adapter->hw);
2076                         if_link_state_change(ifp, LINK_STATE_UP);
2077                 }
2078         } else { /* Link down */
2079                 if (adapter->link_active == TRUE) {
2080                         if (bootverbose)
2081                                 device_printf(dev,"Link is Down\n");
2082                         if_link_state_change(ifp, LINK_STATE_DOWN);
2083                         adapter->link_active = FALSE;
2084                         for (int i = 0; i < adapter->num_queues;
2085                             i++, txr++)
2086                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2087                 }
2088         }
2089
2090         return;
2091 }
2092
2093
2094 /*********************************************************************
2095  *
2096  *  This routine disables all traffic on the adapter by issuing a
2097  *  global reset on the MAC and deallocates TX/RX buffers.
2098  *
2099  **********************************************************************/
2100
2101 static void
2102 ixgbe_stop(void *arg)
2103 {
2104         struct ifnet   *ifp;
2105         struct adapter *adapter = arg;
2106         struct ixgbe_hw *hw = &adapter->hw;
2107         ifp = adapter->ifp;
2108
2109         mtx_assert(&adapter->core_mtx, MA_OWNED);
2110
2111         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2112         ixgbe_disable_intr(adapter);
2113         callout_stop(&adapter->timer);
2114
2115         /* Let the stack know...*/
2116         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2117         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2118
2119         ixgbe_reset_hw(hw);
2120         hw->adapter_stopped = FALSE;
2121         ixgbe_stop_adapter(hw);
2122         /* Turn off the laser */
2123         if (hw->phy.multispeed_fiber)
2124                 ixgbe_disable_tx_laser(hw);
2125
2126         /* reprogram the RAR[0] in case user changed it. */
2127         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2128
2129         return;
2130 }
2131
2132
2133 /*********************************************************************
2134  *
2135  *  Determine hardware revision.
2136  *
2137  **********************************************************************/
2138 static void
2139 ixgbe_identify_hardware(struct adapter *adapter)
2140 {
2141         device_t        dev = adapter->dev;
2142         struct ixgbe_hw *hw = &adapter->hw;
2143
2144         /* Save off the information about this board */
2145         hw->vendor_id = pci_get_vendor(dev);
2146         hw->device_id = pci_get_device(dev);
2147         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2148         hw->subsystem_vendor_id =
2149             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2150         hw->subsystem_device_id =
2151             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2152
2153         /* We need this here to set the num_segs below */
2154         ixgbe_set_mac_type(hw);
2155
2156         /* Pick up the 82599 and VF settings */
2157         if (hw->mac.type != ixgbe_mac_82598EB) {
2158                 hw->phy.smart_speed = ixgbe_smart_speed;
2159                 adapter->num_segs = IXGBE_82599_SCATTER;
2160         } else
2161                 adapter->num_segs = IXGBE_82598_SCATTER;
2162
2163         return;
2164 }
2165
2166 /*********************************************************************
2167  *
2168  *  Determine optic type
2169  *
2170  **********************************************************************/
2171 static void
2172 ixgbe_setup_optics(struct adapter *adapter)
2173 {
2174         struct ixgbe_hw *hw = &adapter->hw;
2175         int             layer;
2176
2177         layer = ixgbe_get_supported_physical_layer(hw);
2178
2179         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2180                 adapter->optics = IFM_10G_T;
2181                 return;
2182         }
2183
2184         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2185                 adapter->optics = IFM_1000_T;
2186                 return;
2187         }
2188
2189         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2190             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2191                 adapter->optics = IFM_10G_LR;
2192                 return;
2193         }
2194
2195         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2196                 adapter->optics = IFM_10G_SR;
2197                 return;
2198         }
2199
2200         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2201                 adapter->optics = IFM_10G_TWINAX;
2202                 return;
2203         }
2204
2205         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2206             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2207                 adapter->optics = IFM_10G_CX4;
2208                 return;
2209         }
2210
2211         /* If we get here just set the default */
2212         adapter->optics = IFM_ETHER | IFM_AUTO;
2213         return;
2214 }
2215
2216 /*********************************************************************
2217  *
2218  *  Setup the Legacy or MSI Interrupt handler
2219  *
2220  **********************************************************************/
2221 static int
2222 ixgbe_allocate_legacy(struct adapter *adapter)
2223 {
2224         device_t dev = adapter->dev;
2225         struct          ix_queue *que = adapter->queues;
2226         int error, rid = 0;
2227
2228         /* MSI RID at 1 */
2229         if (adapter->msix == 1)
2230                 rid = 1;
2231
2232         /* We allocate a single interrupt resource */
2233         adapter->res = bus_alloc_resource_any(dev,
2234             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2235         if (adapter->res == NULL) {
2236                 device_printf(dev, "Unable to allocate bus resource: "
2237                     "interrupt\n");
2238                 return (ENXIO);
2239         }
2240
2241         /*
2242          * Try allocating a fast interrupt and the associated deferred
2243          * processing contexts.
2244          */
2245         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2246         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2247             taskqueue_thread_enqueue, &que->tq);
2248         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2249             device_get_nameunit(adapter->dev));
2250
2251         /* Tasklets for Link, SFP and Multispeed Fiber */
2252         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2253         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2254         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2255 #ifdef IXGBE_FDIR
2256         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2257 #endif
2258         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2259             taskqueue_thread_enqueue, &adapter->tq);
2260         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2261             device_get_nameunit(adapter->dev));
2262
2263         if ((error = bus_setup_intr(dev, adapter->res,
2264             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2265             que, &adapter->tag)) != 0) {
2266                 device_printf(dev, "Failed to register fast interrupt "
2267                     "handler: %d\n", error);
2268                 taskqueue_free(que->tq);
2269                 taskqueue_free(adapter->tq);
2270                 que->tq = NULL;
2271                 adapter->tq = NULL;
2272                 return (error);
2273         }
2274         /* For simplicity in the handlers */
2275         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2276
2277         return (0);
2278 }
2279
2280
2281 /*********************************************************************
2282  *
2283  *  Setup MSIX Interrupt resources and handlers
2284  *
2285  **********************************************************************/
2286 static int
2287 ixgbe_allocate_msix(struct adapter *adapter)
2288 {
2289         device_t        dev = adapter->dev;
2290         struct          ix_queue *que = adapter->queues;
2291         int             error, rid, vector = 0;
2292
2293         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2294                 rid = vector + 1;
2295                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2296                     RF_SHAREABLE | RF_ACTIVE);
2297                 if (que->res == NULL) {
2298                         device_printf(dev,"Unable to allocate"
2299                             " bus resource: que interrupt [%d]\n", vector);
2300                         return (ENXIO);
2301                 }
2302                 /* Set the handler function */
2303                 error = bus_setup_intr(dev, que->res,
2304                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2305                     ixgbe_msix_que, que, &que->tag);
2306                 if (error) {
2307                         que->res = NULL;
2308                         device_printf(dev, "Failed to register QUE handler");
2309                         return (error);
2310                 }
2311 #if __FreeBSD_version >= 800504
2312                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2313 #endif
2314                 que->msix = vector;
2315                 adapter->que_mask |= (u64)(1 << que->msix);
2316                 /*
2317                 ** Bind the msix vector, and thus the
2318                 ** ring to the corresponding cpu.
2319                 */
2320                 if (adapter->num_queues > 1)
2321                         bus_bind_intr(dev, que->res, i);
2322
2323                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2324                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2325                     taskqueue_thread_enqueue, &que->tq);
2326                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2327                     device_get_nameunit(adapter->dev));
2328         }
2329
2330         /* and Link */
2331         rid = vector + 1;
2332         adapter->res = bus_alloc_resource_any(dev,
2333             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2334         if (!adapter->res) {
2335                 device_printf(dev,"Unable to allocate"
2336             " bus resource: Link interrupt [%d]\n", rid);
2337                 return (ENXIO);
2338         }
2339         /* Set the link handler function */
2340         error = bus_setup_intr(dev, adapter->res,
2341             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2342             ixgbe_msix_link, adapter, &adapter->tag);
2343         if (error) {
2344                 adapter->res = NULL;
2345                 device_printf(dev, "Failed to register LINK handler");
2346                 return (error);
2347         }
2348 #if __FreeBSD_version >= 800504
2349         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2350 #endif
2351         adapter->linkvec = vector;
2352         /* Tasklets for Link, SFP and Multispeed Fiber */
2353         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2354         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2355         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2356 #ifdef IXGBE_FDIR
2357         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2358 #endif
2359         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2360             taskqueue_thread_enqueue, &adapter->tq);
2361         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2362             device_get_nameunit(adapter->dev));
2363
2364         return (0);
2365 }
2366
2367 /*
2368  * Setup Either MSI/X or MSI
2369  */
2370 static int
2371 ixgbe_setup_msix(struct adapter *adapter)
2372 {
2373         device_t dev = adapter->dev;
2374         int rid, want, queues, msgs;
2375
2376         /* Override by tuneable */
2377         if (ixgbe_enable_msix == 0)
2378                 goto msi;
2379
2380         /* First try MSI/X */
2381         rid = PCIR_BAR(MSIX_82598_BAR);
2382         adapter->msix_mem = bus_alloc_resource_any(dev,
2383             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2384         if (!adapter->msix_mem) {
2385                 rid += 4;       /* 82599 maps in higher BAR */
2386                 adapter->msix_mem = bus_alloc_resource_any(dev,
2387                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2388         }
2389         if (!adapter->msix_mem) {
2390                 /* May not be enabled */
2391                 device_printf(adapter->dev,
2392                     "Unable to map MSIX table \n");
2393                 goto msi;
2394         }
2395
2396         msgs = pci_msix_count(dev);
2397         if (msgs == 0) { /* system has msix disabled */
2398                 bus_release_resource(dev, SYS_RES_MEMORY,
2399                     rid, adapter->msix_mem);
2400                 adapter->msix_mem = NULL;
2401                 goto msi;
2402         }
2403
2404         /* Figure out a reasonable auto config value */
2405         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2406
2407         if (ixgbe_num_queues != 0)
2408                 queues = ixgbe_num_queues;
2409         /* Set max queues to 8 when autoconfiguring */
2410         else if ((ixgbe_num_queues == 0) && (queues > 8))
2411                 queues = 8;
2412
2413         /*
2414         ** Want one vector (RX/TX pair) per queue
2415         ** plus an additional for Link.
2416         */
2417         want = queues + 1;
2418         if (msgs >= want)
2419                 msgs = want;
2420         else {
2421                 device_printf(adapter->dev,
2422                     "MSIX Configuration Problem, "
2423                     "%d vectors but %d queues wanted!\n",
2424                     msgs, want);
2425                 return (0); /* Will go to Legacy setup */
2426         }
2427         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2428                 device_printf(adapter->dev,
2429                     "Using MSIX interrupts with %d vectors\n", msgs);
2430                 adapter->num_queues = queues;
2431                 return (msgs);
2432         }
2433 msi:
2434         msgs = pci_msi_count(dev);
2435         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2436                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2437         else
2438                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2439         return (msgs);
2440 }
2441
2442
2443 static int
2444 ixgbe_allocate_pci_resources(struct adapter *adapter)
2445 {
2446         int             rid;
2447         device_t        dev = adapter->dev;
2448
2449         rid = PCIR_BAR(0);
2450         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2451             &rid, RF_ACTIVE);
2452
2453         if (!(adapter->pci_mem)) {
2454                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2455                 return (ENXIO);
2456         }
2457
2458         adapter->osdep.mem_bus_space_tag =
2459                 rman_get_bustag(adapter->pci_mem);
2460         adapter->osdep.mem_bus_space_handle =
2461                 rman_get_bushandle(adapter->pci_mem);
2462         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2463
2464         /* Legacy defaults */
2465         adapter->num_queues = 1;
2466         adapter->hw.back = &adapter->osdep;
2467
2468         /*
2469         ** Now setup MSI or MSI/X, should
2470         ** return us the number of supported
2471         ** vectors. (Will be 1 for MSI)
2472         */
2473         adapter->msix = ixgbe_setup_msix(adapter);
2474         return (0);
2475 }
2476
2477 static void
2478 ixgbe_free_pci_resources(struct adapter * adapter)
2479 {
2480         struct          ix_queue *que = adapter->queues;
2481         device_t        dev = adapter->dev;
2482         int             rid, memrid;
2483
2484         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2485                 memrid = PCIR_BAR(MSIX_82598_BAR);
2486         else
2487                 memrid = PCIR_BAR(MSIX_82599_BAR);
2488
2489         /*
2490         ** There is a slight possibility of a failure mode
2491         ** in attach that will result in entering this function
2492         ** before interrupt resources have been initialized, and
2493         ** in that case we do not want to execute the loops below
2494         ** We can detect this reliably by the state of the adapter
2495         ** res pointer.
2496         */
2497         if (adapter->res == NULL)
2498                 goto mem;
2499
2500         /*
2501         **  Release all msix queue resources:
2502         */
2503         for (int i = 0; i < adapter->num_queues; i++, que++) {
2504                 rid = que->msix + 1;
2505                 if (que->tag != NULL) {
2506                         bus_teardown_intr(dev, que->res, que->tag);
2507                         que->tag = NULL;
2508                 }
2509                 if (que->res != NULL)
2510                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2511         }
2512
2513
2514         /* Clean the Legacy or Link interrupt last */
2515         if (adapter->linkvec) /* we are doing MSIX */
2516                 rid = adapter->linkvec + 1;
2517         else
2518                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2519
2520         if (adapter->tag != NULL) {
2521                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2522                 adapter->tag = NULL;
2523         }
2524         if (adapter->res != NULL)
2525                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2526
2527 mem:
2528         if (adapter->msix)
2529                 pci_release_msi(dev);
2530
2531         if (adapter->msix_mem != NULL)
2532                 bus_release_resource(dev, SYS_RES_MEMORY,
2533                     memrid, adapter->msix_mem);
2534
2535         if (adapter->pci_mem != NULL)
2536                 bus_release_resource(dev, SYS_RES_MEMORY,
2537                     PCIR_BAR(0), adapter->pci_mem);
2538
2539         return;
2540 }
2541
2542 /*********************************************************************
2543  *
2544  *  Setup networking device structure and register an interface.
2545  *
2546  **********************************************************************/
2547 static int
2548 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2549 {
2550         struct ixgbe_hw *hw = &adapter->hw;
2551         struct ifnet   *ifp;
2552
2553         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2554
2555         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2556         if (ifp == NULL) {
2557                 device_printf(dev, "can not allocate ifnet structure\n");
2558                 return (-1);
2559         }
2560         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2561         ifp->if_baudrate = 1000000000;
2562         ifp->if_init = ixgbe_init;
2563         ifp->if_softc = adapter;
2564         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2565         ifp->if_ioctl = ixgbe_ioctl;
2566         ifp->if_start = ixgbe_start;
2567 #if __FreeBSD_version >= 800000
2568         ifp->if_transmit = ixgbe_mq_start;
2569         ifp->if_qflush = ixgbe_qflush;
2570 #endif
2571         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2572
2573         ether_ifattach(ifp, adapter->hw.mac.addr);
2574
2575         adapter->max_frame_size =
2576             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2577
2578         /*
2579          * Tell the upper layer(s) we support long frames.
2580          */
2581         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2582
2583         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM;
2584         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2585         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2586                              |  IFCAP_VLAN_HWTSO
2587                              |  IFCAP_VLAN_MTU;
2588         ifp->if_capenable = ifp->if_capabilities;
2589
2590         /* Don't enable LRO by default */
2591         ifp->if_capabilities |= IFCAP_LRO;
2592
2593         /*
2594         ** Don't turn this on by default, if vlans are
2595         ** created on another pseudo device (eg. lagg)
2596         ** then vlan events are not passed thru, breaking
2597         ** operation, but with HW FILTER off it works. If
2598         ** using vlans directly on the ixgbe driver you can
2599         ** enable this and get full hardware tag filtering.
2600         */
2601         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2602
2603         /*
2604          * Specify the media types supported by this adapter and register
2605          * callbacks to update media and link information
2606          */
2607         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2608                      ixgbe_media_status);
2609         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2610         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2611         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2612                 ifmedia_add(&adapter->media,
2613                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2614                 ifmedia_add(&adapter->media,
2615                     IFM_ETHER | IFM_1000_T, 0, NULL);
2616         }
2617         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2618         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2619
2620         return (0);
2621 }
2622
2623 static void
2624 ixgbe_config_link(struct adapter *adapter)
2625 {
2626         struct ixgbe_hw *hw = &adapter->hw;
2627         u32     autoneg, err = 0;
2628         bool    sfp, negotiate;
2629
2630         sfp = ixgbe_is_sfp(hw);
2631
2632         if (sfp) {
2633                 if (hw->phy.multispeed_fiber) {
2634                         hw->mac.ops.setup_sfp(hw);
2635                         ixgbe_enable_tx_laser(hw);
2636                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2637                 } else
2638                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2639         } else {
2640                 if (hw->mac.ops.check_link)
2641                         err = ixgbe_check_link(hw, &autoneg,
2642                             &adapter->link_up, FALSE);
2643                 if (err)
2644                         goto out;
2645                 autoneg = hw->phy.autoneg_advertised;
2646                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2647                         err  = hw->mac.ops.get_link_capabilities(hw,
2648                             &autoneg, &negotiate);
2649                 if (err)
2650                         goto out;
2651                 if (hw->mac.ops.setup_link)
2652                         err = hw->mac.ops.setup_link(hw, autoneg,
2653                             negotiate, adapter->link_up);
2654         }
2655 out:
2656         return;
2657 }
2658
2659 /********************************************************************
2660  * Manage DMA'able memory.
2661  *******************************************************************/
2662 static void
2663 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2664 {
2665         if (error)
2666                 return;
2667         *(bus_addr_t *) arg = segs->ds_addr;
2668         return;
2669 }
2670
2671 static int
2672 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2673                 struct ixgbe_dma_alloc *dma, int mapflags)
2674 {
2675         device_t dev = adapter->dev;
2676         int             r;
2677
2678         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2679                                DBA_ALIGN, 0,    /* alignment, bounds */
2680                                BUS_SPACE_MAXADDR,       /* lowaddr */
2681                                BUS_SPACE_MAXADDR,       /* highaddr */
2682                                NULL, NULL,      /* filter, filterarg */
2683                                size,    /* maxsize */
2684                                1,       /* nsegments */
2685                                size,    /* maxsegsize */
2686                                BUS_DMA_ALLOCNOW,        /* flags */
2687                                NULL,    /* lockfunc */
2688                                NULL,    /* lockfuncarg */
2689                                &dma->dma_tag);
2690         if (r != 0) {
2691                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2692                        "error %u\n", r);
2693                 goto fail_0;
2694         }
2695         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2696                              BUS_DMA_NOWAIT, &dma->dma_map);
2697         if (r != 0) {
2698                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2699                        "error %u\n", r);
2700                 goto fail_1;
2701         }
2702         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2703                             size,
2704                             ixgbe_dmamap_cb,
2705                             &dma->dma_paddr,
2706                             mapflags | BUS_DMA_NOWAIT);
2707         if (r != 0) {
2708                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2709                        "error %u\n", r);
2710                 goto fail_2;
2711         }
2712         dma->dma_size = size;
2713         return (0);
2714 fail_2:
2715         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2716 fail_1:
2717         bus_dma_tag_destroy(dma->dma_tag);
2718 fail_0:
2719         dma->dma_map = NULL;
2720         dma->dma_tag = NULL;
2721         return (r);
2722 }
2723
2724 static void
2725 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2726 {
2727         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2728             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2729         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2730         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2731         bus_dma_tag_destroy(dma->dma_tag);
2732 }
2733
2734
2735 /*********************************************************************
2736  *
2737  *  Allocate memory for the transmit and receive rings, and then
2738  *  the descriptors associated with each, called only once at attach.
2739  *
2740  **********************************************************************/
2741 static int
2742 ixgbe_allocate_queues(struct adapter *adapter)
2743 {
2744         device_t        dev = adapter->dev;
2745         struct ix_queue *que;
2746         struct tx_ring  *txr;
2747         struct rx_ring  *rxr;
2748         int rsize, tsize, error = IXGBE_SUCCESS;
2749         int txconf = 0, rxconf = 0;
2750
2751         /* First allocate the top level queue structs */
2752         if (!(adapter->queues =
2753             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2754             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2755                 device_printf(dev, "Unable to allocate queue memory\n");
2756                 error = ENOMEM;
2757                 goto fail;
2758         }
2759
2760         /* First allocate the TX ring struct memory */
2761         if (!(adapter->tx_rings =
2762             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2763             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2764                 device_printf(dev, "Unable to allocate TX ring memory\n");
2765                 error = ENOMEM;
2766                 goto tx_fail;
2767         }
2768
2769         /* Next allocate the RX */
2770         if (!(adapter->rx_rings =
2771             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2772             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2773                 device_printf(dev, "Unable to allocate RX ring memory\n");
2774                 error = ENOMEM;
2775                 goto rx_fail;
2776         }
2777
2778         /* For the ring itself */
2779         tsize = roundup2(adapter->num_tx_desc *
2780             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2781
2782         /*
2783          * Now set up the TX queues, txconf is needed to handle the
2784          * possibility that things fail midcourse and we need to
2785          * undo memory gracefully
2786          */
2787         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2788                 /* Set up some basics */
2789                 txr = &adapter->tx_rings[i];
2790                 txr->adapter = adapter;
2791                 txr->me = i;
2792
2793                 /* Initialize the TX side lock */
2794                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2795                     device_get_nameunit(dev), txr->me);
2796                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2797
2798                 if (ixgbe_dma_malloc(adapter, tsize,
2799                         &txr->txdma, BUS_DMA_NOWAIT)) {
2800                         device_printf(dev,
2801                             "Unable to allocate TX Descriptor memory\n");
2802                         error = ENOMEM;
2803                         goto err_tx_desc;
2804                 }
2805                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2806                 bzero((void *)txr->tx_base, tsize);
2807
2808                 /* Now allocate transmit buffers for the ring */
2809                 if (ixgbe_allocate_transmit_buffers(txr)) {
2810                         device_printf(dev,
2811                             "Critical Failure setting up transmit buffers\n");
2812                         error = ENOMEM;
2813                         goto err_tx_desc;
2814                 }
2815 #if __FreeBSD_version >= 800000
2816                 /* Allocate a buf ring */
2817                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2818                     M_WAITOK, &txr->tx_mtx);
2819                 if (txr->br == NULL) {
2820                         device_printf(dev,
2821                             "Critical Failure setting up buf ring\n");
2822                         error = ENOMEM;
2823                         goto err_tx_desc;
2824                 }
2825 #endif
2826         }
2827
2828         /*
2829          * Next the RX queues...
2830          */
2831         rsize = roundup2(adapter->num_rx_desc *
2832             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2833         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2834                 rxr = &adapter->rx_rings[i];
2835                 /* Set up some basics */
2836                 rxr->adapter = adapter;
2837                 rxr->me = i;
2838
2839                 /* Initialize the RX side lock */
2840                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2841                     device_get_nameunit(dev), rxr->me);
2842                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2843
2844                 if (ixgbe_dma_malloc(adapter, rsize,
2845                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2846                         device_printf(dev,
2847                             "Unable to allocate RxDescriptor memory\n");
2848                         error = ENOMEM;
2849                         goto err_rx_desc;
2850                 }
2851                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2852                 bzero((void *)rxr->rx_base, rsize);
2853
2854                 /* Allocate receive buffers for the ring*/
2855                 if (ixgbe_allocate_receive_buffers(rxr)) {
2856                         device_printf(dev,
2857                             "Critical Failure setting up receive buffers\n");
2858                         error = ENOMEM;
2859                         goto err_rx_desc;
2860                 }
2861         }
2862
2863         /*
2864         ** Finally set up the queue holding structs
2865         */
2866         for (int i = 0; i < adapter->num_queues; i++) {
2867                 que = &adapter->queues[i];
2868                 que->adapter = adapter;
2869                 que->txr = &adapter->tx_rings[i];
2870                 que->rxr = &adapter->rx_rings[i];
2871         }
2872
2873         return (0);
2874
2875 err_rx_desc:
2876         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2877                 ixgbe_dma_free(adapter, &rxr->rxdma);
2878 err_tx_desc:
2879         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2880                 ixgbe_dma_free(adapter, &txr->txdma);
2881         free(adapter->rx_rings, M_DEVBUF);
2882 rx_fail:
2883         free(adapter->tx_rings, M_DEVBUF);
2884 tx_fail:
2885         free(adapter->queues, M_DEVBUF);
2886 fail:
2887         return (error);
2888 }
2889
2890 /*********************************************************************
2891  *
2892  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2893  *  the information needed to transmit a packet on the wire. This is
2894  *  called only once at attach, setup is done every reset.
2895  *
2896  **********************************************************************/
2897 static int
2898 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2899 {
2900         struct adapter *adapter = txr->adapter;
2901         device_t dev = adapter->dev;
2902         struct ixgbe_tx_buf *txbuf;
2903         int error, i;
2904
2905         /*
2906          * Setup DMA descriptor areas.
2907          */
2908         if ((error = bus_dma_tag_create(
2909                                bus_get_dma_tag(adapter->dev),   /* parent */
2910                                1, 0,            /* alignment, bounds */
2911                                BUS_SPACE_MAXADDR,       /* lowaddr */
2912                                BUS_SPACE_MAXADDR,       /* highaddr */
2913                                NULL, NULL,              /* filter, filterarg */
2914                                IXGBE_TSO_SIZE,          /* maxsize */
2915                                adapter->num_segs,       /* nsegments */
2916                                PAGE_SIZE,               /* maxsegsize */
2917                                0,                       /* flags */
2918                                NULL,                    /* lockfunc */
2919                                NULL,                    /* lockfuncarg */
2920                                &txr->txtag))) {
2921                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2922                 goto fail;
2923         }
2924
2925         if (!(txr->tx_buffers =
2926             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2927             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2928                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2929                 error = ENOMEM;
2930                 goto fail;
2931         }
2932
2933         /* Create the descriptor buffer dma maps */
2934         txbuf = txr->tx_buffers;
2935         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2936                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2937                 if (error != 0) {
2938                         device_printf(dev, "Unable to create TX DMA map\n");
2939                         goto fail;
2940                 }
2941         }
2942
2943         return 0;
2944 fail:
2945         /* We free all, it handles case where we are in the middle */
2946         ixgbe_free_transmit_structures(adapter);
2947         return (error);
2948 }
2949
2950 /*********************************************************************
2951  *
2952  *  Initialize a transmit ring.
2953  *
2954  **********************************************************************/
2955 static void
2956 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2957 {
2958         struct adapter *adapter = txr->adapter;
2959         struct ixgbe_tx_buf *txbuf;
2960         int i;
2961 #ifdef DEV_NETMAP
2962         struct netmap_adapter *na = NA(adapter->ifp);
2963         struct netmap_slot *slot;
2964 #endif /* DEV_NETMAP */
2965
2966         /* Clear the old ring contents */
2967         IXGBE_TX_LOCK(txr);
2968 #ifdef DEV_NETMAP
2969         /*
2970          * (under lock): if in netmap mode, do some consistency
2971          * checks and set slot to entry 0 of the netmap ring.
2972          */
2973         slot = netmap_reset(na, NR_TX, txr->me, 0);
2974 #endif /* DEV_NETMAP */
2975         bzero((void *)txr->tx_base,
2976               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2977         /* Reset indices */
2978         txr->next_avail_desc = 0;
2979         txr->next_to_clean = 0;
2980
2981         /* Free any existing tx buffers. */
2982         txbuf = txr->tx_buffers;
2983         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2984                 if (txbuf->m_head != NULL) {
2985                         bus_dmamap_sync(txr->txtag, txbuf->map,
2986                             BUS_DMASYNC_POSTWRITE);
2987                         bus_dmamap_unload(txr->txtag, txbuf->map);
2988                         m_freem(txbuf->m_head);
2989                         txbuf->m_head = NULL;
2990                 }
2991 #ifdef DEV_NETMAP
2992                 /*
2993                  * In netmap mode, set the map for the packet buffer.
2994                  * NOTE: Some drivers (not this one) also need to set
2995                  * the physical buffer address in the NIC ring.
2996                  * Slots in the netmap ring (indexed by "si") are
2997                  * kring->nkr_hwofs positions "ahead" wrt the
2998                  * corresponding slot in the NIC ring. In some drivers
2999                  * (not here) nkr_hwofs can be negative. Function
3000                  * netmap_idx_n2k() handles wraparounds properly.
3001                  */
3002                 if (slot) {
3003                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3004                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3005                 }
3006 #endif /* DEV_NETMAP */
3007                 /* Clear the EOP index */
3008                 txbuf->eop_index = -1;
3009         }
3010
3011 #ifdef IXGBE_FDIR
3012         /* Set the rate at which we sample packets */
3013         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3014                 txr->atr_sample = atr_sample_rate;
3015 #endif
3016
3017         /* Set number of descriptors available */
3018         txr->tx_avail = adapter->num_tx_desc;
3019
3020         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3021             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3022         IXGBE_TX_UNLOCK(txr);
3023 }
3024
3025 /*********************************************************************
3026  *
3027  *  Initialize all transmit rings.
3028  *
3029  **********************************************************************/
3030 static int
3031 ixgbe_setup_transmit_structures(struct adapter *adapter)
3032 {
3033         struct tx_ring *txr = adapter->tx_rings;
3034
3035         for (int i = 0; i < adapter->num_queues; i++, txr++)
3036                 ixgbe_setup_transmit_ring(txr);
3037
3038         return (0);
3039 }
3040
3041 /*********************************************************************
3042  *
3043  *  Enable transmit unit.
3044  *
3045  **********************************************************************/
3046 static void
3047 ixgbe_initialize_transmit_units(struct adapter *adapter)
3048 {
3049         struct tx_ring  *txr = adapter->tx_rings;
3050         struct ixgbe_hw *hw = &adapter->hw;
3051
3052         /* Setup the Base and Length of the Tx Descriptor Ring */
3053
3054         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3055                 u64     tdba = txr->txdma.dma_paddr;
3056                 u32     txctrl;
3057
3058                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3059                        (tdba & 0x00000000ffffffffULL));
3060                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3061                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3062                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3063
3064                 /* Setup the HW Tx Head and Tail descriptor pointers */
3065                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3066                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3067
3068                 /* Setup Transmit Descriptor Cmd Settings */
3069                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3070                 txr->queue_status = IXGBE_QUEUE_IDLE;
3071
3072                 /* Disable Head Writeback */
3073                 switch (hw->mac.type) {
3074                 case ixgbe_mac_82598EB:
3075                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3076                         break;
3077                 case ixgbe_mac_82599EB:
3078                 case ixgbe_mac_X540:
3079                 default:
3080                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3081                         break;
3082                 }
3083                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3084                 switch (hw->mac.type) {
3085                 case ixgbe_mac_82598EB:
3086                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3087                         break;
3088                 case ixgbe_mac_82599EB:
3089                 case ixgbe_mac_X540:
3090                 default:
3091                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3092                         break;
3093                 }
3094
3095         }
3096
3097         if (hw->mac.type != ixgbe_mac_82598EB) {
3098                 u32 dmatxctl, rttdcs;
3099                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3100                 dmatxctl |= IXGBE_DMATXCTL_TE;
3101                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3102                 /* Disable arbiter to set MTQC */
3103                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3104                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3105                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3106                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3107                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3108                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3109         }
3110
3111         return;
3112 }
3113
3114 /*********************************************************************
3115  *
3116  *  Free all transmit rings.
3117  *
3118  **********************************************************************/
3119 static void
3120 ixgbe_free_transmit_structures(struct adapter *adapter)
3121 {
3122         struct tx_ring *txr = adapter->tx_rings;
3123
3124         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3125                 IXGBE_TX_LOCK(txr);
3126                 ixgbe_free_transmit_buffers(txr);
3127                 ixgbe_dma_free(adapter, &txr->txdma);
3128                 IXGBE_TX_UNLOCK(txr);
3129                 IXGBE_TX_LOCK_DESTROY(txr);
3130         }
3131         free(adapter->tx_rings, M_DEVBUF);
3132 }
3133
3134 /*********************************************************************
3135  *
3136  *  Free transmit ring related data structures.
3137  *
3138  **********************************************************************/
3139 static void
3140 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3141 {
3142         struct adapter *adapter = txr->adapter;
3143         struct ixgbe_tx_buf *tx_buffer;
3144         int             i;
3145
3146         INIT_DEBUGOUT("free_transmit_ring: begin");
3147
3148         if (txr->tx_buffers == NULL)
3149                 return;
3150
3151         tx_buffer = txr->tx_buffers;
3152         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3153                 if (tx_buffer->m_head != NULL) {
3154                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3155                             BUS_DMASYNC_POSTWRITE);
3156                         bus_dmamap_unload(txr->txtag,
3157                             tx_buffer->map);
3158                         m_freem(tx_buffer->m_head);
3159                         tx_buffer->m_head = NULL;
3160                         if (tx_buffer->map != NULL) {
3161                                 bus_dmamap_destroy(txr->txtag,
3162                                     tx_buffer->map);
3163                                 tx_buffer->map = NULL;
3164                         }
3165                 } else if (tx_buffer->map != NULL) {
3166                         bus_dmamap_unload(txr->txtag,
3167                             tx_buffer->map);
3168                         bus_dmamap_destroy(txr->txtag,
3169                             tx_buffer->map);
3170                         tx_buffer->map = NULL;
3171                 }
3172         }
3173 #if __FreeBSD_version >= 800000
3174         if (txr->br != NULL)
3175                 buf_ring_free(txr->br, M_DEVBUF);
3176 #endif
3177         if (txr->tx_buffers != NULL) {
3178                 free(txr->tx_buffers, M_DEVBUF);
3179                 txr->tx_buffers = NULL;
3180         }
3181         if (txr->txtag != NULL) {
3182                 bus_dma_tag_destroy(txr->txtag);
3183                 txr->txtag = NULL;
3184         }
3185         return;
3186 }
3187
3188 /*********************************************************************
3189  *
3190  *  Advanced Context Descriptor setup for VLAN or CSUM
3191  *
3192  **********************************************************************/
3193
3194 static bool
3195 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3196 {
3197         struct adapter *adapter = txr->adapter;
3198         struct ixgbe_adv_tx_context_desc *TXD;
3199         struct ixgbe_tx_buf        *tx_buffer;
3200         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3201         struct ether_vlan_header *eh;
3202         struct ip *ip;
3203         struct ip6_hdr *ip6;
3204         int  ehdrlen, ip_hlen = 0;
3205         u16     etype;
3206         u8      ipproto = 0;
3207         bool    offload = TRUE;
3208         int ctxd = txr->next_avail_desc;
3209         u16 vtag = 0;
3210
3211
3212         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3213                 offload = FALSE;
3214
3215         tx_buffer = &txr->tx_buffers[ctxd];
3216         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3217
3218         /*
3219         ** In advanced descriptors the vlan tag must
3220         ** be placed into the descriptor itself.
3221         */
3222         if (mp->m_flags & M_VLANTAG) {
3223                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3224                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3225         } else if (offload == FALSE)
3226                 return FALSE;
3227
3228         /*
3229          * Determine where frame payload starts.
3230          * Jump over vlan headers if already present,
3231          * helpful for QinQ too.
3232          */
3233         eh = mtod(mp, struct ether_vlan_header *);
3234         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3235                 etype = ntohs(eh->evl_proto);
3236                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3237         } else {
3238                 etype = ntohs(eh->evl_encap_proto);
3239                 ehdrlen = ETHER_HDR_LEN;
3240         }
3241
3242         /* Set the ether header length */
3243         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3244
3245         switch (etype) {
3246                 case ETHERTYPE_IP:
3247                         ip = (struct ip *)(mp->m_data + ehdrlen);
3248                         ip_hlen = ip->ip_hl << 2;
3249                         ipproto = ip->ip_p;
3250                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3251                         break;
3252                 case ETHERTYPE_IPV6:
3253                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3254                         ip_hlen = sizeof(struct ip6_hdr);
3255                         ipproto = ip6->ip6_nxt;
3256                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3257                         break;
3258                 default:
3259                         offload = FALSE;
3260                         break;
3261         }
3262
3263         vlan_macip_lens |= ip_hlen;
3264         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3265
3266         switch (ipproto) {
3267                 case IPPROTO_TCP:
3268                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3269                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3270                         break;
3271
3272                 case IPPROTO_UDP:
3273                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3274                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3275                         break;
3276
3277 #if __FreeBSD_version >= 800000
3278                 case IPPROTO_SCTP:
3279                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3280                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3281                         break;
3282 #endif
3283                 default:
3284                         offload = FALSE;
3285                         break;
3286         }
3287
3288         /* Now copy bits into descriptor */
3289         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3290         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3291         TXD->seqnum_seed = htole32(0);
3292         TXD->mss_l4len_idx = htole32(0);
3293
3294         tx_buffer->m_head = NULL;
3295         tx_buffer->eop_index = -1;
3296
3297         /* We've consumed the first desc, adjust counters */
3298         if (++ctxd == adapter->num_tx_desc)
3299                 ctxd = 0;
3300         txr->next_avail_desc = ctxd;
3301         --txr->tx_avail;
3302
3303         return (offload);
3304 }
3305
3306 /**********************************************************************
3307  *
3308  *  Setup work for hardware segmentation offload (TSO) on
3309  *  adapters using advanced tx descriptors
3310  *
3311  **********************************************************************/
3312 static bool
3313 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
3314 {
3315         struct adapter *adapter = txr->adapter;
3316         struct ixgbe_adv_tx_context_desc *TXD;
3317         struct ixgbe_tx_buf        *tx_buffer;
3318         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3319         u32 mss_l4len_idx = 0;
3320         u16 vtag = 0;
3321         int ctxd, ehdrlen,  hdrlen, ip_hlen, tcp_hlen;
3322         struct ether_vlan_header *eh;
3323         struct ip *ip;
3324         struct tcphdr *th;
3325
3326
3327         /*
3328          * Determine where frame payload starts.
3329          * Jump over vlan headers if already present
3330          */
3331         eh = mtod(mp, struct ether_vlan_header *);
3332         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3333                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3334         else
3335                 ehdrlen = ETHER_HDR_LEN;
3336
3337         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3338         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3339                 return FALSE;
3340
3341         ctxd = txr->next_avail_desc;
3342         tx_buffer = &txr->tx_buffers[ctxd];
3343         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3344
3345         ip = (struct ip *)(mp->m_data + ehdrlen);
3346         if (ip->ip_p != IPPROTO_TCP)
3347                 return FALSE;   /* 0 */
3348         ip->ip_sum = 0;
3349         ip_hlen = ip->ip_hl << 2;
3350         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3351         th->th_sum = in_pseudo(ip->ip_src.s_addr,
3352             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3353         tcp_hlen = th->th_off << 2;
3354         hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3355
3356         /* This is used in the transmit desc in encap */
3357         *paylen = mp->m_pkthdr.len - hdrlen;
3358
3359         /* VLAN MACLEN IPLEN */
3360         if (mp->m_flags & M_VLANTAG) {
3361                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3362                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3363         }
3364
3365         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3366         vlan_macip_lens |= ip_hlen;
3367         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3368
3369         /* ADV DTYPE TUCMD */
3370         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3371         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3372         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3373         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3374
3375
3376         /* MSS L4LEN IDX */
3377         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3378         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3379         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3380
3381         TXD->seqnum_seed = htole32(0);
3382         tx_buffer->m_head = NULL;
3383         tx_buffer->eop_index = -1;
3384
3385         if (++ctxd == adapter->num_tx_desc)
3386                 ctxd = 0;
3387
3388         txr->tx_avail--;
3389         txr->next_avail_desc = ctxd;
3390         return TRUE;
3391 }
3392
3393 #ifdef IXGBE_FDIR
3394 /*
3395 ** This routine parses packet headers so that Flow
3396 ** Director can make a hashed filter table entry
3397 ** allowing traffic flows to be identified and kept
3398 ** on the same cpu.  This would be a performance
3399 ** hit, but we only do it at IXGBE_FDIR_RATE of
3400 ** packets.
3401 */
3402 static void
3403 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3404 {
3405         struct adapter                  *adapter = txr->adapter;
3406         struct ix_queue                 *que;
3407         struct ip                       *ip;
3408         struct tcphdr                   *th;
3409         struct udphdr                   *uh;
3410         struct ether_vlan_header        *eh;
3411         union ixgbe_atr_hash_dword      input = {.dword = 0};
3412         union ixgbe_atr_hash_dword      common = {.dword = 0};
3413         int                             ehdrlen, ip_hlen;
3414         u16                             etype;
3415
3416         eh = mtod(mp, struct ether_vlan_header *);
3417         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3418                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3419                 etype = eh->evl_proto;
3420         } else {
3421                 ehdrlen = ETHER_HDR_LEN;
3422                 etype = eh->evl_encap_proto;
3423         }
3424
3425         /* Only handling IPv4 */
3426         if (etype != htons(ETHERTYPE_IP))
3427                 return;
3428
3429         ip = (struct ip *)(mp->m_data + ehdrlen);
3430         ip_hlen = ip->ip_hl << 2;
3431
3432         /* check if we're UDP or TCP */
3433         switch (ip->ip_p) {
3434         case IPPROTO_TCP:
3435                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3436                 /* src and dst are inverted */
3437                 common.port.dst ^= th->th_sport;
3438                 common.port.src ^= th->th_dport;
3439                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3440                 break;
3441         case IPPROTO_UDP:
3442                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3443                 /* src and dst are inverted */
3444                 common.port.dst ^= uh->uh_sport;
3445                 common.port.src ^= uh->uh_dport;
3446                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3447                 break;
3448         default:
3449                 return;
3450         }
3451
3452         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3453         if (mp->m_pkthdr.ether_vtag)
3454                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3455         else
3456                 common.flex_bytes ^= etype;
3457         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3458
3459         que = &adapter->queues[txr->me];
3460         /*
3461         ** This assumes the Rx queue and Tx
3462         ** queue are bound to the same CPU
3463         */
3464         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3465             input, common, que->msix);
3466 }
3467 #endif /* IXGBE_FDIR */
3468
3469 /**********************************************************************
3470  *
3471  *  Examine each tx_buffer in the used queue. If the hardware is done
3472  *  processing the packet then free associated resources. The
3473  *  tx_buffer is put back on the free queue.
3474  *
3475  **********************************************************************/
3476 static bool
3477 ixgbe_txeof(struct tx_ring *txr)
3478 {
3479         struct adapter  *adapter = txr->adapter;
3480         struct ifnet    *ifp = adapter->ifp;
3481         u32     first, last, done, processed;
3482         struct ixgbe_tx_buf *tx_buffer;
3483         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3484
3485         mtx_assert(&txr->tx_mtx, MA_OWNED);
3486
3487 #ifdef DEV_NETMAP
3488         if (ifp->if_capenable & IFCAP_NETMAP) {
3489                 struct netmap_adapter *na = NA(ifp);
3490                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3491
3492                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3493
3494                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3495                     BUS_DMASYNC_POSTREAD);
3496                 /*
3497                  * In netmap mode, all the work is done in the context
3498                  * of the client thread. Interrupt handlers only wake up
3499                  * clients, which may be sleeping on individual rings
3500                  * or on a global resource for all rings.
3501                  * To implement tx interrupt mitigation, we wake up the client
3502                  * thread roughly every half ring, even if the NIC interrupts
3503                  * more frequently. This is implemented as follows:
3504                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3505                  *   the slot that should wake up the thread (nkr_num_slots
3506                  *   means the user thread should not be woken up);
3507                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3508                  *   or the slot has the DD bit set.
3509                  *
3510                  * When the driver has separate locks, we need to
3511                  * release and re-acquire txlock to avoid deadlocks.
3512                  * XXX see if we can find a better way.
3513                  */
3514                 if (!netmap_mitigate ||
3515                     (kring->nr_kflags < kring->nkr_num_slots &&
3516                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3517                         kring->nr_kflags = kring->nkr_num_slots;
3518                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3519                         IXGBE_TX_UNLOCK(txr);
3520                         IXGBE_CORE_LOCK(adapter);
3521                         selwakeuppri(&na->tx_si, PI_NET);
3522                         IXGBE_CORE_UNLOCK(adapter);
3523                         IXGBE_TX_LOCK(txr);
3524                 }
3525                 return FALSE;
3526         }
3527 #endif /* DEV_NETMAP */
3528
3529         if (txr->tx_avail == adapter->num_tx_desc) {
3530                 txr->queue_status = IXGBE_QUEUE_IDLE;
3531                 return FALSE;
3532         }
3533
3534         processed = 0;
3535         first = txr->next_to_clean;
3536         tx_buffer = &txr->tx_buffers[first];
3537         /* For cleanup we just use legacy struct */
3538         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3539         last = tx_buffer->eop_index;
3540         if (last == -1)
3541                 return FALSE;
3542         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3543
3544         /*
3545         ** Get the index of the first descriptor
3546         ** BEYOND the EOP and call that 'done'.
3547         ** I do this so the comparison in the
3548         ** inner while loop below can be simple
3549         */
3550         if (++last == adapter->num_tx_desc) last = 0;
3551         done = last;
3552
3553         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3554             BUS_DMASYNC_POSTREAD);
3555         /*
3556         ** Only the EOP descriptor of a packet now has the DD
3557         ** bit set, this is what we look for...
3558         */
3559         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3560                 /* We clean the range of the packet */
3561                 while (first != done) {
3562                         tx_desc->upper.data = 0;
3563                         tx_desc->lower.data = 0;
3564                         tx_desc->buffer_addr = 0;
3565                         ++txr->tx_avail;
3566                         ++processed;
3567
3568                         if (tx_buffer->m_head) {
3569                                 txr->bytes +=
3570                                     tx_buffer->m_head->m_pkthdr.len;
3571                                 bus_dmamap_sync(txr->txtag,
3572                                     tx_buffer->map,
3573                                     BUS_DMASYNC_POSTWRITE);
3574                                 bus_dmamap_unload(txr->txtag,
3575                                     tx_buffer->map);
3576                                 m_freem(tx_buffer->m_head);
3577                                 tx_buffer->m_head = NULL;
3578                                 tx_buffer->map = NULL;
3579                         }
3580                         tx_buffer->eop_index = -1;
3581                         txr->watchdog_time = ticks;
3582
3583                         if (++first == adapter->num_tx_desc)
3584                                 first = 0;
3585
3586                         tx_buffer = &txr->tx_buffers[first];
3587                         tx_desc =
3588                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3589                 }
3590                 ++txr->packets;
3591                 ++ifp->if_opackets;
3592                 /* See if there is more work now */
3593                 last = tx_buffer->eop_index;
3594                 if (last != -1) {
3595                         eop_desc =
3596                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3597                         /* Get next done point */
3598                         if (++last == adapter->num_tx_desc) last = 0;
3599                         done = last;
3600                 } else
3601                         break;
3602         }
3603         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3604             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3605
3606         txr->next_to_clean = first;
3607
3608         /*
3609         ** Watchdog calculation, we know there's
3610         ** work outstanding or the first return
3611         ** would have been taken, so none processed
3612         ** for too long indicates a hang.
3613         */
3614         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3615                 txr->queue_status = IXGBE_QUEUE_HUNG;
3616
3617         /* With a minimum free clear the depleted state bit.  */
3618         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3619                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3620
3621         if (txr->tx_avail == adapter->num_tx_desc) {
3622                 txr->queue_status = IXGBE_QUEUE_IDLE;
3623                 return (FALSE);
3624         }
3625
3626         return TRUE;
3627 }
3628
3629 /*********************************************************************
3630  *
3631  *  Refresh mbuf buffers for RX descriptor rings
3632  *   - now keeps its own state so discards due to resource
3633  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3634  *     it just returns, keeping its placeholder, thus it can simply
3635  *     be recalled to try again.
3636  *
3637  **********************************************************************/
3638 static void
3639 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3640 {
3641         struct adapter          *adapter = rxr->adapter;
3642         bus_dma_segment_t       hseg[1];
3643         bus_dma_segment_t       pseg[1];
3644         struct ixgbe_rx_buf     *rxbuf;
3645         struct mbuf             *mh, *mp;
3646         int                     i, j, nsegs, error;
3647         bool                    refreshed = FALSE;
3648
3649         i = j = rxr->next_to_refresh;
3650         /* Control the loop with one beyond */
3651         if (++j == adapter->num_rx_desc)
3652                 j = 0;
3653
3654         while (j != limit) {
3655                 rxbuf = &rxr->rx_buffers[i];
3656                 if (rxr->hdr_split == FALSE)
3657                         goto no_split;
3658
3659                 if (rxbuf->m_head == NULL) {
3660                         mh = m_gethdr(M_DONTWAIT, MT_DATA);
3661                         if (mh == NULL)
3662                                 goto update;
3663                 } else
3664                         mh = rxbuf->m_head;
3665
3666                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3667                 mh->m_len = MHLEN;
3668                 mh->m_flags |= M_PKTHDR;
3669                 /* Get the memory mapping */
3670                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3671                     rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
3672                 if (error != 0) {
3673                         printf("Refresh mbufs: hdr dmamap load"
3674                             " failure - %d\n", error);
3675                         m_free(mh);
3676                         rxbuf->m_head = NULL;
3677                         goto update;
3678                 }
3679                 rxbuf->m_head = mh;
3680                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3681                     BUS_DMASYNC_PREREAD);
3682                 rxr->rx_base[i].read.hdr_addr =
3683                     htole64(hseg[0].ds_addr);
3684
3685 no_split:
3686                 if (rxbuf->m_pack == NULL) {
3687                         mp = m_getjcl(M_DONTWAIT, MT_DATA,
3688                             M_PKTHDR, adapter->rx_mbuf_sz);
3689                         if (mp == NULL)
3690                                 goto update;
3691                 } else
3692                         mp = rxbuf->m_pack;
3693
3694                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3695
3696                 /* If we're dealing with an mbuf that was copied rather
3697                  * than replaced, there's no need to go through busdma.
3698                  */
3699                 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
3700                         /* Get the memory mapping */
3701                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3702                             rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
3703                         if (error != 0) {
3704                                 printf("Refresh mbufs: payload dmamap load"
3705                                     " failure - %d\n", error);
3706                                 m_free(mp);
3707                                 rxbuf->m_pack = NULL;
3708                                 goto update;
3709                         }
3710                         rxbuf->m_pack = mp;
3711                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3712                             BUS_DMASYNC_PREREAD);
3713                         rxbuf->paddr = rxr->rx_base[i].read.pkt_addr =
3714                             htole64(pseg[0].ds_addr);
3715                 } else {
3716                         rxr->rx_base[i].read.pkt_addr = rxbuf->paddr;
3717                         rxbuf->flags &= ~IXGBE_RX_COPY;
3718                 }
3719
3720                 refreshed = TRUE;
3721                 /* Next is precalculated */
3722                 i = j;
3723                 rxr->next_to_refresh = i;
3724                 if (++j == adapter->num_rx_desc)
3725                         j = 0;
3726         }
3727 update:
3728         if (refreshed) /* Update hardware tail index */
3729                 IXGBE_WRITE_REG(&adapter->hw,
3730                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3731         return;
3732 }
3733
3734 /*********************************************************************
3735  *
3736  *  Allocate memory for rx_buffer structures. Since we use one
3737  *  rx_buffer per received packet, the maximum number of rx_buffer's
3738  *  that we'll need is equal to the number of receive descriptors
3739  *  that we've allocated.
3740  *
3741  **********************************************************************/
3742 static int
3743 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3744 {
3745         struct  adapter         *adapter = rxr->adapter;
3746         device_t                dev = adapter->dev;
3747         struct ixgbe_rx_buf     *rxbuf;
3748         int                     i, bsize, error;
3749
3750         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3751         if (!(rxr->rx_buffers =
3752             (struct ixgbe_rx_buf *) malloc(bsize,
3753             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3754                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3755                 error = ENOMEM;
3756                 goto fail;
3757         }
3758
3759         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3760                                    1, 0,        /* alignment, bounds */
3761                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3762                                    BUS_SPACE_MAXADDR,   /* highaddr */
3763                                    NULL, NULL,          /* filter, filterarg */
3764                                    MSIZE,               /* maxsize */
3765                                    1,                   /* nsegments */
3766                                    MSIZE,               /* maxsegsize */
3767                                    0,                   /* flags */
3768                                    NULL,                /* lockfunc */
3769                                    NULL,                /* lockfuncarg */
3770                                    &rxr->htag))) {
3771                 device_printf(dev, "Unable to create RX DMA tag\n");
3772                 goto fail;
3773         }
3774
3775         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3776                                    1, 0,        /* alignment, bounds */
3777                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3778                                    BUS_SPACE_MAXADDR,   /* highaddr */
3779                                    NULL, NULL,          /* filter, filterarg */
3780                                    MJUM16BYTES,         /* maxsize */
3781                                    1,                   /* nsegments */
3782                                    MJUM16BYTES,         /* maxsegsize */
3783                                    0,                   /* flags */
3784                                    NULL,                /* lockfunc */
3785                                    NULL,                /* lockfuncarg */
3786                                    &rxr->ptag))) {
3787                 device_printf(dev, "Unable to create RX DMA tag\n");
3788                 goto fail;
3789         }
3790
3791         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3792                 rxbuf = &rxr->rx_buffers[i];
3793                 error = bus_dmamap_create(rxr->htag,
3794                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3795                 if (error) {
3796                         device_printf(dev, "Unable to create RX head map\n");
3797                         goto fail;
3798                 }
3799                 error = bus_dmamap_create(rxr->ptag,
3800                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3801                 if (error) {
3802                         device_printf(dev, "Unable to create RX pkt map\n");
3803                         goto fail;
3804                 }
3805         }
3806
3807         return (0);
3808
3809 fail:
3810         /* Frees all, but can handle partial completion */
3811         ixgbe_free_receive_structures(adapter);
3812         return (error);
3813 }
3814
3815 /*
3816 ** Used to detect a descriptor that has
3817 ** been merged by Hardware RSC.
3818 */
3819 static inline u32
3820 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3821 {
3822         return (le32toh(rx->wb.lower.lo_dword.data) &
3823             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3824 }
3825
3826 /*********************************************************************
3827  *
3828  *  Initialize Hardware RSC (LRO) feature on 82599
3829  *  for an RX ring, this is toggled by the LRO capability
3830  *  even though it is transparent to the stack.
3831  *
3832  **********************************************************************/
3833 static void
3834 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3835 {
3836         struct  adapter         *adapter = rxr->adapter;
3837         struct  ixgbe_hw        *hw = &adapter->hw;
3838         u32                     rscctrl, rdrxctl;
3839
3840         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3841         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3842 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3843         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3844 #endif /* DEV_NETMAP */
3845         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3846         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3847         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3848
3849         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3850         rscctrl |= IXGBE_RSCCTL_RSCEN;
3851         /*
3852         ** Limit the total number of descriptors that
3853         ** can be combined, so it does not exceed 64K
3854         */
3855         if (adapter->rx_mbuf_sz == MCLBYTES)
3856                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3857         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3858                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3859         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3860                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3861         else  /* Using 16K cluster */
3862                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3863
3864         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3865
3866         /* Enable TCP header recognition */
3867         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3868             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3869             IXGBE_PSRTYPE_TCPHDR));
3870
3871         /* Disable RSC for ACK packets */
3872         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3873             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3874
3875         rxr->hw_rsc = TRUE;
3876 }
3877
3878
3879 static void
3880 ixgbe_free_receive_ring(struct rx_ring *rxr)
3881 {
3882         struct  adapter         *adapter;
3883         struct ixgbe_rx_buf       *rxbuf;
3884         int i;
3885
3886         adapter = rxr->adapter;
3887         for (i = 0; i < adapter->num_rx_desc; i++) {
3888                 rxbuf = &rxr->rx_buffers[i];
3889                 if (rxbuf->m_head != NULL) {
3890                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3891                             BUS_DMASYNC_POSTREAD);
3892                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3893                         rxbuf->m_head->m_flags |= M_PKTHDR;
3894                         m_freem(rxbuf->m_head);
3895                 }
3896                 if (rxbuf->m_pack != NULL) {
3897                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3898                             BUS_DMASYNC_POSTREAD);
3899                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3900                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3901                         m_freem(rxbuf->m_pack);
3902                 }
3903                 rxbuf->m_head = NULL;
3904                 rxbuf->m_pack = NULL;
3905         }
3906 }
3907
3908
3909 /*********************************************************************
3910  *
3911  *  Initialize a receive ring and its buffers.
3912  *
3913  **********************************************************************/
3914 static int
3915 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3916 {
3917         struct  adapter         *adapter;
3918         struct ifnet            *ifp;
3919         device_t                dev;
3920         struct ixgbe_rx_buf     *rxbuf;
3921         bus_dma_segment_t       pseg[1], hseg[1];
3922         struct lro_ctrl         *lro = &rxr->lro;
3923         int                     rsize, nsegs, error = 0;
3924 #ifdef DEV_NETMAP
3925         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3926         struct netmap_slot *slot;
3927 #endif /* DEV_NETMAP */
3928
3929         adapter = rxr->adapter;
3930         ifp = adapter->ifp;
3931         dev = adapter->dev;
3932
3933         /* Clear the ring contents */
3934         IXGBE_RX_LOCK(rxr);
3935 #ifdef DEV_NETMAP
3936         /* same as in ixgbe_setup_transmit_ring() */
3937         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3938 #endif /* DEV_NETMAP */
3939         rsize = roundup2(adapter->num_rx_desc *
3940             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3941         bzero((void *)rxr->rx_base, rsize);
3942
3943         /* Free current RX buffer structs and their mbufs */
3944         ixgbe_free_receive_ring(rxr);
3945
3946         /* Configure header split? */
3947         if (ixgbe_header_split)
3948                 rxr->hdr_split = TRUE;
3949
3950         /* Now replenish the mbufs */
3951         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3952                 struct mbuf     *mh, *mp;
3953
3954                 rxbuf = &rxr->rx_buffers[j];
3955 #ifdef DEV_NETMAP
3956                 /*
3957                  * In netmap mode, fill the map and set the buffer
3958                  * address in the NIC ring, considering the offset
3959                  * between the netmap and NIC rings (see comment in
3960                  * ixgbe_setup_transmit_ring() ). No need to allocate
3961                  * an mbuf, so end the block with a continue;
3962                  */
3963                 if (slot) {
3964                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3965                         uint64_t paddr;
3966                         void *addr;
3967
3968                         addr = PNMB(slot + sj, &paddr);
3969                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3970                         /* Update descriptor */
3971                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3972                         continue;
3973                 }
3974 #endif /* DEV_NETMAP */
3975                 /*
3976                 ** Don't allocate mbufs if not
3977                 ** doing header split, its wasteful
3978                 */
3979                 if (rxr->hdr_split == FALSE)
3980                         goto skip_head;
3981
3982                 /* First the header */
3983                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
3984                 if (rxbuf->m_head == NULL) {
3985                         error = ENOBUFS;
3986                         goto fail;
3987                 }
3988                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3989                 mh = rxbuf->m_head;
3990                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3991                 mh->m_flags |= M_PKTHDR;
3992                 /* Get the memory mapping */
3993                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3994                     rxbuf->hmap, rxbuf->m_head, hseg,
3995                     &nsegs, BUS_DMA_NOWAIT);
3996                 if (error != 0) /* Nothing elegant to do here */
3997                         goto fail;
3998                 bus_dmamap_sync(rxr->htag,
3999                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
4000                 /* Update descriptor */
4001                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4002
4003 skip_head:
4004                 /* Now the payload cluster */
4005                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4006                     M_PKTHDR, adapter->rx_mbuf_sz);
4007                 if (rxbuf->m_pack == NULL) {
4008                         error = ENOBUFS;
4009                         goto fail;
4010                 }
4011                 mp = rxbuf->m_pack;
4012                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4013                 /* Get the memory mapping */
4014                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
4015                     rxbuf->pmap, mp, pseg,
4016                     &nsegs, BUS_DMA_NOWAIT);
4017                 if (error != 0)
4018                         goto fail;
4019                 bus_dmamap_sync(rxr->ptag,
4020                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4021                 /* Update descriptor */
4022                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4023         }
4024
4025
4026         /* Setup our descriptor indices */
4027         rxr->next_to_check = 0;
4028         rxr->next_to_refresh = 0;
4029         rxr->lro_enabled = FALSE;
4030         rxr->rx_split_packets = 0;
4031         rxr->rx_copies = 0;
4032         rxr->rx_bytes = 0;
4033         rxr->discard = FALSE;
4034         rxr->vtag_strip = FALSE;
4035
4036         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4037             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4038
4039         /*
4040         ** Now set up the LRO interface:
4041         ** 82598 uses software LRO, the
4042         ** 82599 and X540 use a hardware assist.
4043         */
4044         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4045             (ifp->if_capenable & IFCAP_RXCSUM) &&
4046             (ifp->if_capenable & IFCAP_LRO))
4047                 ixgbe_setup_hw_rsc(rxr);
4048         else if (ifp->if_capenable & IFCAP_LRO) {
4049                 int err = tcp_lro_init(lro);
4050                 if (err) {
4051                         device_printf(dev, "LRO Initialization failed!\n");
4052                         goto fail;
4053                 }
4054                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4055                 rxr->lro_enabled = TRUE;
4056                 lro->ifp = adapter->ifp;
4057         }
4058
4059         IXGBE_RX_UNLOCK(rxr);
4060         return (0);
4061
4062 fail:
4063         ixgbe_free_receive_ring(rxr);
4064         IXGBE_RX_UNLOCK(rxr);
4065         return (error);
4066 }
4067
4068 /*********************************************************************
4069  *
4070  *  Initialize all receive rings.
4071  *
4072  **********************************************************************/
4073 static int
4074 ixgbe_setup_receive_structures(struct adapter *adapter)
4075 {
4076         struct rx_ring *rxr = adapter->rx_rings;
4077         int j;
4078
4079         for (j = 0; j < adapter->num_queues; j++, rxr++)
4080                 if (ixgbe_setup_receive_ring(rxr))
4081                         goto fail;
4082
4083         return (0);
4084 fail:
4085         /*
4086          * Free RX buffers allocated so far, we will only handle
4087          * the rings that completed, the failing case will have
4088          * cleaned up for itself. 'j' failed, so its the terminus.
4089          */
4090         for (int i = 0; i < j; ++i) {
4091                 rxr = &adapter->rx_rings[i];
4092                 ixgbe_free_receive_ring(rxr);
4093         }
4094
4095         return (ENOBUFS);
4096 }
4097
4098 /*********************************************************************
4099  *
4100  *  Setup receive registers and features.
4101  *
4102  **********************************************************************/
4103 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4104
4105 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4106
4107 static void
4108 ixgbe_initialize_receive_units(struct adapter *adapter)
4109 {
4110         struct  rx_ring *rxr = adapter->rx_rings;
4111         struct ixgbe_hw *hw = &adapter->hw;
4112         struct ifnet   *ifp = adapter->ifp;
4113         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4114         u32             reta, mrqc = 0, hlreg, random[10];
4115
4116
4117         /*
4118          * Make sure receives are disabled while
4119          * setting up the descriptor ring
4120          */
4121         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4122         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4123             rxctrl & ~IXGBE_RXCTRL_RXEN);
4124
4125         /* Enable broadcasts */
4126         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4127         fctrl |= IXGBE_FCTRL_BAM;
4128         fctrl |= IXGBE_FCTRL_DPF;
4129         fctrl |= IXGBE_FCTRL_PMCF;
4130         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4131
4132         /* Set for Jumbo Frames? */
4133         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4134         if (ifp->if_mtu > ETHERMTU)
4135                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4136         else
4137                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4138 #ifdef DEV_NETMAP
4139         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4140         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4141                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4142         else
4143                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4144 #endif /* DEV_NETMAP */
4145         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4146
4147         bufsz = (adapter->rx_mbuf_sz +
4148             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4149
4150         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4151                 u64 rdba = rxr->rxdma.dma_paddr;
4152
4153                 /* Setup the Base and Length of the Rx Descriptor Ring */
4154                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4155                                (rdba & 0x00000000ffffffffULL));
4156                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4157                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4158                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4159
4160                 /* Set up the SRRCTL register */
4161                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4162                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4163                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4164                 srrctl |= bufsz;
4165                 if (rxr->hdr_split) {
4166                         /* Use a standard mbuf for the header */
4167                         srrctl |= ((IXGBE_RX_HDR <<
4168                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4169                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4170                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4171                 } else
4172                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4173                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4174
4175                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4176                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4177                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4178         }
4179
4180         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4181                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4182                               IXGBE_PSRTYPE_UDPHDR |
4183                               IXGBE_PSRTYPE_IPV4HDR |
4184                               IXGBE_PSRTYPE_IPV6HDR;
4185                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4186         }
4187
4188         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4189
4190         /* Setup RSS */
4191         if (adapter->num_queues > 1) {
4192                 int i, j;
4193                 reta = 0;
4194
4195                 /* set up random bits */
4196                 arc4rand(&random, sizeof(random), 0);
4197
4198                 /* Set up the redirection table */
4199                 for (i = 0, j = 0; i < 128; i++, j++) {
4200                         if (j == adapter->num_queues) j = 0;
4201                         reta = (reta << 8) | (j * 0x11);
4202                         if ((i & 3) == 3)
4203                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4204                 }
4205
4206                 /* Now fill our hash function seeds */
4207                 for (int i = 0; i < 10; i++)
4208                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4209
4210                 /* Perform hash on these packet types */
4211                 mrqc = IXGBE_MRQC_RSSEN
4212                      | IXGBE_MRQC_RSS_FIELD_IPV4
4213                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4214                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4215                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4216                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4217                      | IXGBE_MRQC_RSS_FIELD_IPV6
4218                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4219                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4220                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4221                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4222
4223                 /* RSS and RX IPP Checksum are mutually exclusive */
4224                 rxcsum |= IXGBE_RXCSUM_PCSD;
4225         }
4226
4227         if (ifp->if_capenable & IFCAP_RXCSUM)
4228                 rxcsum |= IXGBE_RXCSUM_PCSD;
4229
4230         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4231                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4232
4233         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4234
4235         return;
4236 }
4237
4238 /*********************************************************************
4239  *
4240  *  Free all receive rings.
4241  *
4242  **********************************************************************/
4243 static void
4244 ixgbe_free_receive_structures(struct adapter *adapter)
4245 {
4246         struct rx_ring *rxr = adapter->rx_rings;
4247
4248         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4249                 struct lro_ctrl         *lro = &rxr->lro;
4250                 ixgbe_free_receive_buffers(rxr);
4251                 /* Free LRO memory */
4252                 tcp_lro_free(lro);
4253                 /* Free the ring memory as well */
4254                 ixgbe_dma_free(adapter, &rxr->rxdma);
4255         }
4256
4257         free(adapter->rx_rings, M_DEVBUF);
4258 }
4259
4260
4261 /*********************************************************************
4262  *
4263  *  Free receive ring data structures
4264  *
4265  **********************************************************************/
4266 static void
4267 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4268 {
4269         struct adapter          *adapter = rxr->adapter;
4270         struct ixgbe_rx_buf     *rxbuf;
4271
4272         INIT_DEBUGOUT("free_receive_structures: begin");
4273
4274         /* Cleanup any existing buffers */
4275         if (rxr->rx_buffers != NULL) {
4276                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4277                         rxbuf = &rxr->rx_buffers[i];
4278                         if (rxbuf->m_head != NULL) {
4279                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4280                                     BUS_DMASYNC_POSTREAD);
4281                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4282                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4283                                 m_freem(rxbuf->m_head);
4284                         }
4285                         if (rxbuf->m_pack != NULL) {
4286                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4287                                     BUS_DMASYNC_POSTREAD);
4288                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4289                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4290                                 m_freem(rxbuf->m_pack);
4291                         }
4292                         rxbuf->m_head = NULL;
4293                         rxbuf->m_pack = NULL;
4294                         if (rxbuf->hmap != NULL) {
4295                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4296                                 rxbuf->hmap = NULL;
4297                         }
4298                         if (rxbuf->pmap != NULL) {
4299                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4300                                 rxbuf->pmap = NULL;
4301                         }
4302                 }
4303                 if (rxr->rx_buffers != NULL) {
4304                         free(rxr->rx_buffers, M_DEVBUF);
4305                         rxr->rx_buffers = NULL;
4306                 }
4307         }
4308
4309         if (rxr->htag != NULL) {
4310                 bus_dma_tag_destroy(rxr->htag);
4311                 rxr->htag = NULL;
4312         }
4313         if (rxr->ptag != NULL) {
4314                 bus_dma_tag_destroy(rxr->ptag);
4315                 rxr->ptag = NULL;
4316         }
4317
4318         return;
4319 }
4320
4321 static __inline void
4322 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4323 {
4324
4325         /*
4326          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4327          * should be computed by hardware. Also it should not have VLAN tag in
4328          * ethernet header.
4329          */
4330         if (rxr->lro_enabled &&
4331             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4332             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4333             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4334             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) &&
4335             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4336             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4337                 /*
4338                  * Send to the stack if:
4339                  **  - LRO not enabled, or
4340                  **  - no LRO resources, or
4341                  **  - lro enqueue fails
4342                  */
4343                 if (rxr->lro.lro_cnt != 0)
4344                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4345                                 return;
4346         }
4347         IXGBE_RX_UNLOCK(rxr);
4348         (*ifp->if_input)(ifp, m);
4349         IXGBE_RX_LOCK(rxr);
4350 }
4351
4352 static __inline void
4353 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4354 {
4355         struct ixgbe_rx_buf     *rbuf;
4356
4357         rbuf = &rxr->rx_buffers[i];
4358
4359         if (rbuf->fmp != NULL) {/* Partial chain ? */
4360                 rbuf->fmp->m_flags |= M_PKTHDR;
4361                 m_freem(rbuf->fmp);
4362                 rbuf->fmp = NULL;
4363         }
4364
4365         /*
4366         ** With advanced descriptors the writeback
4367         ** clobbers the buffer addrs, so its easier
4368         ** to just free the existing mbufs and take
4369         ** the normal refresh path to get new buffers
4370         ** and mapping.
4371         */
4372         if (rbuf->m_head) {
4373                 m_free(rbuf->m_head);
4374                 rbuf->m_head = NULL;
4375         }
4376
4377         if (rbuf->m_pack) {
4378                 m_free(rbuf->m_pack);
4379                 rbuf->m_pack = NULL;
4380         }
4381
4382         return;
4383 }
4384
4385
4386 /*********************************************************************
4387  *
4388  *  This routine executes in interrupt context. It replenishes
4389  *  the mbufs in the descriptor and sends data which has been
4390  *  dma'ed into host memory to upper layer.
4391  *
4392  *  We loop at most count times if count is > 0, or until done if
4393  *  count < 0.
4394  *
4395  *  Return TRUE for more work, FALSE for all clean.
4396  *********************************************************************/
4397 static bool
4398 ixgbe_rxeof(struct ix_queue *que, int count)
4399 {
4400         struct adapter          *adapter = que->adapter;
4401         struct rx_ring          *rxr = que->rxr;
4402         struct ifnet            *ifp = adapter->ifp;
4403         struct lro_ctrl         *lro = &rxr->lro;
4404         struct lro_entry        *queued;
4405         int                     i, nextp, processed = 0;
4406         u32                     staterr = 0;
4407         union ixgbe_adv_rx_desc *cur;
4408         struct ixgbe_rx_buf     *rbuf, *nbuf;
4409
4410         IXGBE_RX_LOCK(rxr);
4411
4412 #ifdef DEV_NETMAP
4413         if (ifp->if_capenable & IFCAP_NETMAP) {
4414                 /*
4415                  * Same as the txeof routine: only wakeup clients on intr.
4416                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4417                  * mitigation (ixgbe_rxsync() will not look for new packets
4418                  * unless NKR_PENDINTR is set).
4419                  */
4420                 struct netmap_adapter *na = NA(ifp);
4421
4422                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4423                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4424                 IXGBE_RX_UNLOCK(rxr);
4425                 IXGBE_CORE_LOCK(adapter);
4426                 selwakeuppri(&na->rx_si, PI_NET);
4427                 IXGBE_CORE_UNLOCK(adapter);
4428                 return (FALSE);
4429         }
4430 #endif /* DEV_NETMAP */
4431         for (i = rxr->next_to_check; count != 0;) {
4432                 struct mbuf     *sendmp, *mh, *mp;
4433                 u32             rsc, ptype;
4434                 u16             hlen, plen, hdr;
4435                 u16             vtag = 0;
4436                 bool            eop;
4437
4438                 /* Sync the ring. */
4439                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4440                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4441
4442                 cur = &rxr->rx_base[i];
4443                 staterr = le32toh(cur->wb.upper.status_error);
4444
4445                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4446                         break;
4447                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4448                         break;
4449
4450                 count--;
4451                 sendmp = NULL;
4452                 nbuf = NULL;
4453                 rsc = 0;
4454                 cur->wb.upper.status_error = 0;
4455                 rbuf = &rxr->rx_buffers[i];
4456                 mh = rbuf->m_head;
4457                 mp = rbuf->m_pack;
4458
4459                 plen = le16toh(cur->wb.upper.length);
4460                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4461                     IXGBE_RXDADV_PKTTYPE_MASK;
4462                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4463                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4464
4465                 /* Process vlan info */
4466                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4467                         vtag = le16toh(cur->wb.upper.vlan);
4468
4469                 /* Make sure bad packets are discarded */
4470                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4471                     (rxr->discard)) {
4472                         ifp->if_ierrors++;
4473                         rxr->rx_discarded++;
4474                         if (eop)
4475                                 rxr->discard = FALSE;
4476                         else
4477                                 rxr->discard = TRUE;
4478                         ixgbe_rx_discard(rxr, i);
4479                         goto next_desc;
4480                 }
4481
4482                 /*
4483                 ** On 82599 which supports a hardware
4484                 ** LRO (called HW RSC), packets need
4485                 ** not be fragmented across sequential
4486                 ** descriptors, rather the next descriptor
4487                 ** is indicated in bits of the descriptor.
4488                 ** This also means that we might proceses
4489                 ** more than one packet at a time, something
4490                 ** that has never been true before, it
4491                 ** required eliminating global chain pointers
4492                 ** in favor of what we are doing here.  -jfv
4493                 */
4494                 if (!eop) {
4495                         /*
4496                         ** Figure out the next descriptor
4497                         ** of this frame.
4498                         */
4499                         if (rxr->hw_rsc == TRUE) {
4500                                 rsc = ixgbe_rsc_count(cur);
4501                                 rxr->rsc_num += (rsc - 1);
4502                         }
4503                         if (rsc) { /* Get hardware index */
4504                                 nextp = ((staterr &
4505                                     IXGBE_RXDADV_NEXTP_MASK) >>
4506                                     IXGBE_RXDADV_NEXTP_SHIFT);
4507                         } else { /* Just sequential */
4508                                 nextp = i + 1;
4509                                 if (nextp == adapter->num_rx_desc)
4510                                         nextp = 0;
4511                         }
4512                         nbuf = &rxr->rx_buffers[nextp];
4513                         prefetch(nbuf);
4514                 }
4515                 /*
4516                 ** The header mbuf is ONLY used when header
4517                 ** split is enabled, otherwise we get normal
4518                 ** behavior, ie, both header and payload
4519                 ** are DMA'd into the payload buffer.
4520                 **
4521                 ** Rather than using the fmp/lmp global pointers
4522                 ** we now keep the head of a packet chain in the
4523                 ** buffer struct and pass this along from one
4524                 ** descriptor to the next, until we get EOP.
4525                 */
4526                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4527                         /* This must be an initial descriptor */
4528                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4529                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4530                         if (hlen > IXGBE_RX_HDR)
4531                                 hlen = IXGBE_RX_HDR;
4532                         mh->m_len = hlen;
4533                         mh->m_flags |= M_PKTHDR;
4534                         mh->m_next = NULL;
4535                         mh->m_pkthdr.len = mh->m_len;
4536                         /* Null buf pointer so it is refreshed */
4537                         rbuf->m_head = NULL;
4538                         /*
4539                         ** Check the payload length, this
4540                         ** could be zero if its a small
4541                         ** packet.
4542                         */
4543                         if (plen > 0) {
4544                                 mp->m_len = plen;
4545                                 mp->m_next = NULL;
4546                                 mp->m_flags &= ~M_PKTHDR;
4547                                 mh->m_next = mp;
4548                                 mh->m_pkthdr.len += mp->m_len;
4549                                 /* Null buf pointer so it is refreshed */
4550                                 rbuf->m_pack = NULL;
4551                                 rxr->rx_split_packets++;
4552                         }
4553                         /*
4554                         ** Now create the forward
4555                         ** chain so when complete
4556                         ** we wont have to.
4557                         */
4558                         if (eop == 0) {
4559                                 /* stash the chain head */
4560                                 nbuf->fmp = mh;
4561                                 /* Make forward chain */
4562                                 if (plen)
4563                                         mp->m_next = nbuf->m_pack;
4564                                 else
4565                                         mh->m_next = nbuf->m_pack;
4566                         } else {
4567                                 /* Singlet, prepare to send */
4568                                 sendmp = mh;
4569                                 /* If hardware handled vtag */
4570                                 if (vtag) {
4571                                         sendmp->m_pkthdr.ether_vtag = vtag;
4572                                         sendmp->m_flags |= M_VLANTAG;
4573                                 }
4574                         }
4575                 } else {
4576                         /*
4577                         ** Either no header split, or a
4578                         ** secondary piece of a fragmented
4579                         ** split packet.
4580                         */
4581                         mp->m_len = plen;
4582                         /*
4583                         ** See if there is a stored head
4584                         ** that determines what we are
4585                         */
4586                         sendmp = rbuf->fmp;
4587
4588                         if (sendmp != NULL) {  /* secondary frag */
4589                                 rbuf->m_pack = rbuf->fmp = NULL;
4590                                 mp->m_flags &= ~M_PKTHDR;
4591                                 sendmp->m_pkthdr.len += mp->m_len;
4592                         } else {
4593                                 /*
4594                                  * Optimize.  This might be a small packet,
4595                                  * maybe just a TCP ACK.  Do a fast copy that
4596                                  * is cache aligned into a new mbuf, and
4597                                  * leave the old mbuf+cluster for re-use.
4598                                  */
4599                                 if (eop && plen <= IXGBE_RX_COPY_LEN) {
4600                                         sendmp = m_gethdr(M_DONTWAIT, MT_DATA);
4601                                         if (sendmp != NULL) {
4602                                                 sendmp->m_data +=
4603                                                     IXGBE_RX_COPY_ALIGN;
4604                                                 ixgbe_bcopy(mp->m_data,
4605                                                     sendmp->m_data, plen);
4606                                                 sendmp->m_len = plen;
4607                                                 rxr->rx_copies++;
4608                                                 rbuf->flags |= IXGBE_RX_COPY;
4609                                         }
4610                                 }
4611                                 if (sendmp == NULL) {
4612                                         rbuf->m_pack = rbuf->fmp = NULL;
4613                                         sendmp = mp;
4614                                 }
4615
4616                                 /* first desc of a non-ps chain */
4617                                 sendmp->m_flags |= M_PKTHDR;
4618                                 sendmp->m_pkthdr.len = mp->m_len;
4619                                 if (staterr & IXGBE_RXD_STAT_VP) {
4620                                         sendmp->m_pkthdr.ether_vtag = vtag;
4621                                         sendmp->m_flags |= M_VLANTAG;
4622                                 }
4623                         }
4624                         /* Pass the head pointer on */
4625                         if (eop == 0) {
4626                                 nbuf->fmp = sendmp;
4627                                 sendmp = NULL;
4628                                 mp->m_next = nbuf->m_pack;
4629                         }
4630                 }
4631                 ++processed;
4632                 /* Sending this frame? */
4633                 if (eop) {
4634                         sendmp->m_pkthdr.rcvif = ifp;
4635                         ifp->if_ipackets++;
4636                         rxr->rx_packets++;
4637                         /* capture data for AIM */
4638                         rxr->bytes += sendmp->m_pkthdr.len;
4639                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4640                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4641                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4642 #if __FreeBSD_version >= 800000
4643                         sendmp->m_pkthdr.flowid = que->msix;
4644                         sendmp->m_flags |= M_FLOWID;
4645 #endif
4646                 }
4647 next_desc:
4648                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4649                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4650
4651                 /* Advance our pointers to the next descriptor. */
4652                 if (++i == adapter->num_rx_desc)
4653                         i = 0;
4654
4655                 /* Now send to the stack or do LRO */
4656                 if (sendmp != NULL) {
4657                         rxr->next_to_check = i;
4658                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4659                         i = rxr->next_to_check;
4660                 }
4661
4662                /* Every 8 descriptors we go to refresh mbufs */
4663                 if (processed == 8) {
4664                         ixgbe_refresh_mbufs(rxr, i);
4665                         processed = 0;
4666                 }
4667         }
4668
4669         /* Refresh any remaining buf structs */
4670         if (ixgbe_rx_unrefreshed(rxr))
4671                 ixgbe_refresh_mbufs(rxr, i);
4672
4673         rxr->next_to_check = i;
4674
4675         /*
4676          * Flush any outstanding LRO work
4677          */
4678         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4679                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4680                 tcp_lro_flush(lro, queued);
4681         }
4682
4683         IXGBE_RX_UNLOCK(rxr);
4684
4685         /*
4686         ** We still have cleaning to do?
4687         ** Schedule another interrupt if so.
4688         */
4689         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4690                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4691                 return (TRUE);
4692         }
4693
4694         return (FALSE);
4695 }
4696
4697
4698 /*********************************************************************
4699  *
4700  *  Verify that the hardware indicated that the checksum is valid.
4701  *  Inform the stack about the status of checksum so that stack
4702  *  doesn't spend time verifying the checksum.
4703  *
4704  *********************************************************************/
4705 static void
4706 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4707 {
4708         u16     status = (u16) staterr;
4709         u8      errors = (u8) (staterr >> 24);
4710         bool    sctp = FALSE;
4711
4712         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4713             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4714                 sctp = TRUE;
4715
4716         if (status & IXGBE_RXD_STAT_IPCS) {
4717                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4718                         /* IP Checksum Good */
4719                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4720                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4721
4722                 } else
4723                         mp->m_pkthdr.csum_flags = 0;
4724         }
4725         if (status & IXGBE_RXD_STAT_L4CS) {
4726                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4727 #if __FreeBSD_version >= 800000
4728                 if (sctp)
4729                         type = CSUM_SCTP_VALID;
4730 #endif
4731                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4732                         mp->m_pkthdr.csum_flags |= type;
4733                         if (!sctp)
4734                                 mp->m_pkthdr.csum_data = htons(0xffff);
4735                 }
4736         }
4737         return;
4738 }
4739
4740
4741 /*
4742 ** This routine is run via an vlan config EVENT,
4743 ** it enables us to use the HW Filter table since
4744 ** we can get the vlan id. This just creates the
4745 ** entry in the soft version of the VFTA, init will
4746 ** repopulate the real table.
4747 */
4748 static void
4749 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4750 {
4751         struct adapter  *adapter = ifp->if_softc;
4752         u16             index, bit;
4753
4754         if (ifp->if_softc !=  arg)   /* Not our event */
4755                 return;
4756
4757         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4758                 return;
4759
4760         IXGBE_CORE_LOCK(adapter);
4761         index = (vtag >> 5) & 0x7F;
4762         bit = vtag & 0x1F;
4763         adapter->shadow_vfta[index] |= (1 << bit);
4764         ++adapter->num_vlans;
4765         ixgbe_init_locked(adapter);
4766         IXGBE_CORE_UNLOCK(adapter);
4767 }
4768
4769 /*
4770 ** This routine is run via an vlan
4771 ** unconfig EVENT, remove our entry
4772 ** in the soft vfta.
4773 */
4774 static void
4775 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4776 {
4777         struct adapter  *adapter = ifp->if_softc;
4778         u16             index, bit;
4779
4780         if (ifp->if_softc !=  arg)
4781                 return;
4782
4783         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4784                 return;
4785
4786         IXGBE_CORE_LOCK(adapter);
4787         index = (vtag >> 5) & 0x7F;
4788         bit = vtag & 0x1F;
4789         adapter->shadow_vfta[index] &= ~(1 << bit);
4790         --adapter->num_vlans;
4791         /* Re-init to load the changes */
4792         ixgbe_init_locked(adapter);
4793         IXGBE_CORE_UNLOCK(adapter);
4794 }
4795
4796 static void
4797 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4798 {
4799         struct ifnet    *ifp = adapter->ifp;
4800         struct ixgbe_hw *hw = &adapter->hw;
4801         struct rx_ring  *rxr;
4802         u32             ctrl;
4803
4804
4805         /*
4806         ** We get here thru init_locked, meaning
4807         ** a soft reset, this has already cleared
4808         ** the VFTA and other state, so if there
4809         ** have been no vlan's registered do nothing.
4810         */
4811         if (adapter->num_vlans == 0)
4812                 return;
4813
4814         /*
4815         ** A soft reset zero's out the VFTA, so
4816         ** we need to repopulate it now.
4817         */
4818         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4819                 if (adapter->shadow_vfta[i] != 0)
4820                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4821                             adapter->shadow_vfta[i]);
4822
4823         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4824         /* Enable the Filter Table if enabled */
4825         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4826                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4827                 ctrl |= IXGBE_VLNCTRL_VFE;
4828         }
4829         if (hw->mac.type == ixgbe_mac_82598EB)
4830                 ctrl |= IXGBE_VLNCTRL_VME;
4831         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4832
4833         /* Setup the queues for vlans */
4834         for (int i = 0; i < adapter->num_queues; i++) {
4835                 rxr = &adapter->rx_rings[i];
4836                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4837                 if (hw->mac.type != ixgbe_mac_82598EB) {
4838                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4839                         ctrl |= IXGBE_RXDCTL_VME;
4840                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4841                 }
4842                 rxr->vtag_strip = TRUE;
4843         }
4844 }
4845
4846 static void
4847 ixgbe_enable_intr(struct adapter *adapter)
4848 {
4849         struct ixgbe_hw *hw = &adapter->hw;
4850         struct ix_queue *que = adapter->queues;
4851         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4852
4853
4854         /* Enable Fan Failure detection */
4855         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4856                     mask |= IXGBE_EIMS_GPI_SDP1;
4857         else {
4858                     mask |= IXGBE_EIMS_ECC;
4859                     mask |= IXGBE_EIMS_GPI_SDP0;
4860                     mask |= IXGBE_EIMS_GPI_SDP1;
4861                     mask |= IXGBE_EIMS_GPI_SDP2;
4862 #ifdef IXGBE_FDIR
4863                     mask |= IXGBE_EIMS_FLOW_DIR;
4864 #endif
4865         }
4866
4867         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4868
4869         /* With RSS we use auto clear */
4870         if (adapter->msix_mem) {
4871                 mask = IXGBE_EIMS_ENABLE_MASK;
4872                 /* Don't autoclear Link */
4873                 mask &= ~IXGBE_EIMS_OTHER;
4874                 mask &= ~IXGBE_EIMS_LSC;
4875                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4876         }
4877
4878         /*
4879         ** Now enable all queues, this is done separately to
4880         ** allow for handling the extended (beyond 32) MSIX
4881         ** vectors that can be used by 82599
4882         */
4883         for (int i = 0; i < adapter->num_queues; i++, que++)
4884                 ixgbe_enable_queue(adapter, que->msix);
4885
4886         IXGBE_WRITE_FLUSH(hw);
4887
4888         return;
4889 }
4890
4891 static void
4892 ixgbe_disable_intr(struct adapter *adapter)
4893 {
4894         if (adapter->msix_mem)
4895                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4896         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4897                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4898         } else {
4899                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4900                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4901                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4902         }
4903         IXGBE_WRITE_FLUSH(&adapter->hw);
4904         return;
4905 }
4906
4907 u16
4908 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4909 {
4910         u16 value;
4911
4912         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4913             reg, 2);
4914
4915         return (value);
4916 }
4917
4918 void
4919 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4920 {
4921         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4922             reg, value, 2);
4923
4924         return;
4925 }
4926
4927 /*
4928 ** Setup the correct IVAR register for a particular MSIX interrupt
4929 **   (yes this is all very magic and confusing :)
4930 **  - entry is the register array entry
4931 **  - vector is the MSIX vector for this queue
4932 **  - type is RX/TX/MISC
4933 */
4934 static void
4935 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4936 {
4937         struct ixgbe_hw *hw = &adapter->hw;
4938         u32 ivar, index;
4939
4940         vector |= IXGBE_IVAR_ALLOC_VAL;
4941
4942         switch (hw->mac.type) {
4943
4944         case ixgbe_mac_82598EB:
4945                 if (type == -1)
4946                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4947                 else
4948                         entry += (type * 64);
4949                 index = (entry >> 2) & 0x1F;
4950                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4951                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4952                 ivar |= (vector << (8 * (entry & 0x3)));
4953                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4954                 break;
4955
4956         case ixgbe_mac_82599EB:
4957         case ixgbe_mac_X540:
4958                 if (type == -1) { /* MISC IVAR */
4959                         index = (entry & 1) * 8;
4960                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4961                         ivar &= ~(0xFF << index);
4962                         ivar |= (vector << index);
4963                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4964                 } else {        /* RX/TX IVARS */
4965                         index = (16 * (entry & 1)) + (8 * type);
4966                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4967                         ivar &= ~(0xFF << index);
4968                         ivar |= (vector << index);
4969                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4970                 }
4971
4972         default:
4973                 break;
4974         }
4975 }
4976
4977 static void
4978 ixgbe_configure_ivars(struct adapter *adapter)
4979 {
4980         struct  ix_queue *que = adapter->queues;
4981         u32 newitr;
4982
4983         if (ixgbe_max_interrupt_rate > 0)
4984                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
4985         else
4986                 newitr = 0;
4987
4988         for (int i = 0; i < adapter->num_queues; i++, que++) {
4989                 /* First the RX queue entry */
4990                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4991                 /* ... and the TX */
4992                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4993                 /* Set an Initial EITR value */
4994                 IXGBE_WRITE_REG(&adapter->hw,
4995                     IXGBE_EITR(que->msix), newitr);
4996         }
4997
4998         /* For the Link interrupt */
4999         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5000 }
5001
5002 /*
5003 ** ixgbe_sfp_probe - called in the local timer to
5004 ** determine if a port had optics inserted.
5005 */
5006 static bool ixgbe_sfp_probe(struct adapter *adapter)
5007 {
5008         struct ixgbe_hw *hw = &adapter->hw;
5009         device_t        dev = adapter->dev;
5010         bool            result = FALSE;
5011
5012         if ((hw->phy.type == ixgbe_phy_nl) &&
5013             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5014                 s32 ret = hw->phy.ops.identify_sfp(hw);
5015                 if (ret)
5016                         goto out;
5017                 ret = hw->phy.ops.reset(hw);
5018                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5019                         device_printf(dev,"Unsupported SFP+ module detected!");
5020                         printf(" Reload driver with supported module.\n");
5021                         adapter->sfp_probe = FALSE;
5022                         goto out;
5023                 } else
5024                         device_printf(dev,"SFP+ module detected!\n");
5025                 /* We now have supported optics */
5026                 adapter->sfp_probe = FALSE;
5027                 /* Set the optics type so system reports correctly */
5028                 ixgbe_setup_optics(adapter);
5029                 result = TRUE;
5030         }
5031 out:
5032         return (result);
5033 }
5034
5035 /*
5036 ** Tasklet handler for MSIX Link interrupts
5037 **  - do outside interrupt since it might sleep
5038 */
5039 static void
5040 ixgbe_handle_link(void *context, int pending)
5041 {
5042         struct adapter  *adapter = context;
5043
5044         ixgbe_check_link(&adapter->hw,
5045             &adapter->link_speed, &adapter->link_up, 0);
5046         ixgbe_update_link_status(adapter);
5047 }
5048
5049 /*
5050 ** Tasklet for handling SFP module interrupts
5051 */
5052 static void
5053 ixgbe_handle_mod(void *context, int pending)
5054 {
5055         struct adapter  *adapter = context;
5056         struct ixgbe_hw *hw = &adapter->hw;
5057         device_t        dev = adapter->dev;
5058         u32 err;
5059
5060         err = hw->phy.ops.identify_sfp(hw);
5061         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5062                 device_printf(dev,
5063                     "Unsupported SFP+ module type was detected.\n");
5064                 return;
5065         }
5066         err = hw->mac.ops.setup_sfp(hw);
5067         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5068                 device_printf(dev,
5069                     "Setup failure - unsupported SFP+ module type.\n");
5070                 return;
5071         }
5072         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5073         return;
5074 }
5075
5076
5077 /*
5078 ** Tasklet for handling MSF (multispeed fiber) interrupts
5079 */
5080 static void
5081 ixgbe_handle_msf(void *context, int pending)
5082 {
5083         struct adapter  *adapter = context;
5084         struct ixgbe_hw *hw = &adapter->hw;
5085         u32 autoneg;
5086         bool negotiate;
5087
5088         autoneg = hw->phy.autoneg_advertised;
5089         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5090                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5091         if (hw->mac.ops.setup_link)
5092                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5093         return;
5094 }
5095
5096 #ifdef IXGBE_FDIR
5097 /*
5098 ** Tasklet for reinitializing the Flow Director filter table
5099 */
5100 static void
5101 ixgbe_reinit_fdir(void *context, int pending)
5102 {
5103         struct adapter  *adapter = context;
5104         struct ifnet   *ifp = adapter->ifp;
5105
5106         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5107                 return;
5108         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5109         adapter->fdir_reinit = 0;
5110         /* re-enable flow director interrupts */
5111         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5112         /* Restart the interface */
5113         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5114         return;
5115 }
5116 #endif
5117
5118 /**********************************************************************
5119  *
5120  *  Update the board statistics counters.
5121  *
5122  **********************************************************************/
5123 static void
5124 ixgbe_update_stats_counters(struct adapter *adapter)
5125 {
5126         struct ifnet   *ifp = adapter->ifp;
5127         struct ixgbe_hw *hw = &adapter->hw;
5128         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5129         u64  total_missed_rx = 0;
5130
5131         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5132         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5133         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5134         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5135
5136         for (int i = 0; i < 8; i++) {
5137                 u32 mp;
5138                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5139                 /* missed_rx tallies misses for the gprc workaround */
5140                 missed_rx += mp;
5141                 /* global total per queue */
5142                 adapter->stats.mpc[i] += mp;
5143                 /* Running comprehensive total for stats display */
5144                 total_missed_rx += adapter->stats.mpc[i];
5145                 if (hw->mac.type == ixgbe_mac_82598EB)
5146                         adapter->stats.rnbc[i] +=
5147                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5148                 adapter->stats.pxontxc[i] +=
5149                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5150                 adapter->stats.pxonrxc[i] +=
5151                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5152                 adapter->stats.pxofftxc[i] +=
5153                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5154                 adapter->stats.pxoffrxc[i] +=
5155                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5156                 adapter->stats.pxon2offc[i] +=
5157                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5158         }
5159         for (int i = 0; i < 16; i++) {
5160                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5161                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5162                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5163                 adapter->stats.qbrc[i] +=
5164                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5165                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5166                 adapter->stats.qbtc[i] +=
5167                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5168                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5169         }
5170         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5171         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5172         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5173
5174         /* Hardware workaround, gprc counts missed packets */
5175         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5176         adapter->stats.gprc -= missed_rx;
5177
5178         if (hw->mac.type != ixgbe_mac_82598EB) {
5179                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5180                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5181                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5182                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5183                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5184                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5185                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5186                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5187         } else {
5188                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5189                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5190                 /* 82598 only has a counter in the high register */
5191                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5192                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5193                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5194         }
5195
5196         /*
5197          * Workaround: mprc hardware is incorrectly counting
5198          * broadcasts, so for now we subtract those.
5199          */
5200         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5201         adapter->stats.bprc += bprc;
5202         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5203         if (hw->mac.type == ixgbe_mac_82598EB)
5204                 adapter->stats.mprc -= bprc;
5205
5206         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5207         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5208         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5209         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5210         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5211         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5212
5213         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5214         adapter->stats.lxontxc += lxon;
5215         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5216         adapter->stats.lxofftxc += lxoff;
5217         total = lxon + lxoff;
5218
5219         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5220         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5221         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5222         adapter->stats.gptc -= total;
5223         adapter->stats.mptc -= total;
5224         adapter->stats.ptc64 -= total;
5225         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5226
5227         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5228         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5229         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5230         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5231         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5232         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5233         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5234         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5235         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5236         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5237         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5238         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5239         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5240         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5241         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5242         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5243         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5244         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5245         /* Only read FCOE on 82599 */
5246         if (hw->mac.type != ixgbe_mac_82598EB) {
5247                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5248                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5249                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5250                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5251                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5252         }
5253
5254         /* Fill out the OS statistics structure */
5255         ifp->if_ipackets = adapter->stats.gprc;
5256         ifp->if_opackets = adapter->stats.gptc;
5257         ifp->if_ibytes = adapter->stats.gorc;
5258         ifp->if_obytes = adapter->stats.gotc;
5259         ifp->if_imcasts = adapter->stats.mprc;
5260         ifp->if_omcasts = adapter->stats.mptc;
5261         ifp->if_collisions = 0;
5262
5263         /* Rx Errors */
5264         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5265                 adapter->stats.rlec;
5266 }
5267
5268 /** ixgbe_sysctl_tdh_handler - Handler function
5269  *  Retrieves the TDH value from the hardware
5270  */
5271 static int
5272 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5273 {
5274         int error;
5275
5276         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5277         if (!txr) return 0;
5278
5279         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5280         error = sysctl_handle_int(oidp, &val, 0, req);
5281         if (error || !req->newptr)
5282                 return error;
5283         return 0;
5284 }
5285
5286 /** ixgbe_sysctl_tdt_handler - Handler function
5287  *  Retrieves the TDT value from the hardware
5288  */
5289 static int
5290 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5291 {
5292         int error;
5293
5294         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5295         if (!txr) return 0;
5296
5297         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5298         error = sysctl_handle_int(oidp, &val, 0, req);
5299         if (error || !req->newptr)
5300                 return error;
5301         return 0;
5302 }
5303
5304 /** ixgbe_sysctl_rdh_handler - Handler function
5305  *  Retrieves the RDH value from the hardware
5306  */
5307 static int
5308 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5309 {
5310         int error;
5311
5312         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5313         if (!rxr) return 0;
5314
5315         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5316         error = sysctl_handle_int(oidp, &val, 0, req);
5317         if (error || !req->newptr)
5318                 return error;
5319         return 0;
5320 }
5321
5322 /** ixgbe_sysctl_rdt_handler - Handler function
5323  *  Retrieves the RDT value from the hardware
5324  */
5325 static int
5326 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5327 {
5328         int error;
5329
5330         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5331         if (!rxr) return 0;
5332
5333         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5334         error = sysctl_handle_int(oidp, &val, 0, req);
5335         if (error || !req->newptr)
5336                 return error;
5337         return 0;
5338 }
5339
5340 static int
5341 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5342 {
5343         int error;
5344         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5345         unsigned int reg, usec, rate;
5346
5347         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5348         usec = ((reg & 0x0FF8) >> 3);
5349         if (usec > 0)
5350                 rate = 500000 / usec;
5351         else
5352                 rate = 0;
5353         error = sysctl_handle_int(oidp, &rate, 0, req);
5354         if (error || !req->newptr)
5355                 return error;
5356         reg &= ~0xfff; /* default, no limitation */
5357         ixgbe_max_interrupt_rate = 0;
5358         if (rate > 0 && rate < 500000) {
5359                 if (rate < 1000)
5360                         rate = 1000;
5361                 ixgbe_max_interrupt_rate = rate;
5362                 reg |= ((4000000/rate) & 0xff8 );
5363         }
5364         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5365         return 0;
5366 }
5367
5368 /*
5369  * Add sysctl variables, one per statistic, to the system.
5370  */
5371 static void
5372 ixgbe_add_hw_stats(struct adapter *adapter)
5373 {
5374
5375         device_t dev = adapter->dev;
5376
5377         struct tx_ring *txr = adapter->tx_rings;
5378         struct rx_ring *rxr = adapter->rx_rings;
5379
5380         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5381         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5382         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5383         struct ixgbe_hw_stats *stats = &adapter->stats;
5384
5385         struct sysctl_oid *stat_node, *queue_node;
5386         struct sysctl_oid_list *stat_list, *queue_list;
5387
5388 #define QUEUE_NAME_LEN 32
5389         char namebuf[QUEUE_NAME_LEN];
5390
5391         /* Driver Statistics */
5392         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5393                         CTLFLAG_RD, &adapter->dropped_pkts,
5394                         "Driver dropped packets");
5395         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5396                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5397                         "m_defrag() failed");
5398         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5399                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5400                         "Driver tx dma failure in xmit");
5401         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5402                         CTLFLAG_RD, &adapter->watchdog_events,
5403                         "Watchdog timeouts");
5404         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5405                         CTLFLAG_RD, &adapter->tso_tx,
5406                         "TSO");
5407         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5408                         CTLFLAG_RD, &adapter->link_irq,
5409                         "Link MSIX IRQ Handled");
5410
5411         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5412                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5413                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5414                                             CTLFLAG_RD, NULL, "Queue Name");
5415                 queue_list = SYSCTL_CHILDREN(queue_node);
5416
5417                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5418                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5419                                 sizeof(&adapter->queues[i]),
5420                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5421                                 "Interrupt Rate");
5422                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5423                                 CTLFLAG_RD, &(adapter->queues[i].irqs),
5424                                 "irqs on this queue");
5425                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5426                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5427                                 ixgbe_sysctl_tdh_handler, "IU",
5428                                 "Transmit Descriptor Head");
5429                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5430                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5431                                 ixgbe_sysctl_tdt_handler, "IU",
5432                                 "Transmit Descriptor Tail");
5433                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5434                                 CTLFLAG_RD, &txr->no_desc_avail,
5435                                 "Queue No Descriptor Available");
5436                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5437                                 CTLFLAG_RD, &txr->total_packets,
5438                                 "Queue Packets Transmitted");
5439         }
5440
5441         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5442                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5443                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5444                                             CTLFLAG_RD, NULL, "Queue Name");
5445                 queue_list = SYSCTL_CHILDREN(queue_node);
5446
5447                 struct lro_ctrl *lro = &rxr->lro;
5448
5449                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5450                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5451                                             CTLFLAG_RD, NULL, "Queue Name");
5452                 queue_list = SYSCTL_CHILDREN(queue_node);
5453
5454                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5455                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5456                                 ixgbe_sysctl_rdh_handler, "IU",
5457                                 "Receive Descriptor Head");
5458                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5459                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5460                                 ixgbe_sysctl_rdt_handler, "IU",
5461                                 "Receive Descriptor Tail");
5462                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5463                                 CTLFLAG_RD, &rxr->rx_packets,
5464                                 "Queue Packets Received");
5465                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5466                                 CTLFLAG_RD, &rxr->rx_bytes,
5467                                 "Queue Bytes Received");
5468                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
5469                                 CTLFLAG_RD, &rxr->rx_copies,
5470                                 "Copied RX Frames");
5471                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5472                                 CTLFLAG_RD, &lro->lro_queued, 0,
5473                                 "LRO Queued");
5474                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5475                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5476                                 "LRO Flushed");
5477         }
5478
5479         /* MAC stats get the own sub node */
5480
5481         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5482                                     CTLFLAG_RD, NULL, "MAC Statistics");
5483         stat_list = SYSCTL_CHILDREN(stat_node);
5484
5485         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5486                         CTLFLAG_RD, &stats->crcerrs,
5487                         "CRC Errors");
5488         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5489                         CTLFLAG_RD, &stats->illerrc,
5490                         "Illegal Byte Errors");
5491         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5492                         CTLFLAG_RD, &stats->errbc,
5493                         "Byte Errors");
5494         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5495                         CTLFLAG_RD, &stats->mspdc,
5496                         "MAC Short Packets Discarded");
5497         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5498                         CTLFLAG_RD, &stats->mlfc,
5499                         "MAC Local Faults");
5500         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5501                         CTLFLAG_RD, &stats->mrfc,
5502                         "MAC Remote Faults");
5503         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5504                         CTLFLAG_RD, &stats->rlec,
5505                         "Receive Length Errors");
5506         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5507                         CTLFLAG_RD, &stats->lxontxc,
5508                         "Link XON Transmitted");
5509         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5510                         CTLFLAG_RD, &stats->lxonrxc,
5511                         "Link XON Received");
5512         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5513                         CTLFLAG_RD, &stats->lxofftxc,
5514                         "Link XOFF Transmitted");
5515         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5516                         CTLFLAG_RD, &stats->lxoffrxc,
5517                         "Link XOFF Received");
5518
5519         /* Packet Reception Stats */
5520         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5521                         CTLFLAG_RD, &stats->tor,
5522                         "Total Octets Received");
5523         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5524                         CTLFLAG_RD, &stats->gorc,
5525                         "Good Octets Received");
5526         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5527                         CTLFLAG_RD, &stats->tpr,
5528                         "Total Packets Received");
5529         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5530                         CTLFLAG_RD, &stats->gprc,
5531                         "Good Packets Received");
5532         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5533                         CTLFLAG_RD, &stats->mprc,
5534                         "Multicast Packets Received");
5535         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5536                         CTLFLAG_RD, &stats->bprc,
5537                         "Broadcast Packets Received");
5538         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5539                         CTLFLAG_RD, &stats->prc64,
5540                         "64 byte frames received ");
5541         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5542                         CTLFLAG_RD, &stats->prc127,
5543                         "65-127 byte frames received");
5544         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5545                         CTLFLAG_RD, &stats->prc255,
5546                         "128-255 byte frames received");
5547         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5548                         CTLFLAG_RD, &stats->prc511,
5549                         "256-511 byte frames received");
5550         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5551                         CTLFLAG_RD, &stats->prc1023,
5552                         "512-1023 byte frames received");
5553         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5554                         CTLFLAG_RD, &stats->prc1522,
5555                         "1023-1522 byte frames received");
5556         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5557                         CTLFLAG_RD, &stats->ruc,
5558                         "Receive Undersized");
5559         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5560                         CTLFLAG_RD, &stats->rfc,
5561                         "Fragmented Packets Received ");
5562         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5563                         CTLFLAG_RD, &stats->roc,
5564                         "Oversized Packets Received");
5565         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5566                         CTLFLAG_RD, &stats->rjc,
5567                         "Received Jabber");
5568         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5569                         CTLFLAG_RD, &stats->mngprc,
5570                         "Management Packets Received");
5571         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5572                         CTLFLAG_RD, &stats->mngptc,
5573                         "Management Packets Dropped");
5574         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5575                         CTLFLAG_RD, &stats->xec,
5576                         "Checksum Errors");
5577
5578         /* Packet Transmission Stats */
5579         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5580                         CTLFLAG_RD, &stats->gotc,
5581                         "Good Octets Transmitted");
5582         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5583                         CTLFLAG_RD, &stats->tpt,
5584                         "Total Packets Transmitted");
5585         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5586                         CTLFLAG_RD, &stats->gptc,
5587                         "Good Packets Transmitted");
5588         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5589                         CTLFLAG_RD, &stats->bptc,
5590                         "Broadcast Packets Transmitted");
5591         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5592                         CTLFLAG_RD, &stats->mptc,
5593                         "Multicast Packets Transmitted");
5594         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5595                         CTLFLAG_RD, &stats->mngptc,
5596                         "Management Packets Transmitted");
5597         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5598                         CTLFLAG_RD, &stats->ptc64,
5599                         "64 byte frames transmitted ");
5600         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5601                         CTLFLAG_RD, &stats->ptc127,
5602                         "65-127 byte frames transmitted");
5603         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5604                         CTLFLAG_RD, &stats->ptc255,
5605                         "128-255 byte frames transmitted");
5606         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5607                         CTLFLAG_RD, &stats->ptc511,
5608                         "256-511 byte frames transmitted");
5609         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5610                         CTLFLAG_RD, &stats->ptc1023,
5611                         "512-1023 byte frames transmitted");
5612         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5613                         CTLFLAG_RD, &stats->ptc1522,
5614                         "1024-1522 byte frames transmitted");
5615
5616         /* FC Stats */
5617         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5618                 CTLFLAG_RD, &stats->fccrc,
5619                 "FC CRC Errors");
5620         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5621                 CTLFLAG_RD, &stats->fclast,
5622                 "FC Last Error");
5623         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5624                 CTLFLAG_RD, &stats->fcoerpdc,
5625                 "FCoE Packets Dropped");
5626         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5627                 CTLFLAG_RD, &stats->fcoeprc,
5628                 "FCoE Packets Received");
5629         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5630                 CTLFLAG_RD, &stats->fcoeptc,
5631                 "FCoE Packets Transmitted");
5632         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5633                 CTLFLAG_RD, &stats->fcoedwrc,
5634                 "FCoE DWords Received");
5635         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5636                 CTLFLAG_RD, &stats->fcoedwtc,
5637                 "FCoE DWords Transmitted");
5638 }
5639
5640 /*
5641 ** Set flow control using sysctl:
5642 ** Flow control values:
5643 **      0 - off
5644 **      1 - rx pause
5645 **      2 - tx pause
5646 **      3 - full
5647 */
5648 static int
5649 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5650 {
5651         int error, last;
5652         struct adapter *adapter = (struct adapter *) arg1;
5653
5654         last = adapter->fc;
5655         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5656         if ((error) || (req->newptr == NULL))
5657                 return (error);
5658
5659         /* Don't bother if it's not changed */
5660         if (adapter->fc == last)
5661                 return (0);
5662
5663         switch (adapter->fc) {
5664                 case ixgbe_fc_rx_pause:
5665                 case ixgbe_fc_tx_pause:
5666                 case ixgbe_fc_full:
5667                         adapter->hw.fc.requested_mode = adapter->fc;
5668                         break;
5669                 case ixgbe_fc_none:
5670                 default:
5671                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5672         }
5673         /* Don't autoneg if forcing a value */
5674         adapter->hw.fc.disable_fc_autoneg = TRUE;
5675         ixgbe_fc_enable(&adapter->hw);
5676         return error;
5677 }
5678
5679 static void
5680 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5681         const char *description, int *limit, int value)
5682 {
5683         *limit = value;
5684         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5685             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5686             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5687 }
5688
5689 /*
5690 ** Control link advertise speed:
5691 **      1 - advertise only 1G
5692 **      2 - advertise 100Mb
5693 **      3 - advertise normal
5694 */
5695 static int
5696 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5697 {
5698         int                     error = 0;
5699         struct adapter          *adapter;
5700         device_t                dev;
5701         struct ixgbe_hw         *hw;
5702         ixgbe_link_speed        speed, last;
5703
5704         adapter = (struct adapter *) arg1;
5705         dev = adapter->dev;
5706         hw = &adapter->hw;
5707         last = adapter->advertise;
5708
5709         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5710         if ((error) || (adapter->advertise == -1))
5711                 return (error);
5712
5713         if (adapter->advertise == last) /* no change */
5714                 return (0);
5715
5716         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5717             (hw->phy.multispeed_fiber)))
5718                 return (error);
5719
5720         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5721                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5722                 return (error);
5723         }
5724
5725         if (adapter->advertise == 1)
5726                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5727         else if (adapter->advertise == 2)
5728                 speed = IXGBE_LINK_SPEED_100_FULL;
5729         else if (adapter->advertise == 3)
5730                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5731                         IXGBE_LINK_SPEED_10GB_FULL;
5732         else /* bogus value */
5733                 return (error);
5734
5735         hw->mac.autotry_restart = TRUE;
5736         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5737
5738         return (error);
5739 }
5740
5741 /*
5742 ** Thermal Shutdown Trigger
5743 **   - cause a Thermal Overtemp IRQ
5744 */
5745 static int
5746 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5747 {
5748         int             error, fire = 0;
5749         struct adapter  *adapter = (struct adapter *) arg1;
5750         struct ixgbe_hw *hw = &adapter->hw;
5751
5752
5753         if (hw->mac.type != ixgbe_mac_X540)
5754                 return (0);
5755
5756         error = sysctl_handle_int(oidp, &fire, 0, req);
5757         if ((error) || (req->newptr == NULL))
5758                 return (error);
5759
5760         if (fire) {
5761                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5762                 reg |= IXGBE_EICR_TS;
5763                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5764         }
5765
5766         return (0);
5767 }