sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2010, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #endif
  38
  39 #include "ixgbe.h"
  40
  41 /*********************************************************************
  42  *  Set this to one to display debug statistics
  43  *********************************************************************/
  44 int             ixgbe_display_debug_stats = 0;
  45
  46 /*********************************************************************
  47  *  Driver version
  48  *********************************************************************/
  49 char ixgbe_driver_version[] = "2.3.8";
  50
  51 /*********************************************************************
  52  *  PCI Device ID Table
  53  *
  54  *  Used by probe to select devices to load on
  55  *  Last field stores an index into ixgbe_strings
  56  *  Last entry must be all 0s
  57  *
  58  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  59  *********************************************************************/
  60
  61 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  62 {
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         /* required last entry */
  84         {0, 0, 0, 0, 0}
  85 };
  86
  87 /*********************************************************************
  88  *  Table of branding strings
  89  *********************************************************************/
  90
  91 static char    *ixgbe_strings[] = {
  92         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  93 };
  94
  95 /*********************************************************************
  96  *  Function prototypes
  97  *********************************************************************/
  98 static int      ixgbe_probe(device_t);
  99 static int      ixgbe_attach(device_t);
 100 static int      ixgbe_detach(device_t);
 101 static int      ixgbe_shutdown(device_t);
 102 static void     ixgbe_start(struct ifnet *);
 103 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 104 #if __FreeBSD_version >= 800000
 105 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 106 static int      ixgbe_mq_start_locked(struct ifnet *,
 107                     struct tx_ring *, struct mbuf *);
 108 static void     ixgbe_qflush(struct ifnet *);
 109 #endif
 110 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 111 static void     ixgbe_init(void *);
 112 static void     ixgbe_init_locked(struct adapter *);
 113 static void     ixgbe_stop(void *);
 114 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 115 static int      ixgbe_media_change(struct ifnet *);
 116 static void     ixgbe_identify_hardware(struct adapter *);
 117 static int      ixgbe_allocate_pci_resources(struct adapter *);
 118 static int      ixgbe_allocate_msix(struct adapter *);
 119 static int      ixgbe_allocate_legacy(struct adapter *);
 120 static int      ixgbe_allocate_queues(struct adapter *);
 121 static int      ixgbe_setup_msix(struct adapter *);
 122 static void     ixgbe_free_pci_resources(struct adapter *);
 123 static void     ixgbe_local_timer(void *);
 124 static int      ixgbe_setup_interface(device_t, struct adapter *);
 125 static void     ixgbe_config_link(struct adapter *);
 126
 127 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 128 static int      ixgbe_setup_transmit_structures(struct adapter *);
 129 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 130 static void     ixgbe_initialize_transmit_units(struct adapter *);
 131 static void     ixgbe_free_transmit_structures(struct adapter *);
 132 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 133
 134 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 135 static int      ixgbe_setup_receive_structures(struct adapter *);
 136 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 137 static void     ixgbe_initialize_receive_units(struct adapter *);
 138 static void     ixgbe_free_receive_structures(struct adapter *);
 139 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 140 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 141
 142 static void     ixgbe_enable_intr(struct adapter *);
 143 static void     ixgbe_disable_intr(struct adapter *);
 144 static void     ixgbe_update_stats_counters(struct adapter *);
 145 static bool     ixgbe_txeof(struct tx_ring *);
 146 static bool     ixgbe_rxeof(struct ix_queue *, int);
 147 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 148 static void     ixgbe_set_promisc(struct adapter *);
 149 static void     ixgbe_set_multi(struct adapter *);
 150 static void     ixgbe_update_link_status(struct adapter *);
 151 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 152 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 153 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 154 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 155 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 156                     struct ixgbe_dma_alloc *, int);
 157 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 158 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 159                     const char *, int *, int);
 160 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 161 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 162 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 163 static void     ixgbe_configure_ivars(struct adapter *);
 164 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 165
 166 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 167 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 168 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 169
 170 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 171
 172 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 173 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 174                     struct mbuf *, u32);
 175
 176 /* Support for pluggable optic modules */
 177 static bool     ixgbe_sfp_probe(struct adapter *);
 178 static void     ixgbe_setup_optics(struct adapter *);
 179
 180 /* Legacy (single vector interrupt handler */
 181 static void     ixgbe_legacy_irq(void *);
 182
 183 /* The MSI/X Interrupt handlers */
 184 static void     ixgbe_msix_que(void *);
 185 static void     ixgbe_msix_link(void *);
 186
 187 /* Deferred interrupt tasklets */
 188 static void     ixgbe_handle_que(void *, int);
 189 static void     ixgbe_handle_link(void *, int);
 190 static void     ixgbe_handle_msf(void *, int);
 191 static void     ixgbe_handle_mod(void *, int);
 192
 193 #ifdef IXGBE_FDIR
 194 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 195 static void     ixgbe_reinit_fdir(void *, int);
 196 #endif
 197
 198 /*********************************************************************
 199  *  FreeBSD Device Interface Entry Points
 200  *********************************************************************/
 201
 202 static device_method_t ixgbe_methods[] = {
 203         /* Device interface */
 204         DEVMETHOD(device_probe, ixgbe_probe),
 205         DEVMETHOD(device_attach, ixgbe_attach),
 206         DEVMETHOD(device_detach, ixgbe_detach),
 207         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 208         {0, 0}
 209 };
 210
 211 static driver_t ixgbe_driver = {
 212         "ix", ixgbe_methods, sizeof(struct adapter),
 213 };
 214
 215 devclass_t ixgbe_devclass;
 216 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 217
 218 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 219 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 220
 221 /*
 222 ** TUNEABLE PARAMETERS:
 223 */
 224
 225 /*
 226 ** AIM: Adaptive Interrupt Moderation
 227 ** which means that the interrupt rate
 228 ** is varied over time based on the
 229 ** traffic for that interrupt vector
 230 */
 231 static int ixgbe_enable_aim = TRUE;
 232 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 233
 234 static int ixgbe_max_interrupt_rate = (8000000 / IXGBE_LOW_LATENCY);
 235 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 236
 237 /* How many packets rxeof tries to clean at a time */
 238 static int ixgbe_rx_process_limit = 128;
 239 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 240
 241 /* Flow control setting, default to full */
 242 static int ixgbe_flow_control = ixgbe_fc_full;
 243 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
 244
 245 /*
 246 ** Smart speed setting, default to on
 247 ** this only works as a compile option
 248 ** right now as its during attach, set
 249 ** this to 'ixgbe_smart_speed_off' to
 250 ** disable.
 251 */
 252 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 253
 254 /*
 255  * MSIX should be the default for best performance,
 256  * but this allows it to be forced off for testing.
 257  */
 258 static int ixgbe_enable_msix = 1;
 259 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 260
 261 /*
 262  * Header split: this causes the hardware to DMA
 263  * the header into a separate mbuf from the payload,
 264  * it can be a performance win in some workloads, but
 265  * in others it actually hurts, its off by default.
 266  */
 267 static bool ixgbe_header_split = FALSE;
 268 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 269
 270 /*
 271  * Number of Queues, can be set to 0,
 272  * it then autoconfigures based on the
 273  * number of cpus. Each queue is a pair
 274  * of RX and TX rings with a msix vector
 275  */
 276 static int ixgbe_num_queues = 0;
 277 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 278
 279 /*
 280 ** Number of TX descriptors per ring,
 281 ** setting higher than RX as this seems
 282 ** the better performing choice.
 283 */
 284 static int ixgbe_txd = PERFORM_TXD;
 285 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 286
 287 /* Number of RX descriptors per ring */
 288 static int ixgbe_rxd = PERFORM_RXD;
 289 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 290
 291 /* Keep running tab on them for sanity check */
 292 static int ixgbe_total_ports;
 293
 294 #ifdef IXGBE_FDIR
 295 /*
 296 ** For Flow Director: this is the
 297 ** number of TX packets we sample
 298 ** for the filter pool, this means
 299 ** every 20th packet will be probed.
 300 **
 301 ** This feature can be disabled by
 302 ** setting this to 0.
 303 */
 304 static int atr_sample_rate = 20;
 305 /*
 306 ** Flow Director actually 'steals'
 307 ** part of the packet buffer as its
 308 ** filter pool, this variable controls
 309 ** how much it uses:
 310 **  0 = 64K, 1 = 128K, 2 = 256K
 311 */
 312 static int fdir_pballoc = 1;
 313 #endif
 314
 315 /*********************************************************************
 316  *  Device identification routine
 317  *
 318  *  ixgbe_probe determines if the driver should be loaded on
 319  *  adapter based on PCI vendor/device id of the adapter.
 320  *
 321  *  return 0 on success, positive on failure
 322  *********************************************************************/
 323
 324 static int
 325 ixgbe_probe(device_t dev)
 326 {
 327         ixgbe_vendor_info_t *ent;
 328
 329         u16     pci_vendor_id = 0;
 330         u16     pci_device_id = 0;
 331         u16     pci_subvendor_id = 0;
 332         u16     pci_subdevice_id = 0;
 333         char    adapter_name[256];
 334
 335         INIT_DEBUGOUT("ixgbe_probe: begin");
 336
 337         pci_vendor_id = pci_get_vendor(dev);
 338         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 339                 return (ENXIO);
 340
 341         pci_device_id = pci_get_device(dev);
 342         pci_subvendor_id = pci_get_subvendor(dev);
 343         pci_subdevice_id = pci_get_subdevice(dev);
 344
 345         ent = ixgbe_vendor_info_array;
 346         while (ent->vendor_id != 0) {
 347                 if ((pci_vendor_id == ent->vendor_id) &&
 348                     (pci_device_id == ent->device_id) &&
 349
 350                     ((pci_subvendor_id == ent->subvendor_id) ||
 351                      (ent->subvendor_id == 0)) &&
 352
 353                     ((pci_subdevice_id == ent->subdevice_id) ||
 354                      (ent->subdevice_id == 0))) {
 355                         sprintf(adapter_name, "%s, Version - %s",
 356                                 ixgbe_strings[ent->index],
 357                                 ixgbe_driver_version);
 358                         device_set_desc_copy(dev, adapter_name);
 359                         ++ixgbe_total_ports;
 360                         return (0);
 361                 }
 362                 ent++;
 363         }
 364         return (ENXIO);
 365 }
 366
 367 /*********************************************************************
 368  *  Device initialization routine
 369  *
 370  *  The attach entry point is called when the driver is being loaded.
 371  *  This routine identifies the type of hardware, allocates all resources
 372  *  and initializes the hardware.
 373  *
 374  *  return 0 on success, positive on failure
 375  *********************************************************************/
 376
 377 static int
 378 ixgbe_attach(device_t dev)
 379 {
 380         struct adapter *adapter;
 381         struct ixgbe_hw *hw;
 382         int             error = 0;
 383         u16             csum;
 384         u32             ctrl_ext;
 385
 386         INIT_DEBUGOUT("ixgbe_attach: begin");
 387
 388         /* Allocate, clear, and link in our adapter structure */
 389         adapter = device_get_softc(dev);
 390         adapter->dev = adapter->osdep.dev = dev;
 391         hw = &adapter->hw;
 392
 393         /* Core Lock Init*/
 394         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 395
 396         /* SYSCTL APIs */
 397
 398         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 399                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 400                         OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW,
 401                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 402
 403         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 404                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 405                         OID_AUTO, "advertise_gig", CTLTYPE_INT | CTLFLAG_RW,
 406                         adapter, 0, ixgbe_set_advertise, "I", "1G Link");
 407
 408         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 409                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 410                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 411                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 412
 413         /* Set up the timer callout */
 414         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 415
 416         /* Determine hardware revision */
 417         ixgbe_identify_hardware(adapter);
 418
 419         /* Do base PCI setup - map BAR0 */
 420         if (ixgbe_allocate_pci_resources(adapter)) {
 421                 device_printf(dev, "Allocation of PCI resources failed\n");
 422                 error = ENXIO;
 423                 goto err_out;
 424         }
 425
 426         /* Do descriptor calc and sanity checks */
 427         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 428             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 429                 device_printf(dev, "TXD config issue, using default!\n");
 430                 adapter->num_tx_desc = DEFAULT_TXD;
 431         } else
 432                 adapter->num_tx_desc = ixgbe_txd;
 433
 434         /*
 435         ** With many RX rings it is easy to exceed the
 436         ** system mbuf allocation. Tuning nmbclusters
 437         ** can alleviate this.
 438         */
 439         if (nmbclusters > 0 ) {
 440                 int s;
 441                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 442                 if (s > nmbclusters) {
 443                         device_printf(dev, "RX Descriptors exceed "
 444                             "system mbuf max, using default instead!\n");
 445                         ixgbe_rxd = DEFAULT_RXD;
 446                 }
 447         }
 448
 449         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 450             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 451                 device_printf(dev, "RXD config issue, using default!\n");
 452                 adapter->num_rx_desc = DEFAULT_RXD;
 453         } else
 454                 adapter->num_rx_desc = ixgbe_rxd;
 455
 456         /* Allocate our TX/RX Queues */
 457         if (ixgbe_allocate_queues(adapter)) {
 458                 error = ENOMEM;
 459                 goto err_out;
 460         }
 461
 462         /* Allocate multicast array memory. */
 463         adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 464             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 465         if (adapter->mta == NULL) {
 466                 device_printf(dev, "Can not allocate multicast setup array\n");
 467                 error = ENOMEM;
 468                 goto err_late;
 469         }
 470
 471         /* Initialize the shared code */
 472         error = ixgbe_init_shared_code(hw);
 473         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 474                 /*
 475                 ** No optics in this port, set up
 476                 ** so the timer routine will probe
 477                 ** for later insertion.
 478                 */
 479                 adapter->sfp_probe = TRUE;
 480                 error = 0;
 481         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 482                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 483                 error = EIO;
 484                 goto err_late;
 485         } else if (error) {
 486                 device_printf(dev,"Unable to initialize the shared code\n");
 487                 error = EIO;
 488                 goto err_late;
 489         }
 490
 491         /* Make sure we have a good EEPROM before we read from it */
 492         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 493                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 494                 error = EIO;
 495                 goto err_late;
 496         }
 497
 498         /* Get Hardware Flow Control setting */
 499         hw->fc.requested_mode = ixgbe_fc_full;
 500         hw->fc.pause_time = IXGBE_FC_PAUSE;
 501         hw->fc.low_water = IXGBE_FC_LO;
 502         hw->fc.high_water = IXGBE_FC_HI;
 503         hw->fc.send_xon = TRUE;
 504
 505         error = ixgbe_init_hw(hw);
 506         if (error == IXGBE_ERR_EEPROM_VERSION) {
 507                 device_printf(dev, "This device is a pre-production adapter/"
 508                     "LOM.  Please be aware there may be issues associated "
 509                     "with your hardware.\n If you are experiencing problems "
 510                     "please contact your Intel or hardware representative "
 511                     "who provided you with this hardware.\n");
 512         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED)
 513                 device_printf(dev,"Unsupported SFP+ Module\n");
 514
 515         if (error) {
 516                 error = EIO;
 517                 device_printf(dev,"Hardware Initialization Failure\n");
 518                 goto err_late;
 519         }
 520
 521         /* Detect and set physical type */
 522         ixgbe_setup_optics(adapter);
 523
 524         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 525                 error = ixgbe_allocate_msix(adapter);
 526         else
 527                 error = ixgbe_allocate_legacy(adapter);
 528         if (error)
 529                 goto err_late;
 530
 531         /* Setup OS specific network interface */
 532         if (ixgbe_setup_interface(dev, adapter) != 0)
 533                 goto err_late;
 534
 535         /* Sysctl for limiting the amount of work done in the taskqueue */
 536         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 537             "max number of rx packets to process", &adapter->rx_process_limit,
 538             ixgbe_rx_process_limit);
 539
 540         /* Initialize statistics */
 541         ixgbe_update_stats_counters(adapter);
 542
 543         /* Register for VLAN events */
 544         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 545             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 546         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 547             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 548
 549         /* Print PCIE bus type/speed/width info */
 550         ixgbe_get_bus_info(hw);
 551         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 552             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 553             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 554             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 555             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 556             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 557             ("Unknown"));
 558
 559         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 560             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 561                 device_printf(dev, "PCI-Express bandwidth available"
 562                     " for this card\n     is not sufficient for"
 563                     " optimal performance.\n");
 564                 device_printf(dev, "For optimal performance a x8 "
 565                     "PCIE, or x4 PCIE 2 slot is required.\n");
 566         }
 567
 568         /* let hardware know driver is loaded */
 569         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 570         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 571         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 572
 573         ixgbe_add_hw_stats(adapter);
 574
 575         INIT_DEBUGOUT("ixgbe_attach: end");
 576         return (0);
 577 err_late:
 578         ixgbe_free_transmit_structures(adapter);
 579         ixgbe_free_receive_structures(adapter);
 580 err_out:
 581         if (adapter->ifp != NULL)
 582                 if_free(adapter->ifp);
 583         ixgbe_free_pci_resources(adapter);
 584         free(adapter->mta, M_DEVBUF);
 585         return (error);
 586
 587 }
 588
 589 /*********************************************************************
 590  *  Device removal routine
 591  *
 592  *  The detach entry point is called when the driver is being removed.
 593  *  This routine stops the adapter and deallocates all the resources
 594  *  that were allocated for driver operation.
 595  *
 596  *  return 0 on success, positive on failure
 597  *********************************************************************/
 598
 599 static int
 600 ixgbe_detach(device_t dev)
 601 {
 602         struct adapter *adapter = device_get_softc(dev);
 603         struct ix_queue *que = adapter->queues;
 604         u32     ctrl_ext;
 605
 606         INIT_DEBUGOUT("ixgbe_detach: begin");
 607
 608         /* Make sure VLANS are not using driver */
 609         if (adapter->ifp->if_vlantrunk != NULL) {
 610                 device_printf(dev,"Vlan in use, detach first\n");
 611                 return (EBUSY);
 612         }
 613
 614         IXGBE_CORE_LOCK(adapter);
 615         ixgbe_stop(adapter);
 616         IXGBE_CORE_UNLOCK(adapter);
 617
 618         for (int i = 0; i < adapter->num_queues; i++, que++) {
 619                 if (que->tq) {
 620                         taskqueue_drain(que->tq, &que->que_task);
 621                         taskqueue_free(que->tq);
 622                 }
 623         }
 624
 625         /* Drain the Link queue */
 626         if (adapter->tq) {
 627                 taskqueue_drain(adapter->tq, &adapter->link_task);
 628                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 629                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 630 #ifdef IXGBE_FDIR
 631                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 632 #endif
 633                 taskqueue_free(adapter->tq);
 634         }
 635
 636         /* let hardware know driver is unloading */
 637         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 638         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 639         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 640
 641         /* Unregister VLAN events */
 642         if (adapter->vlan_attach != NULL)
 643                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 644         if (adapter->vlan_detach != NULL)
 645                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 646
 647         ether_ifdetach(adapter->ifp);
 648         callout_drain(&adapter->timer);
 649         ixgbe_free_pci_resources(adapter);
 650         bus_generic_detach(dev);
 651         if_free(adapter->ifp);
 652
 653         ixgbe_free_transmit_structures(adapter);
 654         ixgbe_free_receive_structures(adapter);
 655         free(adapter->mta, M_DEVBUF);
 656
 657         IXGBE_CORE_LOCK_DESTROY(adapter);
 658         return (0);
 659 }
 660
 661 /*********************************************************************
 662  *
 663  *  Shutdown entry point
 664  *
 665  **********************************************************************/
 666
 667 static int
 668 ixgbe_shutdown(device_t dev)
 669 {
 670         struct adapter *adapter = device_get_softc(dev);
 671         IXGBE_CORE_LOCK(adapter);
 672         ixgbe_stop(adapter);
 673         IXGBE_CORE_UNLOCK(adapter);
 674         return (0);
 675 }
 676
 677
 678 /*********************************************************************
 679  *  Transmit entry point
 680  *
 681  *  ixgbe_start is called by the stack to initiate a transmit.
 682  *  The driver will remain in this routine as long as there are
 683  *  packets to transmit and transmit resources are available.
 684  *  In case resources are not available stack is notified and
 685  *  the packet is requeued.
 686  **********************************************************************/
 687
 688 static void
 689 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 690 {
 691         struct mbuf    *m_head;
 692         struct adapter *adapter = txr->adapter;
 693
 694         IXGBE_TX_LOCK_ASSERT(txr);
 695
 696         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 697             IFF_DRV_RUNNING)
 698                 return;
 699         if (!adapter->link_active)
 700                 return;
 701
 702         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 703
 704                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 705                 if (m_head == NULL)
 706                         break;
 707
 708                 if (ixgbe_xmit(txr, &m_head)) {
 709                         if (m_head == NULL)
 710                                 break;
 711                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 712                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 713                         break;
 714                 }
 715                 /* Send a copy of the frame to the BPF listener */
 716                 ETHER_BPF_MTAP(ifp, m_head);
 717
 718                 /* Set watchdog on */
 719                 txr->watchdog_time = ticks;
 720                 txr->queue_status = IXGBE_QUEUE_WORKING;
 721
 722         }
 723         return;
 724 }
 725
 726 /*
 727  * Legacy TX start - called by the stack, this
 728  * always uses the first tx ring, and should
 729  * not be used with multiqueue tx enabled.
 730  */
 731 static void
 732 ixgbe_start(struct ifnet *ifp)
 733 {
 734         struct adapter *adapter = ifp->if_softc;
 735         struct tx_ring  *txr = adapter->tx_rings;
 736
 737         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 738                 IXGBE_TX_LOCK(txr);
 739                 ixgbe_start_locked(txr, ifp);
 740                 IXGBE_TX_UNLOCK(txr);
 741         }
 742         return;
 743 }
 744
 745 #if __FreeBSD_version >= 800000
 746 /*
 747 ** Multiqueue Transmit driver
 748 **
 749 */
 750 static int
 751 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 752 {
 753         struct adapter  *adapter = ifp->if_softc;
 754         struct ix_queue *que;
 755         struct tx_ring  *txr;
 756         int             i = 0, err = 0;
 757
 758         /* Which queue to use */
 759         if ((m->m_flags & M_FLOWID) != 0)
 760                 i = m->m_pkthdr.flowid % adapter->num_queues;
 761
 762         txr = &adapter->tx_rings[i];
 763         que = &adapter->queues[i];
 764
 765         if (IXGBE_TX_TRYLOCK(txr)) {
 766                 err = ixgbe_mq_start_locked(ifp, txr, m);
 767                 IXGBE_TX_UNLOCK(txr);
 768         } else {
 769                 err = drbr_enqueue(ifp, txr->br, m);
 770                 taskqueue_enqueue(que->tq, &que->que_task);
 771         }
 772
 773         return (err);
 774 }
 775
 776 static int
 777 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 778 {
 779         struct adapter  *adapter = txr->adapter;
 780         struct mbuf     *next;
 781         int             enqueued, err = 0;
 782
 783         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 784             IFF_DRV_RUNNING || adapter->link_active == 0) {
 785                 if (m != NULL)
 786                         err = drbr_enqueue(ifp, txr->br, m);
 787                 return (err);
 788         }
 789
 790         /* Call cleanup if number of TX descriptors low */
 791         if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD)
 792                 ixgbe_txeof(txr);
 793
 794         enqueued = 0;
 795         if (m == NULL) {
 796                 next = drbr_dequeue(ifp, txr->br);
 797         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 798                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 799                         return (err);
 800                 next = drbr_dequeue(ifp, txr->br);
 801         } else
 802                 next = m;
 803
 804         /* Process the queue */
 805         while (next != NULL) {
 806                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 807                         if (next != NULL)
 808                                 err = drbr_enqueue(ifp, txr->br, next);
 809                         break;
 810                 }
 811                 enqueued++;
 812                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 813                 /* Send a copy of the frame to the BPF listener */
 814                 ETHER_BPF_MTAP(ifp, next);
 815                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 816                         break;
 817                 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) {
 818                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 819                         break;
 820                 }
 821                 next = drbr_dequeue(ifp, txr->br);
 822         }
 823
 824         if (enqueued > 0) {
 825                 /* Set watchdog on */
 826                 txr->queue_status = IXGBE_QUEUE_WORKING;
 827                 txr->watchdog_time = ticks;
 828         }
 829
 830         return (err);
 831 }
 832
 833 /*
 834 ** Flush all ring buffers
 835 */
 836 static void
 837 ixgbe_qflush(struct ifnet *ifp)
 838 {
 839         struct adapter  *adapter = ifp->if_softc;
 840         struct tx_ring  *txr = adapter->tx_rings;
 841         struct mbuf     *m;
 842
 843         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 844                 IXGBE_TX_LOCK(txr);
 845                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 846                         m_freem(m);
 847                 IXGBE_TX_UNLOCK(txr);
 848         }
 849         if_qflush(ifp);
 850 }
 851 #endif /* __FreeBSD_version >= 800000 */
 852
 853 /*********************************************************************
 854  *  Ioctl entry point
 855  *
 856  *  ixgbe_ioctl is called when the user wants to configure the
 857  *  interface.
 858  *
 859  *  return 0 on success, positive on failure
 860  **********************************************************************/
 861
 862 static int
 863 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 864 {
 865         struct adapter  *adapter = ifp->if_softc;
 866         struct ifreq    *ifr = (struct ifreq *) data;
 867         int             error = 0;
 868
 869         switch (command) {
 870
 871         case SIOCSIFMTU:
 872                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 873                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 874                         error = EINVAL;
 875                 } else {
 876                         IXGBE_CORE_LOCK(adapter);
 877                         ifp->if_mtu = ifr->ifr_mtu;
 878                         adapter->max_frame_size =
 879                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 880                         ixgbe_init_locked(adapter);
 881                         IXGBE_CORE_UNLOCK(adapter);
 882                 }
 883                 break;
 884         case SIOCSIFFLAGS:
 885                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 886                 IXGBE_CORE_LOCK(adapter);
 887                 if (ifp->if_flags & IFF_UP) {
 888                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 889                                 if ((ifp->if_flags ^ adapter->if_flags) &
 890                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 891                                         ixgbe_set_promisc(adapter);
 892                                 }
 893                         } else
 894                                 ixgbe_init_locked(adapter);
 895                 } else
 896                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 897                                 ixgbe_stop(adapter);
 898                 adapter->if_flags = ifp->if_flags;
 899                 IXGBE_CORE_UNLOCK(adapter);
 900                 break;
 901         case SIOCADDMULTI:
 902         case SIOCDELMULTI:
 903                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 904                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 905                         IXGBE_CORE_LOCK(adapter);
 906                         ixgbe_disable_intr(adapter);
 907                         ixgbe_set_multi(adapter);
 908                         ixgbe_enable_intr(adapter);
 909                         IXGBE_CORE_UNLOCK(adapter);
 910                 }
 911                 break;
 912         case SIOCSIFMEDIA:
 913         case SIOCGIFMEDIA:
 914                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 915                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 916                 break;
 917         case SIOCSIFCAP:
 918         {
 919                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 920                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 921                 if (mask & IFCAP_HWCSUM)
 922                         ifp->if_capenable ^= IFCAP_HWCSUM;
 923                 if (mask & IFCAP_TSO4)
 924                         ifp->if_capenable ^= IFCAP_TSO4;
 925                 if (mask & IFCAP_LRO)
 926                         ifp->if_capenable ^= IFCAP_LRO;
 927                 if (mask & IFCAP_VLAN_HWTAGGING)
 928                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 929                 if (mask & IFCAP_VLAN_HWFILTER)
 930                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 931                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 932                         IXGBE_CORE_LOCK(adapter);
 933                         ixgbe_init_locked(adapter);
 934                         IXGBE_CORE_UNLOCK(adapter);
 935                 }
 936                 VLAN_CAPABILITIES(ifp);
 937                 break;
 938         }
 939
 940         default:
 941                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 942                 error = ether_ioctl(ifp, command, data);
 943                 break;
 944         }
 945
 946         return (error);
 947 }
 948
 949 /*********************************************************************
 950  *  Init entry point
 951  *
 952  *  This routine is used in two ways. It is used by the stack as
 953  *  init entry point in network interface structure. It is also used
 954  *  by the driver as a hw/sw initialization routine to get to a
 955  *  consistent state.
 956  *
 957  *  return 0 on success, positive on failure
 958  **********************************************************************/
 959 #define IXGBE_MHADD_MFS_SHIFT 16
 960
 961 static void
 962 ixgbe_init_locked(struct adapter *adapter)
 963 {
 964         struct ifnet   *ifp = adapter->ifp;
 965         device_t        dev = adapter->dev;
 966         struct ixgbe_hw *hw = &adapter->hw;
 967         u32             k, txdctl, mhadd, gpie;
 968         u32             rxdctl, rxctrl;
 969
 970         mtx_assert(&adapter->core_mtx, MA_OWNED);
 971         INIT_DEBUGOUT("ixgbe_init: begin");
 972         hw->adapter_stopped = FALSE;
 973         ixgbe_stop_adapter(hw);
 974         callout_stop(&adapter->timer);
 975
 976         /* reprogram the RAR[0] in case user changed it. */
 977         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
 978
 979         /* Get the latest mac address, User can use a LAA */
 980         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
 981               IXGBE_ETH_LENGTH_OF_ADDRESS);
 982         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
 983         hw->addr_ctrl.rar_used_count = 1;
 984
 985         /* Set the various hardware offload abilities */
 986         ifp->if_hwassist = 0;
 987         if (ifp->if_capenable & IFCAP_TSO4)
 988                 ifp->if_hwassist |= CSUM_TSO;
 989         if (ifp->if_capenable & IFCAP_TXCSUM) {
 990                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 991 #if __FreeBSD_version >= 800000
 992                 if (hw->mac.type == ixgbe_mac_82599EB)
 993                         ifp->if_hwassist |= CSUM_SCTP;
 994 #endif
 995         }
 996
 997         /* Prepare transmit descriptors and buffers */
 998         if (ixgbe_setup_transmit_structures(adapter)) {
 999                 device_printf(dev,"Could not setup transmit structures\n");
1000                 ixgbe_stop(adapter);
1001                 return;
1002         }
1003
1004         ixgbe_init_hw(hw);
1005         ixgbe_initialize_transmit_units(adapter);
1006
1007         /* Setup Multicast table */
1008         ixgbe_set_multi(adapter);
1009
1010         /*
1011         ** Determine the correct mbuf pool
1012         ** for doing jumbo/headersplit
1013         */
1014         if (adapter->max_frame_size <= 2048)
1015                 adapter->rx_mbuf_sz = MCLBYTES;
1016         else if (adapter->max_frame_size <= 4096)
1017                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1018         else if (adapter->max_frame_size <= 9216)
1019                 adapter->rx_mbuf_sz = MJUM9BYTES;
1020         else
1021                 adapter->rx_mbuf_sz = MJUM16BYTES;
1022
1023         /* Prepare receive descriptors and buffers */
1024         if (ixgbe_setup_receive_structures(adapter)) {
1025                 device_printf(dev,"Could not setup receive structures\n");
1026                 ixgbe_stop(adapter);
1027                 return;
1028         }
1029
1030         /* Configure RX settings */
1031         ixgbe_initialize_receive_units(adapter);
1032
1033         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1034
1035         if (hw->mac.type == ixgbe_mac_82599EB) {
1036                 gpie |= IXGBE_SDP1_GPIEN;
1037                 gpie |= IXGBE_SDP2_GPIEN;
1038         }
1039
1040         /* Enable Fan Failure Interrupt */
1041         if (hw->device_id == IXGBE_DEV_ID_82598AT)
1042                 gpie |= IXGBE_SDP1_GPIEN;
1043
1044         if (adapter->msix > 1) {
1045                 /* Enable Enhanced MSIX mode */
1046                 gpie |= IXGBE_GPIE_MSIX_MODE;
1047                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1048                     IXGBE_GPIE_OCD;
1049         }
1050         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1051
1052         /* Set MTU size */
1053         if (ifp->if_mtu > ETHERMTU) {
1054                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1055                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1056                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1057                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1058         }
1059
1060         /* Now enable all the queues */
1061
1062         for (int i = 0; i < adapter->num_queues; i++) {
1063                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1064                 txdctl |= IXGBE_TXDCTL_ENABLE;
1065                 /* Set WTHRESH to 8, burst writeback */
1066                 txdctl |= (8 << 16);
1067                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1068         }
1069
1070         for (int i = 0; i < adapter->num_queues; i++) {
1071                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1072                 if (hw->mac.type == ixgbe_mac_82598EB) {
1073                         /*
1074                         ** PTHRESH = 21
1075                         ** HTHRESH = 4
1076                         ** WTHRESH = 8
1077                         */
1078                         rxdctl &= ~0x3FFFFF;
1079                         rxdctl |= 0x080420;
1080                 }
1081                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1082                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1083                 for (k = 0; k < 10; k++) {
1084                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1085                             IXGBE_RXDCTL_ENABLE)
1086                                 break;
1087                         else
1088                                 msec_delay(1);
1089                 }
1090                 wmb();
1091                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1092         }
1093
1094         /* Set up VLAN support and filter */
1095         ixgbe_setup_vlan_hw_support(adapter);
1096
1097         /* Enable Receive engine */
1098         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1099         if (hw->mac.type == ixgbe_mac_82598EB)
1100                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1101         rxctrl |= IXGBE_RXCTRL_RXEN;
1102         ixgbe_enable_rx_dma(hw, rxctrl);
1103
1104         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1105
1106         /* Set up MSI/X routing */
1107         if (ixgbe_enable_msix)  {
1108                 ixgbe_configure_ivars(adapter);
1109                 /* Set up auto-mask */
1110                 if (hw->mac.type == ixgbe_mac_82598EB)
1111                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1112                 else {
1113                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1114                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1115                 }
1116         } else {  /* Simple settings for Legacy/MSI */
1117                 ixgbe_set_ivar(adapter, 0, 0, 0);
1118                 ixgbe_set_ivar(adapter, 0, 0, 1);
1119                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1120         }
1121
1122 #ifdef IXGBE_FDIR
1123         /* Init Flow director */
1124         if (hw->mac.type == ixgbe_mac_82599EB)
1125                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1126 #endif
1127
1128         /*
1129         ** Check on any SFP devices that
1130         ** need to be kick-started
1131         */
1132         if (hw->phy.type == ixgbe_phy_none) {
1133                 int err = hw->phy.ops.identify(hw);
1134                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1135                         device_printf(dev,
1136                             "Unsupported SFP+ module type was detected.\n");
1137                         return;
1138                 }
1139         }
1140
1141         /* Set moderation on the Link interrupt */
1142         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1143
1144         /* Config/Enable Link */
1145         ixgbe_config_link(adapter);
1146
1147         /* And now turn on interrupts */
1148         ixgbe_enable_intr(adapter);
1149
1150         /* Now inform the stack we're ready */
1151         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1152         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1153
1154         return;
1155 }
1156
1157 static void
1158 ixgbe_init(void *arg)
1159 {
1160         struct adapter *adapter = arg;
1161
1162         IXGBE_CORE_LOCK(adapter);
1163         ixgbe_init_locked(adapter);
1164         IXGBE_CORE_UNLOCK(adapter);
1165         return;
1166 }
1167
1168
1169 /*
1170 **
1171 ** MSIX Interrupt Handlers and Tasklets
1172 **
1173 */
1174
1175 static inline void
1176 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1177 {
1178         struct ixgbe_hw *hw = &adapter->hw;
1179         u64     queue = (u64)(1 << vector);
1180         u32     mask;
1181
1182         if (hw->mac.type == ixgbe_mac_82598EB) {
1183                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1184                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1185         } else {
1186                 mask = (queue & 0xFFFFFFFF);
1187                 if (mask)
1188                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1189                 mask = (queue >> 32);
1190                 if (mask)
1191                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1192         }
1193 }
1194
1195 static inline void
1196 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1197 {
1198         struct ixgbe_hw *hw = &adapter->hw;
1199         u64     queue = (u64)(1 << vector);
1200         u32     mask;
1201
1202         if (hw->mac.type == ixgbe_mac_82598EB) {
1203                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1204                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1205         } else {
1206                 mask = (queue & 0xFFFFFFFF);
1207                 if (mask)
1208                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1209                 mask = (queue >> 32);
1210                 if (mask)
1211                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1212         }
1213 }
1214
1215 static inline void
1216 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1217 {
1218         u32 mask;
1219
1220         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1221                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1222                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1223         } else {
1224                 mask = (queues & 0xFFFFFFFF);
1225                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1226                 mask = (queues >> 32);
1227                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1228         }
1229 }
1230
1231
1232 static void
1233 ixgbe_handle_que(void *context, int pending)
1234 {
1235         struct ix_queue *que = context;
1236         struct adapter  *adapter = que->adapter;
1237         struct tx_ring  *txr = que->txr;
1238         struct ifnet    *ifp = adapter->ifp;
1239         bool            more;
1240
1241         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1242                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1243                 IXGBE_TX_LOCK(txr);
1244                 ixgbe_txeof(txr);
1245 #if __FreeBSD_version >= 800000
1246                 if (!drbr_empty(ifp, txr->br))
1247                         ixgbe_mq_start_locked(ifp, txr, NULL);
1248 #else
1249                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1250                         ixgbe_start_locked(txr, ifp);
1251 #endif
1252                 IXGBE_TX_UNLOCK(txr);
1253                 if (more) {
1254                         taskqueue_enqueue(que->tq, &que->que_task);
1255                         return;
1256                 }
1257         }
1258
1259         /* Reenable this interrupt */
1260         ixgbe_enable_queue(adapter, que->msix);
1261         return;
1262 }
1263
1264
1265 /*********************************************************************
1266  *
1267  *  Legacy Interrupt Service routine
1268  *
1269  **********************************************************************/
1270
1271 static void
1272 ixgbe_legacy_irq(void *arg)
1273 {
1274         struct ix_queue *que = arg;
1275         struct adapter  *adapter = que->adapter;
1276         struct ixgbe_hw *hw = &adapter->hw;
1277         struct          tx_ring *txr = adapter->tx_rings;
1278         bool            more_tx, more_rx;
1279         u32             reg_eicr, loop = MAX_LOOP;
1280
1281
1282         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1283
1284         ++que->irqs;
1285         if (reg_eicr == 0) {
1286                 ixgbe_enable_intr(adapter);
1287                 return;
1288         }
1289
1290         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1291
1292         IXGBE_TX_LOCK(txr);
1293         do {
1294                 more_tx = ixgbe_txeof(txr);
1295         } while (loop-- && more_tx);
1296         IXGBE_TX_UNLOCK(txr);
1297
1298         if (more_rx || more_tx)
1299                 taskqueue_enqueue(que->tq, &que->que_task);
1300
1301         /* Check for fan failure */
1302         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1303             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1304                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1305                     "REPLACE IMMEDIATELY!!\n");
1306                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1307         }
1308
1309         /* Link status change */
1310         if (reg_eicr & IXGBE_EICR_LSC)
1311                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1312
1313         ixgbe_enable_intr(adapter);
1314         return;
1315 }
1316
1317
1318 /*********************************************************************
1319  *
1320  *  MSI Queue Interrupt Service routine
1321  *
1322  **********************************************************************/
1323 void
1324 ixgbe_msix_que(void *arg)
1325 {
1326         struct ix_queue *que = arg;
1327         struct adapter  *adapter = que->adapter;
1328         struct tx_ring  *txr = que->txr;
1329         struct rx_ring  *rxr = que->rxr;
1330         bool            more_tx, more_rx;
1331         u32             newitr = 0;
1332
1333         ++que->irqs;
1334
1335         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1336
1337         IXGBE_TX_LOCK(txr);
1338         more_tx = ixgbe_txeof(txr);
1339         IXGBE_TX_UNLOCK(txr);
1340
1341         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1342
1343         /* Do AIM now? */
1344
1345         if (ixgbe_enable_aim == FALSE)
1346                 goto no_calc;
1347         /*
1348         ** Do Adaptive Interrupt Moderation:
1349         **  - Write out last calculated setting
1350         **  - Calculate based on average size over
1351         **    the last interval.
1352         */
1353         if (que->eitr_setting)
1354                 IXGBE_WRITE_REG(&adapter->hw,
1355                     IXGBE_EITR(que->msix), que->eitr_setting);
1356
1357         que->eitr_setting = 0;
1358
1359         /* Idle, do nothing */
1360         if ((txr->bytes == 0) && (rxr->bytes == 0))
1361                 goto no_calc;
1362
1363         if ((txr->bytes) && (txr->packets))
1364                 newitr = txr->bytes/txr->packets;
1365         if ((rxr->bytes) && (rxr->packets))
1366                 newitr = max(newitr,
1367                     (rxr->bytes / rxr->packets));
1368         newitr += 24; /* account for hardware frame, crc */
1369
1370         /* set an upper boundary */
1371         newitr = min(newitr, 3000);
1372
1373         /* Be nice to the mid range */
1374         if ((newitr > 300) && (newitr < 1200))
1375                 newitr = (newitr / 3);
1376         else
1377                 newitr = (newitr / 2);
1378
1379         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1380                 newitr |= newitr << 16;
1381         else
1382                 newitr |= IXGBE_EITR_CNT_WDIS;
1383
1384         /* save for next interrupt */
1385         que->eitr_setting = newitr;
1386
1387         /* Reset state */
1388         txr->bytes = 0;
1389         txr->packets = 0;
1390         rxr->bytes = 0;
1391         rxr->packets = 0;
1392
1393 no_calc:
1394         if (more_tx || more_rx)
1395                 taskqueue_enqueue(que->tq, &que->que_task);
1396         else /* Reenable this interrupt */
1397                 ixgbe_enable_queue(adapter, que->msix);
1398         return;
1399 }
1400
1401
1402 static void
1403 ixgbe_msix_link(void *arg)
1404 {
1405         struct adapter  *adapter = arg;
1406         struct ixgbe_hw *hw = &adapter->hw;
1407         u32             reg_eicr;
1408
1409         ++adapter->link_irq;
1410
1411         /* First get the cause */
1412         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1413         /* Clear interrupt with write */
1414         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1415
1416         /* Link status change */
1417         if (reg_eicr & IXGBE_EICR_LSC)
1418                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1419
1420         if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
1421 #ifdef IXGBE_FDIR
1422                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1423                         /* This is probably overkill :) */
1424                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1425                                 return;
1426                         /* Clear the interrupt */
1427                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR);
1428                         /* Turn off the interface */
1429                         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1430                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1431                 } else
1432 #endif
1433                 if (reg_eicr & IXGBE_EICR_ECC) {
1434                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1435                             "Please Reboot!!\n");
1436                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1437                 } else
1438
1439                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1440                         /* Clear the interrupt */
1441                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1442                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1443                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1444                         /* Clear the interrupt */
1445                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1446                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1447                 }
1448         }
1449
1450         /* Check for fan failure */
1451         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1452             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1453                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1454                     "REPLACE IMMEDIATELY!!\n");
1455                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1456         }
1457
1458         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1459         return;
1460 }
1461
1462 /*********************************************************************
1463  *
1464  *  Media Ioctl callback
1465  *
1466  *  This routine is called whenever the user queries the status of
1467  *  the interface using ifconfig.
1468  *
1469  **********************************************************************/
1470 static void
1471 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1472 {
1473         struct adapter *adapter = ifp->if_softc;
1474
1475         INIT_DEBUGOUT("ixgbe_media_status: begin");
1476         IXGBE_CORE_LOCK(adapter);
1477         ixgbe_update_link_status(adapter);
1478
1479         ifmr->ifm_status = IFM_AVALID;
1480         ifmr->ifm_active = IFM_ETHER;
1481
1482         if (!adapter->link_active) {
1483                 IXGBE_CORE_UNLOCK(adapter);
1484                 return;
1485         }
1486
1487         ifmr->ifm_status |= IFM_ACTIVE;
1488
1489         switch (adapter->link_speed) {
1490                 case IXGBE_LINK_SPEED_1GB_FULL:
1491                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1492                         break;
1493                 case IXGBE_LINK_SPEED_10GB_FULL:
1494                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1495                         break;
1496         }
1497
1498         IXGBE_CORE_UNLOCK(adapter);
1499
1500         return;
1501 }
1502
1503 /*********************************************************************
1504  *
1505  *  Media Ioctl callback
1506  *
1507  *  This routine is called when the user changes speed/duplex using
1508  *  media/mediopt option with ifconfig.
1509  *
1510  **********************************************************************/
1511 static int
1512 ixgbe_media_change(struct ifnet * ifp)
1513 {
1514         struct adapter *adapter = ifp->if_softc;
1515         struct ifmedia *ifm = &adapter->media;
1516
1517         INIT_DEBUGOUT("ixgbe_media_change: begin");
1518
1519         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1520                 return (EINVAL);
1521
1522         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1523         case IFM_AUTO:
1524                 adapter->hw.phy.autoneg_advertised =
1525                     IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL;
1526                 break;
1527         default:
1528                 device_printf(adapter->dev, "Only auto media type\n");
1529                 return (EINVAL);
1530         }
1531
1532         return (0);
1533 }
1534
1535 /*********************************************************************
1536  *
1537  *  This routine maps the mbufs to tx descriptors, allowing the
1538  *  TX engine to transmit the packets.
1539  *      - return 0 on success, positive on failure
1540  *
1541  **********************************************************************/
1542
1543 static int
1544 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1545 {
1546         struct adapter  *adapter = txr->adapter;
1547         u32             olinfo_status = 0, cmd_type_len;
1548         u32             paylen = 0;
1549         int             i, j, error, nsegs;
1550         int             first, last = 0;
1551         struct mbuf     *m_head;
1552         bus_dma_segment_t segs[adapter->num_segs];
1553         bus_dmamap_t    map;
1554         struct ixgbe_tx_buf *txbuf, *txbuf_mapped;
1555         union ixgbe_adv_tx_desc *txd = NULL;
1556
1557         m_head = *m_headp;
1558
1559         /* Basic descriptor defines */
1560         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1561             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1562
1563         if (m_head->m_flags & M_VLANTAG)
1564                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1565
1566         /*
1567          * Important to capture the first descriptor
1568          * used because it will contain the index of
1569          * the one we tell the hardware to report back
1570          */
1571         first = txr->next_avail_desc;
1572         txbuf = &txr->tx_buffers[first];
1573         txbuf_mapped = txbuf;
1574         map = txbuf->map;
1575
1576         /*
1577          * Map the packet for DMA.
1578          */
1579         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1580             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1581
1582         if (error == EFBIG) {
1583                 struct mbuf *m;
1584
1585                 m = m_defrag(*m_headp, M_DONTWAIT);
1586                 if (m == NULL) {
1587                         adapter->mbuf_defrag_failed++;
1588                         m_freem(*m_headp);
1589                         *m_headp = NULL;
1590                         return (ENOBUFS);
1591                 }
1592                 *m_headp = m;
1593
1594                 /* Try it again */
1595                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1596                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1597
1598                 if (error == ENOMEM) {
1599                         adapter->no_tx_dma_setup++;
1600                         return (error);
1601                 } else if (error != 0) {
1602                         adapter->no_tx_dma_setup++;
1603                         m_freem(*m_headp);
1604                         *m_headp = NULL;
1605                         return (error);
1606                 }
1607         } else if (error == ENOMEM) {
1608                 adapter->no_tx_dma_setup++;
1609                 return (error);
1610         } else if (error != 0) {
1611                 adapter->no_tx_dma_setup++;
1612                 m_freem(*m_headp);
1613                 *m_headp = NULL;
1614                 return (error);
1615         }
1616
1617         /* Make certain there are enough descriptors */
1618         if (nsegs > txr->tx_avail - 2) {
1619                 txr->no_desc_avail++;
1620                 error = ENOBUFS;
1621                 goto xmit_fail;
1622         }
1623         m_head = *m_headp;
1624
1625         /*
1626         ** Set up the appropriate offload context
1627         ** this becomes the first descriptor of
1628         ** a packet.
1629         */
1630         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1631                 if (ixgbe_tso_setup(txr, m_head, &paylen)) {
1632                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1633                         olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1634                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1635                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1636                         ++adapter->tso_tx;
1637                 } else
1638                         return (ENXIO);
1639         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1640                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1641
1642 #ifdef IXGBE_IEEE1588
1643         /* This is changing soon to an mtag detection */
1644         if (we detect this mbuf has a TSTAMP mtag)
1645                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1646 #endif
1647
1648 #ifdef IXGBE_FDIR
1649         /* Do the flow director magic */
1650         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1651                 ++txr->atr_count;
1652                 if (txr->atr_count >= atr_sample_rate) {
1653                         ixgbe_atr(txr, m_head);
1654                         txr->atr_count = 0;
1655                 }
1656         }
1657 #endif
1658         /* Record payload length */
1659         if (paylen == 0)
1660                 olinfo_status |= m_head->m_pkthdr.len <<
1661                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1662
1663         i = txr->next_avail_desc;
1664         for (j = 0; j < nsegs; j++) {
1665                 bus_size_t seglen;
1666                 bus_addr_t segaddr;
1667
1668                 txbuf = &txr->tx_buffers[i];
1669                 txd = &txr->tx_base[i];
1670                 seglen = segs[j].ds_len;
1671                 segaddr = htole64(segs[j].ds_addr);
1672
1673                 txd->read.buffer_addr = segaddr;
1674                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1675                     cmd_type_len |seglen);
1676                 txd->read.olinfo_status = htole32(olinfo_status);
1677                 last = i; /* descriptor that will get completion IRQ */
1678
1679                 if (++i == adapter->num_tx_desc)
1680                         i = 0;
1681
1682                 txbuf->m_head = NULL;
1683                 txbuf->eop_index = -1;
1684         }
1685
1686         txd->read.cmd_type_len |=
1687             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1688         txr->tx_avail -= nsegs;
1689         txr->next_avail_desc = i;
1690
1691         txbuf->m_head = m_head;
1692         txbuf->map = map;
1693         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1694
1695         /* Set the index of the descriptor that will be marked done */
1696         txbuf = &txr->tx_buffers[first];
1697         txbuf->eop_index = last;
1698
1699         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1700             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1701         /*
1702          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1703          * hardware that this frame is available to transmit.
1704          */
1705         ++txr->total_packets;
1706         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1707
1708         return (0);
1709
1710 xmit_fail:
1711         bus_dmamap_unload(txr->txtag, txbuf->map);
1712         return (error);
1713
1714 }
1715
1716 static void
1717 ixgbe_set_promisc(struct adapter *adapter)
1718 {
1719         u_int32_t       reg_rctl;
1720         struct ifnet   *ifp = adapter->ifp;
1721
1722         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1723         reg_rctl &= (~IXGBE_FCTRL_UPE);
1724         reg_rctl &= (~IXGBE_FCTRL_MPE);
1725         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1726
1727         if (ifp->if_flags & IFF_PROMISC) {
1728                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1729                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1730         } else if (ifp->if_flags & IFF_ALLMULTI) {
1731                 reg_rctl |= IXGBE_FCTRL_MPE;
1732                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1733                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1734         }
1735         return;
1736 }
1737
1738
1739 /*********************************************************************
1740  *  Multicast Update
1741  *
1742  *  This routine is called whenever multicast address list is updated.
1743  *
1744  **********************************************************************/
1745 #define IXGBE_RAR_ENTRIES 16
1746
1747 static void
1748 ixgbe_set_multi(struct adapter *adapter)
1749 {
1750         u32     fctrl;
1751         u8      *mta;
1752         u8      *update_ptr;
1753         struct  ifmultiaddr *ifma;
1754         int     mcnt = 0;
1755         struct ifnet   *ifp = adapter->ifp;
1756
1757         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1758
1759         mta = adapter->mta;
1760         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1761             MAX_NUM_MULTICAST_ADDRESSES);
1762
1763         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1764         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1765         if (ifp->if_flags & IFF_PROMISC)
1766                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1767         else if (ifp->if_flags & IFF_ALLMULTI) {
1768                 fctrl |= IXGBE_FCTRL_MPE;
1769                 fctrl &= ~IXGBE_FCTRL_UPE;
1770         } else
1771                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1772
1773         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1774
1775 #if __FreeBSD_version < 800000
1776         IF_ADDR_LOCK(ifp);
1777 #else
1778         if_maddr_rlock(ifp);
1779 #endif
1780         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1781                 if (ifma->ifma_addr->sa_family != AF_LINK)
1782                         continue;
1783                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1784                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1785                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1786                 mcnt++;
1787         }
1788 #if __FreeBSD_version < 800000
1789         IF_ADDR_UNLOCK(ifp);
1790 #else
1791         if_maddr_runlock(ifp);
1792 #endif
1793
1794         update_ptr = mta;
1795         ixgbe_update_mc_addr_list(&adapter->hw,
1796             update_ptr, mcnt, ixgbe_mc_array_itr);
1797
1798         return;
1799 }
1800
1801 /*
1802  * This is an iterator function now needed by the multicast
1803  * shared code. It simply feeds the shared code routine the
1804  * addresses in the array of ixgbe_set_multi() one by one.
1805  */
1806 static u8 *
1807 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1808 {
1809         u8 *addr = *update_ptr;
1810         u8 *newptr;
1811         *vmdq = 0;
1812
1813         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1814         *update_ptr = newptr;
1815         return addr;
1816 }
1817
1818
1819 /*********************************************************************
1820  *  Timer routine
1821  *
1822  *  This routine checks for link status,updates statistics,
1823  *  and runs the watchdog check.
1824  *
1825  **********************************************************************/
1826
1827 static void
1828 ixgbe_local_timer(void *arg)
1829 {
1830         struct adapter *adapter = arg;
1831         device_t        dev = adapter->dev;
1832         struct tx_ring *txr = adapter->tx_rings;
1833
1834         mtx_assert(&adapter->core_mtx, MA_OWNED);
1835
1836         /* Check for pluggable optics */
1837         if (adapter->sfp_probe)
1838                 if (!ixgbe_sfp_probe(adapter))
1839                         goto out; /* Nothing to do */
1840
1841         ixgbe_update_link_status(adapter);
1842         ixgbe_update_stats_counters(adapter);
1843
1844         /*
1845          * If the interface has been paused
1846          * then don't do the watchdog check
1847          */
1848         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1849                 goto out;
1850
1851         /*
1852         ** Check status on the TX queues for a hang
1853         */
1854         for (int i = 0; i < adapter->num_queues; i++, txr++)
1855                 if (txr->queue_status == IXGBE_QUEUE_HUNG)
1856                         goto hung;
1857
1858 out:
1859         ixgbe_rearm_queues(adapter, adapter->que_mask);
1860         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1861         return;
1862
1863 hung:
1864         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1865         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1866             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1867             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1868         device_printf(dev,"TX(%d) desc avail = %d,"
1869             "Next TX to Clean = %d\n",
1870             txr->me, txr->tx_avail, txr->next_to_clean);
1871         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1872         adapter->watchdog_events++;
1873         ixgbe_init_locked(adapter);
1874 }
1875
1876 /*
1877 ** Note: this routine updates the OS on the link state
1878 **      the real check of the hardware only happens with
1879 **      a link interrupt.
1880 */
1881 static void
1882 ixgbe_update_link_status(struct adapter *adapter)
1883 {
1884         struct ifnet    *ifp = adapter->ifp;
1885         struct tx_ring *txr = adapter->tx_rings;
1886         device_t dev = adapter->dev;
1887
1888
1889         if (adapter->link_up){
1890                 if (adapter->link_active == FALSE) {
1891                         if (bootverbose)
1892                                 device_printf(dev,"Link is up %d Gbps %s \n",
1893                                     ((adapter->link_speed == 128)? 10:1),
1894                                     "Full Duplex");
1895                         adapter->link_active = TRUE;
1896                         if_link_state_change(ifp, LINK_STATE_UP);
1897                 }
1898         } else { /* Link down */
1899                 if (adapter->link_active == TRUE) {
1900                         if (bootverbose)
1901                                 device_printf(dev,"Link is Down\n");
1902                         if_link_state_change(ifp, LINK_STATE_DOWN);
1903                         adapter->link_active = FALSE;
1904                         for (int i = 0; i < adapter->num_queues;
1905                             i++, txr++)
1906                                 txr->queue_status = IXGBE_QUEUE_IDLE;
1907                 }
1908         }
1909
1910         return;
1911 }
1912
1913
1914 /*********************************************************************
1915  *
1916  *  This routine disables all traffic on the adapter by issuing a
1917  *  global reset on the MAC and deallocates TX/RX buffers.
1918  *
1919  **********************************************************************/
1920
1921 static void
1922 ixgbe_stop(void *arg)
1923 {
1924         struct ifnet   *ifp;
1925         struct adapter *adapter = arg;
1926         struct ixgbe_hw *hw = &adapter->hw;
1927         ifp = adapter->ifp;
1928
1929         mtx_assert(&adapter->core_mtx, MA_OWNED);
1930
1931         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1932         ixgbe_disable_intr(adapter);
1933
1934         /* Tell the stack that the interface is no longer active */
1935         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1936
1937         ixgbe_reset_hw(hw);
1938         hw->adapter_stopped = FALSE;
1939         ixgbe_stop_adapter(hw);
1940         /* Turn off the laser */
1941         if (hw->phy.multispeed_fiber)
1942                 ixgbe_disable_tx_laser(hw);
1943         callout_stop(&adapter->timer);
1944
1945         /* reprogram the RAR[0] in case user changed it. */
1946         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1947
1948         return;
1949 }
1950
1951
1952 /*********************************************************************
1953  *
1954  *  Determine hardware revision.
1955  *
1956  **********************************************************************/
1957 static void
1958 ixgbe_identify_hardware(struct adapter *adapter)
1959 {
1960         device_t        dev = adapter->dev;
1961         struct ixgbe_hw *hw = &adapter->hw;
1962
1963         /* Save off the information about this board */
1964         hw->vendor_id = pci_get_vendor(dev);
1965         hw->device_id = pci_get_device(dev);
1966         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
1967         hw->subsystem_vendor_id =
1968             pci_read_config(dev, PCIR_SUBVEND_0, 2);
1969         hw->subsystem_device_id =
1970             pci_read_config(dev, PCIR_SUBDEV_0, 2);
1971
1972         /* We need this here to set the num_segs below */
1973         ixgbe_set_mac_type(hw);
1974
1975         /* Pick up the 82599 and VF settings */
1976         if (hw->mac.type != ixgbe_mac_82598EB) {
1977                 hw->phy.smart_speed = ixgbe_smart_speed;
1978                 adapter->num_segs = IXGBE_82599_SCATTER;
1979         } else
1980                 adapter->num_segs = IXGBE_82598_SCATTER;
1981
1982         return;
1983 }
1984
1985 /*********************************************************************
1986  *
1987  *  Determine optic type
1988  *
1989  **********************************************************************/
1990 static void
1991 ixgbe_setup_optics(struct adapter *adapter)
1992 {
1993         struct ixgbe_hw *hw = &adapter->hw;
1994         int             layer;
1995
1996         layer = ixgbe_get_supported_physical_layer(hw);
1997         switch (layer) {
1998                 case IXGBE_PHYSICAL_LAYER_10GBASE_T:
1999                         adapter->optics = IFM_10G_T;
2000                         break;
2001                 case IXGBE_PHYSICAL_LAYER_1000BASE_T:
2002                         adapter->optics = IFM_1000_T;
2003                         break;
2004                 case IXGBE_PHYSICAL_LAYER_10GBASE_LR:
2005                 case IXGBE_PHYSICAL_LAYER_10GBASE_LRM:
2006                         adapter->optics = IFM_10G_LR;
2007                         break;
2008                 case IXGBE_PHYSICAL_LAYER_10GBASE_SR:
2009                         adapter->optics = IFM_10G_SR;
2010                         break;
2011                 case IXGBE_PHYSICAL_LAYER_10GBASE_KX4:
2012                 case IXGBE_PHYSICAL_LAYER_10GBASE_CX4:
2013                         adapter->optics = IFM_10G_CX4;
2014                         break;
2015                 case IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU:
2016                         adapter->optics = IFM_10G_TWINAX;
2017                         break;
2018                 case IXGBE_PHYSICAL_LAYER_1000BASE_KX:
2019                 case IXGBE_PHYSICAL_LAYER_10GBASE_KR:
2020                 case IXGBE_PHYSICAL_LAYER_10GBASE_XAUI:
2021                 case IXGBE_PHYSICAL_LAYER_UNKNOWN:
2022                 default:
2023                         adapter->optics = IFM_ETHER | IFM_AUTO;
2024                         break;
2025         }
2026         return;
2027 }
2028
2029 /*********************************************************************
2030  *
2031  *  Setup the Legacy or MSI Interrupt handler
2032  *
2033  **********************************************************************/
2034 static int
2035 ixgbe_allocate_legacy(struct adapter *adapter)
2036 {
2037         device_t dev = adapter->dev;
2038         struct          ix_queue *que = adapter->queues;
2039         int error, rid = 0;
2040
2041         /* MSI RID at 1 */
2042         if (adapter->msix == 1)
2043                 rid = 1;
2044
2045         /* We allocate a single interrupt resource */
2046         adapter->res = bus_alloc_resource_any(dev,
2047             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2048         if (adapter->res == NULL) {
2049                 device_printf(dev, "Unable to allocate bus resource: "
2050                     "interrupt\n");
2051                 return (ENXIO);
2052         }
2053
2054         /*
2055          * Try allocating a fast interrupt and the associated deferred
2056          * processing contexts.
2057          */
2058         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2059         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2060             taskqueue_thread_enqueue, &que->tq);
2061         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2062             device_get_nameunit(adapter->dev));
2063
2064         /* Tasklets for Link, SFP and Multispeed Fiber */
2065         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2066         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2067         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2068 #ifdef IXGBE_FDIR
2069         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2070 #endif
2071         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2072             taskqueue_thread_enqueue, &adapter->tq);
2073         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2074             device_get_nameunit(adapter->dev));
2075
2076         if ((error = bus_setup_intr(dev, adapter->res,
2077             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2078             que, &adapter->tag)) != 0) {
2079                 device_printf(dev, "Failed to register fast interrupt "
2080                     "handler: %d\n", error);
2081                 taskqueue_free(que->tq);
2082                 taskqueue_free(adapter->tq);
2083                 que->tq = NULL;
2084                 adapter->tq = NULL;
2085                 return (error);
2086         }
2087         /* For simplicity in the handlers */
2088         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2089
2090         return (0);
2091 }
2092
2093
2094 /*********************************************************************
2095  *
2096  *  Setup MSIX Interrupt resources and handlers
2097  *
2098  **********************************************************************/
2099 static int
2100 ixgbe_allocate_msix(struct adapter *adapter)
2101 {
2102         device_t        dev = adapter->dev;
2103         struct          ix_queue *que = adapter->queues;
2104         int             error, rid, vector = 0;
2105
2106         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2107                 rid = vector + 1;
2108                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2109                     RF_SHAREABLE | RF_ACTIVE);
2110                 if (que->res == NULL) {
2111                         device_printf(dev,"Unable to allocate"
2112                             " bus resource: que interrupt [%d]\n", vector);
2113                         return (ENXIO);
2114                 }
2115                 /* Set the handler function */
2116                 error = bus_setup_intr(dev, que->res,
2117                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2118                     ixgbe_msix_que, que, &que->tag);
2119                 if (error) {
2120                         que->res = NULL;
2121                         device_printf(dev, "Failed to register QUE handler");
2122                         return (error);
2123                 }
2124 #if __FreeBSD_version >= 800504
2125                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2126 #endif
2127                 que->msix = vector;
2128                 adapter->que_mask |= (u64)(1 << que->msix);
2129                 /*
2130                 ** Bind the msix vector, and thus the
2131                 ** ring to the corresponding cpu.
2132                 */
2133                 if (adapter->num_queues > 1)
2134                         bus_bind_intr(dev, que->res, i);
2135
2136                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2137                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2138                     taskqueue_thread_enqueue, &que->tq);
2139                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2140                     device_get_nameunit(adapter->dev));
2141         }
2142
2143         /* and Link */
2144         rid = vector + 1;
2145         adapter->res = bus_alloc_resource_any(dev,
2146             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2147         if (!adapter->res) {
2148                 device_printf(dev,"Unable to allocate"
2149             " bus resource: Link interrupt [%d]\n", rid);
2150                 return (ENXIO);
2151         }
2152         /* Set the link handler function */
2153         error = bus_setup_intr(dev, adapter->res,
2154             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2155             ixgbe_msix_link, adapter, &adapter->tag);
2156         if (error) {
2157                 adapter->res = NULL;
2158                 device_printf(dev, "Failed to register LINK handler");
2159                 return (error);
2160         }
2161 #if __FreeBSD_version >= 800504
2162         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2163 #endif
2164         adapter->linkvec = vector;
2165         /* Tasklets for Link, SFP and Multispeed Fiber */
2166         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2167         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2168         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2169 #ifdef IXGBE_FDIR
2170         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2171 #endif
2172         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2173             taskqueue_thread_enqueue, &adapter->tq);
2174         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2175             device_get_nameunit(adapter->dev));
2176
2177         return (0);
2178 }
2179
2180 /*
2181  * Setup Either MSI/X or MSI
2182  */
2183 static int
2184 ixgbe_setup_msix(struct adapter *adapter)
2185 {
2186         device_t dev = adapter->dev;
2187         int rid, want, queues, msgs;
2188
2189         /* Override by tuneable */
2190         if (ixgbe_enable_msix == 0)
2191                 goto msi;
2192
2193         /* First try MSI/X */
2194         rid = PCIR_BAR(MSIX_82598_BAR);
2195         adapter->msix_mem = bus_alloc_resource_any(dev,
2196             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2197         if (!adapter->msix_mem) {
2198                 rid += 4;       /* 82599 maps in higher BAR */
2199                 adapter->msix_mem = bus_alloc_resource_any(dev,
2200                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2201         }
2202         if (!adapter->msix_mem) {
2203                 /* May not be enabled */
2204                 device_printf(adapter->dev,
2205                     "Unable to map MSIX table \n");
2206                 goto msi;
2207         }
2208
2209         msgs = pci_msix_count(dev);
2210         if (msgs == 0) { /* system has msix disabled */
2211                 bus_release_resource(dev, SYS_RES_MEMORY,
2212                     rid, adapter->msix_mem);
2213                 adapter->msix_mem = NULL;
2214                 goto msi;
2215         }
2216
2217         /* Figure out a reasonable auto config value */
2218         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2219
2220         if (ixgbe_num_queues != 0)
2221                 queues = ixgbe_num_queues;
2222         /* Set max queues to 8 when autoconfiguring */
2223         else if ((ixgbe_num_queues == 0) && (queues > 8))
2224                 queues = 8;
2225
2226         /*
2227         ** Want one vector (RX/TX pair) per queue
2228         ** plus an additional for Link.
2229         */
2230         want = queues + 1;
2231         if (msgs >= want)
2232                 msgs = want;
2233         else {
2234                 device_printf(adapter->dev,
2235                     "MSIX Configuration Problem, "
2236                     "%d vectors but %d queues wanted!\n",
2237                     msgs, want);
2238                 return (0); /* Will go to Legacy setup */
2239         }
2240         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2241                 device_printf(adapter->dev,
2242                     "Using MSIX interrupts with %d vectors\n", msgs);
2243                 adapter->num_queues = queues;
2244                 return (msgs);
2245         }
2246 msi:
2247         msgs = pci_msi_count(dev);
2248         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2249                 device_printf(adapter->dev,"Using MSI interrupt\n");
2250         return (msgs);
2251 }
2252
2253
2254 static int
2255 ixgbe_allocate_pci_resources(struct adapter *adapter)
2256 {
2257         int             rid;
2258         device_t        dev = adapter->dev;
2259
2260         rid = PCIR_BAR(0);
2261         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2262             &rid, RF_ACTIVE);
2263
2264         if (!(adapter->pci_mem)) {
2265                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2266                 return (ENXIO);
2267         }
2268
2269         adapter->osdep.mem_bus_space_tag =
2270                 rman_get_bustag(adapter->pci_mem);
2271         adapter->osdep.mem_bus_space_handle =
2272                 rman_get_bushandle(adapter->pci_mem);
2273         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2274
2275         /* Legacy defaults */
2276         adapter->num_queues = 1;
2277         adapter->hw.back = &adapter->osdep;
2278
2279         /*
2280         ** Now setup MSI or MSI/X, should
2281         ** return us the number of supported
2282         ** vectors. (Will be 1 for MSI)
2283         */
2284         adapter->msix = ixgbe_setup_msix(adapter);
2285         return (0);
2286 }
2287
2288 static void
2289 ixgbe_free_pci_resources(struct adapter * adapter)
2290 {
2291         struct          ix_queue *que = adapter->queues;
2292         device_t        dev = adapter->dev;
2293         int             rid, memrid;
2294
2295         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2296                 memrid = PCIR_BAR(MSIX_82598_BAR);
2297         else
2298                 memrid = PCIR_BAR(MSIX_82599_BAR);
2299
2300         /*
2301         ** There is a slight possibility of a failure mode
2302         ** in attach that will result in entering this function
2303         ** before interrupt resources have been initialized, and
2304         ** in that case we do not want to execute the loops below
2305         ** We can detect this reliably by the state of the adapter
2306         ** res pointer.
2307         */
2308         if (adapter->res == NULL)
2309                 goto mem;
2310
2311         /*
2312         **  Release all msix queue resources:
2313         */
2314         for (int i = 0; i < adapter->num_queues; i++, que++) {
2315                 rid = que->msix + 1;
2316                 if (que->tag != NULL) {
2317                         bus_teardown_intr(dev, que->res, que->tag);
2318                         que->tag = NULL;
2319                 }
2320                 if (que->res != NULL)
2321                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2322         }
2323
2324
2325         /* Clean the Legacy or Link interrupt last */
2326         if (adapter->linkvec) /* we are doing MSIX */
2327                 rid = adapter->linkvec + 1;
2328         else
2329                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2330
2331         if (adapter->tag != NULL) {
2332                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2333                 adapter->tag = NULL;
2334         }
2335         if (adapter->res != NULL)
2336                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2337
2338 mem:
2339         if (adapter->msix)
2340                 pci_release_msi(dev);
2341
2342         if (adapter->msix_mem != NULL)
2343                 bus_release_resource(dev, SYS_RES_MEMORY,
2344                     memrid, adapter->msix_mem);
2345
2346         if (adapter->pci_mem != NULL)
2347                 bus_release_resource(dev, SYS_RES_MEMORY,
2348                     PCIR_BAR(0), adapter->pci_mem);
2349
2350         return;
2351 }
2352
2353 /*********************************************************************
2354  *
2355  *  Setup networking device structure and register an interface.
2356  *
2357  **********************************************************************/
2358 static int
2359 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2360 {
2361         struct ixgbe_hw *hw = &adapter->hw;
2362         struct ifnet   *ifp;
2363
2364         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2365
2366         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2367         if (ifp == NULL) {
2368                 device_printf(dev, "can not allocate ifnet structure\n");
2369                 return (-1);
2370         }
2371         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2372         ifp->if_mtu = ETHERMTU;
2373         ifp->if_baudrate = 1000000000;
2374         ifp->if_init = ixgbe_init;
2375         ifp->if_softc = adapter;
2376         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2377         ifp->if_ioctl = ixgbe_ioctl;
2378         ifp->if_start = ixgbe_start;
2379 #if __FreeBSD_version >= 800000
2380         ifp->if_transmit = ixgbe_mq_start;
2381         ifp->if_qflush = ixgbe_qflush;
2382 #endif
2383         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2384
2385         ether_ifattach(ifp, adapter->hw.mac.addr);
2386
2387         adapter->max_frame_size =
2388             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2389
2390         /*
2391          * Tell the upper layer(s) we support long frames.
2392          */
2393         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2394
2395         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM;
2396         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2397         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2398         ifp->if_capenable = ifp->if_capabilities;
2399
2400         /* Don't enable LRO by default */
2401         ifp->if_capabilities |= IFCAP_LRO;
2402
2403         /*
2404         ** Dont turn this on by default, if vlans are
2405         ** created on another pseudo device (eg. lagg)
2406         ** then vlan events are not passed thru, breaking
2407         ** operation, but with HW FILTER off it works. If
2408         ** using vlans directly on the em driver you can
2409         ** enable this and get full hardware tag filtering.
2410         */
2411         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2412
2413         /*
2414          * Specify the media types supported by this adapter and register
2415          * callbacks to update media and link information
2416          */
2417         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2418                      ixgbe_media_status);
2419         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2420         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2421         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2422                 ifmedia_add(&adapter->media,
2423                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2424                 ifmedia_add(&adapter->media,
2425                     IFM_ETHER | IFM_1000_T, 0, NULL);
2426         }
2427         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2428         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2429
2430         return (0);
2431 }
2432
2433 static void
2434 ixgbe_config_link(struct adapter *adapter)
2435 {
2436         struct ixgbe_hw *hw = &adapter->hw;
2437         u32     autoneg, err = 0;
2438         bool    sfp, negotiate;
2439
2440         sfp = ixgbe_is_sfp(hw);
2441
2442         if (sfp) {
2443                 if (hw->phy.multispeed_fiber) {
2444                         hw->mac.ops.setup_sfp(hw);
2445                         ixgbe_enable_tx_laser(hw);
2446                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2447                 } else
2448                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2449         } else {
2450                 if (hw->mac.ops.check_link)
2451                         err = ixgbe_check_link(hw, &autoneg,
2452                             &adapter->link_up, FALSE);
2453                 if (err)
2454                         goto out;
2455                 autoneg = hw->phy.autoneg_advertised;
2456                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2457                         err  = hw->mac.ops.get_link_capabilities(hw,
2458                             &autoneg, &negotiate);
2459                 if (err)
2460                         goto out;
2461                 if (hw->mac.ops.setup_link)
2462                         err = hw->mac.ops.setup_link(hw, autoneg,
2463                             negotiate, adapter->link_up);
2464         }
2465 out:
2466         return;
2467 }
2468
2469 /********************************************************************
2470  * Manage DMA'able memory.
2471  *******************************************************************/
2472 static void
2473 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2474 {
2475         if (error)
2476                 return;
2477         *(bus_addr_t *) arg = segs->ds_addr;
2478         return;
2479 }
2480
2481 static int
2482 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2483                 struct ixgbe_dma_alloc *dma, int mapflags)
2484 {
2485         device_t dev = adapter->dev;
2486         int             r;
2487
2488         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2489                                DBA_ALIGN, 0,    /* alignment, bounds */
2490                                BUS_SPACE_MAXADDR,       /* lowaddr */
2491                                BUS_SPACE_MAXADDR,       /* highaddr */
2492                                NULL, NULL,      /* filter, filterarg */
2493                                size,    /* maxsize */
2494                                1,       /* nsegments */
2495                                size,    /* maxsegsize */
2496                                BUS_DMA_ALLOCNOW,        /* flags */
2497                                NULL,    /* lockfunc */
2498                                NULL,    /* lockfuncarg */
2499                                &dma->dma_tag);
2500         if (r != 0) {
2501                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2502                        "error %u\n", r);
2503                 goto fail_0;
2504         }
2505         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2506                              BUS_DMA_NOWAIT, &dma->dma_map);
2507         if (r != 0) {
2508                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2509                        "error %u\n", r);
2510                 goto fail_1;
2511         }
2512         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2513                             size,
2514                             ixgbe_dmamap_cb,
2515                             &dma->dma_paddr,
2516                             mapflags | BUS_DMA_NOWAIT);
2517         if (r != 0) {
2518                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2519                        "error %u\n", r);
2520                 goto fail_2;
2521         }
2522         dma->dma_size = size;
2523         return (0);
2524 fail_2:
2525         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2526 fail_1:
2527         bus_dma_tag_destroy(dma->dma_tag);
2528 fail_0:
2529         dma->dma_map = NULL;
2530         dma->dma_tag = NULL;
2531         return (r);
2532 }
2533
2534 static void
2535 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2536 {
2537         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2538             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2539         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2540         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2541         bus_dma_tag_destroy(dma->dma_tag);
2542 }
2543
2544
2545 /*********************************************************************
2546  *
2547  *  Allocate memory for the transmit and receive rings, and then
2548  *  the descriptors associated with each, called only once at attach.
2549  *
2550  **********************************************************************/
2551 static int
2552 ixgbe_allocate_queues(struct adapter *adapter)
2553 {
2554         device_t        dev = adapter->dev;
2555         struct ix_queue *que;
2556         struct tx_ring  *txr;
2557         struct rx_ring  *rxr;
2558         int rsize, tsize, error = IXGBE_SUCCESS;
2559         int txconf = 0, rxconf = 0;
2560
2561         /* First allocate the top level queue structs */
2562         if (!(adapter->queues =
2563             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2564             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2565                 device_printf(dev, "Unable to allocate queue memory\n");
2566                 error = ENOMEM;
2567                 goto fail;
2568         }
2569
2570         /* First allocate the TX ring struct memory */
2571         if (!(adapter->tx_rings =
2572             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2573             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2574                 device_printf(dev, "Unable to allocate TX ring memory\n");
2575                 error = ENOMEM;
2576                 goto tx_fail;
2577         }
2578
2579         /* Next allocate the RX */
2580         if (!(adapter->rx_rings =
2581             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2582             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2583                 device_printf(dev, "Unable to allocate RX ring memory\n");
2584                 error = ENOMEM;
2585                 goto rx_fail;
2586         }
2587
2588         /* For the ring itself */
2589         tsize = roundup2(adapter->num_tx_desc *
2590             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2591
2592         /*
2593          * Now set up the TX queues, txconf is needed to handle the
2594          * possibility that things fail midcourse and we need to
2595          * undo memory gracefully
2596          */
2597         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2598                 /* Set up some basics */
2599                 txr = &adapter->tx_rings[i];
2600                 txr->adapter = adapter;
2601                 txr->me = i;
2602
2603                 /* Initialize the TX side lock */
2604                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2605                     device_get_nameunit(dev), txr->me);
2606                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2607
2608                 if (ixgbe_dma_malloc(adapter, tsize,
2609                         &txr->txdma, BUS_DMA_NOWAIT)) {
2610                         device_printf(dev,
2611                             "Unable to allocate TX Descriptor memory\n");
2612                         error = ENOMEM;
2613                         goto err_tx_desc;
2614                 }
2615                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2616                 bzero((void *)txr->tx_base, tsize);
2617
2618                 /* Now allocate transmit buffers for the ring */
2619                 if (ixgbe_allocate_transmit_buffers(txr)) {
2620                         device_printf(dev,
2621                             "Critical Failure setting up transmit buffers\n");
2622                         error = ENOMEM;
2623                         goto err_tx_desc;
2624                 }
2625 #if __FreeBSD_version >= 800000
2626                 /* Allocate a buf ring */
2627                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2628                     M_WAITOK, &txr->tx_mtx);
2629                 if (txr->br == NULL) {
2630                         device_printf(dev,
2631                             "Critical Failure setting up buf ring\n");
2632                         error = ENOMEM;
2633                         goto err_tx_desc;
2634                 }
2635 #endif
2636         }
2637
2638         /*
2639          * Next the RX queues...
2640          */
2641         rsize = roundup2(adapter->num_rx_desc *
2642             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2643         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2644                 rxr = &adapter->rx_rings[i];
2645                 /* Set up some basics */
2646                 rxr->adapter = adapter;
2647                 rxr->me = i;
2648
2649                 /* Initialize the RX side lock */
2650                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2651                     device_get_nameunit(dev), rxr->me);
2652                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2653
2654                 if (ixgbe_dma_malloc(adapter, rsize,
2655                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2656                         device_printf(dev,
2657                             "Unable to allocate RxDescriptor memory\n");
2658                         error = ENOMEM;
2659                         goto err_rx_desc;
2660                 }
2661                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2662                 bzero((void *)rxr->rx_base, rsize);
2663
2664                 /* Allocate receive buffers for the ring*/
2665                 if (ixgbe_allocate_receive_buffers(rxr)) {
2666                         device_printf(dev,
2667                             "Critical Failure setting up receive buffers\n");
2668                         error = ENOMEM;
2669                         goto err_rx_desc;
2670                 }
2671         }
2672
2673         /*
2674         ** Finally set up the queue holding structs
2675         */
2676         for (int i = 0; i < adapter->num_queues; i++) {
2677                 que = &adapter->queues[i];
2678                 que->adapter = adapter;
2679                 que->txr = &adapter->tx_rings[i];
2680                 que->rxr = &adapter->rx_rings[i];
2681         }
2682
2683         return (0);
2684
2685 err_rx_desc:
2686         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2687                 ixgbe_dma_free(adapter, &rxr->rxdma);
2688 err_tx_desc:
2689         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2690                 ixgbe_dma_free(adapter, &txr->txdma);
2691         free(adapter->rx_rings, M_DEVBUF);
2692 rx_fail:
2693         free(adapter->tx_rings, M_DEVBUF);
2694 tx_fail:
2695         free(adapter->queues, M_DEVBUF);
2696 fail:
2697         return (error);
2698 }
2699
2700 /*********************************************************************
2701  *
2702  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2703  *  the information needed to transmit a packet on the wire. This is
2704  *  called only once at attach, setup is done every reset.
2705  *
2706  **********************************************************************/
2707 static int
2708 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2709 {
2710         struct adapter *adapter = txr->adapter;
2711         device_t dev = adapter->dev;
2712         struct ixgbe_tx_buf *txbuf;
2713         int error, i;
2714
2715         /*
2716          * Setup DMA descriptor areas.
2717          */
2718         if ((error = bus_dma_tag_create(NULL,           /* parent */
2719                                1, 0,            /* alignment, bounds */
2720                                BUS_SPACE_MAXADDR,       /* lowaddr */
2721                                BUS_SPACE_MAXADDR,       /* highaddr */
2722                                NULL, NULL,              /* filter, filterarg */
2723                                IXGBE_TSO_SIZE,          /* maxsize */
2724                                adapter->num_segs,       /* nsegments */
2725                                PAGE_SIZE,               /* maxsegsize */
2726                                0,                       /* flags */
2727                                NULL,                    /* lockfunc */
2728                                NULL,                    /* lockfuncarg */
2729                                &txr->txtag))) {
2730                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2731                 goto fail;
2732         }
2733
2734         if (!(txr->tx_buffers =
2735             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2736             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2737                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2738                 error = ENOMEM;
2739                 goto fail;
2740         }
2741
2742         /* Create the descriptor buffer dma maps */
2743         txbuf = txr->tx_buffers;
2744         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2745                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2746                 if (error != 0) {
2747                         device_printf(dev, "Unable to create TX DMA map\n");
2748                         goto fail;
2749                 }
2750         }
2751
2752         return 0;
2753 fail:
2754         /* We free all, it handles case where we are in the middle */
2755         ixgbe_free_transmit_structures(adapter);
2756         return (error);
2757 }
2758
2759 /*********************************************************************
2760  *
2761  *  Initialize a transmit ring.
2762  *
2763  **********************************************************************/
2764 static void
2765 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2766 {
2767         struct adapter *adapter = txr->adapter;
2768         struct ixgbe_tx_buf *txbuf;
2769         int i;
2770
2771         /* Clear the old ring contents */
2772         IXGBE_TX_LOCK(txr);
2773         bzero((void *)txr->tx_base,
2774               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2775         /* Reset indices */
2776         txr->next_avail_desc = 0;
2777         txr->next_to_clean = 0;
2778
2779         /* Free any existing tx buffers. */
2780         txbuf = txr->tx_buffers;
2781         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2782                 if (txbuf->m_head != NULL) {
2783                         bus_dmamap_sync(txr->txtag, txbuf->map,
2784                             BUS_DMASYNC_POSTWRITE);
2785                         bus_dmamap_unload(txr->txtag, txbuf->map);
2786                         m_freem(txbuf->m_head);
2787                         txbuf->m_head = NULL;
2788                 }
2789                 /* Clear the EOP index */
2790                 txbuf->eop_index = -1;
2791         }
2792
2793 #ifdef IXGBE_FDIR
2794         /* Set the rate at which we sample packets */
2795         if (adapter->hw.mac.type == ixgbe_mac_82599EB)
2796                 txr->atr_sample = atr_sample_rate;
2797 #endif
2798
2799         /* Set number of descriptors available */
2800         txr->tx_avail = adapter->num_tx_desc;
2801
2802         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2803             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2804         IXGBE_TX_UNLOCK(txr);
2805 }
2806
2807 /*********************************************************************
2808  *
2809  *  Initialize all transmit rings.
2810  *
2811  **********************************************************************/
2812 static int
2813 ixgbe_setup_transmit_structures(struct adapter *adapter)
2814 {
2815         struct tx_ring *txr = adapter->tx_rings;
2816
2817         for (int i = 0; i < adapter->num_queues; i++, txr++)
2818                 ixgbe_setup_transmit_ring(txr);
2819
2820         return (0);
2821 }
2822
2823 /*********************************************************************
2824  *
2825  *  Enable transmit unit.
2826  *
2827  **********************************************************************/
2828 static void
2829 ixgbe_initialize_transmit_units(struct adapter *adapter)
2830 {
2831         struct tx_ring  *txr = adapter->tx_rings;
2832         struct ixgbe_hw *hw = &adapter->hw;
2833
2834         /* Setup the Base and Length of the Tx Descriptor Ring */
2835
2836         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2837                 u64     tdba = txr->txdma.dma_paddr;
2838                 u32     txctrl;
2839
2840                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2841                        (tdba & 0x00000000ffffffffULL));
2842                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2843                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2844                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2845
2846                 /* Setup the HW Tx Head and Tail descriptor pointers */
2847                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2848                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2849
2850                 /* Setup Transmit Descriptor Cmd Settings */
2851                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2852                 txr->queue_status = IXGBE_QUEUE_IDLE;
2853
2854                 /* Disable Head Writeback */
2855                 switch (hw->mac.type) {
2856                 case ixgbe_mac_82598EB:
2857                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2858                         break;
2859                 case ixgbe_mac_82599EB:
2860                 default:
2861                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2862                         break;
2863                 }
2864                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
2865                 switch (hw->mac.type) {
2866                 case ixgbe_mac_82598EB:
2867                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2868                         break;
2869                 case ixgbe_mac_82599EB:
2870                 default:
2871                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2872                         break;
2873                 }
2874
2875         }
2876
2877         if (hw->mac.type == ixgbe_mac_82599EB) {
2878                 u32 dmatxctl, rttdcs;
2879                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2880                 dmatxctl |= IXGBE_DMATXCTL_TE;
2881                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2882                 /* Disable arbiter to set MTQC */
2883                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2884                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2885                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2886                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2887                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2888                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2889         }
2890
2891         return;
2892 }
2893
2894 /*********************************************************************
2895  *
2896  *  Free all transmit rings.
2897  *
2898  **********************************************************************/
2899 static void
2900 ixgbe_free_transmit_structures(struct adapter *adapter)
2901 {
2902         struct tx_ring *txr = adapter->tx_rings;
2903
2904         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2905                 IXGBE_TX_LOCK(txr);
2906                 ixgbe_free_transmit_buffers(txr);
2907                 ixgbe_dma_free(adapter, &txr->txdma);
2908                 IXGBE_TX_UNLOCK(txr);
2909                 IXGBE_TX_LOCK_DESTROY(txr);
2910         }
2911         free(adapter->tx_rings, M_DEVBUF);
2912 }
2913
2914 /*********************************************************************
2915  *
2916  *  Free transmit ring related data structures.
2917  *
2918  **********************************************************************/
2919 static void
2920 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2921 {
2922         struct adapter *adapter = txr->adapter;
2923         struct ixgbe_tx_buf *tx_buffer;
2924         int             i;
2925
2926         INIT_DEBUGOUT("free_transmit_ring: begin");
2927
2928         if (txr->tx_buffers == NULL)
2929                 return;
2930
2931         tx_buffer = txr->tx_buffers;
2932         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2933                 if (tx_buffer->m_head != NULL) {
2934                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2935                             BUS_DMASYNC_POSTWRITE);
2936                         bus_dmamap_unload(txr->txtag,
2937                             tx_buffer->map);
2938                         m_freem(tx_buffer->m_head);
2939                         tx_buffer->m_head = NULL;
2940                         if (tx_buffer->map != NULL) {
2941                                 bus_dmamap_destroy(txr->txtag,
2942                                     tx_buffer->map);
2943                                 tx_buffer->map = NULL;
2944                         }
2945                 } else if (tx_buffer->map != NULL) {
2946                         bus_dmamap_unload(txr->txtag,
2947                             tx_buffer->map);
2948                         bus_dmamap_destroy(txr->txtag,
2949                             tx_buffer->map);
2950                         tx_buffer->map = NULL;
2951                 }
2952         }
2953 #if __FreeBSD_version >= 800000
2954         if (txr->br != NULL)
2955                 buf_ring_free(txr->br, M_DEVBUF);
2956 #endif
2957         if (txr->tx_buffers != NULL) {
2958                 free(txr->tx_buffers, M_DEVBUF);
2959                 txr->tx_buffers = NULL;
2960         }
2961         if (txr->txtag != NULL) {
2962                 bus_dma_tag_destroy(txr->txtag);
2963                 txr->txtag = NULL;
2964         }
2965         return;
2966 }
2967
2968 /*********************************************************************
2969  *
2970  *  Advanced Context Descriptor setup for VLAN or CSUM
2971  *
2972  **********************************************************************/
2973
2974 static boolean_t
2975 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
2976 {
2977         struct adapter *adapter = txr->adapter;
2978         struct ixgbe_adv_tx_context_desc *TXD;
2979         struct ixgbe_tx_buf        *tx_buffer;
2980         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2981         struct ether_vlan_header *eh;
2982         struct ip *ip;
2983         struct ip6_hdr *ip6;
2984         int  ehdrlen, ip_hlen = 0;
2985         u16     etype;
2986         u8      ipproto = 0;
2987         bool    offload = TRUE;
2988         int ctxd = txr->next_avail_desc;
2989         u16 vtag = 0;
2990
2991
2992         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
2993                 offload = FALSE;
2994
2995         tx_buffer = &txr->tx_buffers[ctxd];
2996         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2997
2998         /*
2999         ** In advanced descriptors the vlan tag must
3000         ** be placed into the descriptor itself.
3001         */
3002         if (mp->m_flags & M_VLANTAG) {
3003                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3004                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3005         } else if (offload == FALSE)
3006                 return FALSE;
3007
3008         /*
3009          * Determine where frame payload starts.
3010          * Jump over vlan headers if already present,
3011          * helpful for QinQ too.
3012          */
3013         eh = mtod(mp, struct ether_vlan_header *);
3014         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3015                 etype = ntohs(eh->evl_proto);
3016                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3017         } else {
3018                 etype = ntohs(eh->evl_encap_proto);
3019                 ehdrlen = ETHER_HDR_LEN;
3020         }
3021
3022         /* Set the ether header length */
3023         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3024
3025         switch (etype) {
3026                 case ETHERTYPE_IP:
3027                         ip = (struct ip *)(mp->m_data + ehdrlen);
3028                         ip_hlen = ip->ip_hl << 2;
3029                         ipproto = ip->ip_p;
3030                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3031                         break;
3032                 case ETHERTYPE_IPV6:
3033                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3034                         ip_hlen = sizeof(struct ip6_hdr);
3035                         ipproto = ip6->ip6_nxt;
3036                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3037                         break;
3038                 default:
3039                         offload = FALSE;
3040                         break;
3041         }
3042
3043         vlan_macip_lens |= ip_hlen;
3044         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3045
3046         switch (ipproto) {
3047                 case IPPROTO_TCP:
3048                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3049                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3050                         break;
3051
3052                 case IPPROTO_UDP:
3053                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3054                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3055                         break;
3056
3057 #if __FreeBSD_version >= 800000
3058                 case IPPROTO_SCTP:
3059                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3060                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3061                         break;
3062 #endif
3063                 default:
3064                         offload = FALSE;
3065                         break;
3066         }
3067
3068         /* Now copy bits into descriptor */
3069         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3070         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3071         TXD->seqnum_seed = htole32(0);
3072         TXD->mss_l4len_idx = htole32(0);
3073
3074         tx_buffer->m_head = NULL;
3075         tx_buffer->eop_index = -1;
3076
3077         /* We've consumed the first desc, adjust counters */
3078         if (++ctxd == adapter->num_tx_desc)
3079                 ctxd = 0;
3080         txr->next_avail_desc = ctxd;
3081         --txr->tx_avail;
3082
3083         return (offload);
3084 }
3085
3086 /**********************************************************************
3087  *
3088  *  Setup work for hardware segmentation offload (TSO) on
3089  *  adapters using advanced tx descriptors
3090  *
3091  **********************************************************************/
3092 static boolean_t
3093 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
3094 {
3095         struct adapter *adapter = txr->adapter;
3096         struct ixgbe_adv_tx_context_desc *TXD;
3097         struct ixgbe_tx_buf        *tx_buffer;
3098         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3099         u32 mss_l4len_idx = 0;
3100         u16 vtag = 0;
3101         int ctxd, ehdrlen,  hdrlen, ip_hlen, tcp_hlen;
3102         struct ether_vlan_header *eh;
3103         struct ip *ip;
3104         struct tcphdr *th;
3105
3106
3107         /*
3108          * Determine where frame payload starts.
3109          * Jump over vlan headers if already present
3110          */
3111         eh = mtod(mp, struct ether_vlan_header *);
3112         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3113                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3114         else
3115                 ehdrlen = ETHER_HDR_LEN;
3116
3117         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3118         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3119                 return FALSE;
3120
3121         ctxd = txr->next_avail_desc;
3122         tx_buffer = &txr->tx_buffers[ctxd];
3123         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3124
3125         ip = (struct ip *)(mp->m_data + ehdrlen);
3126         if (ip->ip_p != IPPROTO_TCP)
3127                 return FALSE;   /* 0 */
3128         ip->ip_sum = 0;
3129         ip_hlen = ip->ip_hl << 2;
3130         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3131         th->th_sum = in_pseudo(ip->ip_src.s_addr,
3132             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3133         tcp_hlen = th->th_off << 2;
3134         hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3135
3136         /* This is used in the transmit desc in encap */
3137         *paylen = mp->m_pkthdr.len - hdrlen;
3138
3139         /* VLAN MACLEN IPLEN */
3140         if (mp->m_flags & M_VLANTAG) {
3141                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3142                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3143         }
3144
3145         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3146         vlan_macip_lens |= ip_hlen;
3147         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3148
3149         /* ADV DTYPE TUCMD */
3150         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3151         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3152         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3153         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3154
3155
3156         /* MSS L4LEN IDX */
3157         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3158         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3159         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3160
3161         TXD->seqnum_seed = htole32(0);
3162         tx_buffer->m_head = NULL;
3163         tx_buffer->eop_index = -1;
3164
3165         if (++ctxd == adapter->num_tx_desc)
3166                 ctxd = 0;
3167
3168         txr->tx_avail--;
3169         txr->next_avail_desc = ctxd;
3170         return TRUE;
3171 }
3172
3173 #ifdef IXGBE_FDIR
3174 /*
3175 ** This routine parses packet headers so that Flow
3176 ** Director can make a hashed filter table entry
3177 ** allowing traffic flows to be identified and kept
3178 ** on the same cpu.  This would be a performance
3179 ** hit, but we only do it at IXGBE_FDIR_RATE of
3180 ** packets.
3181 */
3182 static void
3183 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3184 {
3185         struct adapter                  *adapter = txr->adapter;
3186         struct ix_queue                 *que;
3187         struct ip                       *ip;
3188         struct tcphdr                   *th;
3189         struct udphdr                   *uh;
3190         struct ether_vlan_header        *eh;
3191         union ixgbe_atr_hash_dword      input = {.dword = 0};
3192         union ixgbe_atr_hash_dword      common = {.dword = 0};
3193         int                             ehdrlen, ip_hlen;
3194         u16                             etype;
3195
3196         eh = mtod(mp, struct ether_vlan_header *);
3197         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3198                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3199                 etype = eh->evl_proto;
3200         } else {
3201                 ehdrlen = ETHER_HDR_LEN;
3202                 etype = eh->evl_encap_proto;
3203         }
3204
3205         /* Only handling IPv4 */
3206         if (etype != htons(ETHERTYPE_IP))
3207                 return;
3208
3209         ip = (struct ip *)(mp->m_data + ehdrlen);
3210         ip_hlen = ip->ip_hl << 2;
3211
3212         /* check if we're UDP or TCP */
3213         switch (ip->ip_p) {
3214         case IPPROTO_TCP:
3215                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3216                 /* src and dst are inverted */
3217                 common.port.dst ^= th->th_sport;
3218                 common.port.src ^= th->th_dport;
3219                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3220                 break;
3221         case IPPROTO_UDP:
3222                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3223                 /* src and dst are inverted */
3224                 common.port.dst ^= uh->uh_sport;
3225                 common.port.src ^= uh->uh_dport;
3226                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3227                 break;
3228         default:
3229                 return;
3230         }
3231
3232         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3233         if (mp->m_pkthdr.ether_vtag)
3234                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3235         else
3236                 common.flex_bytes ^= etype;
3237         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3238
3239         que = &adapter->queues[txr->me];
3240         /*
3241         ** This assumes the Rx queue and Tx
3242         ** queue are bound to the same CPU
3243         */
3244         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3245             input, common, que->msix);
3246 }
3247 #endif /* IXGBE_FDIR */
3248
3249 /**********************************************************************
3250  *
3251  *  Examine each tx_buffer in the used queue. If the hardware is done
3252  *  processing the packet then free associated resources. The
3253  *  tx_buffer is put back on the free queue.
3254  *
3255  **********************************************************************/
3256 static boolean_t
3257 ixgbe_txeof(struct tx_ring *txr)
3258 {
3259         struct adapter  *adapter = txr->adapter;
3260         struct ifnet    *ifp = adapter->ifp;
3261         u32     first, last, done, processed;
3262         struct ixgbe_tx_buf *tx_buffer;
3263         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3264
3265         mtx_assert(&txr->tx_mtx, MA_OWNED);
3266
3267         if (txr->tx_avail == adapter->num_tx_desc) {
3268                 txr->queue_status = IXGBE_QUEUE_IDLE;
3269                 return FALSE;
3270         }
3271
3272         processed = 0;
3273         first = txr->next_to_clean;
3274         tx_buffer = &txr->tx_buffers[first];
3275         /* For cleanup we just use legacy struct */
3276         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3277         last = tx_buffer->eop_index;
3278         if (last == -1)
3279                 return FALSE;
3280         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3281
3282         /*
3283         ** Get the index of the first descriptor
3284         ** BEYOND the EOP and call that 'done'.
3285         ** I do this so the comparison in the
3286         ** inner while loop below can be simple
3287         */
3288         if (++last == adapter->num_tx_desc) last = 0;
3289         done = last;
3290
3291         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3292             BUS_DMASYNC_POSTREAD);
3293         /*
3294         ** Only the EOP descriptor of a packet now has the DD
3295         ** bit set, this is what we look for...
3296         */
3297         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3298                 /* We clean the range of the packet */
3299                 while (first != done) {
3300                         tx_desc->upper.data = 0;
3301                         tx_desc->lower.data = 0;
3302                         tx_desc->buffer_addr = 0;
3303                         ++txr->tx_avail;
3304                         ++processed;
3305
3306                         if (tx_buffer->m_head) {
3307                                 txr->bytes +=
3308                                     tx_buffer->m_head->m_pkthdr.len;
3309                                 bus_dmamap_sync(txr->txtag,
3310                                     tx_buffer->map,
3311                                     BUS_DMASYNC_POSTWRITE);
3312                                 bus_dmamap_unload(txr->txtag,
3313                                     tx_buffer->map);
3314                                 m_freem(tx_buffer->m_head);
3315                                 tx_buffer->m_head = NULL;
3316                                 tx_buffer->map = NULL;
3317                         }
3318                         tx_buffer->eop_index = -1;
3319                         txr->watchdog_time = ticks;
3320
3321                         if (++first == adapter->num_tx_desc)
3322                                 first = 0;
3323
3324                         tx_buffer = &txr->tx_buffers[first];
3325                         tx_desc =
3326                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3327                 }
3328                 ++txr->packets;
3329                 ++ifp->if_opackets;
3330                 /* See if there is more work now */
3331                 last = tx_buffer->eop_index;
3332                 if (last != -1) {
3333                         eop_desc =
3334                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3335                         /* Get next done point */
3336                         if (++last == adapter->num_tx_desc) last = 0;
3337                         done = last;
3338                 } else
3339                         break;
3340         }
3341         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3342             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3343
3344         txr->next_to_clean = first;
3345
3346         /*
3347         ** Watchdog calculation, we know there's
3348         ** work outstanding or the first return
3349         ** would have been taken, so none processed
3350         ** for too long indicates a hang.
3351         */
3352         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3353                 txr->queue_status = IXGBE_QUEUE_HUNG;
3354
3355         /*
3356          * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that
3357          * it is OK to send packets. If there are no pending descriptors,
3358          * clear the timeout. Otherwise, if some descriptors have been freed,
3359          * restart the timeout.
3360          */
3361         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) {
3362                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3363                 if (txr->tx_avail == adapter->num_tx_desc) {
3364                         txr->queue_status = IXGBE_QUEUE_IDLE;
3365                         return FALSE;
3366                 }
3367         }
3368
3369         return TRUE;
3370 }
3371
3372 /*********************************************************************
3373  *
3374  *  Refresh mbuf buffers for RX descriptor rings
3375  *   - now keeps its own state so discards due to resource
3376  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3377  *     it just returns, keeping its placeholder, thus it can simply
3378  *     be recalled to try again.
3379  *
3380  **********************************************************************/
3381 static void
3382 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3383 {
3384         struct adapter          *adapter = rxr->adapter;
3385         bus_dma_segment_t       hseg[1];
3386         bus_dma_segment_t       pseg[1];
3387         struct ixgbe_rx_buf     *rxbuf;
3388         struct mbuf             *mh, *mp;
3389         int                     i, nsegs, error, cleaned;
3390
3391         i = rxr->next_to_refresh;
3392         cleaned = -1; /* Signify no completions */
3393         while (i != limit) {
3394                 rxbuf = &rxr->rx_buffers[i];
3395                 if (rxr->hdr_split == FALSE)
3396                         goto no_split;
3397
3398                 if (rxbuf->m_head == NULL) {
3399                         mh = m_gethdr(M_DONTWAIT, MT_DATA);
3400                         if (mh == NULL)
3401                                 goto update;
3402                 } else
3403                         mh = rxbuf->m_head;
3404
3405                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3406                 mh->m_len = MHLEN;
3407                 mh->m_flags |= M_PKTHDR;
3408                 /* Get the memory mapping */
3409                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3410                     rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
3411                 if (error != 0) {
3412                         printf("Refresh mbufs: hdr dmamap load"
3413                             " failure - %d\n", error);
3414                         m_free(mh);
3415                         rxbuf->m_head = NULL;
3416                         goto update;
3417                 }
3418                 rxbuf->m_head = mh;
3419                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3420                     BUS_DMASYNC_PREREAD);
3421                 rxr->rx_base[i].read.hdr_addr = htole64(hseg[0].ds_addr);
3422
3423 no_split:
3424                 if (rxbuf->m_pack == NULL) {
3425                         mp = m_getjcl(M_DONTWAIT, MT_DATA,
3426                             M_PKTHDR, adapter->rx_mbuf_sz);
3427                         if (mp == NULL)
3428                                 goto update;
3429                 } else
3430                         mp = rxbuf->m_pack;
3431
3432                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3433                 /* Get the memory mapping */
3434                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3435                     rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
3436                 if (error != 0) {
3437                         printf("Refresh mbufs: payload dmamap load"
3438                             " failure - %d\n", error);
3439                         m_free(mp);
3440                         rxbuf->m_pack = NULL;
3441                         goto update;
3442                 }
3443                 rxbuf->m_pack = mp;
3444                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3445                     BUS_DMASYNC_PREREAD);
3446                 rxr->rx_base[i].read.pkt_addr =
3447                     htole64(pseg[0].ds_addr);
3448
3449                 cleaned = i;
3450                 /* Calculate next index */
3451                 if (++i == adapter->num_rx_desc)
3452                         i = 0;
3453                 /* This is the work marker for refresh */
3454                 rxr->next_to_refresh = i;
3455         }
3456 update:
3457         if (cleaned != -1) /* If we refreshed some, bump tail */
3458                 IXGBE_WRITE_REG(&adapter->hw,
3459                     IXGBE_RDT(rxr->me), cleaned);
3460         return;
3461 }
3462
3463 /*********************************************************************
3464  *
3465  *  Allocate memory for rx_buffer structures. Since we use one
3466  *  rx_buffer per received packet, the maximum number of rx_buffer's
3467  *  that we'll need is equal to the number of receive descriptors
3468  *  that we've allocated.
3469  *
3470  **********************************************************************/
3471 static int
3472 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3473 {
3474         struct  adapter         *adapter = rxr->adapter;
3475         device_t                dev = adapter->dev;
3476         struct ixgbe_rx_buf     *rxbuf;
3477         int                     i, bsize, error;
3478
3479         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3480         if (!(rxr->rx_buffers =
3481             (struct ixgbe_rx_buf *) malloc(bsize,
3482             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3483                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3484                 error = ENOMEM;
3485                 goto fail;
3486         }
3487
3488         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3489                                    1, 0,        /* alignment, bounds */
3490                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3491                                    BUS_SPACE_MAXADDR,   /* highaddr */
3492                                    NULL, NULL,          /* filter, filterarg */
3493                                    MSIZE,               /* maxsize */
3494                                    1,                   /* nsegments */
3495                                    MSIZE,               /* maxsegsize */
3496                                    0,                   /* flags */
3497                                    NULL,                /* lockfunc */
3498                                    NULL,                /* lockfuncarg */
3499                                    &rxr->htag))) {
3500                 device_printf(dev, "Unable to create RX DMA tag\n");
3501                 goto fail;
3502         }
3503
3504         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3505                                    1, 0,        /* alignment, bounds */
3506                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3507                                    BUS_SPACE_MAXADDR,   /* highaddr */
3508                                    NULL, NULL,          /* filter, filterarg */
3509                                    MJUM16BYTES,         /* maxsize */
3510                                    1,                   /* nsegments */
3511                                    MJUM16BYTES,         /* maxsegsize */
3512                                    0,                   /* flags */
3513                                    NULL,                /* lockfunc */
3514                                    NULL,                /* lockfuncarg */
3515                                    &rxr->ptag))) {
3516                 device_printf(dev, "Unable to create RX DMA tag\n");
3517                 goto fail;
3518         }
3519
3520         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3521                 rxbuf = &rxr->rx_buffers[i];
3522                 error = bus_dmamap_create(rxr->htag,
3523                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3524                 if (error) {
3525                         device_printf(dev, "Unable to create RX head map\n");
3526                         goto fail;
3527                 }
3528                 error = bus_dmamap_create(rxr->ptag,
3529                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3530                 if (error) {
3531                         device_printf(dev, "Unable to create RX pkt map\n");
3532                         goto fail;
3533                 }
3534         }
3535
3536         return (0);
3537
3538 fail:
3539         /* Frees all, but can handle partial completion */
3540         ixgbe_free_receive_structures(adapter);
3541         return (error);
3542 }
3543
3544 /*
3545 ** Used to detect a descriptor that has
3546 ** been merged by Hardware RSC.
3547 */
3548 static inline u32
3549 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3550 {
3551         return (le32toh(rx->wb.lower.lo_dword.data) &
3552             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3553 }
3554
3555 /*********************************************************************
3556  *
3557  *  Initialize Hardware RSC (LRO) feature on 82599
3558  *  for an RX ring, this is toggled by the LRO capability
3559  *  even though it is transparent to the stack.
3560  *
3561  **********************************************************************/
3562 static void
3563 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3564 {
3565         struct  adapter         *adapter = rxr->adapter;
3566         struct  ixgbe_hw        *hw = &adapter->hw;
3567         u32                     rscctrl, rdrxctl;
3568
3569         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3570         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3571         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3572         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3573         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3574
3575         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3576         rscctrl |= IXGBE_RSCCTL_RSCEN;
3577         /*
3578         ** Limit the total number of descriptors that
3579         ** can be combined, so it does not exceed 64K
3580         */
3581         if (adapter->rx_mbuf_sz == MCLBYTES)
3582                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3583         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3584                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3585         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3586                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3587         else  /* Using 16K cluster */
3588                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3589
3590         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3591
3592         /* Enable TCP header recognition */
3593         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3594             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3595             IXGBE_PSRTYPE_TCPHDR));
3596
3597         /* Disable RSC for ACK packets */
3598         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3599             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3600
3601         rxr->hw_rsc = TRUE;
3602 }
3603
3604
3605 static void
3606 ixgbe_free_receive_ring(struct rx_ring *rxr)
3607 {
3608         struct  adapter         *adapter;
3609         struct ixgbe_rx_buf       *rxbuf;
3610         int i;
3611
3612         adapter = rxr->adapter;
3613         for (i = 0; i < adapter->num_rx_desc; i++) {
3614                 rxbuf = &rxr->rx_buffers[i];
3615                 if (rxbuf->m_head != NULL) {
3616                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3617                             BUS_DMASYNC_POSTREAD);
3618                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3619                         rxbuf->m_head->m_flags |= M_PKTHDR;
3620                         m_freem(rxbuf->m_head);
3621                 }
3622                 if (rxbuf->m_pack != NULL) {
3623                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3624                             BUS_DMASYNC_POSTREAD);
3625                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3626                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3627                         m_freem(rxbuf->m_pack);
3628                 }
3629                 rxbuf->m_head = NULL;
3630                 rxbuf->m_pack = NULL;
3631         }
3632 }
3633
3634
3635 /*********************************************************************
3636  *
3637  *  Initialize a receive ring and its buffers.
3638  *
3639  **********************************************************************/
3640 static int
3641 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3642 {
3643         struct  adapter         *adapter;
3644         struct ifnet            *ifp;
3645         device_t                dev;
3646         struct ixgbe_rx_buf     *rxbuf;
3647         bus_dma_segment_t       pseg[1], hseg[1];
3648         struct lro_ctrl         *lro = &rxr->lro;
3649         int                     rsize, nsegs, error = 0;
3650
3651         adapter = rxr->adapter;
3652         ifp = adapter->ifp;
3653         dev = adapter->dev;
3654
3655         /* Clear the ring contents */
3656         IXGBE_RX_LOCK(rxr);
3657         rsize = roundup2(adapter->num_rx_desc *
3658             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3659         bzero((void *)rxr->rx_base, rsize);
3660
3661         /* Free current RX buffer structs and their mbufs */
3662         ixgbe_free_receive_ring(rxr);
3663
3664         /* Configure header split? */
3665         if (ixgbe_header_split)
3666                 rxr->hdr_split = TRUE;
3667
3668         /* Now replenish the mbufs */
3669         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3670                 struct mbuf     *mh, *mp;
3671
3672                 rxbuf = &rxr->rx_buffers[j];
3673                 /*
3674                 ** Don't allocate mbufs if not
3675                 ** doing header split, its wasteful
3676                 */
3677                 if (rxr->hdr_split == FALSE)
3678                         goto skip_head;
3679
3680                 /* First the header */
3681                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
3682                 if (rxbuf->m_head == NULL) {
3683                         error = ENOBUFS;
3684                         goto fail;
3685                 }
3686                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3687                 mh = rxbuf->m_head;
3688                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3689                 mh->m_flags |= M_PKTHDR;
3690                 /* Get the memory mapping */
3691                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3692                     rxbuf->hmap, rxbuf->m_head, hseg,
3693                     &nsegs, BUS_DMA_NOWAIT);
3694                 if (error != 0) /* Nothing elegant to do here */
3695                         goto fail;
3696                 bus_dmamap_sync(rxr->htag,
3697                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3698                 /* Update descriptor */
3699                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3700
3701 skip_head:
3702                 /* Now the payload cluster */
3703                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
3704                     M_PKTHDR, adapter->rx_mbuf_sz);
3705                 if (rxbuf->m_pack == NULL) {
3706                         error = ENOBUFS;
3707                         goto fail;
3708                 }
3709                 mp = rxbuf->m_pack;
3710                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3711                 /* Get the memory mapping */
3712                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3713                     rxbuf->pmap, mp, pseg,
3714                     &nsegs, BUS_DMA_NOWAIT);
3715                 if (error != 0)
3716                         goto fail;
3717                 bus_dmamap_sync(rxr->ptag,
3718                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3719                 /* Update descriptor */
3720                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3721         }
3722
3723
3724         /* Setup our descriptor indices */
3725         rxr->next_to_check = 0;
3726         rxr->next_to_refresh = 0;
3727         rxr->lro_enabled = FALSE;
3728         rxr->rx_split_packets = 0;
3729         rxr->rx_bytes = 0;
3730
3731         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3732             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3733
3734         /*
3735         ** Now set up the LRO interface:
3736         ** 82598 uses software LRO, the
3737         ** 82599 uses a hardware assist.
3738         */
3739         if ((adapter->hw.mac.type == ixgbe_mac_82599EB) &&
3740             (ifp->if_capenable & IFCAP_RXCSUM) &&
3741             (ifp->if_capenable & IFCAP_LRO))
3742                 ixgbe_setup_hw_rsc(rxr);
3743         else if (ifp->if_capenable & IFCAP_LRO) {
3744                 int err = tcp_lro_init(lro);
3745                 if (err) {
3746                         device_printf(dev, "LRO Initialization failed!\n");
3747                         goto fail;
3748                 }
3749                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3750                 rxr->lro_enabled = TRUE;
3751                 lro->ifp = adapter->ifp;
3752         }
3753
3754         IXGBE_RX_UNLOCK(rxr);
3755         return (0);
3756
3757 fail:
3758         ixgbe_free_receive_ring(rxr);
3759         IXGBE_RX_UNLOCK(rxr);
3760         return (error);
3761 }
3762
3763 /*********************************************************************
3764  *
3765  *  Initialize all receive rings.
3766  *
3767  **********************************************************************/
3768 static int
3769 ixgbe_setup_receive_structures(struct adapter *adapter)
3770 {
3771         struct rx_ring *rxr = adapter->rx_rings;
3772         int j;
3773
3774         for (j = 0; j < adapter->num_queues; j++, rxr++)
3775                 if (ixgbe_setup_receive_ring(rxr))
3776                         goto fail;
3777
3778         return (0);
3779 fail:
3780         /*
3781          * Free RX buffers allocated so far, we will only handle
3782          * the rings that completed, the failing case will have
3783          * cleaned up for itself. 'j' failed, so its the terminus.
3784          */
3785         for (int i = 0; i < j; ++i) {
3786                 rxr = &adapter->rx_rings[i];
3787                 ixgbe_free_receive_ring(rxr);
3788         }
3789
3790         return (ENOBUFS);
3791 }
3792
3793 /*********************************************************************
3794  *
3795  *  Setup receive registers and features.
3796  *
3797  **********************************************************************/
3798 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3799
3800 static void
3801 ixgbe_initialize_receive_units(struct adapter *adapter)
3802 {
3803         struct  rx_ring *rxr = adapter->rx_rings;
3804         struct ixgbe_hw *hw = &adapter->hw;
3805         struct ifnet   *ifp = adapter->ifp;
3806         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
3807         u32             reta, mrqc = 0, hlreg, random[10];
3808
3809
3810         /*
3811          * Make sure receives are disabled while
3812          * setting up the descriptor ring
3813          */
3814         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3815         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
3816             rxctrl & ~IXGBE_RXCTRL_RXEN);
3817
3818         /* Enable broadcasts */
3819         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3820         fctrl |= IXGBE_FCTRL_BAM;
3821         fctrl |= IXGBE_FCTRL_DPF;
3822         fctrl |= IXGBE_FCTRL_PMCF;
3823         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3824
3825         /* Set for Jumbo Frames? */
3826         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3827         if (ifp->if_mtu > ETHERMTU)
3828                 hlreg |= IXGBE_HLREG0_JUMBOEN;
3829         else
3830                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
3831         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
3832
3833         bufsz = adapter->rx_mbuf_sz  >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3834
3835         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3836                 u64 rdba = rxr->rxdma.dma_paddr;
3837
3838                 /* Setup the Base and Length of the Rx Descriptor Ring */
3839                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
3840                                (rdba & 0x00000000ffffffffULL));
3841                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
3842                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
3843                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
3844
3845                 /* Set up the SRRCTL register */
3846                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
3847                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
3848                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
3849                 srrctl |= bufsz;
3850                 if (rxr->hdr_split) {
3851                         /* Use a standard mbuf for the header */
3852                         srrctl |= ((IXGBE_RX_HDR <<
3853                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
3854                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
3855                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3856                 } else
3857                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
3858                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
3859
3860                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
3861                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
3862                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
3863         }
3864
3865         if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
3866                 /* PSRTYPE must be initialized in 82599 */
3867                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
3868                               IXGBE_PSRTYPE_UDPHDR |
3869                               IXGBE_PSRTYPE_IPV4HDR |
3870                               IXGBE_PSRTYPE_IPV6HDR;
3871                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
3872         }
3873
3874         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3875
3876         /* Setup RSS */
3877         if (adapter->num_queues > 1) {
3878                 int i, j;
3879                 reta = 0;
3880
3881                 /* set up random bits */
3882                 arc4rand(&random, sizeof(random), 0);
3883
3884                 /* Set up the redirection table */
3885                 for (i = 0, j = 0; i < 128; i++, j++) {
3886                         if (j == adapter->num_queues) j = 0;
3887                         reta = (reta << 8) | (j * 0x11);
3888                         if ((i & 3) == 3)
3889                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3890                 }
3891
3892                 /* Now fill our hash function seeds */
3893                 for (int i = 0; i < 10; i++)
3894                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
3895
3896                 /* Perform hash on these packet types */
3897                 mrqc = IXGBE_MRQC_RSSEN
3898                      | IXGBE_MRQC_RSS_FIELD_IPV4
3899                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
3900                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
3901                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
3902                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
3903                      | IXGBE_MRQC_RSS_FIELD_IPV6
3904                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
3905                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
3906                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3907                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3908
3909                 /* RSS and RX IPP Checksum are mutually exclusive */
3910                 rxcsum |= IXGBE_RXCSUM_PCSD;
3911         }
3912
3913         if (ifp->if_capenable & IFCAP_RXCSUM)
3914                 rxcsum |= IXGBE_RXCSUM_PCSD;
3915
3916         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
3917                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
3918
3919         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3920
3921         return;
3922 }
3923
3924 /*********************************************************************
3925  *
3926  *  Free all receive rings.
3927  *
3928  **********************************************************************/
3929 static void
3930 ixgbe_free_receive_structures(struct adapter *adapter)
3931 {
3932         struct rx_ring *rxr = adapter->rx_rings;
3933
3934         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3935                 struct lro_ctrl         *lro = &rxr->lro;
3936                 ixgbe_free_receive_buffers(rxr);
3937                 /* Free LRO memory */
3938                 tcp_lro_free(lro);
3939                 /* Free the ring memory as well */
3940                 ixgbe_dma_free(adapter, &rxr->rxdma);
3941         }
3942
3943         free(adapter->rx_rings, M_DEVBUF);
3944 }
3945
3946
3947 /*********************************************************************
3948  *
3949  *  Free receive ring data structures
3950  *
3951  **********************************************************************/
3952 static void
3953 ixgbe_free_receive_buffers(struct rx_ring *rxr)
3954 {
3955         struct adapter          *adapter = rxr->adapter;
3956         struct ixgbe_rx_buf     *rxbuf;
3957
3958         INIT_DEBUGOUT("free_receive_structures: begin");
3959
3960         /* Cleanup any existing buffers */
3961         if (rxr->rx_buffers != NULL) {
3962                 for (int i = 0; i < adapter->num_rx_desc; i++) {
3963                         rxbuf = &rxr->rx_buffers[i];
3964                         if (rxbuf->m_head != NULL) {
3965                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3966                                     BUS_DMASYNC_POSTREAD);
3967                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3968                                 rxbuf->m_head->m_flags |= M_PKTHDR;
3969                                 m_freem(rxbuf->m_head);
3970                         }
3971                         if (rxbuf->m_pack != NULL) {
3972                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3973                                     BUS_DMASYNC_POSTREAD);
3974                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3975                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
3976                                 m_freem(rxbuf->m_pack);
3977                         }
3978                         rxbuf->m_head = NULL;
3979                         rxbuf->m_pack = NULL;
3980                         if (rxbuf->hmap != NULL) {
3981                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
3982                                 rxbuf->hmap = NULL;
3983                         }
3984                         if (rxbuf->pmap != NULL) {
3985                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
3986                                 rxbuf->pmap = NULL;
3987                         }
3988                 }
3989                 if (rxr->rx_buffers != NULL) {
3990                         free(rxr->rx_buffers, M_DEVBUF);
3991                         rxr->rx_buffers = NULL;
3992                 }
3993         }
3994
3995         if (rxr->htag != NULL) {
3996                 bus_dma_tag_destroy(rxr->htag);
3997                 rxr->htag = NULL;
3998         }
3999         if (rxr->ptag != NULL) {
4000                 bus_dma_tag_destroy(rxr->ptag);
4001                 rxr->ptag = NULL;
4002         }
4003
4004         return;
4005 }
4006
4007 static __inline void
4008 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4009 {
4010
4011         /*
4012          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4013          * should be computed by hardware. Also it should not have VLAN tag in
4014          * ethernet header.
4015          */
4016         if (rxr->lro_enabled &&
4017             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4018             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4019             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4020             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) &&
4021             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4022             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4023                 /*
4024                  * Send to the stack if:
4025                  **  - LRO not enabled, or
4026                  **  - no LRO resources, or
4027                  **  - lro enqueue fails
4028                  */
4029                 if (rxr->lro.lro_cnt != 0)
4030                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4031                                 return;
4032         }
4033         IXGBE_RX_UNLOCK(rxr);
4034         (*ifp->if_input)(ifp, m);
4035         IXGBE_RX_LOCK(rxr);
4036 }
4037
4038 static __inline void
4039 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4040 {
4041         struct ixgbe_rx_buf     *rbuf;
4042
4043         rbuf = &rxr->rx_buffers[i];
4044
4045         if (rbuf->fmp != NULL) {/* Partial chain ? */
4046                 rbuf->fmp->m_flags |= M_PKTHDR;
4047                 m_freem(rbuf->fmp);
4048                 rbuf->fmp = NULL;
4049         }
4050
4051         /*
4052         ** With advanced descriptors the writeback
4053         ** clobbers the buffer addrs, so its easier
4054         ** to just free the existing mbufs and take
4055         ** the normal refresh path to get new buffers
4056         ** and mapping.
4057         */
4058         if (rbuf->m_head) {
4059                 m_free(rbuf->m_head);
4060                 rbuf->m_head = NULL;
4061         }
4062
4063         if (rbuf->m_pack) {
4064                 m_free(rbuf->m_pack);
4065                 rbuf->m_pack = NULL;
4066         }
4067
4068         return;
4069 }
4070
4071
4072 /*********************************************************************
4073  *
4074  *  This routine executes in interrupt context. It replenishes
4075  *  the mbufs in the descriptor and sends data which has been
4076  *  dma'ed into host memory to upper layer.
4077  *
4078  *  We loop at most count times if count is > 0, or until done if
4079  *  count < 0.
4080  *
4081  *  Return TRUE for more work, FALSE for all clean.
4082  *********************************************************************/
4083 static bool
4084 ixgbe_rxeof(struct ix_queue *que, int count)
4085 {
4086         struct adapter          *adapter = que->adapter;
4087         struct rx_ring          *rxr = que->rxr;
4088         struct ifnet            *ifp = adapter->ifp;
4089         struct lro_ctrl         *lro = &rxr->lro;
4090         struct lro_entry        *queued;
4091         int                     i, nextp, processed = 0;
4092         u32                     staterr = 0;
4093         union ixgbe_adv_rx_desc *cur;
4094         struct ixgbe_rx_buf     *rbuf, *nbuf;
4095
4096         IXGBE_RX_LOCK(rxr);
4097
4098         for (i = rxr->next_to_check; count != 0;) {
4099                 struct mbuf     *sendmp, *mh, *mp;
4100                 u32             rsc, ptype;
4101                 u16             hlen, plen, hdr, vtag;
4102                 bool            eop;
4103
4104                 /* Sync the ring. */
4105                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4106                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4107
4108                 cur = &rxr->rx_base[i];
4109                 staterr = le32toh(cur->wb.upper.status_error);
4110
4111                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4112                         break;
4113                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4114                         break;
4115
4116                 count--;
4117                 sendmp = NULL;
4118                 nbuf = NULL;
4119                 rsc = 0;
4120                 cur->wb.upper.status_error = 0;
4121                 rbuf = &rxr->rx_buffers[i];
4122                 mh = rbuf->m_head;
4123                 mp = rbuf->m_pack;
4124
4125                 plen = le16toh(cur->wb.upper.length);
4126                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4127                     IXGBE_RXDADV_PKTTYPE_MASK;
4128                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4129                 vtag = le16toh(cur->wb.upper.vlan);
4130                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4131
4132                 /* Make sure bad packets are discarded */
4133                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4134                     (rxr->discard)) {
4135                         ifp->if_ierrors++;
4136                         rxr->rx_discarded++;
4137                         if (eop)
4138                                 rxr->discard = FALSE;
4139                         else
4140                                 rxr->discard = TRUE;
4141                         ixgbe_rx_discard(rxr, i);
4142                         goto next_desc;
4143                 }
4144
4145                 /*
4146                 ** On 82599 which supports a hardware
4147                 ** LRO (called HW RSC), packets need
4148                 ** not be fragmented across sequential
4149                 ** descriptors, rather the next descriptor
4150                 ** is indicated in bits of the descriptor.
4151                 ** This also means that we might proceses
4152                 ** more than one packet at a time, something
4153                 ** that has never been true before, it
4154                 ** required eliminating global chain pointers
4155                 ** in favor of what we are doing here.  -jfv
4156                 */
4157                 if (!eop) {
4158                         /*
4159                         ** Figure out the next descriptor
4160                         ** of this frame.
4161                         */
4162                         if (rxr->hw_rsc == TRUE) {
4163                                 rsc = ixgbe_rsc_count(cur);
4164                                 rxr->rsc_num += (rsc - 1);
4165                         }
4166                         if (rsc) { /* Get hardware index */
4167                                 nextp = ((staterr &
4168                                     IXGBE_RXDADV_NEXTP_MASK) >>
4169                                     IXGBE_RXDADV_NEXTP_SHIFT);
4170                         } else { /* Just sequential */
4171                                 nextp = i + 1;
4172                                 if (nextp == adapter->num_rx_desc)
4173                                         nextp = 0;
4174                         }
4175                         nbuf = &rxr->rx_buffers[nextp];
4176                         prefetch(nbuf);
4177                 }
4178                 /*
4179                 ** The header mbuf is ONLY used when header
4180                 ** split is enabled, otherwise we get normal
4181                 ** behavior, ie, both header and payload
4182                 ** are DMA'd into the payload buffer.
4183                 **
4184                 ** Rather than using the fmp/lmp global pointers
4185                 ** we now keep the head of a packet chain in the
4186                 ** buffer struct and pass this along from one
4187                 ** descriptor to the next, until we get EOP.
4188                 */
4189                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4190                         /* This must be an initial descriptor */
4191                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4192                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4193                         if (hlen > IXGBE_RX_HDR)
4194                                 hlen = IXGBE_RX_HDR;
4195                         mh->m_len = hlen;
4196                         mh->m_flags |= M_PKTHDR;
4197                         mh->m_next = NULL;
4198                         mh->m_pkthdr.len = mh->m_len;
4199                         /* Null buf pointer so it is refreshed */
4200                         rbuf->m_head = NULL;
4201                         /*
4202                         ** Check the payload length, this
4203                         ** could be zero if its a small
4204                         ** packet.
4205                         */
4206                         if (plen > 0) {
4207                                 mp->m_len = plen;
4208                                 mp->m_next = NULL;
4209                                 mp->m_flags &= ~M_PKTHDR;
4210                                 mh->m_next = mp;
4211                                 mh->m_pkthdr.len += mp->m_len;
4212                                 /* Null buf pointer so it is refreshed */
4213                                 rbuf->m_pack = NULL;
4214                                 rxr->rx_split_packets++;
4215                         }
4216                         /*
4217                         ** Now create the forward
4218                         ** chain so when complete
4219                         ** we wont have to.
4220                         */
4221                         if (eop == 0) {
4222                                 /* stash the chain head */
4223                                 nbuf->fmp = mh;
4224                                 /* Make forward chain */
4225                                 if (plen)
4226                                         mp->m_next = nbuf->m_pack;
4227                                 else
4228                                         mh->m_next = nbuf->m_pack;
4229                         } else {
4230                                 /* Singlet, prepare to send */
4231                                 sendmp = mh;
4232                                 if ((adapter->num_vlans) &&
4233                                   (staterr & IXGBE_RXD_STAT_VP)) {
4234                                         sendmp->m_pkthdr.ether_vtag = vtag;
4235                                         sendmp->m_flags |= M_VLANTAG;
4236                                 }
4237                         }
4238                 } else {
4239                         /*
4240                         ** Either no header split, or a
4241                         ** secondary piece of a fragmented
4242                         ** split packet.
4243                         */
4244                         mp->m_len = plen;
4245                         /*
4246                         ** See if there is a stored head
4247                         ** that determines what we are
4248                         */
4249                         sendmp = rbuf->fmp;
4250                         rbuf->m_pack = rbuf->fmp = NULL;
4251
4252                         if (sendmp != NULL) /* secondary frag */
4253                                 sendmp->m_pkthdr.len += mp->m_len;
4254                         else {
4255                                 /* first desc of a non-ps chain */
4256                                 sendmp = mp;
4257                                 sendmp->m_flags |= M_PKTHDR;
4258                                 sendmp->m_pkthdr.len = mp->m_len;
4259                                 if (staterr & IXGBE_RXD_STAT_VP) {
4260                                         sendmp->m_pkthdr.ether_vtag = vtag;
4261                                         sendmp->m_flags |= M_VLANTAG;
4262                                 }
4263                         }
4264                         /* Pass the head pointer on */
4265                         if (eop == 0) {
4266                                 nbuf->fmp = sendmp;
4267                                 sendmp = NULL;
4268                                 mp->m_next = nbuf->m_pack;
4269                         }
4270                 }
4271                 ++processed;
4272                 /* Sending this frame? */
4273                 if (eop) {
4274                         sendmp->m_pkthdr.rcvif = ifp;
4275                         ifp->if_ipackets++;
4276                         rxr->rx_packets++;
4277                         /* capture data for AIM */
4278                         rxr->bytes += sendmp->m_pkthdr.len;
4279                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4280                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4281                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4282 #if __FreeBSD_version >= 800000
4283                         sendmp->m_pkthdr.flowid = que->msix;
4284                         sendmp->m_flags |= M_FLOWID;
4285 #endif
4286                 }
4287 next_desc:
4288                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4289                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4290
4291                 /* Advance our pointers to the next descriptor. */
4292                 if (++i == adapter->num_rx_desc)
4293                         i = 0;
4294
4295                 /* Now send to the stack or do LRO */
4296                 if (sendmp != NULL) {
4297                         rxr->next_to_check = i;
4298                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4299                         i = rxr->next_to_check;
4300                 }
4301
4302                /* Every 8 descriptors we go to refresh mbufs */
4303                 if (processed == 8) {
4304                         ixgbe_refresh_mbufs(rxr, i);
4305                         processed = 0;
4306                 }
4307         }
4308
4309         /* Refresh any remaining buf structs */
4310         if (processed != 0) {
4311                 ixgbe_refresh_mbufs(rxr, i);
4312                 processed = 0;
4313         }
4314
4315         rxr->next_to_check = i;
4316
4317         /*
4318          * Flush any outstanding LRO work
4319          */
4320         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4321                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4322                 tcp_lro_flush(lro, queued);
4323         }
4324
4325         IXGBE_RX_UNLOCK(rxr);
4326
4327         /*
4328         ** We still have cleaning to do?
4329         ** Schedule another interrupt if so.
4330         */
4331         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4332                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4333                 return (TRUE);
4334         }
4335
4336         return (FALSE);
4337 }
4338
4339
4340 /*********************************************************************
4341  *
4342  *  Verify that the hardware indicated that the checksum is valid.
4343  *  Inform the stack about the status of checksum so that stack
4344  *  doesn't spend time verifying the checksum.
4345  *
4346  *********************************************************************/
4347 static void
4348 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4349 {
4350         u16     status = (u16) staterr;
4351         u8      errors = (u8) (staterr >> 24);
4352         bool    sctp = FALSE;
4353
4354         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4355             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4356                 sctp = TRUE;
4357
4358         if (status & IXGBE_RXD_STAT_IPCS) {
4359                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4360                         /* IP Checksum Good */
4361                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4362                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4363
4364                 } else
4365                         mp->m_pkthdr.csum_flags = 0;
4366         }
4367         if (status & IXGBE_RXD_STAT_L4CS) {
4368                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4369 #if __FreeBSD_version >= 800000
4370                 if (sctp)
4371                         type = CSUM_SCTP_VALID;
4372 #endif
4373                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4374                         mp->m_pkthdr.csum_flags |= type;
4375                         if (!sctp)
4376                                 mp->m_pkthdr.csum_data = htons(0xffff);
4377                 }
4378         }
4379         return;
4380 }
4381
4382
4383 /*
4384 ** This routine is run via an vlan config EVENT,
4385 ** it enables us to use the HW Filter table since
4386 ** we can get the vlan id. This just creates the
4387 ** entry in the soft version of the VFTA, init will
4388 ** repopulate the real table.
4389 */
4390 static void
4391 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4392 {
4393         struct adapter  *adapter = ifp->if_softc;
4394         u16             index, bit;
4395
4396         if (ifp->if_softc !=  arg)   /* Not our event */
4397                 return;
4398
4399         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4400                 return;
4401
4402         IXGBE_CORE_LOCK(adapter);
4403         index = (vtag >> 5) & 0x7F;
4404         bit = vtag & 0x1F;
4405         adapter->shadow_vfta[index] |= (1 << bit);
4406         ++adapter->num_vlans;
4407         ixgbe_init_locked(adapter);
4408         IXGBE_CORE_UNLOCK(adapter);
4409 }
4410
4411 /*
4412 ** This routine is run via an vlan
4413 ** unconfig EVENT, remove our entry
4414 ** in the soft vfta.
4415 */
4416 static void
4417 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4418 {
4419         struct adapter  *adapter = ifp->if_softc;
4420         u16             index, bit;
4421
4422         if (ifp->if_softc !=  arg)
4423                 return;
4424
4425         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4426                 return;
4427
4428         IXGBE_CORE_LOCK(adapter);
4429         index = (vtag >> 5) & 0x7F;
4430         bit = vtag & 0x1F;
4431         adapter->shadow_vfta[index] &= ~(1 << bit);
4432         --adapter->num_vlans;
4433         /* Re-init to load the changes */
4434         ixgbe_init_locked(adapter);
4435         IXGBE_CORE_UNLOCK(adapter);
4436 }
4437
4438 static void
4439 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4440 {
4441         struct ifnet    *ifp = adapter->ifp;
4442         struct ixgbe_hw *hw = &adapter->hw;
4443         u32             ctrl;
4444
4445
4446         /*
4447         ** We get here thru init_locked, meaning
4448         ** a soft reset, this has already cleared
4449         ** the VFTA and other state, so if there
4450         ** have been no vlan's registered do nothing.
4451         */
4452         if (adapter->num_vlans == 0)
4453                 return;
4454
4455         /*
4456         ** A soft reset zero's out the VFTA, so
4457         ** we need to repopulate it now.
4458         */
4459         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4460                 if (adapter->shadow_vfta[i] != 0)
4461                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4462                             adapter->shadow_vfta[i]);
4463
4464         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4465         /* Enable the Filter Table if enabled */
4466         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4467                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4468                 ctrl |= IXGBE_VLNCTRL_VFE;
4469         }
4470         if (hw->mac.type == ixgbe_mac_82598EB)
4471                 ctrl |= IXGBE_VLNCTRL_VME;
4472         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4473
4474         /* On 82599 the VLAN enable is per/queue in RXDCTL */
4475         if (hw->mac.type == ixgbe_mac_82599EB)
4476                 for (int i = 0; i < adapter->num_queues; i++) {
4477                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4478                                 ctrl |= IXGBE_RXDCTL_VME;
4479                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4480                 }
4481 }
4482
4483 static void
4484 ixgbe_enable_intr(struct adapter *adapter)
4485 {
4486         struct ixgbe_hw *hw = &adapter->hw;
4487         struct ix_queue *que = adapter->queues;
4488         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4489
4490
4491         /* Enable Fan Failure detection */
4492         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4493                     mask |= IXGBE_EIMS_GPI_SDP1;
4494
4495         /* 82599 specific interrupts */
4496         if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
4497                     mask |= IXGBE_EIMS_ECC;
4498                     mask |= IXGBE_EIMS_GPI_SDP1;
4499                     mask |= IXGBE_EIMS_GPI_SDP2;
4500 #ifdef IXGBE_FDIR
4501                     mask |= IXGBE_EIMS_FLOW_DIR;
4502 #endif
4503         }
4504
4505         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4506
4507         /* With RSS we use auto clear */
4508         if (adapter->msix_mem) {
4509                 mask = IXGBE_EIMS_ENABLE_MASK;
4510                 /* Don't autoclear Link */
4511                 mask &= ~IXGBE_EIMS_OTHER;
4512                 mask &= ~IXGBE_EIMS_LSC;
4513                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4514         }
4515
4516         /*
4517         ** Now enable all queues, this is done separately to
4518         ** allow for handling the extended (beyond 32) MSIX
4519         ** vectors that can be used by 82599
4520         */
4521         for (int i = 0; i < adapter->num_queues; i++, que++)
4522                 ixgbe_enable_queue(adapter, que->msix);
4523
4524         IXGBE_WRITE_FLUSH(hw);
4525
4526         return;
4527 }
4528
4529 static void
4530 ixgbe_disable_intr(struct adapter *adapter)
4531 {
4532         if (adapter->msix_mem)
4533                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4534         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4535                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4536         } else {
4537                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4538                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4539                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4540         }
4541         IXGBE_WRITE_FLUSH(&adapter->hw);
4542         return;
4543 }
4544
4545 u16
4546 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4547 {
4548         u16 value;
4549
4550         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4551             reg, 2);
4552
4553         return (value);
4554 }
4555
4556 void
4557 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4558 {
4559         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4560             reg, value, 2);
4561
4562         return;
4563 }
4564
4565 /*
4566 ** Setup the correct IVAR register for a particular MSIX interrupt
4567 **   (yes this is all very magic and confusing :)
4568 **  - entry is the register array entry
4569 **  - vector is the MSIX vector for this queue
4570 **  - type is RX/TX/MISC
4571 */
4572 static void
4573 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4574 {
4575         struct ixgbe_hw *hw = &adapter->hw;
4576         u32 ivar, index;
4577
4578         vector |= IXGBE_IVAR_ALLOC_VAL;
4579
4580         switch (hw->mac.type) {
4581
4582         case ixgbe_mac_82598EB:
4583                 if (type == -1)
4584                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4585                 else
4586                         entry += (type * 64);
4587                 index = (entry >> 2) & 0x1F;
4588                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4589                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4590                 ivar |= (vector << (8 * (entry & 0x3)));
4591                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4592                 break;
4593
4594         case ixgbe_mac_82599EB:
4595                 if (type == -1) { /* MISC IVAR */
4596                         index = (entry & 1) * 8;
4597                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4598                         ivar &= ~(0xFF << index);
4599                         ivar |= (vector << index);
4600                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4601                 } else {        /* RX/TX IVARS */
4602                         index = (16 * (entry & 1)) + (8 * type);
4603                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4604                         ivar &= ~(0xFF << index);
4605                         ivar |= (vector << index);
4606                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4607                 }
4608
4609         default:
4610                 break;
4611         }
4612 }
4613
4614 static void
4615 ixgbe_configure_ivars(struct adapter *adapter)
4616 {
4617         struct  ix_queue *que = adapter->queues;
4618         u32 newitr;
4619
4620         if (ixgbe_max_interrupt_rate > 0)
4621                 newitr = (8000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
4622         else
4623                 newitr = 0;
4624
4625         for (int i = 0; i < adapter->num_queues; i++, que++) {
4626                 /* First the RX queue entry */
4627                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4628                 /* ... and the TX */
4629                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4630                 /* Set an Initial EITR value */
4631                 IXGBE_WRITE_REG(&adapter->hw,
4632                     IXGBE_EITR(que->msix), newitr);
4633         }
4634
4635         /* For the Link interrupt */
4636         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4637 }
4638
4639 /*
4640 ** ixgbe_sfp_probe - called in the local timer to
4641 ** determine if a port had optics inserted.
4642 */
4643 static bool ixgbe_sfp_probe(struct adapter *adapter)
4644 {
4645         struct ixgbe_hw *hw = &adapter->hw;
4646         device_t        dev = adapter->dev;
4647         bool            result = FALSE;
4648
4649         if ((hw->phy.type == ixgbe_phy_nl) &&
4650             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4651                 s32 ret = hw->phy.ops.identify_sfp(hw);
4652                 if (ret)
4653                         goto out;
4654                 ret = hw->phy.ops.reset(hw);
4655                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4656                         device_printf(dev,"Unsupported SFP+ module detected!");
4657                         printf(" Reload driver with supported module.\n");
4658                         adapter->sfp_probe = FALSE;
4659                         goto out;
4660                 } else
4661                         device_printf(dev,"SFP+ module detected!\n");
4662                 /* We now have supported optics */
4663                 adapter->sfp_probe = FALSE;
4664                 /* Set the optics type so system reports correctly */
4665                 ixgbe_setup_optics(adapter);
4666                 result = TRUE;
4667         }
4668 out:
4669         return (result);
4670 }
4671
4672 /*
4673 ** Tasklet handler for MSIX Link interrupts
4674 **  - do outside interrupt since it might sleep
4675 */
4676 static void
4677 ixgbe_handle_link(void *context, int pending)
4678 {
4679         struct adapter  *adapter = context;
4680
4681         ixgbe_check_link(&adapter->hw,
4682             &adapter->link_speed, &adapter->link_up, 0);
4683         ixgbe_update_link_status(adapter);
4684 }
4685
4686 /*
4687 ** Tasklet for handling SFP module interrupts
4688 */
4689 static void
4690 ixgbe_handle_mod(void *context, int pending)
4691 {
4692         struct adapter  *adapter = context;
4693         struct ixgbe_hw *hw = &adapter->hw;
4694         device_t        dev = adapter->dev;
4695         u32 err;
4696
4697         err = hw->phy.ops.identify_sfp(hw);
4698         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4699                 device_printf(dev,
4700                     "Unsupported SFP+ module type was detected.\n");
4701                 return;
4702         }
4703         err = hw->mac.ops.setup_sfp(hw);
4704         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4705                 device_printf(dev,
4706                     "Setup failure - unsupported SFP+ module type.\n");
4707                 return;
4708         }
4709         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4710         return;
4711 }
4712
4713
4714 /*
4715 ** Tasklet for handling MSF (multispeed fiber) interrupts
4716 */
4717 static void
4718 ixgbe_handle_msf(void *context, int pending)
4719 {
4720         struct adapter  *adapter = context;
4721         struct ixgbe_hw *hw = &adapter->hw;
4722         u32 autoneg;
4723         bool negotiate;
4724
4725         autoneg = hw->phy.autoneg_advertised;
4726         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4727                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4728         if (hw->mac.ops.setup_link)
4729                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4730         return;
4731 }
4732
4733 #ifdef IXGBE_FDIR
4734 /*
4735 ** Tasklet for reinitializing the Flow Director filter table
4736 */
4737 static void
4738 ixgbe_reinit_fdir(void *context, int pending)
4739 {
4740         struct adapter  *adapter = context;
4741         struct ifnet   *ifp = adapter->ifp;
4742
4743         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4744                 return;
4745         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4746         adapter->fdir_reinit = 0;
4747         /* Restart the interface */
4748         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4749         return;
4750 }
4751 #endif
4752
4753 /**********************************************************************
4754  *
4755  *  Update the board statistics counters.
4756  *
4757  **********************************************************************/
4758 static void
4759 ixgbe_update_stats_counters(struct adapter *adapter)
4760 {
4761         struct ifnet   *ifp = adapter->ifp;
4762         struct ixgbe_hw *hw = &adapter->hw;
4763         u32  missed_rx = 0, bprc, lxon, lxoff, total;
4764         u64  total_missed_rx = 0;
4765
4766         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
4767         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
4768         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
4769         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
4770
4771         for (int i = 0; i < 8; i++) {
4772                 u32 mp;
4773                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
4774                 /* missed_rx tallies misses for the gprc workaround */
4775                 missed_rx += mp;
4776                 /* global total per queue */
4777                 adapter->stats.mpc[i] += mp;
4778                 /* Running comprehensive total for stats display */
4779                 total_missed_rx += adapter->stats.mpc[i];
4780                 if (hw->mac.type == ixgbe_mac_82598EB)
4781                         adapter->stats.rnbc[i] +=
4782                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
4783                 adapter->stats.pxontxc[i] +=
4784                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
4785                 adapter->stats.pxonrxc[i] +=
4786                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
4787                 adapter->stats.pxofftxc[i] +=
4788                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
4789                 adapter->stats.pxoffrxc[i] +=
4790                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
4791                 adapter->stats.pxon2offc[i] +=
4792                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
4793         }
4794         for (int i = 0; i < 16; i++) {
4795                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
4796                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
4797                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
4798                 adapter->stats.qbrc[i] +=
4799                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
4800                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
4801                 adapter->stats.qbtc[i] +=
4802                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
4803                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
4804         }
4805         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
4806         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
4807         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
4808
4809         /* Hardware workaround, gprc counts missed packets */
4810         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
4811         adapter->stats.gprc -= missed_rx;
4812
4813         if (hw->mac.type == ixgbe_mac_82599EB) {
4814                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
4815                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
4816                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
4817                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
4818                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
4819                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
4820                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
4821                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
4822         } else {
4823                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
4824                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
4825                 /* 82598 only has a counter in the high register */
4826                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
4827                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
4828                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
4829         }
4830
4831         /*
4832          * Workaround: mprc hardware is incorrectly counting
4833          * broadcasts, so for now we subtract those.
4834          */
4835         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
4836         adapter->stats.bprc += bprc;
4837         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
4838         if (hw->mac.type == ixgbe_mac_82598EB)
4839                 adapter->stats.mprc -= bprc;
4840
4841         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
4842         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
4843         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
4844         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
4845         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
4846         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
4847
4848         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
4849         adapter->stats.lxontxc += lxon;
4850         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
4851         adapter->stats.lxofftxc += lxoff;
4852         total = lxon + lxoff;
4853
4854         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
4855         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
4856         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
4857         adapter->stats.gptc -= total;
4858         adapter->stats.mptc -= total;
4859         adapter->stats.ptc64 -= total;
4860         adapter->stats.gotc -= total * ETHER_MIN_LEN;
4861
4862         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
4863         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
4864         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
4865         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
4866         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
4867         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
4868         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
4869         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
4870         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
4871         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
4872         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
4873         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
4874         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
4875         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
4876         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
4877         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
4878         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
4879         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
4880         /* Only read FCOE on 82599 */
4881         if (hw->mac.type == ixgbe_mac_82599EB) {
4882                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
4883                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
4884                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
4885                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
4886                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
4887         }
4888
4889         /* Fill out the OS statistics structure */
4890         ifp->if_ipackets = adapter->stats.gprc;
4891         ifp->if_opackets = adapter->stats.gptc;
4892         ifp->if_ibytes = adapter->stats.gorc;
4893         ifp->if_obytes = adapter->stats.gotc;
4894         ifp->if_imcasts = adapter->stats.mprc;
4895         ifp->if_collisions = 0;
4896
4897         /* Rx Errors */
4898         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
4899                 adapter->stats.rlec;
4900 }
4901
4902 /** ixgbe_sysctl_tdh_handler - Handler function
4903  *  Retrieves the TDH value from the hardware
4904  */
4905 static int
4906 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
4907 {
4908         int error;
4909
4910         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
4911         if (!txr) return 0;
4912
4913         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
4914         error = sysctl_handle_int(oidp, &val, 0, req);
4915         if (error || !req->newptr)
4916                 return error;
4917         return 0;
4918 }
4919
4920 /** ixgbe_sysctl_tdt_handler - Handler function
4921  *  Retrieves the TDT value from the hardware
4922  */
4923 static int
4924 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
4925 {
4926         int error;
4927
4928         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
4929         if (!txr) return 0;
4930
4931         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
4932         error = sysctl_handle_int(oidp, &val, 0, req);
4933         if (error || !req->newptr)
4934                 return error;
4935         return 0;
4936 }
4937
4938 /** ixgbe_sysctl_rdh_handler - Handler function
4939  *  Retrieves the RDH value from the hardware
4940  */
4941 static int
4942 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
4943 {
4944         int error;
4945
4946         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
4947         if (!rxr) return 0;
4948
4949         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
4950         error = sysctl_handle_int(oidp, &val, 0, req);
4951         if (error || !req->newptr)
4952                 return error;
4953         return 0;
4954 }
4955
4956 /** ixgbe_sysctl_rdt_handler - Handler function
4957  *  Retrieves the RDT value from the hardware
4958  */
4959 static int
4960 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
4961 {
4962         int error;
4963
4964         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
4965         if (!rxr) return 0;
4966
4967         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
4968         error = sysctl_handle_int(oidp, &val, 0, req);
4969         if (error || !req->newptr)
4970                 return error;
4971         return 0;
4972 }
4973
4974 static int
4975 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
4976 {
4977         int error;
4978         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
4979         unsigned int reg, usec, rate;
4980
4981         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
4982         usec = ((reg & 0x0FF8) >> 3);
4983         if (usec > 0)
4984                 rate = 1000000 / usec;
4985         else
4986                 rate = 0;
4987         error = sysctl_handle_int(oidp, &rate, 0, req);
4988         if (error || !req->newptr)
4989                 return error;
4990         return 0;
4991 }
4992
4993 /*
4994  * Add sysctl variables, one per statistic, to the system.
4995  */
4996 static void
4997 ixgbe_add_hw_stats(struct adapter *adapter)
4998 {
4999
5000         device_t dev = adapter->dev;
5001
5002         struct tx_ring *txr = adapter->tx_rings;
5003         struct rx_ring *rxr = adapter->rx_rings;
5004
5005         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5006         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5007         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5008         struct ixgbe_hw_stats *stats = &adapter->stats;
5009
5010         struct sysctl_oid *stat_node, *queue_node;
5011         struct sysctl_oid_list *stat_list, *queue_list;
5012
5013 #define QUEUE_NAME_LEN 32
5014         char namebuf[QUEUE_NAME_LEN];
5015
5016         /* Driver Statistics */
5017         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5018                         CTLFLAG_RD, &adapter->dropped_pkts,
5019                         "Driver dropped packets");
5020         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5021                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5022                         "m_defrag() failed");
5023         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5024                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5025                         "Driver tx dma failure in xmit");
5026         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5027                         CTLFLAG_RD, &adapter->watchdog_events,
5028                         "Watchdog timeouts");
5029         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5030                         CTLFLAG_RD, &adapter->tso_tx,
5031                         "TSO");
5032         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5033                         CTLFLAG_RD, &adapter->link_irq,
5034                         "Link MSIX IRQ Handled");
5035
5036         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5037                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5038                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5039                                             CTLFLAG_RD, NULL, "Queue Name");
5040                 queue_list = SYSCTL_CHILDREN(queue_node);
5041
5042                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5043                                 CTLFLAG_RD, &adapter->queues[i], sizeof(&adapter->queues[i]),
5044                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5045                                 "Interrupt Rate");
5046                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5047                                 CTLFLAG_RD, txr, sizeof(txr),
5048                                 ixgbe_sysctl_tdh_handler, "IU",
5049                                 "Transmit Descriptor Head");
5050                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5051                                 CTLFLAG_RD, txr, sizeof(txr),
5052                                 ixgbe_sysctl_tdt_handler, "IU",
5053                                 "Transmit Descriptor Tail");
5054                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5055                                 CTLFLAG_RD, &txr->no_desc_avail,
5056                                 "Queue No Descriptor Available");
5057                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5058                                 CTLFLAG_RD, &txr->total_packets,
5059                                 "Queue Packets Transmitted");
5060         }
5061
5062         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5063                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5064                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5065                                             CTLFLAG_RD, NULL, "Queue Name");
5066                 queue_list = SYSCTL_CHILDREN(queue_node);
5067
5068                 struct lro_ctrl *lro = &rxr->lro;
5069
5070                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5071                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5072                                             CTLFLAG_RD, NULL, "Queue Name");
5073                 queue_list = SYSCTL_CHILDREN(queue_node);
5074
5075                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5076                                 CTLFLAG_RD, rxr, sizeof(rxr),
5077                                 ixgbe_sysctl_rdh_handler, "IU",
5078                                 "Receive Descriptor Head");
5079                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5080                                 CTLFLAG_RD, rxr, sizeof(rxr),
5081                                 ixgbe_sysctl_rdt_handler, "IU",
5082                                 "Receive Descriptor Tail");
5083                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5084                                 CTLFLAG_RD, &rxr->rx_packets,
5085                                 "Queue Packets Received");
5086                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5087                                 CTLFLAG_RD, &rxr->rx_bytes,
5088                                 "Queue Bytes Received");
5089                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
5090                                 CTLFLAG_RD, &lro->lro_queued, 0,
5091                                 "LRO Queued");
5092                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
5093                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5094                                 "LRO Flushed");
5095         }
5096
5097         /* MAC stats get the own sub node */
5098
5099         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5100                                     CTLFLAG_RD, NULL, "MAC Statistics");
5101         stat_list = SYSCTL_CHILDREN(stat_node);
5102
5103         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5104                         CTLFLAG_RD, &stats->crcerrs,
5105                         "CRC Errors");
5106         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5107                         CTLFLAG_RD, &stats->illerrc,
5108                         "Illegal Byte Errors");
5109         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5110                         CTLFLAG_RD, &stats->errbc,
5111                         "Byte Errors");
5112         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "short_discards",
5113                         CTLFLAG_RD, &stats->mspdc,
5114                         "MAC Short Packets Discarded");
5115         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "local_faults",
5116                         CTLFLAG_RD, &stats->mlfc,
5117                         "MAC Local Faults");
5118         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5119                         CTLFLAG_RD, &stats->mrfc,
5120                         "MAC Remote Faults");
5121         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5122                         CTLFLAG_RD, &stats->rlec,
5123                         "Receive Length Errors");
5124         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5125                         CTLFLAG_RD, &stats->lxontxc,
5126                         "Link XON Transmitted");
5127         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5128                         CTLFLAG_RD, &stats->lxonrxc,
5129                         "Link XON Received");
5130         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5131                         CTLFLAG_RD, &stats->lxofftxc,
5132                         "Link XOFF Transmitted");
5133         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5134                         CTLFLAG_RD, &stats->lxoffrxc,
5135                         "Link XOFF Received");
5136
5137         /* Packet Reception Stats */
5138         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5139                         CTLFLAG_RD, &stats->tor,
5140                         "Total Octets Received");
5141         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5142                         CTLFLAG_RD, &stats->gorc,
5143                         "Good Octets Received");
5144         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5145                         CTLFLAG_RD, &stats->tpr,
5146                         "Total Packets Received");
5147         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5148                         CTLFLAG_RD, &stats->gprc,
5149                         "Good Packets Received");
5150         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5151                         CTLFLAG_RD, &stats->mprc,
5152                         "Multicast Packets Received");
5153         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5154                         CTLFLAG_RD, &stats->bprc,
5155                         "Broadcast Packets Received");
5156         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5157                         CTLFLAG_RD, &stats->prc64,
5158                         "64 byte frames received ");
5159         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5160                         CTLFLAG_RD, &stats->prc127,
5161                         "65-127 byte frames received");
5162         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5163                         CTLFLAG_RD, &stats->prc255,
5164                         "128-255 byte frames received");
5165         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5166                         CTLFLAG_RD, &stats->prc511,
5167                         "256-511 byte frames received");
5168         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5169                         CTLFLAG_RD, &stats->prc1023,
5170                         "512-1023 byte frames received");
5171         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5172                         CTLFLAG_RD, &stats->prc1522,
5173                         "1023-1522 byte frames received");
5174         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5175                         CTLFLAG_RD, &stats->ruc,
5176                         "Receive Undersized");
5177         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5178                         CTLFLAG_RD, &stats->rfc,
5179                         "Fragmented Packets Received ");
5180         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5181                         CTLFLAG_RD, &stats->roc,
5182                         "Oversized Packets Received");
5183         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5184                         CTLFLAG_RD, &stats->rjc,
5185                         "Received Jabber");
5186         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5187                         CTLFLAG_RD, &stats->mngprc,
5188                         "Management Packets Received");
5189         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5190                         CTLFLAG_RD, &stats->mngptc,
5191                         "Management Packets Dropped");
5192         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5193                         CTLFLAG_RD, &stats->xec,
5194                         "Checksum Errors");
5195
5196         /* Packet Transmission Stats */
5197         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5198                         CTLFLAG_RD, &stats->gotc,
5199                         "Good Octets Transmitted");
5200         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5201                         CTLFLAG_RD, &stats->tpt,
5202                         "Total Packets Transmitted");
5203         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5204                         CTLFLAG_RD, &stats->gptc,
5205                         "Good Packets Transmitted");
5206         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5207                         CTLFLAG_RD, &stats->bptc,
5208                         "Broadcast Packets Transmitted");
5209         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5210                         CTLFLAG_RD, &stats->mptc,
5211                         "Multicast Packets Transmitted");
5212         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5213                         CTLFLAG_RD, &stats->mngptc,
5214                         "Management Packets Transmitted");
5215         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5216                         CTLFLAG_RD, &stats->ptc64,
5217                         "64 byte frames transmitted ");
5218         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5219                         CTLFLAG_RD, &stats->ptc127,
5220                         "65-127 byte frames transmitted");
5221         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5222                         CTLFLAG_RD, &stats->ptc255,
5223                         "128-255 byte frames transmitted");
5224         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5225                         CTLFLAG_RD, &stats->ptc511,
5226                         "256-511 byte frames transmitted");
5227         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5228                         CTLFLAG_RD, &stats->ptc1023,
5229                         "512-1023 byte frames transmitted");
5230         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5231                         CTLFLAG_RD, &stats->ptc1522,
5232                         "1024-1522 byte frames transmitted");
5233
5234         /* FC Stats */
5235         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5236                 CTLFLAG_RD, &stats->fccrc,
5237                 "FC CRC Errors");
5238         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "fc_last",
5239                 CTLFLAG_RD, &stats->fclast,
5240                 "FC Last Error");
5241         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5242                 CTLFLAG_RD, &stats->fcoerpdc,
5243                 "FCoE Packets Dropped");
5244         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5245                 CTLFLAG_RD, &stats->fcoeprc,
5246                 "FCoE Packets Received");
5247         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5248                 CTLFLAG_RD, &stats->fcoeptc,
5249                 "FCoE Packets Transmitted");
5250         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5251                 CTLFLAG_RD, &stats->fcoedwrc,
5252                 "FCoE DWords Received");
5253         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5254                 CTLFLAG_RD, &stats->fcoedwtc,
5255                 "FCoE DWords Transmitted");
5256 }
5257
5258 /*
5259 ** Set flow control using sysctl:
5260 ** Flow control values:
5261 **      0 - off
5262 **      1 - rx pause
5263 **      2 - tx pause
5264 **      3 - full
5265 */
5266 static int
5267 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5268 {
5269         int error;
5270         int last = ixgbe_flow_control;
5271         struct adapter *adapter;
5272
5273         error = sysctl_handle_int(oidp, &ixgbe_flow_control, 0, req);
5274         if (error)
5275                 return (error);
5276
5277         /* Don't bother if it's not changed */
5278         if (ixgbe_flow_control == last)
5279                 return (0);
5280
5281         adapter = (struct adapter *) arg1;
5282         switch (ixgbe_flow_control) {
5283                 case ixgbe_fc_rx_pause:
5284                 case ixgbe_fc_tx_pause:
5285                 case ixgbe_fc_full:
5286                         adapter->hw.fc.requested_mode = ixgbe_flow_control;
5287                         break;
5288                 case ixgbe_fc_none:
5289                 default:
5290                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5291         }
5292
5293         ixgbe_fc_enable(&adapter->hw, 0);
5294         return error;
5295 }
5296
5297 static void
5298 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5299         const char *description, int *limit, int value)
5300 {
5301         *limit = value;
5302         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5303             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5304             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5305 }
5306
5307 /*
5308 ** Control link advertise speed:
5309 **      0 - normal
5310 **      1 - advertise only 1G
5311 */
5312 static int
5313 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5314 {
5315         int                     error;
5316         struct adapter          *adapter;
5317         struct ixgbe_hw         *hw;
5318         ixgbe_link_speed        speed, last;
5319
5320         adapter = (struct adapter *) arg1;
5321         hw = &adapter->hw;
5322         last = hw->phy.autoneg_advertised;
5323
5324         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5325
5326         if ((error) || (adapter->advertise == -1))
5327                 return (error);
5328
5329         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5330             (hw->phy.multispeed_fiber)))
5331                 return (error);
5332
5333         if (adapter->advertise == 1)
5334                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5335         else
5336                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5337                         IXGBE_LINK_SPEED_10GB_FULL;
5338
5339         if (speed == last) /* no change */
5340                 return (error);
5341
5342         hw->mac.autotry_restart = TRUE;
5343         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5344
5345         return (error);
5346 }