sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2010, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #endif
  38
  39 #include "ixgbe.h"
  40
  41 /*********************************************************************
  42  *  Set this to one to display debug statistics
  43  *********************************************************************/
  44 int             ixgbe_display_debug_stats = 0;
  45
  46 /*********************************************************************
  47  *  Driver version
  48  *********************************************************************/
  49 char ixgbe_driver_version[] = "2.2.0";
  50
  51 /*********************************************************************
  52  *  PCI Device ID Table
  53  *
  54  *  Used by probe to select devices to load on
  55  *  Last field stores an index into ixgbe_strings
  56  *  Last entry must be all 0s
  57  *
  58  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  59  *********************************************************************/
  60
  61 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  62 {
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  81         /* required last entry */
  82         {0, 0, 0, 0, 0}
  83 };
  84
  85 /*********************************************************************
  86  *  Table of branding strings
  87  *********************************************************************/
  88
  89 static char    *ixgbe_strings[] = {
  90         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  91 };
  92
  93 /*********************************************************************
  94  *  Function prototypes
  95  *********************************************************************/
  96 static int      ixgbe_probe(device_t);
  97 static int      ixgbe_attach(device_t);
  98 static int      ixgbe_detach(device_t);
  99 static int      ixgbe_shutdown(device_t);
 100 static void     ixgbe_start(struct ifnet *);
 101 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 102 #if __FreeBSD_version >= 800000
 103 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 104 static int      ixgbe_mq_start_locked(struct ifnet *,
 105                     struct tx_ring *, struct mbuf *);
 106 static void     ixgbe_qflush(struct ifnet *);
 107 #endif
 108 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 109 static void     ixgbe_init(void *);
 110 static int      ixgbe_init_locked(struct adapter *);
 111 static void     ixgbe_stop(void *);
 112 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 113 static int      ixgbe_media_change(struct ifnet *);
 114 static void     ixgbe_identify_hardware(struct adapter *);
 115 static int      ixgbe_allocate_pci_resources(struct adapter *);
 116 static int      ixgbe_allocate_msix(struct adapter *);
 117 static int      ixgbe_allocate_legacy(struct adapter *);
 118 static int      ixgbe_allocate_queues(struct adapter *);
 119 static int      ixgbe_setup_msix(struct adapter *);
 120 static void     ixgbe_free_pci_resources(struct adapter *);
 121 static void     ixgbe_local_timer(void *);
 122 static void     ixgbe_setup_interface(device_t, struct adapter *);
 123 static void     ixgbe_config_link(struct adapter *);
 124
 125 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 126 static int      ixgbe_setup_transmit_structures(struct adapter *);
 127 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 128 static void     ixgbe_initialize_transmit_units(struct adapter *);
 129 static void     ixgbe_free_transmit_structures(struct adapter *);
 130 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 131
 132 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 133 static int      ixgbe_setup_receive_structures(struct adapter *);
 134 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 135 static void     ixgbe_initialize_receive_units(struct adapter *);
 136 static void     ixgbe_free_receive_structures(struct adapter *);
 137 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 138 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 139
 140 static void     ixgbe_enable_intr(struct adapter *);
 141 static void     ixgbe_disable_intr(struct adapter *);
 142 static void     ixgbe_update_stats_counters(struct adapter *);
 143 static bool     ixgbe_txeof(struct tx_ring *);
 144 static bool     ixgbe_rxeof(struct ix_queue *, int);
 145 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 146 static void     ixgbe_set_promisc(struct adapter *);
 147 static void     ixgbe_disable_promisc(struct adapter *);
 148 static void     ixgbe_set_multi(struct adapter *);
 149 static void     ixgbe_print_hw_stats(struct adapter *);
 150 static void     ixgbe_print_debug_info(struct adapter *);
 151 static void     ixgbe_update_link_status(struct adapter *);
 152 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 153 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 154 static int      ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS);
 155 static int      ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS);
 156 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 157 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 158                     struct ixgbe_dma_alloc *, int);
 159 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 160 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 161                     const char *, int *, int);
 162 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 163 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 164 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 165 static void     ixgbe_configure_ivars(struct adapter *);
 166 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 167
 168 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 169 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 170 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 171
 172 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 173 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 174                     struct mbuf *, u32);
 175
 176 /* Support for pluggable optic modules */
 177 static bool     ixgbe_sfp_probe(struct adapter *);
 178
 179 /* Legacy (single vector interrupt handler */
 180 static void     ixgbe_legacy_irq(void *);
 181
 182 /* The MSI/X Interrupt handlers */
 183 static void     ixgbe_msix_que(void *);
 184 static void     ixgbe_msix_link(void *);
 185
 186 /* Deferred interrupt tasklets */
 187 static void     ixgbe_handle_que(void *, int);
 188 static void     ixgbe_handle_link(void *, int);
 189 static void     ixgbe_handle_msf(void *, int);
 190 static void     ixgbe_handle_mod(void *, int);
 191
 192 #ifdef IXGBE_FDIR
 193 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 194 static void     ixgbe_reinit_fdir(void *, int);
 195 #endif
 196
 197 /*********************************************************************
 198  *  FreeBSD Device Interface Entry Points
 199  *********************************************************************/
 200
 201 static device_method_t ixgbe_methods[] = {
 202         /* Device interface */
 203         DEVMETHOD(device_probe, ixgbe_probe),
 204         DEVMETHOD(device_attach, ixgbe_attach),
 205         DEVMETHOD(device_detach, ixgbe_detach),
 206         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 207         {0, 0}
 208 };
 209
 210 static driver_t ixgbe_driver = {
 211         "ix", ixgbe_methods, sizeof(struct adapter),
 212 };
 213
 214 static devclass_t ixgbe_devclass;
 215 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 216
 217 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 218 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 219
 220 /*
 221 ** TUNEABLE PARAMETERS:
 222 */
 223
 224 /*
 225 ** AIM: Adaptive Interrupt Moderation
 226 ** which means that the interrupt rate
 227 ** is varied over time based on the
 228 ** traffic for that interrupt vector
 229 */
 230 static int ixgbe_enable_aim = TRUE;
 231 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 232
 233 /* How many packets rxeof tries to clean at a time */
 234 static int ixgbe_rx_process_limit = 128;
 235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 236
 237 /* Flow control setting, default to full */
 238 static int ixgbe_flow_control = ixgbe_fc_full;
 239 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
 240
 241 /*
 242 ** Smart speed setting, default to on
 243 ** this only works as a compile option
 244 ** right now as its during attach, set
 245 ** this to 'ixgbe_smart_speed_off' to
 246 ** disable.
 247 */
 248 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 249
 250 /*
 251  * MSIX should be the default for best performance,
 252  * but this allows it to be forced off for testing.
 253  */
 254 static int ixgbe_enable_msix = 1;
 255 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 256
 257 /*
 258  * Header split has seemed to be beneficial in
 259  * most circumstances tested, so its on by default
 260  * however this variable will allow it to be disabled
 261  * for some debug purposes.
 262  */
 263 static bool ixgbe_header_split = FALSE;
 264 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 265
 266 /*
 267  * Number of Queues, can be set to 0,
 268  * it then autoconfigures based on the
 269  * number of cpus. Each queue is a pair
 270  * of RX and TX rings with a msix vector
 271  */
 272 static int ixgbe_num_queues = 0;
 273 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 274
 275 /*
 276 ** Number of TX descriptors per ring,
 277 ** setting higher than RX as this seems
 278 ** the better performing choice.
 279 */
 280 static int ixgbe_txd = PERFORM_TXD;
 281 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 282
 283 /* Number of RX descriptors per ring */
 284 static int ixgbe_rxd = PERFORM_RXD;
 285 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 286
 287 /* Keep running tab on them for sanity check */
 288 static int ixgbe_total_ports;
 289
 290 /*
 291 ** Shadow VFTA table, this is needed because
 292 ** the real filter table gets cleared during
 293 ** a soft reset and we need to repopulate it.
 294 */
 295 static u32 ixgbe_shadow_vfta[IXGBE_VFTA_SIZE];
 296
 297 /*
 298 ** The number of scatter-gather segments
 299 ** differs for 82598 and 82599, default to
 300 ** the former.
 301 */
 302 static int ixgbe_num_segs = IXGBE_82598_SCATTER;
 303
 304 #ifdef IXGBE_FDIR
 305 /*
 306 ** For Flow Director: this is the
 307 ** number of TX packets we sample
 308 ** for the filter pool, this means
 309 ** every 20th packet will be probed.
 310 **
 311 ** This feature can be disabled by
 312 ** setting this to 0.
 313 */
 314 static int atr_sample_rate = 20;
 315 /*
 316 ** Flow Director actually 'steals'
 317 ** part of the packet buffer as its
 318 ** filter pool, this variable controls
 319 ** how much it uses:
 320 **  0 = 64K, 1 = 128K, 2 = 256K
 321 */
 322 static int fdir_pballoc = 1;
 323 #endif
 324
 325 /*********************************************************************
 326  *  Device identification routine
 327  *
 328  *  ixgbe_probe determines if the driver should be loaded on
 329  *  adapter based on PCI vendor/device id of the adapter.
 330  *
 331  *  return 0 on success, positive on failure
 332  *********************************************************************/
 333
 334 static int
 335 ixgbe_probe(device_t dev)
 336 {
 337         ixgbe_vendor_info_t *ent;
 338
 339         u16     pci_vendor_id = 0;
 340         u16     pci_device_id = 0;
 341         u16     pci_subvendor_id = 0;
 342         u16     pci_subdevice_id = 0;
 343         char    adapter_name[256];
 344
 345         INIT_DEBUGOUT("ixgbe_probe: begin");
 346
 347         pci_vendor_id = pci_get_vendor(dev);
 348         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 349                 return (ENXIO);
 350
 351         pci_device_id = pci_get_device(dev);
 352         pci_subvendor_id = pci_get_subvendor(dev);
 353         pci_subdevice_id = pci_get_subdevice(dev);
 354
 355         ent = ixgbe_vendor_info_array;
 356         while (ent->vendor_id != 0) {
 357                 if ((pci_vendor_id == ent->vendor_id) &&
 358                     (pci_device_id == ent->device_id) &&
 359
 360                     ((pci_subvendor_id == ent->subvendor_id) ||
 361                      (ent->subvendor_id == 0)) &&
 362
 363                     ((pci_subdevice_id == ent->subdevice_id) ||
 364                      (ent->subdevice_id == 0))) {
 365                         sprintf(adapter_name, "%s, Version - %s",
 366                                 ixgbe_strings[ent->index],
 367                                 ixgbe_driver_version);
 368                         device_set_desc_copy(dev, adapter_name);
 369                         ++ixgbe_total_ports;
 370                         return (0);
 371                 }
 372                 ent++;
 373         }
 374         return (ENXIO);
 375 }
 376
 377 /*********************************************************************
 378  *  Device initialization routine
 379  *
 380  *  The attach entry point is called when the driver is being loaded.
 381  *  This routine identifies the type of hardware, allocates all resources
 382  *  and initializes the hardware.
 383  *
 384  *  return 0 on success, positive on failure
 385  *********************************************************************/
 386
 387 static int
 388 ixgbe_attach(device_t dev)
 389 {
 390         struct adapter *adapter;
 391         struct ixgbe_hw *hw;
 392         int             error = 0;
 393         u16             pci_device_id, csum;
 394         u32             ctrl_ext;
 395
 396         INIT_DEBUGOUT("ixgbe_attach: begin");
 397
 398         /* Allocate, clear, and link in our adapter structure */
 399         adapter = device_get_softc(dev);
 400         adapter->dev = adapter->osdep.dev = dev;
 401         hw = &adapter->hw;
 402
 403         /* Core Lock Init*/
 404         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 405
 406         /* Keep track of optics */
 407         pci_device_id = pci_get_device(dev);
 408         switch (pci_device_id) {
 409                 case IXGBE_DEV_ID_82598_CX4_DUAL_PORT:
 410                 case IXGBE_DEV_ID_82598EB_CX4:
 411                         adapter->optics = IFM_10G_CX4;
 412                         break;
 413                 case IXGBE_DEV_ID_82598:
 414                 case IXGBE_DEV_ID_82598AF_DUAL_PORT:
 415                 case IXGBE_DEV_ID_82598AF_SINGLE_PORT:
 416                 case IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM:
 417                 case IXGBE_DEV_ID_82598EB_SFP_LOM:
 418                 case IXGBE_DEV_ID_82598AT:
 419                         adapter->optics = IFM_10G_SR;
 420                         break;
 421                 case IXGBE_DEV_ID_82598AT2:
 422                         adapter->optics = IFM_10G_T;
 423                         break;
 424                 case IXGBE_DEV_ID_82598EB_XF_LR:
 425                         adapter->optics = IFM_10G_LR;
 426                         break;
 427                 case IXGBE_DEV_ID_82599_SFP:
 428                         adapter->optics = IFM_10G_SR;
 429                         ixgbe_num_segs = IXGBE_82599_SCATTER;
 430                         break;
 431                 case IXGBE_DEV_ID_82598_DA_DUAL_PORT :
 432                         adapter->optics = IFM_10G_TWINAX;
 433                         break;
 434                 case IXGBE_DEV_ID_82599_KX4:
 435                 case IXGBE_DEV_ID_82599_KX4_MEZZ:
 436                 case IXGBE_DEV_ID_82599_CX4:
 437                         adapter->optics = IFM_10G_CX4;
 438                         ixgbe_num_segs = IXGBE_82599_SCATTER;
 439                         break;
 440                 case IXGBE_DEV_ID_82599_XAUI_LOM:
 441                 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
 442                         ixgbe_num_segs = IXGBE_82599_SCATTER;
 443                         break;
 444                 case IXGBE_DEV_ID_82599_T3_LOM:
 445                         ixgbe_num_segs = IXGBE_82599_SCATTER;
 446                         adapter->optics = IFM_10G_T;
 447                 default:
 448                         break;
 449         }
 450
 451         /* SYSCTL APIs */
 452         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 453                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 454                         OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW,
 455                         adapter, 0, ixgbe_sysctl_stats, "I", "Statistics");
 456
 457         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 458                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 459                         OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW,
 460                         adapter, 0, ixgbe_sysctl_debug, "I", "Debug Info");
 461
 462         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 463                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 464                         OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW,
 465                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 466
 467         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 468                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 469                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 470                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 471
 472         /* Set up the timer callout */
 473         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 474
 475         /* Determine hardware revision */
 476         ixgbe_identify_hardware(adapter);
 477
 478         /* Do base PCI setup - map BAR0 */
 479         if (ixgbe_allocate_pci_resources(adapter)) {
 480                 device_printf(dev, "Allocation of PCI resources failed\n");
 481                 error = ENXIO;
 482                 goto err_out;
 483         }
 484
 485         /* Do descriptor calc and sanity checks */
 486         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 487             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 488                 device_printf(dev, "TXD config issue, using default!\n");
 489                 adapter->num_tx_desc = DEFAULT_TXD;
 490         } else
 491                 adapter->num_tx_desc = ixgbe_txd;
 492
 493         /*
 494         ** With many RX rings it is easy to exceed the
 495         ** system mbuf allocation. Tuning nmbclusters
 496         ** can alleviate this.
 497         */
 498         if (nmbclusters > 0 ) {
 499                 int s;
 500                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 501                 if (s > nmbclusters) {
 502                         device_printf(dev, "RX Descriptors exceed "
 503                             "system mbuf max, using default instead!\n");
 504                         ixgbe_rxd = DEFAULT_RXD;
 505                 }
 506         }
 507
 508         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 509             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 510                 device_printf(dev, "RXD config issue, using default!\n");
 511                 adapter->num_rx_desc = DEFAULT_RXD;
 512         } else
 513                 adapter->num_rx_desc = ixgbe_rxd;
 514
 515         /* Allocate our TX/RX Queues */
 516         if (ixgbe_allocate_queues(adapter)) {
 517                 error = ENOMEM;
 518                 goto err_out;
 519         }
 520
 521         /* Initialize the shared code */
 522         error = ixgbe_init_shared_code(hw);
 523         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 524                 /*
 525                 ** No optics in this port, set up
 526                 ** so the timer routine will probe
 527                 ** for later insertion.
 528                 */
 529                 adapter->sfp_probe = TRUE;
 530                 error = 0;
 531         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 532                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 533                 error = EIO;
 534                 goto err_late;
 535         } else if (error) {
 536                 device_printf(dev,"Unable to initialize the shared code\n");
 537                 error = EIO;
 538                 goto err_late;
 539         }
 540
 541         /* Make sure we have a good EEPROM before we read from it */
 542         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 543                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 544                 error = EIO;
 545                 goto err_late;
 546         }
 547
 548         /* Pick up the smart speed setting */
 549         if (hw->mac.type == ixgbe_mac_82599EB)
 550                 hw->phy.smart_speed = ixgbe_smart_speed;
 551
 552         /* Get Hardware Flow Control setting */
 553         hw->fc.requested_mode = ixgbe_fc_full;
 554         hw->fc.pause_time = IXGBE_FC_PAUSE;
 555         hw->fc.low_water = IXGBE_FC_LO;
 556         hw->fc.high_water = IXGBE_FC_HI;
 557         hw->fc.send_xon = TRUE;
 558
 559         error = ixgbe_init_hw(hw);
 560         if (error == IXGBE_ERR_EEPROM_VERSION) {
 561                 device_printf(dev, "This device is a pre-production adapter/"
 562                     "LOM.  Please be aware there may be issues associated "
 563                     "with your hardware.\n If you are experiencing problems "
 564                     "please contact your Intel or hardware representative "
 565                     "who provided you with this hardware.\n");
 566         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED)
 567                 device_printf(dev,"Unsupported SFP+ Module\n");
 568
 569         if (error) {
 570                 error = EIO;
 571                 device_printf(dev,"Hardware Initialization Failure\n");
 572                 goto err_late;
 573         }
 574
 575         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 576                 error = ixgbe_allocate_msix(adapter);
 577         else
 578                 error = ixgbe_allocate_legacy(adapter);
 579         if (error)
 580                 goto err_late;
 581
 582         /* Setup OS specific network interface */
 583         ixgbe_setup_interface(dev, adapter);
 584
 585         /* Sysctl for limiting the amount of work done in the taskqueue */
 586         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 587             "max number of rx packets to process", &adapter->rx_process_limit,
 588             ixgbe_rx_process_limit);
 589
 590         /* Initialize statistics */
 591         ixgbe_update_stats_counters(adapter);
 592
 593         /* Register for VLAN events */
 594         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 595             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 596         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 597             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 598
 599         /* Print PCIE bus type/speed/width info */
 600         ixgbe_get_bus_info(hw);
 601         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 602             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 603             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 604             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 605             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 606             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 607             ("Unknown"));
 608
 609         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 610             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 611                 device_printf(dev, "PCI-Express bandwidth available"
 612                     " for this card\n     is not sufficient for"
 613                     " optimal performance.\n");
 614                 device_printf(dev, "For optimal performance a x8 "
 615                     "PCIE, or x4 PCIE 2 slot is required.\n");
 616         }
 617
 618         /* let hardware know driver is loaded */
 619         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 620         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 621         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 622
 623         INIT_DEBUGOUT("ixgbe_attach: end");
 624         return (0);
 625 err_late:
 626         ixgbe_free_transmit_structures(adapter);
 627         ixgbe_free_receive_structures(adapter);
 628 err_out:
 629         ixgbe_free_pci_resources(adapter);
 630         return (error);
 631
 632 }
 633
 634 /*********************************************************************
 635  *  Device removal routine
 636  *
 637  *  The detach entry point is called when the driver is being removed.
 638  *  This routine stops the adapter and deallocates all the resources
 639  *  that were allocated for driver operation.
 640  *
 641  *  return 0 on success, positive on failure
 642  *********************************************************************/
 643
 644 static int
 645 ixgbe_detach(device_t dev)
 646 {
 647         struct adapter *adapter = device_get_softc(dev);
 648         struct ix_queue *que = adapter->queues;
 649         u32     ctrl_ext;
 650
 651         INIT_DEBUGOUT("ixgbe_detach: begin");
 652
 653         /* Make sure VLANS are not using driver */
 654         if (adapter->ifp->if_vlantrunk != NULL) {
 655                 device_printf(dev,"Vlan in use, detach first\n");
 656                 return (EBUSY);
 657         }
 658
 659         IXGBE_CORE_LOCK(adapter);
 660         ixgbe_stop(adapter);
 661         IXGBE_CORE_UNLOCK(adapter);
 662
 663         for (int i = 0; i < adapter->num_queues; i++, que++) {
 664                 if (que->tq) {
 665                         taskqueue_drain(que->tq, &que->que_task);
 666                         taskqueue_free(que->tq);
 667                 }
 668         }
 669
 670         /* Drain the Link queue */
 671         if (adapter->tq) {
 672                 taskqueue_drain(adapter->tq, &adapter->link_task);
 673                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 674                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 675 #ifdef IXGBE_FDIR
 676                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 677 #endif
 678                 taskqueue_free(adapter->tq);
 679         }
 680
 681         /* let hardware know driver is unloading */
 682         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 683         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 684         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 685
 686         /* Unregister VLAN events */
 687         if (adapter->vlan_attach != NULL)
 688                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 689         if (adapter->vlan_detach != NULL)
 690                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 691
 692         ether_ifdetach(adapter->ifp);
 693         callout_drain(&adapter->timer);
 694         ixgbe_free_pci_resources(adapter);
 695         bus_generic_detach(dev);
 696         if_free(adapter->ifp);
 697
 698         ixgbe_free_transmit_structures(adapter);
 699         ixgbe_free_receive_structures(adapter);
 700
 701         IXGBE_CORE_LOCK_DESTROY(adapter);
 702         return (0);
 703 }
 704
 705 /*********************************************************************
 706  *
 707  *  Shutdown entry point
 708  *
 709  **********************************************************************/
 710
 711 static int
 712 ixgbe_shutdown(device_t dev)
 713 {
 714         struct adapter *adapter = device_get_softc(dev);
 715         IXGBE_CORE_LOCK(adapter);
 716         ixgbe_stop(adapter);
 717         IXGBE_CORE_UNLOCK(adapter);
 718         return (0);
 719 }
 720
 721
 722 /*********************************************************************
 723  *  Transmit entry point
 724  *
 725  *  ixgbe_start is called by the stack to initiate a transmit.
 726  *  The driver will remain in this routine as long as there are
 727  *  packets to transmit and transmit resources are available.
 728  *  In case resources are not available stack is notified and
 729  *  the packet is requeued.
 730  **********************************************************************/
 731
 732 static void
 733 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 734 {
 735         struct mbuf    *m_head;
 736         struct adapter *adapter = txr->adapter;
 737
 738         IXGBE_TX_LOCK_ASSERT(txr);
 739
 740         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 741             IFF_DRV_RUNNING)
 742                 return;
 743         if (!adapter->link_active)
 744                 return;
 745
 746         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 747
 748                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 749                 if (m_head == NULL)
 750                         break;
 751
 752                 if (ixgbe_xmit(txr, &m_head)) {
 753                         if (m_head == NULL)
 754                                 break;
 755                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 756                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 757                         break;
 758                 }
 759                 /* Send a copy of the frame to the BPF listener */
 760                 ETHER_BPF_MTAP(ifp, m_head);
 761
 762                 /* Set watchdog on */
 763                 txr->watchdog_check = TRUE;
 764                 txr->watchdog_time = ticks;
 765
 766         }
 767         return;
 768 }
 769
 770 /*
 771  * Legacy TX start - called by the stack, this
 772  * always uses the first tx ring, and should
 773  * not be used with multiqueue tx enabled.
 774  */
 775 static void
 776 ixgbe_start(struct ifnet *ifp)
 777 {
 778         struct adapter *adapter = ifp->if_softc;
 779         struct tx_ring  *txr = adapter->tx_rings;
 780
 781         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 782                 IXGBE_TX_LOCK(txr);
 783                 ixgbe_start_locked(txr, ifp);
 784                 IXGBE_TX_UNLOCK(txr);
 785         }
 786         return;
 787 }
 788
 789 #if __FreeBSD_version >= 800000
 790 /*
 791 ** Multiqueue Transmit driver
 792 **
 793 */
 794 static int
 795 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 796 {
 797         struct adapter  *adapter = ifp->if_softc;
 798         struct tx_ring  *txr;
 799         int             i = 0, err = 0;
 800
 801         /* Which queue to use */
 802         if ((m->m_flags & M_FLOWID) != 0)
 803                 i = m->m_pkthdr.flowid % adapter->num_queues;
 804
 805         txr = &adapter->tx_rings[i];
 806
 807         if (IXGBE_TX_TRYLOCK(txr)) {
 808                 err = ixgbe_mq_start_locked(ifp, txr, m);
 809                 IXGBE_TX_UNLOCK(txr);
 810         } else
 811                 err = drbr_enqueue(ifp, txr->br, m);
 812
 813         return (err);
 814 }
 815
 816 static int
 817 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 818 {
 819         struct adapter  *adapter = txr->adapter;
 820         struct mbuf     *next;
 821         int             enqueued, err = 0;
 822
 823         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 824             IFF_DRV_RUNNING || adapter->link_active == 0) {
 825                 if (m != NULL)
 826                         err = drbr_enqueue(ifp, txr->br, m);
 827                 return (err);
 828         }
 829
 830         enqueued = 0;
 831         if (m == NULL) {
 832                 next = drbr_dequeue(ifp, txr->br);
 833         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 834                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 835                         return (err);
 836                 next = drbr_dequeue(ifp, txr->br);
 837         } else
 838                 next = m;
 839
 840         /* Process the queue */
 841         while (next != NULL) {
 842                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 843                         if (next != NULL)
 844                                 err = drbr_enqueue(ifp, txr->br, next);
 845                         break;
 846                 }
 847                 enqueued++;
 848                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 849                 /* Send a copy of the frame to the BPF listener */
 850                 ETHER_BPF_MTAP(ifp, next);
 851                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 852                         break;
 853                 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) {
 854                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 855                         break;
 856                 }
 857                 next = drbr_dequeue(ifp, txr->br);
 858         }
 859
 860         if (enqueued > 0) {
 861                 /* Set watchdog on */
 862                 txr->watchdog_check = TRUE;
 863                 txr->watchdog_time = ticks;
 864         }
 865
 866         return (err);
 867 }
 868
 869 /*
 870 ** Flush all ring buffers
 871 */
 872 static void
 873 ixgbe_qflush(struct ifnet *ifp)
 874 {
 875         struct adapter  *adapter = ifp->if_softc;
 876         struct tx_ring  *txr = adapter->tx_rings;
 877         struct mbuf     *m;
 878
 879         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 880                 IXGBE_TX_LOCK(txr);
 881                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 882                         m_freem(m);
 883                 IXGBE_TX_UNLOCK(txr);
 884         }
 885         if_qflush(ifp);
 886 }
 887 #endif /* __FreeBSD_version >= 800000 */
 888
 889 /*********************************************************************
 890  *  Ioctl entry point
 891  *
 892  *  ixgbe_ioctl is called when the user wants to configure the
 893  *  interface.
 894  *
 895  *  return 0 on success, positive on failure
 896  **********************************************************************/
 897
 898 static int
 899 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 900 {
 901         struct adapter  *adapter = ifp->if_softc;
 902         struct ifreq    *ifr = (struct ifreq *) data;
 903         int             error = 0;
 904
 905         switch (command) {
 906
 907         case SIOCSIFMTU:
 908                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 909                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 910                         error = EINVAL;
 911                 } else {
 912                         IXGBE_CORE_LOCK(adapter);
 913                         ifp->if_mtu = ifr->ifr_mtu;
 914                         adapter->max_frame_size =
 915                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 916                         error = ixgbe_init_locked(adapter);
 917                         IXGBE_CORE_UNLOCK(adapter);
 918                 }
 919                 break;
 920         case SIOCSIFFLAGS:
 921                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 922                 IXGBE_CORE_LOCK(adapter);
 923                 if (ifp->if_flags & IFF_UP) {
 924                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 925                                 if ((ifp->if_flags ^ adapter->if_flags) &
 926                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 927                                         ixgbe_disable_promisc(adapter);
 928                                         ixgbe_set_promisc(adapter);
 929                                 }
 930                         } else
 931                                 error = ixgbe_init_locked(adapter);
 932                 } else
 933                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 934                                 ixgbe_stop(adapter);
 935                 adapter->if_flags = ifp->if_flags;
 936                 IXGBE_CORE_UNLOCK(adapter);
 937                 break;
 938         case SIOCADDMULTI:
 939         case SIOCDELMULTI:
 940                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 941                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 942                         IXGBE_CORE_LOCK(adapter);
 943                         ixgbe_disable_intr(adapter);
 944                         ixgbe_set_multi(adapter);
 945                         ixgbe_enable_intr(adapter);
 946                         IXGBE_CORE_UNLOCK(adapter);
 947                 }
 948                 break;
 949         case SIOCSIFMEDIA:
 950         case SIOCGIFMEDIA:
 951                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 952                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 953                 break;
 954         case SIOCSIFCAP:
 955         {
 956                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 957                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 958                 if (mask & IFCAP_HWCSUM)
 959                         ifp->if_capenable ^= IFCAP_HWCSUM;
 960                 if (mask & IFCAP_TSO4)
 961                         ifp->if_capenable ^= IFCAP_TSO4;
 962                 if (mask & IFCAP_LRO)
 963                         ifp->if_capenable ^= IFCAP_LRO;
 964                 if (mask & IFCAP_VLAN_HWTAGGING)
 965                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 966                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 967                         IXGBE_CORE_LOCK(adapter);
 968                         error = ixgbe_init_locked(adapter);
 969                         IXGBE_CORE_UNLOCK(adapter);
 970                 }
 971                 VLAN_CAPABILITIES(ifp);
 972                 break;
 973         }
 974
 975         default:
 976                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 977                 error = ether_ioctl(ifp, command, data);
 978                 break;
 979         }
 980
 981         return (error);
 982 }
 983
 984 /*********************************************************************
 985  *  Init entry point
 986  *
 987  *  This routine is used in two ways. It is used by the stack as
 988  *  init entry point in network interface structure. It is also used
 989  *  by the driver as a hw/sw initialization routine to get to a
 990  *  consistent state.
 991  *
 992  *  return 0 on success, positive on failure
 993  **********************************************************************/
 994 #define IXGBE_MHADD_MFS_SHIFT 16
 995
 996 static int
 997 ixgbe_init_locked(struct adapter *adapter)
 998 {
 999         struct ifnet   *ifp = adapter->ifp;
1000         device_t        dev = adapter->dev;
1001         struct ixgbe_hw *hw = &adapter->hw;
1002         u32             k, txdctl, mhadd, gpie;
1003         u32             rxdctl, rxctrl;
1004
1005         mtx_assert(&adapter->core_mtx, MA_OWNED);
1006         INIT_DEBUGOUT("ixgbe_init: begin");
1007         ixgbe_reset_hw(hw);
1008         hw->adapter_stopped = FALSE;
1009         ixgbe_stop_adapter(hw);
1010         callout_stop(&adapter->timer);
1011
1012         /* reprogram the RAR[0] in case user changed it. */
1013         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1014
1015         /* Get the latest mac address, User can use a LAA */
1016         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1017               IXGBE_ETH_LENGTH_OF_ADDRESS);
1018         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1019         hw->addr_ctrl.rar_used_count = 1;
1020
1021         /* Prepare transmit descriptors and buffers */
1022         if (ixgbe_setup_transmit_structures(adapter)) {
1023                 device_printf(dev,"Could not setup transmit structures\n");
1024                 ixgbe_stop(adapter);
1025                 return (ENOMEM);
1026         }
1027
1028         ixgbe_init_hw(hw);
1029         ixgbe_initialize_transmit_units(adapter);
1030
1031         /* Setup Multicast table */
1032         ixgbe_set_multi(adapter);
1033
1034         /*
1035         ** Determine the correct mbuf pool
1036         ** for doing jumbo/headersplit
1037         */
1038         if (ifp->if_mtu > ETHERMTU)
1039                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1040         else
1041                 adapter->rx_mbuf_sz = MCLBYTES;
1042
1043         /* Prepare receive descriptors and buffers */
1044         if (ixgbe_setup_receive_structures(adapter)) {
1045                 device_printf(dev,"Could not setup receive structures\n");
1046                 ixgbe_stop(adapter);
1047                 return (ENOMEM);
1048         }
1049
1050         /* Configure RX settings */
1051         ixgbe_initialize_receive_units(adapter);
1052
1053         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1054
1055         if (hw->mac.type == ixgbe_mac_82599EB) {
1056                 gpie |= IXGBE_SDP1_GPIEN;
1057                 gpie |= IXGBE_SDP2_GPIEN;
1058         }
1059
1060         /* Enable Fan Failure Interrupt */
1061         if (hw->device_id == IXGBE_DEV_ID_82598AT)
1062                 gpie |= IXGBE_SDP1_GPIEN;
1063
1064         if (adapter->msix > 1) {
1065                 /* Enable Enhanced MSIX mode */
1066                 gpie |= IXGBE_GPIE_MSIX_MODE;
1067                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1068                     IXGBE_GPIE_OCD;
1069         }
1070         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1071
1072         /* Set the various hardware offload abilities */
1073         ifp->if_hwassist = 0;
1074         if (ifp->if_capenable & IFCAP_TSO4)
1075                 ifp->if_hwassist |= CSUM_TSO;
1076         if (ifp->if_capenable & IFCAP_TXCSUM)
1077                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1078 #if __FreeBSD_version >= 800000
1079         if (hw->mac.type == ixgbe_mac_82599EB)
1080                 ifp->if_hwassist |= CSUM_SCTP;
1081 #endif
1082         /* Set MTU size */
1083         if (ifp->if_mtu > ETHERMTU) {
1084                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1085                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1086                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1087                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1088         }
1089
1090         /* Now enable all the queues */
1091
1092         for (int i = 0; i < adapter->num_queues; i++) {
1093                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1094                 txdctl |= IXGBE_TXDCTL_ENABLE;
1095                 /* Set WTHRESH to 8, burst writeback */
1096                 txdctl |= (8 << 16);
1097                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1098         }
1099
1100         for (int i = 0; i < adapter->num_queues; i++) {
1101                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1102                 if (hw->mac.type == ixgbe_mac_82598EB) {
1103                         /*
1104                         ** PTHRESH = 21
1105                         ** HTHRESH = 4
1106                         ** WTHRESH = 8
1107                         */
1108                         rxdctl &= ~0x3FFFFF;
1109                         rxdctl |= 0x080420;
1110                 }
1111                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1112                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1113                 for (k = 0; k < 10; k++) {
1114                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1115                             IXGBE_RXDCTL_ENABLE)
1116                                 break;
1117                         else
1118                                 msec_delay(1);
1119                 }
1120                 wmb();
1121                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1122         }
1123
1124         /* Set up VLAN offloads and filter */
1125         ixgbe_setup_vlan_hw_support(adapter);
1126
1127         /* Enable Receive engine */
1128         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1129         if (hw->mac.type == ixgbe_mac_82598EB)
1130                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1131         rxctrl |= IXGBE_RXCTRL_RXEN;
1132         ixgbe_enable_rx_dma(hw, rxctrl);
1133
1134         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1135
1136         /* Set up MSI/X routing */
1137         if (ixgbe_enable_msix)  {
1138                 ixgbe_configure_ivars(adapter);
1139                 /* Set up auto-mask */
1140                 if (hw->mac.type == ixgbe_mac_82598EB)
1141                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1142                 else {
1143                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1144                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1145                 }
1146         } else {  /* Simple settings for Legacy/MSI */
1147                 ixgbe_set_ivar(adapter, 0, 0, 0);
1148                 ixgbe_set_ivar(adapter, 0, 0, 1);
1149                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1150         }
1151
1152 #ifdef IXGBE_FDIR
1153         /* Init Flow director */
1154         if (hw->mac.type == ixgbe_mac_82599EB)
1155                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1156 #endif
1157
1158         /*
1159         ** Check on any SFP devices that
1160         ** need to be kick-started
1161         */
1162         if (hw->phy.type == ixgbe_phy_none) {
1163                 int err = hw->phy.ops.identify(hw);
1164                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1165                         device_printf(dev,
1166                             "Unsupported SFP+ module type was detected.\n");
1167                         return (EIO);
1168                 }
1169         }
1170
1171         /* Set moderation on the Link interrupt */
1172         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1173
1174         /* Config/Enable Link */
1175         ixgbe_config_link(adapter);
1176
1177         /* And now turn on interrupts */
1178         ixgbe_enable_intr(adapter);
1179
1180         /* Now inform the stack we're ready */
1181         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1182         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1183
1184         return (0);
1185 }
1186
1187 static void
1188 ixgbe_init(void *arg)
1189 {
1190         struct adapter *adapter = arg;
1191
1192         IXGBE_CORE_LOCK(adapter);
1193         ixgbe_init_locked(adapter);
1194         IXGBE_CORE_UNLOCK(adapter);
1195         return;
1196 }
1197
1198
1199 /*
1200 **
1201 ** MSIX Interrupt Handlers and Tasklets
1202 **
1203 */
1204
1205 static inline void
1206 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1207 {
1208         struct ixgbe_hw *hw = &adapter->hw;
1209         u64     queue = (u64)(1 << vector);
1210         u32     mask;
1211
1212         if (hw->mac.type == ixgbe_mac_82598EB) {
1213                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1214                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1215         } else {
1216                 mask = (queue & 0xFFFFFFFF);
1217                 if (mask)
1218                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1219                 mask = (queue >> 32);
1220                 if (mask)
1221                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1222         }
1223 }
1224
1225 static inline void
1226 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1227 {
1228         struct ixgbe_hw *hw = &adapter->hw;
1229         u64     queue = (u64)(1 << vector);
1230         u32     mask;
1231
1232         if (hw->mac.type == ixgbe_mac_82598EB) {
1233                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1234                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1235         } else {
1236                 mask = (queue & 0xFFFFFFFF);
1237                 if (mask)
1238                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1239                 mask = (queue >> 32);
1240                 if (mask)
1241                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1242         }
1243 }
1244
1245 static inline void
1246 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1247 {
1248         u32 mask;
1249
1250         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1251                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1252                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1253         } else {
1254                 mask = (queues & 0xFFFFFFFF);
1255                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1256                 mask = (queues >> 32);
1257                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1258         }
1259 }
1260
1261
1262 static void
1263 ixgbe_handle_que(void *context, int pending)
1264 {
1265         struct ix_queue *que = context;
1266         struct adapter  *adapter = que->adapter;
1267         struct tx_ring  *txr = que->txr;
1268         struct ifnet    *ifp = adapter->ifp;
1269         bool            more;
1270
1271         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1272                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1273                 IXGBE_TX_LOCK(txr);
1274                 ixgbe_txeof(txr);
1275 #if __FreeBSD_version >= 800000
1276                 if (!drbr_empty(ifp, txr->br))
1277                         ixgbe_mq_start_locked(ifp, txr, NULL);
1278 #else
1279                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1280                         ixgbe_start_locked(txr, ifp);
1281 #endif
1282                 IXGBE_TX_UNLOCK(txr);
1283                 if (more) {
1284                         taskqueue_enqueue(que->tq, &que->que_task);
1285                         return;
1286                 }
1287         }
1288
1289         /* Reenable this interrupt */
1290         ixgbe_enable_queue(adapter, que->msix);
1291         return;
1292 }
1293
1294
1295 /*********************************************************************
1296  *
1297  *  Legacy Interrupt Service routine
1298  *
1299  **********************************************************************/
1300
1301 static void
1302 ixgbe_legacy_irq(void *arg)
1303 {
1304         struct ix_queue *que = arg;
1305         struct adapter  *adapter = que->adapter;
1306         struct ixgbe_hw *hw = &adapter->hw;
1307         struct          tx_ring *txr = adapter->tx_rings;
1308         bool            more_tx, more_rx;
1309         u32             reg_eicr, loop = MAX_LOOP;
1310
1311
1312         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1313
1314         ++que->irqs;
1315         if (reg_eicr == 0) {
1316                 ixgbe_enable_intr(adapter);
1317                 return;
1318         }
1319
1320         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1321
1322         IXGBE_TX_LOCK(txr);
1323         do {
1324                 more_tx = ixgbe_txeof(txr);
1325         } while (loop-- && more_tx);
1326         IXGBE_TX_UNLOCK(txr);
1327
1328         if (more_rx || more_tx)
1329                 taskqueue_enqueue(que->tq, &que->que_task);
1330
1331         /* Check for fan failure */
1332         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1333             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1334                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1335                     "REPLACE IMMEDIATELY!!\n");
1336                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1337         }
1338
1339         /* Link status change */
1340         if (reg_eicr & IXGBE_EICR_LSC)
1341                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1342
1343         ixgbe_enable_intr(adapter);
1344         return;
1345 }
1346
1347
1348 /*********************************************************************
1349  *
1350  *  MSI Queue Interrupt Service routine
1351  *
1352  **********************************************************************/
1353 void
1354 ixgbe_msix_que(void *arg)
1355 {
1356         struct ix_queue *que = arg;
1357         struct adapter  *adapter = que->adapter;
1358         struct tx_ring  *txr = que->txr;
1359         struct rx_ring  *rxr = que->rxr;
1360         bool            more_tx, more_rx;
1361         u32             newitr = 0;
1362
1363         ixgbe_disable_queue(adapter, que->msix);
1364         ++que->irqs;
1365
1366         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1367
1368         IXGBE_TX_LOCK(txr);
1369         more_tx = ixgbe_txeof(txr);
1370         IXGBE_TX_UNLOCK(txr);
1371
1372         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1373
1374         /* Do AIM now? */
1375
1376         if (ixgbe_enable_aim == FALSE)
1377                 goto no_calc;
1378         /*
1379         ** Do Adaptive Interrupt Moderation:
1380         **  - Write out last calculated setting
1381         **  - Calculate based on average size over
1382         **    the last interval.
1383         */
1384         if (que->eitr_setting)
1385                 IXGBE_WRITE_REG(&adapter->hw,
1386                     IXGBE_EITR(que->msix), que->eitr_setting);
1387
1388         que->eitr_setting = 0;
1389
1390         /* Idle, do nothing */
1391         if ((txr->bytes == 0) && (rxr->bytes == 0))
1392                 goto no_calc;
1393
1394         if ((txr->bytes) && (txr->packets))
1395                 newitr = txr->bytes/txr->packets;
1396         if ((rxr->bytes) && (rxr->packets))
1397                 newitr = max(newitr,
1398                     (rxr->bytes / rxr->packets));
1399         newitr += 24; /* account for hardware frame, crc */
1400
1401         /* set an upper boundary */
1402         newitr = min(newitr, 3000);
1403
1404         /* Be nice to the mid range */
1405         if ((newitr > 300) && (newitr < 1200))
1406                 newitr = (newitr / 3);
1407         else
1408                 newitr = (newitr / 2);
1409
1410         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1411                 newitr |= newitr << 16;
1412         else
1413                 newitr |= IXGBE_EITR_CNT_WDIS;
1414
1415         /* save for next interrupt */
1416         que->eitr_setting = newitr;
1417
1418         /* Reset state */
1419         txr->bytes = 0;
1420         txr->packets = 0;
1421         rxr->bytes = 0;
1422         rxr->packets = 0;
1423
1424 no_calc:
1425         if (more_tx || more_rx)
1426                 taskqueue_enqueue(que->tq, &que->que_task);
1427         else /* Reenable this interrupt */
1428                 ixgbe_enable_queue(adapter, que->msix);
1429         return;
1430 }
1431
1432
1433 static void
1434 ixgbe_msix_link(void *arg)
1435 {
1436         struct adapter  *adapter = arg;
1437         struct ixgbe_hw *hw = &adapter->hw;
1438         u32             reg_eicr;
1439
1440         ++adapter->link_irq;
1441
1442         /* First get the cause */
1443         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1444         /* Clear interrupt with write */
1445         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1446
1447         /* Link status change */
1448         if (reg_eicr & IXGBE_EICR_LSC)
1449                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1450
1451         if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
1452 #ifdef IXGBE_FDIR
1453                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1454                         /* This is probably overkill :) */
1455                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1456                                 return;
1457                         /* Clear the interrupt */
1458                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR);
1459                         /* Turn off the interface */
1460                         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1461                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1462                 } else
1463 #endif
1464                 if (reg_eicr & IXGBE_EICR_ECC) {
1465                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1466                             "Please Reboot!!\n");
1467                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1468                 } else
1469
1470                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1471                         /* Clear the interrupt */
1472                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1473                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1474                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1475                         /* Clear the interrupt */
1476                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1477                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1478                 }
1479         }
1480
1481         /* Check for fan failure */
1482         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1483             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1484                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1485                     "REPLACE IMMEDIATELY!!\n");
1486                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1487         }
1488
1489         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1490         return;
1491 }
1492
1493 /*********************************************************************
1494  *
1495  *  Media Ioctl callback
1496  *
1497  *  This routine is called whenever the user queries the status of
1498  *  the interface using ifconfig.
1499  *
1500  **********************************************************************/
1501 static void
1502 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1503 {
1504         struct adapter *adapter = ifp->if_softc;
1505
1506         INIT_DEBUGOUT("ixgbe_media_status: begin");
1507         IXGBE_CORE_LOCK(adapter);
1508         ixgbe_update_link_status(adapter);
1509
1510         ifmr->ifm_status = IFM_AVALID;
1511         ifmr->ifm_active = IFM_ETHER;
1512
1513         if (!adapter->link_active) {
1514                 IXGBE_CORE_UNLOCK(adapter);
1515                 return;
1516         }
1517
1518         ifmr->ifm_status |= IFM_ACTIVE;
1519
1520         switch (adapter->link_speed) {
1521                 case IXGBE_LINK_SPEED_1GB_FULL:
1522                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1523                         break;
1524                 case IXGBE_LINK_SPEED_10GB_FULL:
1525                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1526                         break;
1527         }
1528
1529         IXGBE_CORE_UNLOCK(adapter);
1530
1531         return;
1532 }
1533
1534 /*********************************************************************
1535  *
1536  *  Media Ioctl callback
1537  *
1538  *  This routine is called when the user changes speed/duplex using
1539  *  media/mediopt option with ifconfig.
1540  *
1541  **********************************************************************/
1542 static int
1543 ixgbe_media_change(struct ifnet * ifp)
1544 {
1545         struct adapter *adapter = ifp->if_softc;
1546         struct ifmedia *ifm = &adapter->media;
1547
1548         INIT_DEBUGOUT("ixgbe_media_change: begin");
1549
1550         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1551                 return (EINVAL);
1552
1553         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1554         case IFM_AUTO:
1555                 adapter->hw.phy.autoneg_advertised =
1556                     IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL;
1557                 break;
1558         default:
1559                 device_printf(adapter->dev, "Only auto media type\n");
1560                 return (EINVAL);
1561         }
1562
1563         return (0);
1564 }
1565
1566 /*********************************************************************
1567  *
1568  *  This routine maps the mbufs to tx descriptors, allowing the
1569  *  TX engine to transmit the packets.
1570  *      - return 0 on success, positive on failure
1571  *
1572  **********************************************************************/
1573
1574 static int
1575 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1576 {
1577         struct adapter  *adapter = txr->adapter;
1578         u32             olinfo_status = 0, cmd_type_len;
1579         u32             paylen = 0;
1580         int             i, j, error, nsegs;
1581         int             first, last = 0;
1582         struct mbuf     *m_head;
1583         bus_dma_segment_t segs[ixgbe_num_segs];
1584         bus_dmamap_t    map;
1585         struct ixgbe_tx_buf *txbuf, *txbuf_mapped;
1586         union ixgbe_adv_tx_desc *txd = NULL;
1587
1588         m_head = *m_headp;
1589
1590         /* Basic descriptor defines */
1591         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1592             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1593
1594         if (m_head->m_flags & M_VLANTAG)
1595                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1596
1597         /*
1598          * Important to capture the first descriptor
1599          * used because it will contain the index of
1600          * the one we tell the hardware to report back
1601          */
1602         first = txr->next_avail_desc;
1603         txbuf = &txr->tx_buffers[first];
1604         txbuf_mapped = txbuf;
1605         map = txbuf->map;
1606
1607         /*
1608          * Map the packet for DMA.
1609          */
1610         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1611             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1612
1613         if (error == EFBIG) {
1614                 struct mbuf *m;
1615
1616                 m = m_defrag(*m_headp, M_DONTWAIT);
1617                 if (m == NULL) {
1618                         adapter->mbuf_defrag_failed++;
1619                         m_freem(*m_headp);
1620                         *m_headp = NULL;
1621                         return (ENOBUFS);
1622                 }
1623                 *m_headp = m;
1624
1625                 /* Try it again */
1626                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1627                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1628
1629                 if (error == ENOMEM) {
1630                         adapter->no_tx_dma_setup++;
1631                         return (error);
1632                 } else if (error != 0) {
1633                         adapter->no_tx_dma_setup++;
1634                         m_freem(*m_headp);
1635                         *m_headp = NULL;
1636                         return (error);
1637                 }
1638         } else if (error == ENOMEM) {
1639                 adapter->no_tx_dma_setup++;
1640                 return (error);
1641         } else if (error != 0) {
1642                 adapter->no_tx_dma_setup++;
1643                 m_freem(*m_headp);
1644                 *m_headp = NULL;
1645                 return (error);
1646         }
1647
1648         /* Make certain there are enough descriptors */
1649         if (nsegs > txr->tx_avail - 2) {
1650                 txr->no_desc_avail++;
1651                 error = ENOBUFS;
1652                 goto xmit_fail;
1653         }
1654         m_head = *m_headp;
1655
1656         /*
1657         ** Set up the appropriate offload context
1658         ** this becomes the first descriptor of
1659         ** a packet.
1660         */
1661         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1662                 if (ixgbe_tso_setup(txr, m_head, &paylen)) {
1663                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1664                         olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1665                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1666                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1667                         ++adapter->tso_tx;
1668                 } else
1669                         return (ENXIO);
1670         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1671                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1672
1673 #ifdef IXGBE_IEEE1588
1674         /* This is changing soon to an mtag detection */
1675         if (we detect this mbuf has a TSTAMP mtag)
1676                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1677 #endif
1678
1679 #ifdef IXGBE_FDIR
1680         /* Do the flow director magic */
1681         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1682                 ++txr->atr_count;
1683                 if (txr->atr_count >= atr_sample_rate) {
1684                         ixgbe_atr(txr, m_head);
1685                         txr->atr_count = 0;
1686                 }
1687         }
1688 #endif
1689         /* Record payload length */
1690         if (paylen == 0)
1691                 olinfo_status |= m_head->m_pkthdr.len <<
1692                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1693
1694         i = txr->next_avail_desc;
1695         for (j = 0; j < nsegs; j++) {
1696                 bus_size_t seglen;
1697                 bus_addr_t segaddr;
1698
1699                 txbuf = &txr->tx_buffers[i];
1700                 txd = &txr->tx_base[i];
1701                 seglen = segs[j].ds_len;
1702                 segaddr = htole64(segs[j].ds_addr);
1703
1704                 txd->read.buffer_addr = segaddr;
1705                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1706                     cmd_type_len |seglen);
1707                 txd->read.olinfo_status = htole32(olinfo_status);
1708                 last = i; /* descriptor that will get completion IRQ */
1709
1710                 if (++i == adapter->num_tx_desc)
1711                         i = 0;
1712
1713                 txbuf->m_head = NULL;
1714                 txbuf->eop_index = -1;
1715         }
1716
1717         txd->read.cmd_type_len |=
1718             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1719         txr->tx_avail -= nsegs;
1720         txr->next_avail_desc = i;
1721
1722         txbuf->m_head = m_head;
1723         txbuf->map = map;
1724         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1725
1726         /* Set the index of the descriptor that will be marked done */
1727         txbuf = &txr->tx_buffers[first];
1728         txbuf->eop_index = last;
1729
1730         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1731             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1732         /*
1733          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1734          * hardware that this frame is available to transmit.
1735          */
1736         ++txr->total_packets;
1737         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1738
1739         /* Do a clean if descriptors are low */
1740         if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD)
1741                 ixgbe_txeof(txr);
1742
1743         return (0);
1744
1745 xmit_fail:
1746         bus_dmamap_unload(txr->txtag, txbuf->map);
1747         return (error);
1748
1749 }
1750
1751 static void
1752 ixgbe_set_promisc(struct adapter *adapter)
1753 {
1754
1755         u_int32_t       reg_rctl;
1756         struct ifnet   *ifp = adapter->ifp;
1757
1758         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1759
1760         if (ifp->if_flags & IFF_PROMISC) {
1761                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1762                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1763         } else if (ifp->if_flags & IFF_ALLMULTI) {
1764                 reg_rctl |= IXGBE_FCTRL_MPE;
1765                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1766                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1767         }
1768         return;
1769 }
1770
1771 static void
1772 ixgbe_disable_promisc(struct adapter * adapter)
1773 {
1774         u_int32_t       reg_rctl;
1775
1776         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1777
1778         reg_rctl &= (~IXGBE_FCTRL_UPE);
1779         reg_rctl &= (~IXGBE_FCTRL_MPE);
1780         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1781
1782         return;
1783 }
1784
1785
1786 /*********************************************************************
1787  *  Multicast Update
1788  *
1789  *  This routine is called whenever multicast address list is updated.
1790  *
1791  **********************************************************************/
1792 #define IXGBE_RAR_ENTRIES 16
1793
1794 static void
1795 ixgbe_set_multi(struct adapter *adapter)
1796 {
1797         u32     fctrl;
1798         u8      mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
1799         u8      *update_ptr;
1800         struct  ifmultiaddr *ifma;
1801         int     mcnt = 0;
1802         struct ifnet   *ifp = adapter->ifp;
1803
1804         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1805
1806         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1807         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1808         if (ifp->if_flags & IFF_PROMISC)
1809                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1810         else if (ifp->if_flags & IFF_ALLMULTI) {
1811                 fctrl |= IXGBE_FCTRL_MPE;
1812                 fctrl &= ~IXGBE_FCTRL_UPE;
1813         } else
1814                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1815
1816         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1817
1818 #if __FreeBSD_version < 800000
1819         IF_ADDR_LOCK(ifp);
1820 #else
1821         if_maddr_rlock(ifp);
1822 #endif
1823         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1824                 if (ifma->ifma_addr->sa_family != AF_LINK)
1825                         continue;
1826                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1827                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1828                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1829                 mcnt++;
1830         }
1831 #if __FreeBSD_version < 800000
1832         IF_ADDR_UNLOCK(ifp);
1833 #else
1834         if_maddr_runlock(ifp);
1835 #endif
1836
1837         update_ptr = mta;
1838         ixgbe_update_mc_addr_list(&adapter->hw,
1839             update_ptr, mcnt, ixgbe_mc_array_itr);
1840
1841         return;
1842 }
1843
1844 /*
1845  * This is an iterator function now needed by the multicast
1846  * shared code. It simply feeds the shared code routine the
1847  * addresses in the array of ixgbe_set_multi() one by one.
1848  */
1849 static u8 *
1850 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1851 {
1852         u8 *addr = *update_ptr;
1853         u8 *newptr;
1854         *vmdq = 0;
1855
1856         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1857         *update_ptr = newptr;
1858         return addr;
1859 }
1860
1861
1862 /*********************************************************************
1863  *  Timer routine
1864  *
1865  *  This routine checks for link status,updates statistics,
1866  *  and runs the watchdog check.
1867  *
1868  **********************************************************************/
1869
1870 static void
1871 ixgbe_local_timer(void *arg)
1872 {
1873         struct adapter *adapter = arg;
1874         struct ifnet   *ifp = adapter->ifp;
1875         device_t        dev = adapter->dev;
1876         struct tx_ring *txr = adapter->tx_rings;
1877
1878         mtx_assert(&adapter->core_mtx, MA_OWNED);
1879
1880         /* Check for pluggable optics */
1881         if (adapter->sfp_probe)
1882                 if (!ixgbe_sfp_probe(adapter))
1883                         goto out; /* Nothing to do */
1884
1885         ixgbe_update_link_status(adapter);
1886         ixgbe_update_stats_counters(adapter);
1887
1888         /* Debug display */
1889         if (ixgbe_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
1890                 ixgbe_print_hw_stats(adapter);
1891
1892         /*
1893          * If the interface has been paused
1894          * then don't do the watchdog check
1895          */
1896         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1897                 goto out;
1898         /*
1899         ** Check for time since any descriptor was cleaned
1900         */
1901         for (int i = 0; i < adapter->num_queues; i++, txr++) {
1902                 IXGBE_TX_LOCK(txr);
1903                 if (txr->watchdog_check == FALSE) {
1904                         IXGBE_TX_UNLOCK(txr);
1905                         continue;
1906                 }
1907                 if ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG)
1908                         goto hung;
1909                 IXGBE_TX_UNLOCK(txr);
1910         }
1911 out:
1912         ixgbe_rearm_queues(adapter, adapter->que_mask);
1913         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1914         return;
1915
1916 hung:
1917         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1918         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1919             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1920             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1921         device_printf(dev,"TX(%d) desc avail = %d,"
1922             "Next TX to Clean = %d\n",
1923             txr->me, txr->tx_avail, txr->next_to_clean);
1924         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1925         adapter->watchdog_events++;
1926         IXGBE_TX_UNLOCK(txr);
1927         ixgbe_init_locked(adapter);
1928 }
1929
1930 /*
1931 ** Note: this routine updates the OS on the link state
1932 **      the real check of the hardware only happens with
1933 **      a link interrupt.
1934 */
1935 static void
1936 ixgbe_update_link_status(struct adapter *adapter)
1937 {
1938         struct ifnet    *ifp = adapter->ifp;
1939         struct tx_ring *txr = adapter->tx_rings;
1940         device_t dev = adapter->dev;
1941
1942
1943         if (adapter->link_up){
1944                 if (adapter->link_active == FALSE) {
1945                         if (bootverbose)
1946                                 device_printf(dev,"Link is up %d Gbps %s \n",
1947                                     ((adapter->link_speed == 128)? 10:1),
1948                                     "Full Duplex");
1949                         adapter->link_active = TRUE;
1950                         if_link_state_change(ifp, LINK_STATE_UP);
1951                 }
1952         } else { /* Link down */
1953                 if (adapter->link_active == TRUE) {
1954                         if (bootverbose)
1955                                 device_printf(dev,"Link is Down\n");
1956                         if_link_state_change(ifp, LINK_STATE_DOWN);
1957                         adapter->link_active = FALSE;
1958                         for (int i = 0; i < adapter->num_queues;
1959                             i++, txr++)
1960                                 txr->watchdog_check = FALSE;
1961                 }
1962         }
1963
1964         return;
1965 }
1966
1967
1968 /*********************************************************************
1969  *
1970  *  This routine disables all traffic on the adapter by issuing a
1971  *  global reset on the MAC and deallocates TX/RX buffers.
1972  *
1973  **********************************************************************/
1974
1975 static void
1976 ixgbe_stop(void *arg)
1977 {
1978         struct ifnet   *ifp;
1979         struct adapter *adapter = arg;
1980         ifp = adapter->ifp;
1981
1982         mtx_assert(&adapter->core_mtx, MA_OWNED);
1983
1984         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1985         ixgbe_disable_intr(adapter);
1986
1987         /* Tell the stack that the interface is no longer active */
1988         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1989
1990         ixgbe_reset_hw(&adapter->hw);
1991         adapter->hw.adapter_stopped = FALSE;
1992         ixgbe_stop_adapter(&adapter->hw);
1993         callout_stop(&adapter->timer);
1994
1995         /* reprogram the RAR[0] in case user changed it. */
1996         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1997
1998         return;
1999 }
2000
2001
2002 /*********************************************************************
2003  *
2004  *  Determine hardware revision.
2005  *
2006  **********************************************************************/
2007 static void
2008 ixgbe_identify_hardware(struct adapter *adapter)
2009 {
2010         device_t        dev = adapter->dev;
2011
2012         /* Save off the information about this board */
2013         adapter->hw.vendor_id = pci_get_vendor(dev);
2014         adapter->hw.device_id = pci_get_device(dev);
2015         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2016         adapter->hw.subsystem_vendor_id =
2017             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2018         adapter->hw.subsystem_device_id =
2019             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2020
2021         return;
2022 }
2023
2024 /*********************************************************************
2025  *
2026  *  Setup the Legacy or MSI Interrupt handler
2027  *
2028  **********************************************************************/
2029 static int
2030 ixgbe_allocate_legacy(struct adapter *adapter)
2031 {
2032         device_t dev = adapter->dev;
2033         struct          ix_queue *que = adapter->queues;
2034         int error, rid = 0;
2035
2036         /* MSI RID at 1 */
2037         if (adapter->msix == 1)
2038                 rid = 1;
2039
2040         /* We allocate a single interrupt resource */
2041         adapter->res = bus_alloc_resource_any(dev,
2042             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2043         if (adapter->res == NULL) {
2044                 device_printf(dev, "Unable to allocate bus resource: "
2045                     "interrupt\n");
2046                 return (ENXIO);
2047         }
2048
2049         /*
2050          * Try allocating a fast interrupt and the associated deferred
2051          * processing contexts.
2052          */
2053         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2054         que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2055             taskqueue_thread_enqueue, &que->tq);
2056         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s ixq",
2057             device_get_nameunit(adapter->dev));
2058
2059         /* Tasklets for Link, SFP and Multispeed Fiber */
2060         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2061         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2062         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2063 #ifdef IXGBE_FDIR
2064         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2065 #endif
2066         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2067             taskqueue_thread_enqueue, &adapter->tq);
2068         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2069             device_get_nameunit(adapter->dev));
2070
2071         if ((error = bus_setup_intr(dev, adapter->res,
2072             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
2073             que, &adapter->tag)) != 0) {
2074                 device_printf(dev, "Failed to register fast interrupt "
2075                     "handler: %d\n", error);
2076                 taskqueue_free(que->tq);
2077                 taskqueue_free(adapter->tq);
2078                 que->tq = NULL;
2079                 adapter->tq = NULL;
2080                 return (error);
2081         }
2082         /* For simplicity in the handlers */
2083         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2084
2085         return (0);
2086 }
2087
2088
2089 /*********************************************************************
2090  *
2091  *  Setup MSIX Interrupt resources and handlers
2092  *
2093  **********************************************************************/
2094 static int
2095 ixgbe_allocate_msix(struct adapter *adapter)
2096 {
2097         device_t        dev = adapter->dev;
2098         struct          ix_queue *que = adapter->queues;
2099         int             error, rid, vector = 0;
2100
2101         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2102                 rid = vector + 1;
2103                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2104                     RF_SHAREABLE | RF_ACTIVE);
2105                 if (que->res == NULL) {
2106                         device_printf(dev,"Unable to allocate"
2107                             " bus resource: que interrupt [%d]\n", vector);
2108                         return (ENXIO);
2109                 }
2110                 /* Set the handler function */
2111                 error = bus_setup_intr(dev, que->res,
2112                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2113                     ixgbe_msix_que, que, &que->tag);
2114                 if (error) {
2115                         que->res = NULL;
2116                         device_printf(dev, "Failed to register QUE handler");
2117                         return (error);
2118                 }
2119                 que->msix = vector;
2120                 adapter->que_mask |= (u64)(1 << que->msix);
2121                 /*
2122                 ** Bind the msix vector, and thus the
2123                 ** ring to the corresponding cpu.
2124                 */
2125                 if (adapter->num_queues > 1)
2126                         bus_bind_intr(dev, que->res, i);
2127
2128                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2129                 que->tq = taskqueue_create_fast("ixgbe_que", M_NOWAIT,
2130                     taskqueue_thread_enqueue, &que->tq);
2131                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2132                     device_get_nameunit(adapter->dev));
2133         }
2134
2135         /* and Link */
2136         rid = vector + 1;
2137         adapter->res = bus_alloc_resource_any(dev,
2138             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2139         if (!adapter->res) {
2140                 device_printf(dev,"Unable to allocate"
2141             " bus resource: Link interrupt [%d]\n", rid);
2142                 return (ENXIO);
2143         }
2144         /* Set the link handler function */
2145         error = bus_setup_intr(dev, adapter->res,
2146             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2147             ixgbe_msix_link, adapter, &adapter->tag);
2148         if (error) {
2149                 adapter->res = NULL;
2150                 device_printf(dev, "Failed to register LINK handler");
2151                 return (error);
2152         }
2153         adapter->linkvec = vector;
2154         /* Tasklets for Link, SFP and Multispeed Fiber */
2155         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2156         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2157         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2158 #ifdef IXGBE_FDIR
2159         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2160 #endif
2161         adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
2162             taskqueue_thread_enqueue, &adapter->tq);
2163         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2164             device_get_nameunit(adapter->dev));
2165
2166         return (0);
2167 }
2168
2169 /*
2170  * Setup Either MSI/X or MSI
2171  */
2172 static int
2173 ixgbe_setup_msix(struct adapter *adapter)
2174 {
2175         device_t dev = adapter->dev;
2176         int rid, want, queues, msgs;
2177
2178         /* Override by tuneable */
2179         if (ixgbe_enable_msix == 0)
2180                 goto msi;
2181
2182         /* First try MSI/X */
2183         rid = PCIR_BAR(MSIX_82598_BAR);
2184         adapter->msix_mem = bus_alloc_resource_any(dev,
2185             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2186         if (!adapter->msix_mem) {
2187                 rid += 4;       /* 82599 maps in higher BAR */
2188                 adapter->msix_mem = bus_alloc_resource_any(dev,
2189                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2190         }
2191         if (!adapter->msix_mem) {
2192                 /* May not be enabled */
2193                 device_printf(adapter->dev,
2194                     "Unable to map MSIX table \n");
2195                 goto msi;
2196         }
2197
2198         msgs = pci_msix_count(dev);
2199         if (msgs == 0) { /* system has msix disabled */
2200                 bus_release_resource(dev, SYS_RES_MEMORY,
2201                     rid, adapter->msix_mem);
2202                 adapter->msix_mem = NULL;
2203                 goto msi;
2204         }
2205
2206         /* Figure out a reasonable auto config value */
2207         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2208
2209         if (ixgbe_num_queues != 0)
2210                 queues = ixgbe_num_queues;
2211
2212         /*
2213         ** Want one vector (RX/TX pair) per queue
2214         ** plus an additional for Link.
2215         */
2216         want = queues + 1;
2217         if (msgs >= want)
2218                 msgs = want;
2219         else {
2220                 device_printf(adapter->dev,
2221                     "MSIX Configuration Problem, "
2222                     "%d vectors but %d queues wanted!\n",
2223                     msgs, want);
2224                 return (0); /* Will go to Legacy setup */
2225         }
2226         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2227                 device_printf(adapter->dev,
2228                     "Using MSIX interrupts with %d vectors\n", msgs);
2229                 adapter->num_queues = queues;
2230                 return (msgs);
2231         }
2232 msi:
2233         msgs = pci_msi_count(dev);
2234         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2235                 device_printf(adapter->dev,"Using MSI interrupt\n");
2236         return (msgs);
2237 }
2238
2239
2240 static int
2241 ixgbe_allocate_pci_resources(struct adapter *adapter)
2242 {
2243         int             rid;
2244         device_t        dev = adapter->dev;
2245
2246         rid = PCIR_BAR(0);
2247         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2248             &rid, RF_ACTIVE);
2249
2250         if (!(adapter->pci_mem)) {
2251                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2252                 return (ENXIO);
2253         }
2254
2255         adapter->osdep.mem_bus_space_tag =
2256                 rman_get_bustag(adapter->pci_mem);
2257         adapter->osdep.mem_bus_space_handle =
2258                 rman_get_bushandle(adapter->pci_mem);
2259         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2260
2261         /* Legacy defaults */
2262         adapter->num_queues = 1;
2263         adapter->hw.back = &adapter->osdep;
2264
2265         /*
2266         ** Now setup MSI or MSI/X, should
2267         ** return us the number of supported
2268         ** vectors. (Will be 1 for MSI)
2269         */
2270         adapter->msix = ixgbe_setup_msix(adapter);
2271         return (0);
2272 }
2273
2274 static void
2275 ixgbe_free_pci_resources(struct adapter * adapter)
2276 {
2277         struct          ix_queue *que = adapter->queues;
2278         device_t        dev = adapter->dev;
2279         int             rid, memrid;
2280
2281         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2282                 memrid = PCIR_BAR(MSIX_82598_BAR);
2283         else
2284                 memrid = PCIR_BAR(MSIX_82599_BAR);
2285
2286         /*
2287         ** There is a slight possibility of a failure mode
2288         ** in attach that will result in entering this function
2289         ** before interrupt resources have been initialized, and
2290         ** in that case we do not want to execute the loops below
2291         ** We can detect this reliably by the state of the adapter
2292         ** res pointer.
2293         */
2294         if (adapter->res == NULL)
2295                 goto mem;
2296
2297         /*
2298         **  Release all msix queue resources:
2299         */
2300         for (int i = 0; i < adapter->num_queues; i++, que++) {
2301                 rid = que->msix + 1;
2302                 if (que->tag != NULL) {
2303                         bus_teardown_intr(dev, que->res, que->tag);
2304                         que->tag = NULL;
2305                 }
2306                 if (que->res != NULL)
2307                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2308         }
2309
2310
2311         /* Clean the Legacy or Link interrupt last */
2312         if (adapter->linkvec) /* we are doing MSIX */
2313                 rid = adapter->linkvec + 1;
2314         else
2315                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2316
2317         if (adapter->tag != NULL) {
2318                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2319                 adapter->tag = NULL;
2320         }
2321         if (adapter->res != NULL)
2322                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2323
2324 mem:
2325         if (adapter->msix)
2326                 pci_release_msi(dev);
2327
2328         if (adapter->msix_mem != NULL)
2329                 bus_release_resource(dev, SYS_RES_MEMORY,
2330                     memrid, adapter->msix_mem);
2331
2332         if (adapter->pci_mem != NULL)
2333                 bus_release_resource(dev, SYS_RES_MEMORY,
2334                     PCIR_BAR(0), adapter->pci_mem);
2335
2336         return;
2337 }
2338
2339 /*********************************************************************
2340  *
2341  *  Setup networking device structure and register an interface.
2342  *
2343  **********************************************************************/
2344 static void
2345 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2346 {
2347         struct ixgbe_hw *hw = &adapter->hw;
2348         struct ifnet   *ifp;
2349
2350         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2351
2352         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2353         if (ifp == NULL)
2354                 panic("%s: can not if_alloc()\n", device_get_nameunit(dev));
2355         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2356         ifp->if_mtu = ETHERMTU;
2357         ifp->if_baudrate = 1000000000;
2358         ifp->if_init = ixgbe_init;
2359         ifp->if_softc = adapter;
2360         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2361         ifp->if_ioctl = ixgbe_ioctl;
2362         ifp->if_start = ixgbe_start;
2363 #if __FreeBSD_version >= 800000
2364         ifp->if_transmit = ixgbe_mq_start;
2365         ifp->if_qflush = ixgbe_qflush;
2366 #endif
2367         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2368
2369         ether_ifattach(ifp, adapter->hw.mac.addr);
2370
2371         adapter->max_frame_size =
2372             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2373
2374         /*
2375          * Tell the upper layer(s) we support long frames.
2376          */
2377         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2378
2379         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM;
2380         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2381         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_LRO;
2382
2383         ifp->if_capenable = ifp->if_capabilities;
2384
2385         /*
2386          * Specify the media types supported by this adapter and register
2387          * callbacks to update media and link information
2388          */
2389         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2390                      ixgbe_media_status);
2391         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics |
2392             IFM_FDX, 0, NULL);
2393         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2394                 ifmedia_add(&adapter->media,
2395                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2396                 ifmedia_add(&adapter->media,
2397                     IFM_ETHER | IFM_1000_T, 0, NULL);
2398         }
2399         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2400         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2401
2402         return;
2403 }
2404
2405 static void
2406 ixgbe_config_link(struct adapter *adapter)
2407 {
2408         struct ixgbe_hw *hw = &adapter->hw;
2409         u32     autoneg, err = 0;
2410         bool    sfp, negotiate;
2411
2412         sfp = ixgbe_is_sfp(hw);
2413
2414         if (sfp) {
2415                 if (hw->phy.multispeed_fiber) {
2416                         hw->mac.ops.setup_sfp(hw);
2417                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2418                 } else
2419                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2420         } else {
2421                 if (hw->mac.ops.check_link)
2422                         err = ixgbe_check_link(hw, &autoneg,
2423                             &adapter->link_up, FALSE);
2424                 if (err)
2425                         goto out;
2426                 autoneg = hw->phy.autoneg_advertised;
2427                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2428                         err  = hw->mac.ops.get_link_capabilities(hw,
2429                             &autoneg, &negotiate);
2430                 if (err)
2431                         goto out;
2432                 if (hw->mac.ops.setup_link)
2433                         err = hw->mac.ops.setup_link(hw, autoneg,
2434                             negotiate, adapter->link_up);
2435         }
2436 out:
2437         return;
2438 }
2439
2440 /********************************************************************
2441  * Manage DMA'able memory.
2442  *******************************************************************/
2443 static void
2444 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2445 {
2446         if (error)
2447                 return;
2448         *(bus_addr_t *) arg = segs->ds_addr;
2449         return;
2450 }
2451
2452 static int
2453 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2454                 struct ixgbe_dma_alloc *dma, int mapflags)
2455 {
2456         device_t dev = adapter->dev;
2457         int             r;
2458
2459         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
2460                                DBA_ALIGN, 0,    /* alignment, bounds */
2461                                BUS_SPACE_MAXADDR,       /* lowaddr */
2462                                BUS_SPACE_MAXADDR,       /* highaddr */
2463                                NULL, NULL,      /* filter, filterarg */
2464                                size,    /* maxsize */
2465                                1,       /* nsegments */
2466                                size,    /* maxsegsize */
2467                                BUS_DMA_ALLOCNOW,        /* flags */
2468                                NULL,    /* lockfunc */
2469                                NULL,    /* lockfuncarg */
2470                                &dma->dma_tag);
2471         if (r != 0) {
2472                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2473                        "error %u\n", r);
2474                 goto fail_0;
2475         }
2476         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2477                              BUS_DMA_NOWAIT, &dma->dma_map);
2478         if (r != 0) {
2479                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2480                        "error %u\n", r);
2481                 goto fail_1;
2482         }
2483         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2484                             size,
2485                             ixgbe_dmamap_cb,
2486                             &dma->dma_paddr,
2487                             mapflags | BUS_DMA_NOWAIT);
2488         if (r != 0) {
2489                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2490                        "error %u\n", r);
2491                 goto fail_2;
2492         }
2493         dma->dma_size = size;
2494         return (0);
2495 fail_2:
2496         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2497 fail_1:
2498         bus_dma_tag_destroy(dma->dma_tag);
2499 fail_0:
2500         dma->dma_map = NULL;
2501         dma->dma_tag = NULL;
2502         return (r);
2503 }
2504
2505 static void
2506 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2507 {
2508         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2509             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2510         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2511         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2512         bus_dma_tag_destroy(dma->dma_tag);
2513 }
2514
2515
2516 /*********************************************************************
2517  *
2518  *  Allocate memory for the transmit and receive rings, and then
2519  *  the descriptors associated with each, called only once at attach.
2520  *
2521  **********************************************************************/
2522 static int
2523 ixgbe_allocate_queues(struct adapter *adapter)
2524 {
2525         device_t        dev = adapter->dev;
2526         struct ix_queue *que;
2527         struct tx_ring  *txr;
2528         struct rx_ring  *rxr;
2529         int rsize, tsize, error = IXGBE_SUCCESS;
2530         int txconf = 0, rxconf = 0;
2531
2532         /* First allocate the top level queue structs */
2533         if (!(adapter->queues =
2534             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2535             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2536                 device_printf(dev, "Unable to allocate queue memory\n");
2537                 error = ENOMEM;
2538                 goto fail;
2539         }
2540
2541         /* First allocate the TX ring struct memory */
2542         if (!(adapter->tx_rings =
2543             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2544             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2545                 device_printf(dev, "Unable to allocate TX ring memory\n");
2546                 error = ENOMEM;
2547                 goto tx_fail;
2548         }
2549
2550         /* Next allocate the RX */
2551         if (!(adapter->rx_rings =
2552             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2553             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2554                 device_printf(dev, "Unable to allocate RX ring memory\n");
2555                 error = ENOMEM;
2556                 goto rx_fail;
2557         }
2558
2559         /* For the ring itself */
2560         tsize = roundup2(adapter->num_tx_desc *
2561             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2562
2563         /*
2564          * Now set up the TX queues, txconf is needed to handle the
2565          * possibility that things fail midcourse and we need to
2566          * undo memory gracefully
2567          */
2568         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2569                 /* Set up some basics */
2570                 txr = &adapter->tx_rings[i];
2571                 txr->adapter = adapter;
2572                 txr->me = i;
2573
2574                 /* Initialize the TX side lock */
2575                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2576                     device_get_nameunit(dev), txr->me);
2577                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2578
2579                 if (ixgbe_dma_malloc(adapter, tsize,
2580                         &txr->txdma, BUS_DMA_NOWAIT)) {
2581                         device_printf(dev,
2582                             "Unable to allocate TX Descriptor memory\n");
2583                         error = ENOMEM;
2584                         goto err_tx_desc;
2585                 }
2586                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2587                 bzero((void *)txr->tx_base, tsize);
2588
2589                 /* Now allocate transmit buffers for the ring */
2590                 if (ixgbe_allocate_transmit_buffers(txr)) {
2591                         device_printf(dev,
2592                             "Critical Failure setting up transmit buffers\n");
2593                         error = ENOMEM;
2594                         goto err_tx_desc;
2595                 }
2596 #if __FreeBSD_version >= 800000
2597                 /* Allocate a buf ring */
2598                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2599                     M_WAITOK, &txr->tx_mtx);
2600                 if (txr->br == NULL) {
2601                         device_printf(dev,
2602                             "Critical Failure setting up buf ring\n");
2603                         error = ENOMEM;
2604                         goto err_tx_desc;
2605                 }
2606 #endif
2607         }
2608
2609         /*
2610          * Next the RX queues...
2611          */
2612         rsize = roundup2(adapter->num_rx_desc *
2613             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2614         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2615                 rxr = &adapter->rx_rings[i];
2616                 /* Set up some basics */
2617                 rxr->adapter = adapter;
2618                 rxr->me = i;
2619
2620                 /* Initialize the RX side lock */
2621                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2622                     device_get_nameunit(dev), rxr->me);
2623                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2624
2625                 if (ixgbe_dma_malloc(adapter, rsize,
2626                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2627                         device_printf(dev,
2628                             "Unable to allocate RxDescriptor memory\n");
2629                         error = ENOMEM;
2630                         goto err_rx_desc;
2631                 }
2632                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2633                 bzero((void *)rxr->rx_base, rsize);
2634
2635                 /* Allocate receive buffers for the ring*/
2636                 if (ixgbe_allocate_receive_buffers(rxr)) {
2637                         device_printf(dev,
2638                             "Critical Failure setting up receive buffers\n");
2639                         error = ENOMEM;
2640                         goto err_rx_desc;
2641                 }
2642         }
2643
2644         /*
2645         ** Finally set up the queue holding structs
2646         */
2647         for (int i = 0; i < adapter->num_queues; i++) {
2648                 que = &adapter->queues[i];
2649                 que->adapter = adapter;
2650                 que->txr = &adapter->tx_rings[i];
2651                 que->rxr = &adapter->rx_rings[i];
2652         }
2653
2654         return (0);
2655
2656 err_rx_desc:
2657         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2658                 ixgbe_dma_free(adapter, &rxr->rxdma);
2659 err_tx_desc:
2660         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2661                 ixgbe_dma_free(adapter, &txr->txdma);
2662         free(adapter->rx_rings, M_DEVBUF);
2663 rx_fail:
2664         free(adapter->tx_rings, M_DEVBUF);
2665 tx_fail:
2666         free(adapter->queues, M_DEVBUF);
2667 fail:
2668         return (error);
2669 }
2670
2671 /*********************************************************************
2672  *
2673  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2674  *  the information needed to transmit a packet on the wire. This is
2675  *  called only once at attach, setup is done every reset.
2676  *
2677  **********************************************************************/
2678 static int
2679 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2680 {
2681         struct adapter *adapter = txr->adapter;
2682         device_t dev = adapter->dev;
2683         struct ixgbe_tx_buf *txbuf;
2684         int error, i;
2685
2686         /*
2687          * Setup DMA descriptor areas.
2688          */
2689         if ((error = bus_dma_tag_create(NULL,           /* parent */
2690                                1, 0,            /* alignment, bounds */
2691                                BUS_SPACE_MAXADDR,       /* lowaddr */
2692                                BUS_SPACE_MAXADDR,       /* highaddr */
2693                                NULL, NULL,              /* filter, filterarg */
2694                                IXGBE_TSO_SIZE,          /* maxsize */
2695                                ixgbe_num_segs,          /* nsegments */
2696                                PAGE_SIZE,               /* maxsegsize */
2697                                0,                       /* flags */
2698                                NULL,                    /* lockfunc */
2699                                NULL,                    /* lockfuncarg */
2700                                &txr->txtag))) {
2701                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2702                 goto fail;
2703         }
2704
2705         if (!(txr->tx_buffers =
2706             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2707             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2708                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2709                 error = ENOMEM;
2710                 goto fail;
2711         }
2712
2713         /* Create the descriptor buffer dma maps */
2714         txbuf = txr->tx_buffers;
2715         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2716                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2717                 if (error != 0) {
2718                         device_printf(dev, "Unable to create TX DMA map\n");
2719                         goto fail;
2720                 }
2721         }
2722
2723         return 0;
2724 fail:
2725         /* We free all, it handles case where we are in the middle */
2726         ixgbe_free_transmit_structures(adapter);
2727         return (error);
2728 }
2729
2730 /*********************************************************************
2731  *
2732  *  Initialize a transmit ring.
2733  *
2734  **********************************************************************/
2735 static void
2736 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2737 {
2738         struct adapter *adapter = txr->adapter;
2739         struct ixgbe_tx_buf *txbuf;
2740         int i;
2741
2742         /* Clear the old ring contents */
2743         IXGBE_TX_LOCK(txr);
2744         bzero((void *)txr->tx_base,
2745               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2746         /* Reset indices */
2747         txr->next_avail_desc = 0;
2748         txr->next_to_clean = 0;
2749
2750         /* Free any existing tx buffers. */
2751         txbuf = txr->tx_buffers;
2752         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2753                 if (txbuf->m_head != NULL) {
2754                         bus_dmamap_sync(txr->txtag, txbuf->map,
2755                             BUS_DMASYNC_POSTWRITE);
2756                         bus_dmamap_unload(txr->txtag, txbuf->map);
2757                         m_freem(txbuf->m_head);
2758                         txbuf->m_head = NULL;
2759                 }
2760                 /* Clear the EOP index */
2761                 txbuf->eop_index = -1;
2762         }
2763
2764 #ifdef IXGBE_FDIR
2765         /* Set the rate at which we sample packets */
2766         if (adapter->hw.mac.type == ixgbe_mac_82599EB)
2767                 txr->atr_sample = atr_sample_rate;
2768 #endif
2769
2770         /* Set number of descriptors available */
2771         txr->tx_avail = adapter->num_tx_desc;
2772
2773         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2774             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2775         IXGBE_TX_UNLOCK(txr);
2776 }
2777
2778 /*********************************************************************
2779  *
2780  *  Initialize all transmit rings.
2781  *
2782  **********************************************************************/
2783 static int
2784 ixgbe_setup_transmit_structures(struct adapter *adapter)
2785 {
2786         struct tx_ring *txr = adapter->tx_rings;
2787
2788         for (int i = 0; i < adapter->num_queues; i++, txr++)
2789                 ixgbe_setup_transmit_ring(txr);
2790
2791         return (0);
2792 }
2793
2794 /*********************************************************************
2795  *
2796  *  Enable transmit unit.
2797  *
2798  **********************************************************************/
2799 static void
2800 ixgbe_initialize_transmit_units(struct adapter *adapter)
2801 {
2802         struct tx_ring  *txr = adapter->tx_rings;
2803         struct ixgbe_hw *hw = &adapter->hw;
2804
2805         /* Setup the Base and Length of the Tx Descriptor Ring */
2806
2807         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2808                 u64     tdba = txr->txdma.dma_paddr;
2809                 u32     txctrl;
2810
2811                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2812                        (tdba & 0x00000000ffffffffULL));
2813                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2814                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2815                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2816
2817                 /* Setup the HW Tx Head and Tail descriptor pointers */
2818                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2819                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2820
2821                 /* Setup Transmit Descriptor Cmd Settings */
2822                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2823                 txr->watchdog_check = FALSE;
2824
2825                 /* Disable Head Writeback */
2826                 switch (hw->mac.type) {
2827                 case ixgbe_mac_82598EB:
2828                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2829                         break;
2830                 case ixgbe_mac_82599EB:
2831                 default:
2832                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2833                         break;
2834                 }
2835                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
2836                 switch (hw->mac.type) {
2837                 case ixgbe_mac_82598EB:
2838                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2839                         break;
2840                 case ixgbe_mac_82599EB:
2841                 default:
2842                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2843                         break;
2844                 }
2845
2846         }
2847
2848         if (hw->mac.type == ixgbe_mac_82599EB) {
2849                 u32 dmatxctl, rttdcs;
2850                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2851                 dmatxctl |= IXGBE_DMATXCTL_TE;
2852                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2853                 /* Disable arbiter to set MTQC */
2854                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2855                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2856                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2857                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2858                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2859                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2860         }
2861
2862         return;
2863 }
2864
2865 /*********************************************************************
2866  *
2867  *  Free all transmit rings.
2868  *
2869  **********************************************************************/
2870 static void
2871 ixgbe_free_transmit_structures(struct adapter *adapter)
2872 {
2873         struct tx_ring *txr = adapter->tx_rings;
2874
2875         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2876                 IXGBE_TX_LOCK(txr);
2877                 ixgbe_free_transmit_buffers(txr);
2878                 ixgbe_dma_free(adapter, &txr->txdma);
2879                 IXGBE_TX_UNLOCK(txr);
2880                 IXGBE_TX_LOCK_DESTROY(txr);
2881         }
2882         free(adapter->tx_rings, M_DEVBUF);
2883 }
2884
2885 /*********************************************************************
2886  *
2887  *  Free transmit ring related data structures.
2888  *
2889  **********************************************************************/
2890 static void
2891 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2892 {
2893         struct adapter *adapter = txr->adapter;
2894         struct ixgbe_tx_buf *tx_buffer;
2895         int             i;
2896
2897         INIT_DEBUGOUT("free_transmit_ring: begin");
2898
2899         if (txr->tx_buffers == NULL)
2900                 return;
2901
2902         tx_buffer = txr->tx_buffers;
2903         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2904                 if (tx_buffer->m_head != NULL) {
2905                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2906                             BUS_DMASYNC_POSTWRITE);
2907                         bus_dmamap_unload(txr->txtag,
2908                             tx_buffer->map);
2909                         m_freem(tx_buffer->m_head);
2910                         tx_buffer->m_head = NULL;
2911                         if (tx_buffer->map != NULL) {
2912                                 bus_dmamap_destroy(txr->txtag,
2913                                     tx_buffer->map);
2914                                 tx_buffer->map = NULL;
2915                         }
2916                 } else if (tx_buffer->map != NULL) {
2917                         bus_dmamap_unload(txr->txtag,
2918                             tx_buffer->map);
2919                         bus_dmamap_destroy(txr->txtag,
2920                             tx_buffer->map);
2921                         tx_buffer->map = NULL;
2922                 }
2923         }
2924 #if __FreeBSD_version >= 800000
2925         if (txr->br != NULL)
2926                 buf_ring_free(txr->br, M_DEVBUF);
2927 #endif
2928         if (txr->tx_buffers != NULL) {
2929                 free(txr->tx_buffers, M_DEVBUF);
2930                 txr->tx_buffers = NULL;
2931         }
2932         if (txr->txtag != NULL) {
2933                 bus_dma_tag_destroy(txr->txtag);
2934                 txr->txtag = NULL;
2935         }
2936         return;
2937 }
2938
2939 /*********************************************************************
2940  *
2941  *  Advanced Context Descriptor setup for VLAN or CSUM
2942  *
2943  **********************************************************************/
2944
2945 static boolean_t
2946 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
2947 {
2948         struct adapter *adapter = txr->adapter;
2949         struct ixgbe_adv_tx_context_desc *TXD;
2950         struct ixgbe_tx_buf        *tx_buffer;
2951         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2952         struct ether_vlan_header *eh;
2953         struct ip *ip;
2954         struct ip6_hdr *ip6;
2955         int  ehdrlen, ip_hlen = 0;
2956         u16     etype;
2957         u8      ipproto = 0;
2958         bool    offload = TRUE;
2959         int ctxd = txr->next_avail_desc;
2960         u16 vtag = 0;
2961
2962
2963         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
2964                 offload = FALSE;
2965
2966         tx_buffer = &txr->tx_buffers[ctxd];
2967         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2968
2969         /*
2970         ** In advanced descriptors the vlan tag must
2971         ** be placed into the descriptor itself.
2972         */
2973         if (mp->m_flags & M_VLANTAG) {
2974                 vtag = htole16(mp->m_pkthdr.ether_vtag);
2975                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2976         } else if (offload == FALSE)
2977                 return FALSE;
2978
2979         /*
2980          * Determine where frame payload starts.
2981          * Jump over vlan headers if already present,
2982          * helpful for QinQ too.
2983          */
2984         eh = mtod(mp, struct ether_vlan_header *);
2985         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2986                 etype = ntohs(eh->evl_proto);
2987                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2988         } else {
2989                 etype = ntohs(eh->evl_encap_proto);
2990                 ehdrlen = ETHER_HDR_LEN;
2991         }
2992
2993         /* Set the ether header length */
2994         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2995
2996         switch (etype) {
2997                 case ETHERTYPE_IP:
2998                         ip = (struct ip *)(mp->m_data + ehdrlen);
2999                         ip_hlen = ip->ip_hl << 2;
3000                         if (mp->m_len < ehdrlen + ip_hlen)
3001                                 return (FALSE);
3002                         ipproto = ip->ip_p;
3003                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3004                         break;
3005                 case ETHERTYPE_IPV6:
3006                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3007                         ip_hlen = sizeof(struct ip6_hdr);
3008                         if (mp->m_len < ehdrlen + ip_hlen)
3009                                 return (FALSE);
3010                         ipproto = ip6->ip6_nxt;
3011                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3012                         break;
3013                 default:
3014                         offload = FALSE;
3015                         break;
3016         }
3017
3018         vlan_macip_lens |= ip_hlen;
3019         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3020
3021         switch (ipproto) {
3022                 case IPPROTO_TCP:
3023                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3024                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3025                         break;
3026
3027                 case IPPROTO_UDP:
3028                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3029                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3030                         break;
3031
3032 #if __FreeBSD_version >= 800000
3033                 case IPPROTO_SCTP:
3034                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3035                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3036                         break;
3037 #endif
3038                 default:
3039                         offload = FALSE;
3040                         break;
3041         }
3042
3043         /* Now copy bits into descriptor */
3044         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3045         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3046         TXD->seqnum_seed = htole32(0);
3047         TXD->mss_l4len_idx = htole32(0);
3048
3049         tx_buffer->m_head = NULL;
3050         tx_buffer->eop_index = -1;
3051
3052         /* We've consumed the first desc, adjust counters */
3053         if (++ctxd == adapter->num_tx_desc)
3054                 ctxd = 0;
3055         txr->next_avail_desc = ctxd;
3056         --txr->tx_avail;
3057
3058         return (offload);
3059 }
3060
3061 /**********************************************************************
3062  *
3063  *  Setup work for hardware segmentation offload (TSO) on
3064  *  adapters using advanced tx descriptors
3065  *
3066  **********************************************************************/
3067 static boolean_t
3068 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
3069 {
3070         struct adapter *adapter = txr->adapter;
3071         struct ixgbe_adv_tx_context_desc *TXD;
3072         struct ixgbe_tx_buf        *tx_buffer;
3073         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3074         u32 mss_l4len_idx = 0;
3075         u16 vtag = 0;
3076         int ctxd, ehdrlen,  hdrlen, ip_hlen, tcp_hlen;
3077         struct ether_vlan_header *eh;
3078         struct ip *ip;
3079         struct tcphdr *th;
3080
3081
3082         /*
3083          * Determine where frame payload starts.
3084          * Jump over vlan headers if already present
3085          */
3086         eh = mtod(mp, struct ether_vlan_header *);
3087         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3088                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3089         else
3090                 ehdrlen = ETHER_HDR_LEN;
3091
3092         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3093         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3094                 return FALSE;
3095
3096         ctxd = txr->next_avail_desc;
3097         tx_buffer = &txr->tx_buffers[ctxd];
3098         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3099
3100         ip = (struct ip *)(mp->m_data + ehdrlen);
3101         if (ip->ip_p != IPPROTO_TCP)
3102                 return FALSE;   /* 0 */
3103         ip->ip_sum = 0;
3104         ip_hlen = ip->ip_hl << 2;
3105         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3106         th->th_sum = in_pseudo(ip->ip_src.s_addr,
3107             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3108         tcp_hlen = th->th_off << 2;
3109         hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3110
3111         /* This is used in the transmit desc in encap */
3112         *paylen = mp->m_pkthdr.len - hdrlen;
3113
3114         /* VLAN MACLEN IPLEN */
3115         if (mp->m_flags & M_VLANTAG) {
3116                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3117                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3118         }
3119
3120         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3121         vlan_macip_lens |= ip_hlen;
3122         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3123
3124         /* ADV DTYPE TUCMD */
3125         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3126         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3127         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3128         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3129
3130
3131         /* MSS L4LEN IDX */
3132         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3133         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3134         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3135
3136         TXD->seqnum_seed = htole32(0);
3137         tx_buffer->m_head = NULL;
3138         tx_buffer->eop_index = -1;
3139
3140         if (++ctxd == adapter->num_tx_desc)
3141                 ctxd = 0;
3142
3143         txr->tx_avail--;
3144         txr->next_avail_desc = ctxd;
3145         return TRUE;
3146 }
3147
3148 #ifdef IXGBE_FDIR
3149 /*
3150 ** This routine parses packet headers so that Flow
3151 ** Director can make a hashed filter table entry
3152 ** allowing traffic flows to be identified and kept
3153 ** on the same cpu.  This would be a performance
3154 ** hit, but we only do it at IXGBE_FDIR_RATE of
3155 ** packets.
3156 */
3157 static void
3158 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3159 {
3160         struct adapter                  *adapter = txr->adapter;
3161         struct ix_queue                 *que;
3162         struct ixgbe_atr_input          atr_input;
3163         struct ip                       *ip;
3164         struct tcphdr                   *th;
3165         struct udphdr                   *uh;
3166         struct ether_vlan_header        *eh;
3167         int                             ehdrlen, ip_hlen;
3168         u16 etype, vlan_id, src_port, dst_port, flex_bytes;
3169         u32 src_ipv4_addr, dst_ipv4_addr;
3170         u8 l4type = 0, ipproto = 0;
3171
3172         eh = mtod(mp, struct ether_vlan_header *);
3173         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3174                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3175         else
3176                 ehdrlen = ETHER_HDR_LEN;
3177         etype = ntohs(eh->evl_proto);
3178
3179         /* Only handling IPv4 */
3180         if (etype != ETHERTYPE_IP)
3181                 return;
3182
3183         ip = (struct ip *)(mp->m_data + ehdrlen);
3184         ipproto = ip->ip_p;
3185         ip_hlen = ip->ip_hl << 2;
3186         src_port = dst_port = 0;
3187
3188         /* check if we're UDP or TCP */
3189         switch (ipproto) {
3190         case IPPROTO_TCP:
3191                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3192                 src_port = th->th_sport;
3193                 dst_port = th->th_dport;
3194                 l4type |= IXGBE_ATR_L4TYPE_TCP;
3195                 break;
3196         case IPPROTO_UDP:
3197                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3198                 src_port = uh->uh_sport;
3199                 dst_port = uh->uh_dport;
3200                 l4type |= IXGBE_ATR_L4TYPE_UDP;
3201                 break;
3202         default:
3203                 return;
3204         }
3205
3206         memset(&atr_input, 0, sizeof(struct ixgbe_atr_input));
3207
3208         vlan_id = htole16(mp->m_pkthdr.ether_vtag);
3209         src_ipv4_addr = ip->ip_src.s_addr;
3210         dst_ipv4_addr = ip->ip_dst.s_addr;
3211         flex_bytes = etype;
3212         que = &adapter->queues[txr->me];
3213
3214         ixgbe_atr_set_vlan_id_82599(&atr_input, vlan_id);
3215         ixgbe_atr_set_src_port_82599(&atr_input, dst_port);
3216         ixgbe_atr_set_dst_port_82599(&atr_input, src_port);
3217         ixgbe_atr_set_flex_byte_82599(&atr_input, flex_bytes);
3218         ixgbe_atr_set_l4type_82599(&atr_input, l4type);
3219         /* src and dst are inverted, think how the receiver sees them */
3220         ixgbe_atr_set_src_ipv4_82599(&atr_input, dst_ipv4_addr);
3221         ixgbe_atr_set_dst_ipv4_82599(&atr_input, src_ipv4_addr);
3222
3223         /* This assumes the Rx queue and Tx queue are bound to the same CPU */
3224         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3225             &atr_input, que->msix);
3226 }
3227 #endif
3228
3229 /**********************************************************************
3230  *
3231  *  Examine each tx_buffer in the used queue. If the hardware is done
3232  *  processing the packet then free associated resources. The
3233  *  tx_buffer is put back on the free queue.
3234  *
3235  **********************************************************************/
3236 static boolean_t
3237 ixgbe_txeof(struct tx_ring *txr)
3238 {
3239         struct adapter  *adapter = txr->adapter;
3240         struct ifnet    *ifp = adapter->ifp;
3241         u32     first, last, done;
3242         struct ixgbe_tx_buf *tx_buffer;
3243         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3244
3245         mtx_assert(&txr->tx_mtx, MA_OWNED);
3246
3247         if (txr->tx_avail == adapter->num_tx_desc)
3248                 return FALSE;
3249
3250         first = txr->next_to_clean;
3251         tx_buffer = &txr->tx_buffers[first];
3252         /* For cleanup we just use legacy struct */
3253         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3254         last = tx_buffer->eop_index;
3255         if (last == -1)
3256                 return FALSE;
3257         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3258
3259         /*
3260         ** Get the index of the first descriptor
3261         ** BEYOND the EOP and call that 'done'.
3262         ** I do this so the comparison in the
3263         ** inner while loop below can be simple
3264         */
3265         if (++last == adapter->num_tx_desc) last = 0;
3266         done = last;
3267
3268         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3269             BUS_DMASYNC_POSTREAD);
3270         /*
3271         ** Only the EOP descriptor of a packet now has the DD
3272         ** bit set, this is what we look for...
3273         */
3274         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3275                 /* We clean the range of the packet */
3276                 while (first != done) {
3277                         tx_desc->upper.data = 0;
3278                         tx_desc->lower.data = 0;
3279                         tx_desc->buffer_addr = 0;
3280                         ++txr->tx_avail;
3281
3282                         if (tx_buffer->m_head) {
3283                                 txr->bytes +=
3284                                     tx_buffer->m_head->m_pkthdr.len;
3285                                 bus_dmamap_sync(txr->txtag,
3286                                     tx_buffer->map,
3287                                     BUS_DMASYNC_POSTWRITE);
3288                                 bus_dmamap_unload(txr->txtag,
3289                                     tx_buffer->map);
3290                                 m_freem(tx_buffer->m_head);
3291                                 tx_buffer->m_head = NULL;
3292                                 tx_buffer->map = NULL;
3293                         }
3294                         tx_buffer->eop_index = -1;
3295                         txr->watchdog_time = ticks;
3296
3297                         if (++first == adapter->num_tx_desc)
3298                                 first = 0;
3299
3300                         tx_buffer = &txr->tx_buffers[first];
3301                         tx_desc =
3302                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3303                 }
3304                 ++txr->packets;
3305                 ++ifp->if_opackets;
3306                 /* See if there is more work now */
3307                 last = tx_buffer->eop_index;
3308                 if (last != -1) {
3309                         eop_desc =
3310                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3311                         /* Get next done point */
3312                         if (++last == adapter->num_tx_desc) last = 0;
3313                         done = last;
3314                 } else
3315                         break;
3316         }
3317         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3318             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3319
3320         txr->next_to_clean = first;
3321
3322         /*
3323          * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that
3324          * it is OK to send packets. If there are no pending descriptors,
3325          * clear the timeout. Otherwise, if some descriptors have been freed,
3326          * restart the timeout.
3327          */
3328         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) {
3329                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3330                 if (txr->tx_avail == adapter->num_tx_desc) {
3331                         txr->watchdog_check = FALSE;
3332                         return FALSE;
3333                 }
3334         }
3335
3336         return TRUE;
3337 }
3338
3339 /*********************************************************************
3340  *
3341  *  Refresh mbuf buffers for RX descriptor rings
3342  *   - now keeps its own state so discards due to resource
3343  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3344  *     it just returns, keeping its placeholder, thus it can simply
3345  *     be recalled to try again.
3346  *
3347  **********************************************************************/
3348 static void
3349 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3350 {
3351         struct adapter          *adapter = rxr->adapter;
3352         bus_dma_segment_t       hseg[1];
3353         bus_dma_segment_t       pseg[1];
3354         struct ixgbe_rx_buf     *rxbuf;
3355         struct mbuf             *mh, *mp;
3356         int                     i, nsegs, error, cleaned;
3357
3358         i = rxr->next_to_refresh;
3359         cleaned = -1; /* Signify no completions */
3360         while (i != limit) {
3361                 rxbuf = &rxr->rx_buffers[i];
3362                 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) {
3363                         mh = m_gethdr(M_DONTWAIT, MT_DATA);
3364                         if (mh == NULL)
3365                                 goto update;
3366                         mh->m_pkthdr.len = mh->m_len = MHLEN;
3367                         mh->m_len = MHLEN;
3368                         mh->m_flags |= M_PKTHDR;
3369                         m_adj(mh, ETHER_ALIGN);
3370                         /* Get the memory mapping */
3371                         error = bus_dmamap_load_mbuf_sg(rxr->htag,
3372                             rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
3373                         if (error != 0) {
3374                                 printf("GET BUF: dmamap load"
3375                                     " failure - %d\n", error);
3376                                 m_free(mh);
3377                                 goto update;
3378                         }
3379                         rxbuf->m_head = mh;
3380                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3381                             BUS_DMASYNC_PREREAD);
3382                         rxr->rx_base[i].read.hdr_addr =
3383                             htole64(hseg[0].ds_addr);
3384                 }
3385
3386                 if (rxbuf->m_pack == NULL) {
3387                         mp = m_getjcl(M_DONTWAIT, MT_DATA,
3388                             M_PKTHDR, adapter->rx_mbuf_sz);
3389                         if (mp == NULL)
3390                                 goto update;
3391                         mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3392                         /* Get the memory mapping */
3393                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3394                             rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
3395                         if (error != 0) {
3396                                 printf("GET BUF: dmamap load"
3397                                     " failure - %d\n", error);
3398                                 m_free(mp);
3399                                 goto update;
3400                         }
3401                         rxbuf->m_pack = mp;
3402                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3403                             BUS_DMASYNC_PREREAD);
3404                         rxr->rx_base[i].read.pkt_addr =
3405                             htole64(pseg[0].ds_addr);
3406                 }
3407
3408                 cleaned = i;
3409                 /* Calculate next index */
3410                 if (++i == adapter->num_rx_desc)
3411                         i = 0;
3412                 /* This is the work marker for refresh */
3413                 rxr->next_to_refresh = i;
3414         }
3415 update:
3416         if (cleaned != -1) /* If we refreshed some, bump tail */
3417                 IXGBE_WRITE_REG(&adapter->hw,
3418                     IXGBE_RDT(rxr->me), cleaned);
3419         return;
3420 }
3421
3422 /*********************************************************************
3423  *
3424  *  Allocate memory for rx_buffer structures. Since we use one
3425  *  rx_buffer per received packet, the maximum number of rx_buffer's
3426  *  that we'll need is equal to the number of receive descriptors
3427  *  that we've allocated.
3428  *
3429  **********************************************************************/
3430 static int
3431 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3432 {
3433         struct  adapter         *adapter = rxr->adapter;
3434         device_t                dev = adapter->dev;
3435         struct ixgbe_rx_buf     *rxbuf;
3436         int                     i, bsize, error;
3437
3438         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3439         if (!(rxr->rx_buffers =
3440             (struct ixgbe_rx_buf *) malloc(bsize,
3441             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3442                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3443                 error = ENOMEM;
3444                 goto fail;
3445         }
3446
3447         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3448                                    1, 0,        /* alignment, bounds */
3449                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3450                                    BUS_SPACE_MAXADDR,   /* highaddr */
3451                                    NULL, NULL,          /* filter, filterarg */
3452                                    MSIZE,               /* maxsize */
3453                                    1,                   /* nsegments */
3454                                    MSIZE,               /* maxsegsize */
3455                                    0,                   /* flags */
3456                                    NULL,                /* lockfunc */
3457                                    NULL,                /* lockfuncarg */
3458                                    &rxr->htag))) {
3459                 device_printf(dev, "Unable to create RX DMA tag\n");
3460                 goto fail;
3461         }
3462
3463         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
3464                                    1, 0,        /* alignment, bounds */
3465                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3466                                    BUS_SPACE_MAXADDR,   /* highaddr */
3467                                    NULL, NULL,          /* filter, filterarg */
3468                                    MJUMPAGESIZE,        /* maxsize */
3469                                    1,                   /* nsegments */
3470                                    MJUMPAGESIZE,        /* maxsegsize */
3471                                    0,                   /* flags */
3472                                    NULL,                /* lockfunc */
3473                                    NULL,                /* lockfuncarg */
3474                                    &rxr->ptag))) {
3475                 device_printf(dev, "Unable to create RX DMA tag\n");
3476                 goto fail;
3477         }
3478
3479         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3480                 rxbuf = &rxr->rx_buffers[i];
3481                 error = bus_dmamap_create(rxr->htag,
3482                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3483                 if (error) {
3484                         device_printf(dev, "Unable to create RX head map\n");
3485                         goto fail;
3486                 }
3487                 error = bus_dmamap_create(rxr->ptag,
3488                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3489                 if (error) {
3490                         device_printf(dev, "Unable to create RX pkt map\n");
3491                         goto fail;
3492                 }
3493         }
3494
3495         return (0);
3496
3497 fail:
3498         /* Frees all, but can handle partial completion */
3499         ixgbe_free_receive_structures(adapter);
3500         return (error);
3501 }
3502
3503 /*
3504 ** Used to detect a descriptor that has
3505 ** been merged by Hardware RSC.
3506 */
3507 static inline u32
3508 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3509 {
3510         return (le32toh(rx->wb.lower.lo_dword.data) &
3511             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3512 }
3513
3514 /*********************************************************************
3515  *
3516  *  Initialize Hardware RSC (LRO) feature on 82599
3517  *  for an RX ring, this is toggled by the LRO capability
3518  *  even though it is transparent to the stack.
3519  *
3520  **********************************************************************/
3521 static void
3522 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3523 {
3524         struct  adapter         *adapter = rxr->adapter;
3525         struct  ixgbe_hw        *hw = &adapter->hw;
3526         u32                     rscctrl, rdrxctl;
3527
3528         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3529         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3530         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3531         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3532         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3533
3534         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3535         rscctrl |= IXGBE_RSCCTL_RSCEN;
3536         /*
3537         ** Limit the total number of descriptors that
3538         ** can be combined, so it does not exceed 64K
3539         */
3540         if (adapter->rx_mbuf_sz == MCLBYTES)
3541                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3542         else  /* using 4K clusters */
3543                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3544         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3545
3546         /* Enable TCP header recognition */
3547         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3548             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3549             IXGBE_PSRTYPE_TCPHDR));
3550
3551         /* Disable RSC for ACK packets */
3552         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3553             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3554
3555         rxr->hw_rsc = TRUE;
3556 }
3557
3558
3559 static void
3560 ixgbe_free_receive_ring(struct rx_ring *rxr)
3561 {
3562         struct  adapter         *adapter;
3563         struct ixgbe_rx_buf       *rxbuf;
3564         int i;
3565
3566         adapter = rxr->adapter;
3567         for (i = 0; i < adapter->num_rx_desc; i++) {
3568                 rxbuf = &rxr->rx_buffers[i];
3569                 if (rxbuf->m_head != NULL) {
3570                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3571                             BUS_DMASYNC_POSTREAD);
3572                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3573                         rxbuf->m_head->m_flags |= M_PKTHDR;
3574                         m_freem(rxbuf->m_head);
3575                 }
3576                 if (rxbuf->m_pack != NULL) {
3577                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3578                             BUS_DMASYNC_POSTREAD);
3579                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3580                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3581                         m_freem(rxbuf->m_pack);
3582                 }
3583                 rxbuf->m_head = NULL;
3584                 rxbuf->m_pack = NULL;
3585         }
3586 }
3587
3588
3589 /*********************************************************************
3590  *
3591  *  Initialize a receive ring and its buffers.
3592  *
3593  **********************************************************************/
3594 static int
3595 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3596 {
3597         struct  adapter         *adapter;
3598         struct ifnet            *ifp;
3599         device_t                dev;
3600         struct ixgbe_rx_buf     *rxbuf;
3601         bus_dma_segment_t       pseg[1], hseg[1];
3602         struct lro_ctrl         *lro = &rxr->lro;
3603         int                     rsize, nsegs, error = 0;
3604
3605         adapter = rxr->adapter;
3606         ifp = adapter->ifp;
3607         dev = adapter->dev;
3608
3609         /* Clear the ring contents */
3610         IXGBE_RX_LOCK(rxr);
3611         rsize = roundup2(adapter->num_rx_desc *
3612             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3613         bzero((void *)rxr->rx_base, rsize);
3614
3615         /* Free current RX buffer structs and their mbufs */
3616         ixgbe_free_receive_ring(rxr);
3617
3618         /* Configure header split? */
3619         if (ixgbe_header_split)
3620                 rxr->hdr_split = TRUE;
3621
3622         /* Now replenish the mbufs */
3623         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3624                 struct mbuf     *mh, *mp;
3625
3626                 rxbuf = &rxr->rx_buffers[j];
3627                 /*
3628                 ** Dont allocate mbufs if not
3629                 ** doing header split, its wasteful
3630                 */
3631                 if (rxr->hdr_split == FALSE)
3632                         goto skip_head;
3633
3634                 /* First the header */
3635                 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA);
3636                 if (rxbuf->m_head == NULL)
3637                         goto fail;
3638                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3639                 mh = rxbuf->m_head;
3640                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3641                 mh->m_flags |= M_PKTHDR;
3642                 /* Get the memory mapping */
3643                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3644                     rxbuf->hmap, rxbuf->m_head, hseg,
3645                     &nsegs, BUS_DMA_NOWAIT);
3646                 if (error != 0) /* Nothing elegant to do here */
3647                         goto fail;
3648                 bus_dmamap_sync(rxr->htag,
3649                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3650                 /* Update descriptor */
3651                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3652
3653 skip_head:
3654                 /* Now the payload cluster */
3655                 rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA,
3656                     M_PKTHDR, adapter->rx_mbuf_sz);
3657                 if (rxbuf->m_pack == NULL)
3658                         goto fail;
3659                 mp = rxbuf->m_pack;
3660                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3661                 /* Get the memory mapping */
3662                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3663                     rxbuf->pmap, mp, pseg,
3664                     &nsegs, BUS_DMA_NOWAIT);
3665                 if (error != 0)
3666                         goto fail;
3667                 bus_dmamap_sync(rxr->ptag,
3668                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3669                 /* Update descriptor */
3670                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3671         }
3672
3673
3674         /* Setup our descriptor indices */
3675         rxr->next_to_check = 0;
3676         rxr->next_to_refresh = 0;
3677         rxr->lro_enabled = FALSE;
3678         rxr->rx_split_packets = 0;
3679         rxr->rx_bytes = 0;
3680
3681         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3682             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3683
3684         /*
3685         ** Now set up the LRO interface:
3686         ** 82598 uses software LRO, the
3687         ** 82599 uses a hardware assist.
3688         */
3689         if ((adapter->hw.mac.type == ixgbe_mac_82599EB) &&
3690             (ifp->if_capenable & IFCAP_RXCSUM) &&
3691             (ifp->if_capenable & IFCAP_LRO))
3692                 ixgbe_setup_hw_rsc(rxr);
3693         else if (ifp->if_capenable & IFCAP_LRO) {
3694                 int err = tcp_lro_init(lro);
3695                 if (err) {
3696                         device_printf(dev, "LRO Initialization failed!\n");
3697                         goto fail;
3698                 }
3699                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3700                 rxr->lro_enabled = TRUE;
3701                 lro->ifp = adapter->ifp;
3702         }
3703
3704         IXGBE_RX_UNLOCK(rxr);
3705         return (0);
3706
3707 fail:
3708         ixgbe_free_receive_ring(rxr);
3709         IXGBE_RX_UNLOCK(rxr);
3710         return (error);
3711 }
3712
3713 /*********************************************************************
3714  *
3715  *  Initialize all receive rings.
3716  *
3717  **********************************************************************/
3718 static int
3719 ixgbe_setup_receive_structures(struct adapter *adapter)
3720 {
3721         struct rx_ring *rxr = adapter->rx_rings;
3722         int j;
3723
3724         for (j = 0; j < adapter->num_queues; j++, rxr++)
3725                 if (ixgbe_setup_receive_ring(rxr))
3726                         goto fail;
3727
3728         return (0);
3729 fail:
3730         /*
3731          * Free RX buffers allocated so far, we will only handle
3732          * the rings that completed, the failing case will have
3733          * cleaned up for itself. 'j' failed, so its the terminus.
3734          */
3735         for (int i = 0; i < j; ++i) {
3736                 rxr = &adapter->rx_rings[i];
3737                 ixgbe_free_receive_ring(rxr);
3738         }
3739
3740         return (ENOBUFS);
3741 }
3742
3743 /*********************************************************************
3744  *
3745  *  Setup receive registers and features.
3746  *
3747  **********************************************************************/
3748 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3749
3750 static void
3751 ixgbe_initialize_receive_units(struct adapter *adapter)
3752 {
3753         struct  rx_ring *rxr = adapter->rx_rings;
3754         struct ixgbe_hw *hw = &adapter->hw;
3755         struct ifnet   *ifp = adapter->ifp;
3756         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
3757         u32             reta, mrqc = 0, hlreg, random[10];
3758
3759
3760         /*
3761          * Make sure receives are disabled while
3762          * setting up the descriptor ring
3763          */
3764         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3765         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
3766             rxctrl & ~IXGBE_RXCTRL_RXEN);
3767
3768         /* Enable broadcasts */
3769         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3770         fctrl |= IXGBE_FCTRL_BAM;
3771         fctrl |= IXGBE_FCTRL_DPF;
3772         fctrl |= IXGBE_FCTRL_PMCF;
3773         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3774
3775         /* Set for Jumbo Frames? */
3776         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3777         if (ifp->if_mtu > ETHERMTU) {
3778                 hlreg |= IXGBE_HLREG0_JUMBOEN;
3779                 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3780         } else {
3781                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
3782                 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3783         }
3784         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
3785
3786         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3787                 u64 rdba = rxr->rxdma.dma_paddr;
3788
3789                 /* Setup the Base and Length of the Rx Descriptor Ring */
3790                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
3791                                (rdba & 0x00000000ffffffffULL));
3792                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
3793                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
3794                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
3795
3796                 /* Set up the SRRCTL register */
3797                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
3798                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
3799                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
3800                 srrctl |= bufsz;
3801                 if (rxr->hdr_split) {
3802                         /* Use a standard mbuf for the header */
3803                         srrctl |= ((IXGBE_RX_HDR <<
3804                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
3805                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
3806                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3807                 } else
3808                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
3809                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
3810
3811                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
3812                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
3813                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
3814         }
3815
3816         if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
3817                 /* PSRTYPE must be initialized in 82599 */
3818                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
3819                               IXGBE_PSRTYPE_UDPHDR |
3820                               IXGBE_PSRTYPE_IPV4HDR |
3821                               IXGBE_PSRTYPE_IPV6HDR;
3822                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
3823         }
3824
3825         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3826
3827         /* Setup RSS */
3828         if (adapter->num_queues > 1) {
3829                 int i, j;
3830                 reta = 0;
3831
3832                 /* set up random bits */
3833                 arc4rand(&random, sizeof(random), 0);
3834
3835                 /* Set up the redirection table */
3836                 for (i = 0, j = 0; i < 128; i++, j++) {
3837                         if (j == adapter->num_queues) j = 0;
3838                         reta = (reta << 8) | (j * 0x11);
3839                         if ((i & 3) == 3)
3840                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3841                 }
3842
3843                 /* Now fill our hash function seeds */
3844                 for (int i = 0; i < 10; i++)
3845                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
3846
3847                 /* Perform hash on these packet types */
3848                 mrqc = IXGBE_MRQC_RSSEN
3849                      | IXGBE_MRQC_RSS_FIELD_IPV4
3850                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
3851                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
3852                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
3853                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
3854                      | IXGBE_MRQC_RSS_FIELD_IPV6
3855                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
3856                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
3857                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3858                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3859
3860                 /* RSS and RX IPP Checksum are mutually exclusive */
3861                 rxcsum |= IXGBE_RXCSUM_PCSD;
3862         }
3863
3864         if (ifp->if_capenable & IFCAP_RXCSUM)
3865                 rxcsum |= IXGBE_RXCSUM_PCSD;
3866
3867         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
3868                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
3869
3870         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3871
3872         return;
3873 }
3874
3875 /*********************************************************************
3876  *
3877  *  Free all receive rings.
3878  *
3879  **********************************************************************/
3880 static void
3881 ixgbe_free_receive_structures(struct adapter *adapter)
3882 {
3883         struct rx_ring *rxr = adapter->rx_rings;
3884
3885         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3886                 struct lro_ctrl         *lro = &rxr->lro;
3887                 ixgbe_free_receive_buffers(rxr);
3888                 /* Free LRO memory */
3889                 tcp_lro_free(lro);
3890                 /* Free the ring memory as well */
3891                 ixgbe_dma_free(adapter, &rxr->rxdma);
3892         }
3893
3894         free(adapter->rx_rings, M_DEVBUF);
3895 }
3896
3897
3898 /*********************************************************************
3899  *
3900  *  Free receive ring data structures
3901  *
3902  **********************************************************************/
3903 static void
3904 ixgbe_free_receive_buffers(struct rx_ring *rxr)
3905 {
3906         struct adapter          *adapter = rxr->adapter;
3907         struct ixgbe_rx_buf     *rxbuf;
3908
3909         INIT_DEBUGOUT("free_receive_structures: begin");
3910
3911         /* Cleanup any existing buffers */
3912         if (rxr->rx_buffers != NULL) {
3913                 for (int i = 0; i < adapter->num_rx_desc; i++) {
3914                         rxbuf = &rxr->rx_buffers[i];
3915                         if (rxbuf->m_head != NULL) {
3916                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3917                                     BUS_DMASYNC_POSTREAD);
3918                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3919                                 rxbuf->m_head->m_flags |= M_PKTHDR;
3920                                 m_freem(rxbuf->m_head);
3921                         }
3922                         if (rxbuf->m_pack != NULL) {
3923                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3924                                     BUS_DMASYNC_POSTREAD);
3925                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3926                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
3927                                 m_freem(rxbuf->m_pack);
3928                         }
3929                         rxbuf->m_head = NULL;
3930                         rxbuf->m_pack = NULL;
3931                         if (rxbuf->hmap != NULL) {
3932                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
3933                                 rxbuf->hmap = NULL;
3934                         }
3935                         if (rxbuf->pmap != NULL) {
3936                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
3937                                 rxbuf->pmap = NULL;
3938                         }
3939                 }
3940                 if (rxr->rx_buffers != NULL) {
3941                         free(rxr->rx_buffers, M_DEVBUF);
3942                         rxr->rx_buffers = NULL;
3943                 }
3944         }
3945
3946         if (rxr->htag != NULL) {
3947                 bus_dma_tag_destroy(rxr->htag);
3948                 rxr->htag = NULL;
3949         }
3950         if (rxr->ptag != NULL) {
3951                 bus_dma_tag_destroy(rxr->ptag);
3952                 rxr->ptag = NULL;
3953         }
3954
3955         return;
3956 }
3957
3958 static __inline void
3959 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
3960 {
3961
3962         /*
3963          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
3964          * should be computed by hardware. Also it should not have VLAN tag in
3965          * ethernet header.
3966          */
3967         if (rxr->lro_enabled &&
3968             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
3969             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
3970             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
3971             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) &&
3972             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
3973             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
3974                 /*
3975                  * Send to the stack if:
3976                  **  - LRO not enabled, or
3977                  **  - no LRO resources, or
3978                  **  - lro enqueue fails
3979                  */
3980                 if (rxr->lro.lro_cnt != 0)
3981                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
3982                                 return;
3983         }
3984         (*ifp->if_input)(ifp, m);
3985 }
3986
3987 static __inline void
3988 ixgbe_rx_discard(struct rx_ring *rxr, int i)
3989 {
3990         struct adapter          *adapter = rxr->adapter;
3991         struct ixgbe_rx_buf     *rbuf;
3992         struct mbuf             *mh, *mp;
3993
3994         rbuf = &rxr->rx_buffers[i];
3995         if (rbuf->fmp != NULL) /* Partial chain ? */
3996                 m_freem(rbuf->fmp);
3997
3998         mh = rbuf->m_head;
3999         mp = rbuf->m_pack;
4000
4001         /* Reuse loaded DMA map and just update mbuf chain */
4002         mh->m_len = MHLEN;
4003         mh->m_flags |= M_PKTHDR;
4004         mh->m_next = NULL;
4005
4006         mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz;
4007         mp->m_data = mp->m_ext.ext_buf;
4008         mp->m_next = NULL;
4009         return;
4010 }
4011
4012
4013 /*********************************************************************
4014  *
4015  *  This routine executes in interrupt context. It replenishes
4016  *  the mbufs in the descriptor and sends data which has been
4017  *  dma'ed into host memory to upper layer.
4018  *
4019  *  We loop at most count times if count is > 0, or until done if
4020  *  count < 0.
4021  *
4022  *  Return TRUE for more work, FALSE for all clean.
4023  *********************************************************************/
4024 static bool
4025 ixgbe_rxeof(struct ix_queue *que, int count)
4026 {
4027         struct adapter          *adapter = que->adapter;
4028         struct rx_ring          *rxr = que->rxr;
4029         struct ifnet            *ifp = adapter->ifp;
4030         struct lro_ctrl         *lro = &rxr->lro;
4031         struct lro_entry        *queued;
4032         int                     i, nextp, processed = 0;
4033         u32                     staterr = 0;
4034         union ixgbe_adv_rx_desc *cur;
4035         struct ixgbe_rx_buf     *rbuf, *nbuf;
4036
4037         IXGBE_RX_LOCK(rxr);
4038
4039         /* Sync the ring. */
4040         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4041             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4042
4043         for (i = rxr->next_to_check; count != 0;) {
4044                 struct mbuf     *sendmp, *mh, *mp;
4045                 u32             rsc, ptype;
4046                 u16             hlen, plen, hdr, vtag;
4047                 bool            eop;
4048
4049                 cur = &rxr->rx_base[i];
4050                 staterr = le32toh(cur->wb.upper.status_error);
4051
4052                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4053                         break;
4054                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4055                         break;
4056
4057                 count--;
4058                 sendmp = NULL;
4059                 nbuf = NULL;
4060                 rsc = 0;
4061                 cur->wb.upper.status_error = 0;
4062                 rbuf = &rxr->rx_buffers[i];
4063                 mh = rbuf->m_head;
4064                 mp = rbuf->m_pack;
4065
4066                 plen = le16toh(cur->wb.upper.length);
4067                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4068                     IXGBE_RXDADV_PKTTYPE_MASK;
4069                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4070                 vtag = le16toh(cur->wb.upper.vlan);
4071                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4072
4073                 /* Make sure all parts of a bad packet are discarded */
4074                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4075                     (rxr->discard)) {
4076                         ifp->if_ierrors++;
4077                         rxr->rx_discarded++;
4078                         if (!eop)
4079                                 rxr->discard = TRUE;
4080                         else
4081                                 rxr->discard = FALSE;
4082                         ixgbe_rx_discard(rxr, i);
4083                         goto next_desc;
4084                 }
4085
4086                 /*
4087                 ** On 82599 which supports a hardware
4088                 ** LRO (called HW RSC), packets need
4089                 ** not be fragmented across sequential
4090                 ** descriptors, rather the next descriptor
4091                 ** is indicated in bits of the descriptor.
4092                 ** This also means that we might proceses
4093                 ** more than one packet at a time, something
4094                 ** that has never been true before, it
4095                 ** required eliminating global chain pointers
4096                 ** in favor of what we are doing here.  -jfv
4097                 */
4098                 if (!eop) {
4099                         /*
4100                         ** Figure out the next descriptor
4101                         ** of this frame.
4102                         */
4103                         if (rxr->hw_rsc == TRUE) {
4104                                 rsc = ixgbe_rsc_count(cur);
4105                                 rxr->rsc_num += (rsc - 1);
4106                         }
4107                         if (rsc) { /* Get hardware index */
4108                                 nextp = ((staterr &
4109                                     IXGBE_RXDADV_NEXTP_MASK) >>
4110                                     IXGBE_RXDADV_NEXTP_SHIFT);
4111                         } else { /* Just sequential */
4112                                 nextp = i + 1;
4113                                 if (nextp == adapter->num_rx_desc)
4114                                         nextp = 0;
4115                         }
4116                         nbuf = &rxr->rx_buffers[nextp];
4117                         prefetch(nbuf);
4118                 }
4119                 /*
4120                 ** The header mbuf is ONLY used when header
4121                 ** split is enabled, otherwise we get normal
4122                 ** behavior, ie, both header and payload
4123                 ** are DMA'd into the payload buffer.
4124                 **
4125                 ** Rather than using the fmp/lmp global pointers
4126                 ** we now keep the head of a packet chain in the
4127                 ** buffer struct and pass this along from one
4128                 ** descriptor to the next, until we get EOP.
4129                 */
4130                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4131                         /* This must be an initial descriptor */
4132                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4133                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4134                         if (hlen > IXGBE_RX_HDR)
4135                                 hlen = IXGBE_RX_HDR;
4136                         mh->m_len = hlen;
4137                         mh->m_flags |= M_PKTHDR;
4138                         mh->m_next = NULL;
4139                         mh->m_pkthdr.len = mh->m_len;
4140                         /* Null buf pointer so it is refreshed */
4141                         rbuf->m_head = NULL;
4142                         /*
4143                         ** Check the payload length, this
4144                         ** could be zero if its a small
4145                         ** packet.
4146                         */
4147                         if (plen > 0) {
4148                                 mp->m_len = plen;
4149                                 mp->m_next = NULL;
4150                                 mp->m_flags &= ~M_PKTHDR;
4151                                 mh->m_next = mp;
4152                                 mh->m_pkthdr.len += mp->m_len;
4153                                 /* Null buf pointer so it is refreshed */
4154                                 rbuf->m_pack = NULL;
4155                                 rxr->rx_split_packets++;
4156                         }
4157                         /*
4158                         ** Now create the forward
4159                         ** chain so when complete
4160                         ** we wont have to.
4161                         */
4162                         if (eop == 0) {
4163                                 /* stash the chain head */
4164                                 nbuf->fmp = mh;
4165                                 /* Make forward chain */
4166                                 if (plen)
4167                                         mp->m_next = nbuf->m_pack;
4168                                 else
4169                                         mh->m_next = nbuf->m_pack;
4170                         } else {
4171                                 /* Singlet, prepare to send */
4172                                 sendmp = mh;
4173                                 if (staterr & IXGBE_RXD_STAT_VP) {
4174                                         sendmp->m_pkthdr.ether_vtag = vtag;
4175                                         sendmp->m_flags |= M_VLANTAG;
4176                                 }
4177                         }
4178                 } else {
4179                         /*
4180                         ** Either no header split, or a
4181                         ** secondary piece of a fragmented
4182                         ** split packet.
4183                         */
4184                         mp->m_len = plen;
4185                         /*
4186                         ** See if there is a stored head
4187                         ** that determines what we are
4188                         */
4189                         sendmp = rbuf->fmp;
4190                         rbuf->m_pack = rbuf->fmp = NULL;
4191
4192                         if (sendmp != NULL) /* secondary frag */
4193                                 sendmp->m_pkthdr.len += mp->m_len;
4194                         else {
4195                                 /* first desc of a non-ps chain */
4196                                 sendmp = mp;
4197                                 sendmp->m_flags |= M_PKTHDR;
4198                                 sendmp->m_pkthdr.len = mp->m_len;
4199                                 if (staterr & IXGBE_RXD_STAT_VP) {
4200                                         sendmp->m_pkthdr.ether_vtag = vtag;
4201                                         sendmp->m_flags |= M_VLANTAG;
4202                                 }
4203                         }
4204                         /* Pass the head pointer on */
4205                         if (eop == 0) {
4206                                 nbuf->fmp = sendmp;
4207                                 sendmp = NULL;
4208                                 mp->m_next = nbuf->m_pack;
4209                         }
4210                 }
4211                 ++processed;
4212                 /* Sending this frame? */
4213                 if (eop) {
4214                         sendmp->m_pkthdr.rcvif = ifp;
4215                         ifp->if_ipackets++;
4216                         rxr->rx_packets++;
4217                         /* capture data for AIM */
4218                         rxr->bytes += sendmp->m_pkthdr.len;
4219                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4220                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4221                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4222 #if __FreeBSD_version >= 800000
4223                         sendmp->m_pkthdr.flowid = que->msix;
4224                         sendmp->m_flags |= M_FLOWID;
4225 #endif
4226                 }
4227 next_desc:
4228                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4229                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4230
4231                 /* Advance our pointers to the next descriptor. */
4232                 if (++i == adapter->num_rx_desc)
4233                         i = 0;
4234
4235                 /* Now send to the stack or do LRO */
4236                 if (sendmp != NULL)
4237                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4238
4239                /* Every 8 descriptors we go to refresh mbufs */
4240                 if (processed == 8) {
4241                         ixgbe_refresh_mbufs(rxr, i);
4242                         processed = 0;
4243                 }
4244         }
4245
4246         /* Refresh any remaining buf structs */
4247         if (processed != 0) {
4248                 ixgbe_refresh_mbufs(rxr, i);
4249                 processed = 0;
4250         }
4251
4252         rxr->next_to_check = i;
4253
4254         /*
4255          * Flush any outstanding LRO work
4256          */
4257         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4258                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4259                 tcp_lro_flush(lro, queued);
4260         }
4261
4262         IXGBE_RX_UNLOCK(rxr);
4263
4264         /*
4265         ** We still have cleaning to do?
4266         ** Schedule another interrupt if so.
4267         */
4268         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4269                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4270                 return (TRUE);
4271         }
4272
4273         return (FALSE);
4274 }
4275
4276
4277 /*********************************************************************
4278  *
4279  *  Verify that the hardware indicated that the checksum is valid.
4280  *  Inform the stack about the status of checksum so that stack
4281  *  doesn't spend time verifying the checksum.
4282  *
4283  *********************************************************************/
4284 static void
4285 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4286 {
4287         u16     status = (u16) staterr;
4288         u8      errors = (u8) (staterr >> 24);
4289         bool    sctp = FALSE;
4290
4291         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4292             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4293                 sctp = TRUE;
4294
4295         if (status & IXGBE_RXD_STAT_IPCS) {
4296                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4297                         /* IP Checksum Good */
4298                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4299                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4300
4301                 } else
4302                         mp->m_pkthdr.csum_flags = 0;
4303         }
4304         if (status & IXGBE_RXD_STAT_L4CS) {
4305                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4306 #if __FreeBSD_version >= 800000
4307                 if (sctp)
4308                         type = CSUM_SCTP_VALID;
4309 #endif
4310                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4311                         mp->m_pkthdr.csum_flags |= type;
4312                         if (!sctp)
4313                                 mp->m_pkthdr.csum_data = htons(0xffff);
4314                 }
4315         }
4316         return;
4317 }
4318
4319
4320 /*
4321 ** This routine is run via an vlan config EVENT,
4322 ** it enables us to use the HW Filter table since
4323 ** we can get the vlan id. This just creates the
4324 ** entry in the soft version of the VFTA, init will
4325 ** repopulate the real table.
4326 */
4327 static void
4328 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4329 {
4330         struct adapter  *adapter = ifp->if_softc;
4331         u16             index, bit;
4332
4333         if (ifp->if_softc !=  arg)   /* Not our event */
4334                 return;
4335
4336         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4337                 return;
4338
4339         index = (vtag >> 5) & 0x7F;
4340         bit = vtag & 0x1F;
4341         ixgbe_shadow_vfta[index] |= (1 << bit);
4342         ++adapter->num_vlans;
4343         /* Re-init to load the changes */
4344         ixgbe_init(adapter);
4345 }
4346
4347 /*
4348 ** This routine is run via an vlan
4349 ** unconfig EVENT, remove our entry
4350 ** in the soft vfta.
4351 */
4352 static void
4353 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4354 {
4355         struct adapter  *adapter = ifp->if_softc;
4356         u16             index, bit;
4357
4358         if (ifp->if_softc !=  arg)
4359                 return;
4360
4361         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4362                 return;
4363
4364         index = (vtag >> 5) & 0x7F;
4365         bit = vtag & 0x1F;
4366         ixgbe_shadow_vfta[index] &= ~(1 << bit);
4367         --adapter->num_vlans;
4368         /* Re-init to load the changes */
4369         ixgbe_init(adapter);
4370 }
4371
4372 static void
4373 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4374 {
4375         struct ixgbe_hw *hw = &adapter->hw;
4376         u32             ctrl;
4377
4378
4379         /*
4380         ** We get here thru init_locked, meaning
4381         ** a soft reset, this has already cleared
4382         ** the VFTA and other state, so if there
4383         ** have been no vlan's registered do nothing.
4384         */
4385         if (adapter->num_vlans == 0)
4386                 return;
4387
4388         /*
4389         ** A soft reset zero's out the VFTA, so
4390         ** we need to repopulate it now.
4391         */
4392         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4393                 if (ixgbe_shadow_vfta[i] != 0)
4394                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4395                             ixgbe_shadow_vfta[i]);
4396
4397         /* Enable the Filter Table */
4398         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4399         ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4400         ctrl |= IXGBE_VLNCTRL_VFE;
4401         if (hw->mac.type == ixgbe_mac_82598EB)
4402                 ctrl |= IXGBE_VLNCTRL_VME;
4403         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4404
4405         /* On 82599 the VLAN enable is per/queue in RXDCTL */
4406         if (hw->mac.type == ixgbe_mac_82599EB)
4407                 for (int i = 0; i < adapter->num_queues; i++) {
4408                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4409                                 ctrl |= IXGBE_RXDCTL_VME;
4410                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4411                 }
4412 }
4413
4414 static void
4415 ixgbe_enable_intr(struct adapter *adapter)
4416 {
4417         struct ixgbe_hw *hw = &adapter->hw;
4418         struct ix_queue *que = adapter->queues;
4419         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4420
4421
4422         /* Enable Fan Failure detection */
4423         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4424                     mask |= IXGBE_EIMS_GPI_SDP1;
4425
4426         /* 82599 specific interrupts */
4427         if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
4428                     mask |= IXGBE_EIMS_ECC;
4429                     mask |= IXGBE_EIMS_GPI_SDP1;
4430                     mask |= IXGBE_EIMS_GPI_SDP2;
4431 #ifdef IXGBE_FDIR
4432                     mask |= IXGBE_EIMS_FLOW_DIR;
4433 #endif
4434         }
4435
4436         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4437
4438         /* With RSS we use auto clear */
4439         if (adapter->msix_mem) {
4440                 mask = IXGBE_EIMS_ENABLE_MASK;
4441                 /* Dont autoclear Link */
4442                 mask &= ~IXGBE_EIMS_OTHER;
4443                 mask &= ~IXGBE_EIMS_LSC;
4444                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4445         }
4446
4447         /*
4448         ** Now enable all queues, this is done seperately to
4449         ** allow for handling the extended (beyond 32) MSIX
4450         ** vectors that can be used by 82599
4451         */
4452         for (int i = 0; i < adapter->num_queues; i++, que++)
4453                 ixgbe_enable_queue(adapter, que->msix);
4454
4455         IXGBE_WRITE_FLUSH(hw);
4456
4457         return;
4458 }
4459
4460 static void
4461 ixgbe_disable_intr(struct adapter *adapter)
4462 {
4463         if (adapter->msix_mem)
4464                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4465         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4466                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4467         } else {
4468                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4469                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4470                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4471         }
4472         IXGBE_WRITE_FLUSH(&adapter->hw);
4473         return;
4474 }
4475
4476 u16
4477 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4478 {
4479         u16 value;
4480
4481         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4482             reg, 2);
4483
4484         return (value);
4485 }
4486
4487 void
4488 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4489 {
4490         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4491             reg, value, 2);
4492
4493         return;
4494 }
4495
4496 /*
4497 ** Setup the correct IVAR register for a particular MSIX interrupt
4498 **   (yes this is all very magic and confusing :)
4499 **  - entry is the register array entry
4500 **  - vector is the MSIX vector for this queue
4501 **  - type is RX/TX/MISC
4502 */
4503 static void
4504 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4505 {
4506         struct ixgbe_hw *hw = &adapter->hw;
4507         u32 ivar, index;
4508
4509         vector |= IXGBE_IVAR_ALLOC_VAL;
4510
4511         switch (hw->mac.type) {
4512
4513         case ixgbe_mac_82598EB:
4514                 if (type == -1)
4515                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4516                 else
4517                         entry += (type * 64);
4518                 index = (entry >> 2) & 0x1F;
4519                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4520                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4521                 ivar |= (vector << (8 * (entry & 0x3)));
4522                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4523                 break;
4524
4525         case ixgbe_mac_82599EB:
4526                 if (type == -1) { /* MISC IVAR */
4527                         index = (entry & 1) * 8;
4528                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4529                         ivar &= ~(0xFF << index);
4530                         ivar |= (vector << index);
4531                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4532                 } else {        /* RX/TX IVARS */
4533                         index = (16 * (entry & 1)) + (8 * type);
4534                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4535                         ivar &= ~(0xFF << index);
4536                         ivar |= (vector << index);
4537                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4538                 }
4539
4540         default:
4541                 break;
4542         }
4543 }
4544
4545 static void
4546 ixgbe_configure_ivars(struct adapter *adapter)
4547 {
4548         struct  ix_queue *que = adapter->queues;
4549
4550         for (int i = 0; i < adapter->num_queues; i++, que++) {
4551                 /* First the RX queue entry */
4552                 ixgbe_set_ivar(adapter, i, que->msix, 0);
4553                 /* ... and the TX */
4554                 ixgbe_set_ivar(adapter, i, que->msix, 1);
4555                 /* Set an Initial EITR value */
4556                 IXGBE_WRITE_REG(&adapter->hw,
4557                     IXGBE_EITR(que->msix), IXGBE_LOW_LATENCY);
4558         }
4559
4560         /* For the Link interrupt */
4561         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4562 }
4563
4564 /*
4565 ** ixgbe_sfp_probe - called in the local timer to
4566 ** determine if a port had optics inserted.
4567 */
4568 static bool ixgbe_sfp_probe(struct adapter *adapter)
4569 {
4570         struct ixgbe_hw *hw = &adapter->hw;
4571         device_t        dev = adapter->dev;
4572         bool            result = FALSE;
4573
4574         if ((hw->phy.type == ixgbe_phy_nl) &&
4575             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4576                 s32 ret = hw->phy.ops.identify_sfp(hw);
4577                 if (ret)
4578                         goto out;
4579                 ret = hw->phy.ops.reset(hw);
4580                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4581                         device_printf(dev,"Unsupported SFP+ module detected!");
4582                         printf(" Reload driver with supported module.\n");
4583                         adapter->sfp_probe = FALSE;
4584                         goto out;
4585                 } else
4586                         device_printf(dev,"SFP+ module detected!\n");
4587                 /* We now have supported optics */
4588                 adapter->sfp_probe = FALSE;
4589                 result = TRUE;
4590         }
4591 out:
4592         return (result);
4593 }
4594
4595 /*
4596 ** Tasklet handler for MSIX Link interrupts
4597 **  - do outside interrupt since it might sleep
4598 */
4599 static void
4600 ixgbe_handle_link(void *context, int pending)
4601 {
4602         struct adapter  *adapter = context;
4603
4604         ixgbe_check_link(&adapter->hw,
4605             &adapter->link_speed, &adapter->link_up, 0);
4606         ixgbe_update_link_status(adapter);
4607 }
4608
4609 /*
4610 ** Tasklet for handling SFP module interrupts
4611 */
4612 static void
4613 ixgbe_handle_mod(void *context, int pending)
4614 {
4615         struct adapter  *adapter = context;
4616         struct ixgbe_hw *hw = &adapter->hw;
4617         device_t        dev = adapter->dev;
4618         u32 err;
4619
4620         err = hw->phy.ops.identify_sfp(hw);
4621         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4622                 device_printf(dev,
4623                     "Unsupported SFP+ module type was detected.\n");
4624                 return;
4625         }
4626         err = hw->mac.ops.setup_sfp(hw);
4627         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4628                 device_printf(dev,
4629                     "Setup failure - unsupported SFP+ module type.\n");
4630                 return;
4631         }
4632         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4633         return;
4634 }
4635
4636
4637 /*
4638 ** Tasklet for handling MSF (multispeed fiber) interrupts
4639 */
4640 static void
4641 ixgbe_handle_msf(void *context, int pending)
4642 {
4643         struct adapter  *adapter = context;
4644         struct ixgbe_hw *hw = &adapter->hw;
4645         u32 autoneg;
4646         bool negotiate;
4647
4648         autoneg = hw->phy.autoneg_advertised;
4649         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4650                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4651         if (hw->mac.ops.setup_link)
4652                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4653 #if 0
4654         ixgbe_check_link(&adapter->hw, &speed, &adapter->link_up, 0);
4655         ixgbe_update_link_status(adapter);
4656 #endif
4657         return;
4658 }
4659
4660 #ifdef IXGBE_FDIR
4661 /*
4662 ** Tasklet for reinitializing the Flow Director filter table
4663 */
4664 static void
4665 ixgbe_reinit_fdir(void *context, int pending)
4666 {
4667         struct adapter  *adapter = context;
4668         struct ifnet   *ifp = adapter->ifp;
4669
4670         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4671                 return;
4672         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4673         adapter->fdir_reinit = 0;
4674         /* Restart the interface */
4675         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4676         return;
4677 }
4678 #endif
4679
4680 /**********************************************************************
4681  *
4682  *  Update the board statistics counters.
4683  *
4684  **********************************************************************/
4685 static void
4686 ixgbe_update_stats_counters(struct adapter *adapter)
4687 {
4688         struct ifnet   *ifp = adapter->ifp;
4689         struct ixgbe_hw *hw = &adapter->hw;
4690         u32  missed_rx = 0, bprc, lxon, lxoff, total;
4691         u64  total_missed_rx = 0;
4692
4693         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
4694
4695         for (int i = 0; i < 8; i++) {
4696                 u32 mp;
4697                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
4698                 /* missed_rx tallies misses for the gprc workaround */
4699                 missed_rx += mp;
4700                 /* global total per queue */
4701                 adapter->stats.mpc[i] += mp;
4702                 /* Running comprehensive total for stats display */
4703                 total_missed_rx += adapter->stats.mpc[i];
4704                 if (hw->mac.type == ixgbe_mac_82598EB)
4705                         adapter->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
4706         }
4707
4708         /* Hardware workaround, gprc counts missed packets */
4709         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
4710         adapter->stats.gprc -= missed_rx;
4711
4712         if (hw->mac.type == ixgbe_mac_82599EB) {
4713                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
4714                 IXGBE_READ_REG(hw, IXGBE_GORCH); /* clears register */
4715                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
4716                 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* clears register */
4717                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL);
4718                 IXGBE_READ_REG(hw, IXGBE_TORH); /* clears register */
4719                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
4720                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
4721         } else {
4722                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
4723                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
4724                 /* 82598 only has a counter in the high register */
4725                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
4726                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
4727                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
4728         }
4729
4730         /*
4731          * Workaround: mprc hardware is incorrectly counting
4732          * broadcasts, so for now we subtract those.
4733          */
4734         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
4735         adapter->stats.bprc += bprc;
4736         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
4737         adapter->stats.mprc -= bprc;
4738
4739         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
4740         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
4741         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
4742         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
4743         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
4744         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
4745         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
4746         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
4747
4748         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
4749         adapter->stats.lxontxc += lxon;
4750         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
4751         adapter->stats.lxofftxc += lxoff;
4752         total = lxon + lxoff;
4753
4754         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
4755         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
4756         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
4757         adapter->stats.gptc -= total;
4758         adapter->stats.mptc -= total;
4759         adapter->stats.ptc64 -= total;
4760         adapter->stats.gotc -= total * ETHER_MIN_LEN;
4761
4762         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
4763         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
4764         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
4765         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
4766         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
4767         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
4768         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
4769         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
4770         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
4771         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
4772
4773
4774         /* Fill out the OS statistics structure */
4775         ifp->if_ipackets = adapter->stats.gprc;
4776         ifp->if_opackets = adapter->stats.gptc;
4777         ifp->if_ibytes = adapter->stats.gorc;
4778         ifp->if_obytes = adapter->stats.gotc;
4779         ifp->if_imcasts = adapter->stats.mprc;
4780         ifp->if_collisions = 0;
4781
4782         /* Rx Errors */
4783         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
4784                 adapter->stats.rlec;
4785 }
4786
4787
4788 /**********************************************************************
4789  *
4790  *  This routine is called only when ixgbe_display_debug_stats is enabled.
4791  *  This routine provides a way to take a look at important statistics
4792  *  maintained by the driver and hardware.
4793  *
4794  **********************************************************************/
4795 static void
4796 ixgbe_print_hw_stats(struct adapter * adapter)
4797 {
4798         device_t dev = adapter->dev;
4799
4800
4801         device_printf(dev,"Std Mbuf Failed = %lu\n",
4802                adapter->mbuf_defrag_failed);
4803         device_printf(dev,"Missed Packets = %llu\n",
4804                (long long)adapter->stats.mpc[0]);
4805         device_printf(dev,"Receive length errors = %llu\n",
4806                ((long long)adapter->stats.roc +
4807                (long long)adapter->stats.ruc));
4808         device_printf(dev,"Crc errors = %llu\n",
4809                (long long)adapter->stats.crcerrs);
4810         device_printf(dev,"Driver dropped packets = %lu\n",
4811                adapter->dropped_pkts);
4812         device_printf(dev, "watchdog timeouts = %ld\n",
4813                adapter->watchdog_events);
4814
4815         device_printf(dev,"XON Rcvd = %llu\n",
4816                (long long)adapter->stats.lxonrxc);
4817         device_printf(dev,"XON Xmtd = %llu\n",
4818                (long long)adapter->stats.lxontxc);
4819         device_printf(dev,"XOFF Rcvd = %llu\n",
4820                (long long)adapter->stats.lxoffrxc);
4821         device_printf(dev,"XOFF Xmtd = %llu\n",
4822                (long long)adapter->stats.lxofftxc);
4823
4824         device_printf(dev,"Total Packets Rcvd = %llu\n",
4825                (long long)adapter->stats.tpr);
4826         device_printf(dev,"Good Packets Rcvd = %llu\n",
4827                (long long)adapter->stats.gprc);
4828         device_printf(dev,"Good Packets Xmtd = %llu\n",
4829                (long long)adapter->stats.gptc);
4830         device_printf(dev,"TSO Transmissions = %lu\n",
4831                adapter->tso_tx);
4832
4833         return;
4834 }
4835
4836 /**********************************************************************
4837  *
4838  *  This routine is called only when em_display_debug_stats is enabled.
4839  *  This routine provides a way to take a look at important statistics
4840  *  maintained by the driver and hardware.
4841  *
4842  **********************************************************************/
4843 static void
4844 ixgbe_print_debug_info(struct adapter *adapter)
4845 {
4846         device_t dev = adapter->dev;
4847         struct ixgbe_hw         *hw = &adapter->hw;
4848         struct ix_queue         *que = adapter->queues;
4849         struct rx_ring          *rxr;
4850         struct tx_ring          *txr;
4851         struct lro_ctrl         *lro;
4852
4853         device_printf(dev,"Error Byte Count = %u \n",
4854             IXGBE_READ_REG(hw, IXGBE_ERRBC));
4855
4856         for (int i = 0; i < adapter->num_queues; i++, que++) {
4857                 txr = que->txr;
4858                 rxr = que->rxr;
4859                 lro = &rxr->lro;
4860                 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n",
4861                     que->msix, (long)que->irqs);
4862                 device_printf(dev,"RX[%d]: rdh = %d, hw rdt = %d\n",
4863                     i, IXGBE_READ_REG(hw, IXGBE_RDH(i)),
4864                     IXGBE_READ_REG(hw, IXGBE_RDT(i)));
4865                 device_printf(dev,"TX[%d] tdh = %d, hw tdt = %d\n", i,
4866                     IXGBE_READ_REG(hw, IXGBE_TDH(i)),
4867                     IXGBE_READ_REG(hw, IXGBE_TDT(i)));
4868                 device_printf(dev,"RX(%d) Packets Received: %lld\n",
4869                     rxr->me, (long long)rxr->rx_packets);
4870                 device_printf(dev,"RX(%d) Split RX Packets: %lld\n",
4871                     rxr->me, (long long)rxr->rx_split_packets);
4872                 device_printf(dev,"RX(%d) Bytes Received: %lu\n",
4873                     rxr->me, (long)rxr->rx_bytes);
4874                 device_printf(dev,"RX(%d) LRO Queued= %d\n",
4875                     rxr->me, lro->lro_queued);
4876                 device_printf(dev,"RX(%d) LRO Flushed= %d\n",
4877                     rxr->me, lro->lro_flushed);
4878                 device_printf(dev,"RX(%d) HW LRO Merges= %lu\n",
4879                     rxr->me, (long)rxr->rsc_num);
4880                 device_printf(dev,"TX(%d) Packets Sent: %lu\n",
4881                     txr->me, (long)txr->total_packets);
4882                 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n",
4883                     txr->me, (long)txr->no_desc_avail);
4884         }
4885
4886         device_printf(dev,"Link IRQ Handled: %lu\n",
4887             (long)adapter->link_irq);
4888         return;
4889 }
4890
4891 static int
4892 ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS)
4893 {
4894         int             error;
4895         int             result;
4896         struct adapter *adapter;
4897
4898         result = -1;
4899         error = sysctl_handle_int(oidp, &result, 0, req);
4900
4901         if (error || !req->newptr)
4902                 return (error);
4903
4904         if (result == 1) {
4905                 adapter = (struct adapter *) arg1;
4906                 ixgbe_print_hw_stats(adapter);
4907         }
4908         return error;
4909 }
4910
4911 static int
4912 ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS)
4913 {
4914         int error, result;
4915         struct adapter *adapter;
4916
4917         result = -1;
4918         error = sysctl_handle_int(oidp, &result, 0, req);
4919
4920         if (error || !req->newptr)
4921                 return (error);
4922
4923         if (result == 1) {
4924                 adapter = (struct adapter *) arg1;
4925                 ixgbe_print_debug_info(adapter);
4926         }
4927         return error;
4928 }
4929
4930 /*
4931 ** Set flow control using sysctl:
4932 ** Flow control values:
4933 **      0 - off
4934 **      1 - rx pause
4935 **      2 - tx pause
4936 **      3 - full
4937 */
4938 static int
4939 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
4940 {
4941         int error;
4942         struct adapter *adapter;
4943
4944         error = sysctl_handle_int(oidp, &ixgbe_flow_control, 0, req);
4945
4946         if (error)
4947                 return (error);
4948
4949         adapter = (struct adapter *) arg1;
4950         switch (ixgbe_flow_control) {
4951                 case ixgbe_fc_rx_pause:
4952                 case ixgbe_fc_tx_pause:
4953                 case ixgbe_fc_full:
4954                         adapter->hw.fc.requested_mode = ixgbe_flow_control;
4955                         break;
4956                 case ixgbe_fc_none:
4957                 default:
4958                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
4959         }
4960
4961         ixgbe_fc_enable(&adapter->hw, 0);
4962         return error;
4963 }
4964
4965 static void
4966 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
4967         const char *description, int *limit, int value)
4968 {
4969         *limit = value;
4970         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4971             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4972             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
4973 }