sys/dev/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2008, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #endif
  38
  39 /* Undefine this if not using CURRENT */
  40 #define IXGBE_VLAN_EVENTS
  41
  42 #include "ixgbe.h"
  43
  44 /*********************************************************************
  45  *  Set this to one to display debug statistics
  46  *********************************************************************/
  47 int             ixgbe_display_debug_stats = 0;
  48
  49 /*********************************************************************
  50  *  Driver version
  51  *********************************************************************/
  52 char ixgbe_driver_version[] = "1.4.7";
  53
  54 /*********************************************************************
  55  *  PCI Device ID Table
  56  *
  57  *  Used by probe to select devices to load on
  58  *  Last field stores an index into ixgbe_strings
  59  *  Last entry must be all 0s
  60  *
  61  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  62  *********************************************************************/
  63
  64 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  65 {
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  73         /* required last entry */
  74         {0, 0, 0, 0, 0}
  75 };
  76
  77 /*********************************************************************
  78  *  Table of branding strings
  79  *********************************************************************/
  80
  81 static char    *ixgbe_strings[] = {
  82         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  83 };
  84
  85 /*********************************************************************
  86  *  Function prototypes
  87  *********************************************************************/
  88 static int      ixgbe_probe(device_t);
  89 static int      ixgbe_attach(device_t);
  90 static int      ixgbe_detach(device_t);
  91 static int      ixgbe_shutdown(device_t);
  92 static void     ixgbe_start(struct ifnet *);
  93 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
  94 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
  95 static void     ixgbe_watchdog(struct adapter *);
  96 static void     ixgbe_init(void *);
  97 static void     ixgbe_init_locked(struct adapter *);
  98 static void     ixgbe_stop(void *);
  99 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 100 static int      ixgbe_media_change(struct ifnet *);
 101 static void     ixgbe_identify_hardware(struct adapter *);
 102 static int      ixgbe_allocate_pci_resources(struct adapter *);
 103 static int      ixgbe_allocate_msix(struct adapter *);
 104 static int      ixgbe_allocate_legacy(struct adapter *);
 105 static int      ixgbe_allocate_queues(struct adapter *);
 106 static int      ixgbe_setup_msix(struct adapter *);
 107 static void     ixgbe_free_pci_resources(struct adapter *);
 108 static void     ixgbe_local_timer(void *);
 109 static int      ixgbe_hardware_init(struct adapter *);
 110 static void     ixgbe_setup_interface(device_t, struct adapter *);
 111
 112 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 113 static int      ixgbe_setup_transmit_structures(struct adapter *);
 114 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 115 static void     ixgbe_initialize_transmit_units(struct adapter *);
 116 static void     ixgbe_free_transmit_structures(struct adapter *);
 117 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 118
 119 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 120 static int      ixgbe_setup_receive_structures(struct adapter *);
 121 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 122 static void     ixgbe_initialize_receive_units(struct adapter *);
 123 static void     ixgbe_free_receive_structures(struct adapter *);
 124 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 125
 126 static void     ixgbe_enable_intr(struct adapter *);
 127 static void     ixgbe_disable_intr(struct adapter *);
 128 static void     ixgbe_update_stats_counters(struct adapter *);
 129 static bool     ixgbe_txeof(struct tx_ring *);
 130 static bool     ixgbe_rxeof(struct rx_ring *, int);
 131 static void     ixgbe_rx_checksum(struct adapter *, u32, struct mbuf *);
 132 static void     ixgbe_set_promisc(struct adapter *);
 133 static void     ixgbe_disable_promisc(struct adapter *);
 134 static void     ixgbe_set_multi(struct adapter *);
 135 static void     ixgbe_print_hw_stats(struct adapter *);
 136 static void     ixgbe_print_debug_info(struct adapter *);
 137 static void     ixgbe_update_link_status(struct adapter *);
 138 static int      ixgbe_get_buf(struct rx_ring *, int);
 139 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 140 static int      ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS);
 141 static int      ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS);
 142 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 143 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 144                     struct ixgbe_dma_alloc *, int);
 145 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 146 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 147                     const char *, int *, int);
 148 static boolean_t ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 149 static boolean_t ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 150 static void     ixgbe_set_ivar(struct adapter *, u16, u8);
 151 static void     ixgbe_configure_ivars(struct adapter *);
 152 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 153
 154 #ifdef IXGBE_VLAN_EVENTS
 155 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 156 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 157 #endif
 158
 159 /* Legacy (single vector interrupt handler */
 160 static void     ixgbe_legacy_irq(void *);
 161
 162 /* The MSI/X Interrupt handlers */
 163 static void     ixgbe_msix_tx(void *);
 164 static void     ixgbe_msix_rx(void *);
 165 static void     ixgbe_msix_link(void *);
 166
 167 /* Legacy interrupts use deferred handlers */
 168 static void     ixgbe_handle_tx(void *context, int pending);
 169 static void     ixgbe_handle_rx(void *context, int pending);
 170
 171 #ifndef NO_82598_A0_SUPPORT
 172 static void     desc_flip(void *);
 173 #endif
 174
 175 /*********************************************************************
 176  *  FreeBSD Device Interface Entry Points
 177  *********************************************************************/
 178
 179 static device_method_t ixgbe_methods[] = {
 180         /* Device interface */
 181         DEVMETHOD(device_probe, ixgbe_probe),
 182         DEVMETHOD(device_attach, ixgbe_attach),
 183         DEVMETHOD(device_detach, ixgbe_detach),
 184         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 185         {0, 0}
 186 };
 187
 188 static driver_t ixgbe_driver = {
 189         "ix", ixgbe_methods, sizeof(struct adapter),
 190 };
 191
 192 static devclass_t ixgbe_devclass;
 193 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 194
 195 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 196 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 197
 198 /*
 199 ** TUNEABLE PARAMETERS:
 200 */
 201
 202 /* How many packets rxeof tries to clean at a time */
 203 static int ixgbe_rx_process_limit = 100;
 204 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 205
 206 /* Flow control setting, default to full */
 207 static int ixgbe_flow_control = 3;
 208 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
 209
 210 /*
 211  * Should the driver do LRO on the RX end
 212  *  this can be toggled on the fly, but the
 213  *  interface must be reset (down/up) for it
 214  *  to take effect.
 215  */
 216 static int ixgbe_enable_lro = 0;
 217 TUNABLE_INT("hw.ixgbe.enable_lro", &ixgbe_enable_lro);
 218
 219 /*
 220  * MSIX should be the default for best performance,
 221  * but this allows it to be forced off for testing.
 222  */
 223 static int ixgbe_enable_msix = 1;
 224 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 225
 226 /*
 227  * Number of TX/RX Queues, with 0 setting
 228  * it autoconfigures to the number of cpus.
 229  */
 230 static int ixgbe_tx_queues = 1;
 231 TUNABLE_INT("hw.ixgbe.tx_queues", &ixgbe_tx_queues);
 232 static int ixgbe_rx_queues = 4;
 233 TUNABLE_INT("hw.ixgbe.rx_queues", &ixgbe_rx_queues);
 234
 235 /* Number of TX descriptors per ring */
 236 static int ixgbe_txd = DEFAULT_TXD;
 237 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 238
 239 /* Number of RX descriptors per ring */
 240 static int ixgbe_rxd = DEFAULT_RXD;
 241 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 242
 243 /* Total number of Interfaces - need for config sanity check */
 244 static int ixgbe_total_ports;
 245
 246 /* Optics type of this interface */
 247 static int ixgbe_optics;
 248
 249 /*********************************************************************
 250  *  Device identification routine
 251  *
 252  *  ixgbe_probe determines if the driver should be loaded on
 253  *  adapter based on PCI vendor/device id of the adapter.
 254  *
 255  *  return 0 on success, positive on failure
 256  *********************************************************************/
 257
 258 static int
 259 ixgbe_probe(device_t dev)
 260 {
 261         ixgbe_vendor_info_t *ent;
 262
 263         u_int16_t       pci_vendor_id = 0;
 264         u_int16_t       pci_device_id = 0;
 265         u_int16_t       pci_subvendor_id = 0;
 266         u_int16_t       pci_subdevice_id = 0;
 267         char            adapter_name[128];
 268
 269         INIT_DEBUGOUT("ixgbe_probe: begin");
 270
 271         pci_vendor_id = pci_get_vendor(dev);
 272         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 273                 return (ENXIO);
 274
 275         pci_device_id = pci_get_device(dev);
 276         pci_subvendor_id = pci_get_subvendor(dev);
 277         pci_subdevice_id = pci_get_subdevice(dev);
 278
 279         ent = ixgbe_vendor_info_array;
 280         while (ent->vendor_id != 0) {
 281                 if ((pci_vendor_id == ent->vendor_id) &&
 282                     (pci_device_id == ent->device_id) &&
 283
 284                     ((pci_subvendor_id == ent->subvendor_id) ||
 285                      (ent->subvendor_id == 0)) &&
 286
 287                     ((pci_subdevice_id == ent->subdevice_id) ||
 288                      (ent->subdevice_id == 0))) {
 289                         sprintf(adapter_name, "%s, Version - %s",
 290                                 ixgbe_strings[ent->index],
 291                                 ixgbe_driver_version);
 292                         switch (pci_device_id) {
 293                                 case IXGBE_DEV_ID_82598AT_DUAL_PORT :
 294                                         ixgbe_total_ports += 2;
 295                                         break;
 296                                 case IXGBE_DEV_ID_82598_CX4_DUAL_PORT :
 297                                         ixgbe_optics = IFM_10G_CX4;
 298                                         ixgbe_total_ports += 2;
 299                                         break;
 300                                 case IXGBE_DEV_ID_82598AF_DUAL_PORT :
 301                                         ixgbe_optics = IFM_10G_SR;
 302                                         ixgbe_total_ports += 2;
 303                                         break;
 304                                 case IXGBE_DEV_ID_82598AF_SINGLE_PORT :
 305                                         ixgbe_optics = IFM_10G_SR;
 306                                         ixgbe_total_ports += 1;
 307                                         break;
 308                                 case IXGBE_DEV_ID_82598EB_XF_LR :
 309                                         ixgbe_optics = IFM_10G_LR;
 310                                         ixgbe_total_ports += 1;
 311                                         break;
 312                                 case IXGBE_DEV_ID_82598EB_CX4 :
 313                                         ixgbe_optics = IFM_10G_CX4;
 314                                         ixgbe_total_ports += 1;
 315                                         break;
 316                                 case IXGBE_DEV_ID_82598AT :
 317                                         ixgbe_total_ports += 1;
 318                                 default:
 319                                         break;
 320                         }
 321                         device_set_desc_copy(dev, adapter_name);
 322                         return (0);
 323                 }
 324                 ent++;
 325         }
 326
 327         return (ENXIO);
 328 }
 329
 330 /*********************************************************************
 331  *  Device initialization routine
 332  *
 333  *  The attach entry point is called when the driver is being loaded.
 334  *  This routine identifies the type of hardware, allocates all resources
 335  *  and initializes the hardware.
 336  *
 337  *  return 0 on success, positive on failure
 338  *********************************************************************/
 339
 340 static int
 341 ixgbe_attach(device_t dev)
 342 {
 343         struct adapter *adapter;
 344         int             error = 0;
 345         u32     ctrl_ext;
 346
 347         INIT_DEBUGOUT("ixgbe_attach: begin");
 348
 349         /* Allocate, clear, and link in our adapter structure */
 350         adapter = device_get_softc(dev);
 351         adapter->dev = adapter->osdep.dev = dev;
 352
 353         /* Core Lock Init*/
 354         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 355
 356         /* SYSCTL APIs */
 357         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 358                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 359                         OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW,
 360                         adapter, 0, ixgbe_sysctl_stats, "I", "Statistics");
 361
 362         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 363                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 364                         OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW,
 365                         adapter, 0, ixgbe_sysctl_debug, "I", "Debug Info");
 366
 367         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 368                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 369                         OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW,
 370                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 371
 372         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 373                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 374                         OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW,
 375                         &ixgbe_enable_lro, 1, "Large Receive Offload");
 376
 377         /* Set up the timer callout */
 378         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 379
 380         /* Determine hardware revision */
 381         ixgbe_identify_hardware(adapter);
 382
 383         /* Indicate to RX setup to use Jumbo Clusters */
 384         adapter->bigbufs = TRUE;
 385
 386         /* Do base PCI setup - map BAR0 */
 387         if (ixgbe_allocate_pci_resources(adapter)) {
 388                 device_printf(dev, "Allocation of PCI resources failed\n");
 389                 error = ENXIO;
 390                 goto err_out;
 391         }
 392
 393         /* Do descriptor calc and sanity checks */
 394         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 395             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 396                 device_printf(dev, "TXD config issue, using default!\n");
 397                 adapter->num_tx_desc = DEFAULT_TXD;
 398         } else
 399                 adapter->num_tx_desc = ixgbe_txd;
 400
 401         /*
 402         ** With many RX rings it is easy to exceed the
 403         ** system mbuf allocation. Tuning nmbclusters
 404         ** can alleviate this.
 405         */
 406         if ((adapter->num_rx_queues > 1) && (nmbclusters > 0 )){
 407                 int s;
 408                 /* Calculate the total RX mbuf needs */
 409                 s = (ixgbe_rxd * adapter->num_rx_queues) * ixgbe_total_ports;
 410                 if (s > nmbclusters) {
 411                         device_printf(dev, "RX Descriptors exceed "
 412                             "system mbuf max, using default instead!\n");
 413                         ixgbe_rxd = DEFAULT_RXD;
 414                 }
 415         }
 416
 417         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 418             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 419                 device_printf(dev, "RXD config issue, using default!\n");
 420                 adapter->num_rx_desc = DEFAULT_RXD;
 421         } else
 422                 adapter->num_rx_desc = ixgbe_rxd;
 423
 424         /* Allocate our TX/RX Queues */
 425         if (ixgbe_allocate_queues(adapter)) {
 426                 error = ENOMEM;
 427                 goto err_out;
 428         }
 429
 430         /* Initialize the shared code */
 431         if (ixgbe_init_shared_code(&adapter->hw)) {
 432                 device_printf(dev,"Unable to initialize the shared code\n");
 433                 error = EIO;
 434                 goto err_late;
 435         }
 436
 437         /* Initialize the hardware */
 438         if (ixgbe_hardware_init(adapter)) {
 439                 device_printf(dev,"Unable to initialize the hardware\n");
 440                 error = EIO;
 441                 goto err_late;
 442         }
 443
 444         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 445                 error = ixgbe_allocate_msix(adapter);
 446         else
 447                 error = ixgbe_allocate_legacy(adapter);
 448         if (error)
 449                 goto err_late;
 450
 451         /* Setup OS specific network interface */
 452         ixgbe_setup_interface(dev, adapter);
 453
 454         /* Sysctl for limiting the amount of work done in the taskqueue */
 455         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 456             "max number of rx packets to process", &adapter->rx_process_limit,
 457             ixgbe_rx_process_limit);
 458
 459         /* Initialize statistics */
 460         ixgbe_update_stats_counters(adapter);
 461
 462 #ifdef IXGBE_VLAN_EVENTS
 463         /* Register for VLAN events */
 464         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 465             ixgbe_register_vlan, 0, EVENTHANDLER_PRI_FIRST);
 466         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 467             ixgbe_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST);
 468 #endif
 469
 470         /* let hardware know driver is loaded */
 471         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 472         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 473         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 474
 475         INIT_DEBUGOUT("ixgbe_attach: end");
 476         return (0);
 477 err_late:
 478         ixgbe_free_transmit_structures(adapter);
 479         ixgbe_free_receive_structures(adapter);
 480 err_out:
 481         ixgbe_free_pci_resources(adapter);
 482         return (error);
 483
 484 }
 485
 486 /*********************************************************************
 487  *  Device removal routine
 488  *
 489  *  The detach entry point is called when the driver is being removed.
 490  *  This routine stops the adapter and deallocates all the resources
 491  *  that were allocated for driver operation.
 492  *
 493  *  return 0 on success, positive on failure
 494  *********************************************************************/
 495
 496 static int
 497 ixgbe_detach(device_t dev)
 498 {
 499         struct adapter *adapter = device_get_softc(dev);
 500         struct tx_ring *txr = adapter->tx_rings;
 501         struct rx_ring *rxr = adapter->rx_rings;
 502         u32     ctrl_ext;
 503
 504         INIT_DEBUGOUT("ixgbe_detach: begin");
 505
 506         /* Make sure VLANS are not using driver */
 507 #if __FreeBSD_version >= 700000
 508         if (adapter->ifp->if_vlantrunk != NULL) {
 509 #else
 510         if (adapter->ifp->if_nvlans != 0) {
 511 #endif
 512                 device_printf(dev,"Vlan in use, detach first\n");
 513                 return (EBUSY);
 514         }
 515
 516         IXGBE_CORE_LOCK(adapter);
 517         ixgbe_stop(adapter);
 518         IXGBE_CORE_UNLOCK(adapter);
 519
 520         for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
 521                 if (txr->tq) {
 522                         taskqueue_drain(txr->tq, &txr->tx_task);
 523                         taskqueue_free(txr->tq);
 524                         txr->tq = NULL;
 525                 }
 526         }
 527
 528         for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
 529                 if (rxr->tq) {
 530                         taskqueue_drain(rxr->tq, &rxr->rx_task);
 531                         taskqueue_free(rxr->tq);
 532                         rxr->tq = NULL;
 533                 }
 534         }
 535
 536 #ifdef IXGBE_VLAN_EVENTS
 537         /* Unregister VLAN events */
 538         if (adapter->vlan_attach != NULL)
 539                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 540         if (adapter->vlan_detach != NULL)
 541                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 542 #endif
 543
 544         /* let hardware know driver is unloading */
 545         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 546         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 547         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 548
 549         ether_ifdetach(adapter->ifp);
 550         callout_drain(&adapter->timer);
 551         ixgbe_free_pci_resources(adapter);
 552         bus_generic_detach(dev);
 553         if_free(adapter->ifp);
 554
 555         ixgbe_free_transmit_structures(adapter);
 556         ixgbe_free_receive_structures(adapter);
 557
 558         IXGBE_CORE_LOCK_DESTROY(adapter);
 559         return (0);
 560 }
 561
 562 /*********************************************************************
 563  *
 564  *  Shutdown entry point
 565  *
 566  **********************************************************************/
 567
 568 static int
 569 ixgbe_shutdown(device_t dev)
 570 {
 571         struct adapter *adapter = device_get_softc(dev);
 572         IXGBE_CORE_LOCK(adapter);
 573         ixgbe_stop(adapter);
 574         IXGBE_CORE_UNLOCK(adapter);
 575         return (0);
 576 }
 577
 578
 579 /*********************************************************************
 580  *  Transmit entry point
 581  *
 582  *  ixgbe_start is called by the stack to initiate a transmit.
 583  *  The driver will remain in this routine as long as there are
 584  *  packets to transmit and transmit resources are available.
 585  *  In case resources are not available stack is notified and
 586  *  the packet is requeued.
 587  **********************************************************************/
 588
 589 static void
 590 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 591 {
 592         struct mbuf    *m_head;
 593         struct adapter *adapter = txr->adapter;
 594
 595         IXGBE_TX_LOCK_ASSERT(txr);
 596
 597         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 598             IFF_DRV_RUNNING)
 599                 return;
 600         if (!adapter->link_active)
 601                 return;
 602
 603         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 604
 605                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 606                 if (m_head == NULL)
 607                         break;
 608
 609                 if (ixgbe_xmit(txr, &m_head)) {
 610                         if (m_head == NULL)
 611                                 break;
 612                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 613                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 614                         break;
 615                 }
 616                 /* Send a copy of the frame to the BPF listener */
 617                 ETHER_BPF_MTAP(ifp, m_head);
 618
 619                 /* Set timeout in case hardware has problems transmitting */
 620                 txr->watchdog_timer = IXGBE_TX_TIMEOUT;
 621
 622         }
 623         return;
 624 }
 625
 626
 627 static void
 628 ixgbe_start(struct ifnet *ifp)
 629 {
 630         struct adapter *adapter = ifp->if_softc;
 631         struct tx_ring  *txr = adapter->tx_rings;
 632         u32 queue = 0;
 633
 634         /*
 635         ** This is really just here for testing
 636         ** TX multiqueue, ultimately what is
 637         ** needed is the flow support in the stack
 638         ** and appropriate logic here to deal with
 639         ** it. -jfv
 640         */
 641         if (adapter->num_tx_queues > 1)
 642                 queue = (curcpu % adapter->num_tx_queues);
 643
 644         txr = &adapter->tx_rings[queue];
 645
 646         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 647                 IXGBE_TX_LOCK(txr);
 648                 ixgbe_start_locked(txr, ifp);
 649                 IXGBE_TX_UNLOCK(txr);
 650         }
 651         return;
 652 }
 653
 654 /*********************************************************************
 655  *  Ioctl entry point
 656  *
 657  *  ixgbe_ioctl is called when the user wants to configure the
 658  *  interface.
 659  *
 660  *  return 0 on success, positive on failure
 661  **********************************************************************/
 662
 663 static int
 664 ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 665 {
 666         int             error = 0;
 667         struct ifreq   *ifr = (struct ifreq *) data;
 668         struct ifaddr   *ifa = (struct ifaddr *) data;
 669         struct adapter *adapter = ifp->if_softc;
 670
 671         switch (command) {
 672         case SIOCSIFADDR:
 673                 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)");
 674                 if (ifa->ifa_addr->sa_family == AF_INET) {
 675                         ifp->if_flags |= IFF_UP;
 676                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 677                                 IXGBE_CORE_LOCK(adapter);
 678                                 ixgbe_init_locked(adapter);
 679                                 IXGBE_CORE_UNLOCK(adapter);
 680                         }
 681                         arp_ifinit(ifp, ifa);
 682                 } else
 683                         ether_ioctl(ifp, command, data);
 684                 break;
 685         case SIOCSIFMTU:
 686                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 687                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 688                         error = EINVAL;
 689                 } else {
 690                         IXGBE_CORE_LOCK(adapter);
 691                         ifp->if_mtu = ifr->ifr_mtu;
 692                         adapter->max_frame_size =
 693                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 694                         ixgbe_init_locked(adapter);
 695                         IXGBE_CORE_UNLOCK(adapter);
 696                 }
 697                 break;
 698         case SIOCSIFFLAGS:
 699                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 700                 IXGBE_CORE_LOCK(adapter);
 701                 if (ifp->if_flags & IFF_UP) {
 702                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 703                                 if ((ifp->if_flags ^ adapter->if_flags) &
 704                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 705                                         ixgbe_disable_promisc(adapter);
 706                                         ixgbe_set_promisc(adapter);
 707                                 }
 708                         } else
 709                                 ixgbe_init_locked(adapter);
 710                 } else
 711                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 712                                 ixgbe_stop(adapter);
 713                 adapter->if_flags = ifp->if_flags;
 714                 IXGBE_CORE_UNLOCK(adapter);
 715                 break;
 716         case SIOCADDMULTI:
 717         case SIOCDELMULTI:
 718                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 719                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 720                         IXGBE_CORE_LOCK(adapter);
 721                         ixgbe_disable_intr(adapter);
 722                         ixgbe_set_multi(adapter);
 723                         ixgbe_enable_intr(adapter);
 724                         IXGBE_CORE_UNLOCK(adapter);
 725                 }
 726                 break;
 727         case SIOCSIFMEDIA:
 728         case SIOCGIFMEDIA:
 729                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 730                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 731                 break;
 732         case SIOCSIFCAP:
 733         {
 734                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 735                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 736                 if (mask & IFCAP_HWCSUM)
 737                         ifp->if_capenable ^= IFCAP_HWCSUM;
 738                 if (mask & IFCAP_TSO4)
 739                         ifp->if_capenable ^= IFCAP_TSO4;
 740                 if (mask & IFCAP_VLAN_HWTAGGING)
 741                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 742                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 743                         ixgbe_init(adapter);
 744 #if __FreeBSD_version >= 700000
 745                 VLAN_CAPABILITIES(ifp);
 746 #endif
 747                 break;
 748         }
 749         default:
 750                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 751                 error = ether_ioctl(ifp, command, data);
 752                 break;
 753         }
 754
 755         return (error);
 756 }
 757
 758 /*********************************************************************
 759  *  Watchdog entry point
 760  *
 761  *  This routine is called by the local timer
 762  *  to detect hardware hangs .
 763  *
 764  **********************************************************************/
 765
 766 static void
 767 ixgbe_watchdog(struct adapter *adapter)
 768 {
 769         device_t        dev = adapter->dev;
 770         struct tx_ring *txr = adapter->tx_rings;
 771         struct ixgbe_hw *hw = &adapter->hw;
 772         bool            tx_hang = FALSE;
 773
 774         IXGBE_CORE_LOCK_ASSERT(adapter);
 775
 776         /*
 777          * The timer is set to 5 every time ixgbe_start() queues a packet.
 778          * Then ixgbe_txeof() keeps resetting to 5 as long as it cleans at
 779          * least one descriptor.
 780          * Finally, anytime all descriptors are clean the timer is
 781          * set to 0.
 782          */
 783         for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
 784                 u32 head, tail;
 785
 786                 IXGBE_TX_LOCK(txr);
 787                 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) {
 788                         IXGBE_TX_UNLOCK(txr);
 789                         continue;
 790                 } else {
 791                         head = IXGBE_READ_REG(hw, IXGBE_TDH(i));
 792                         tail = IXGBE_READ_REG(hw, IXGBE_TDT(i));
 793                         if (head == tail) { /* last minute check */
 794                                 IXGBE_TX_UNLOCK(txr);
 795                                 continue;
 796                         }
 797                         /* Well, seems something is really hung */
 798                         tx_hang = TRUE;
 799                         IXGBE_TX_UNLOCK(txr);
 800                         break;
 801                 }
 802         }
 803         if (tx_hang == FALSE)
 804                 return;
 805
 806         /*
 807          * If we are in this routine because of pause frames, then don't
 808          * reset the hardware.
 809          */
 810         if (IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) {
 811                 txr = adapter->tx_rings;        /* reset pointer */
 812                 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
 813                         IXGBE_TX_LOCK(txr);
 814                         txr->watchdog_timer = IXGBE_TX_TIMEOUT;
 815                         IXGBE_TX_UNLOCK(txr);
 816                 }
 817                 return;
 818         }
 819
 820
 821         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
 822         for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
 823                 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i,
 824                     IXGBE_READ_REG(hw, IXGBE_TDH(i)),
 825                     IXGBE_READ_REG(hw, IXGBE_TDT(i)));
 826                 device_printf(dev,"TX(%d) desc avail = %d,"
 827                     "Next TX to Clean = %d\n",
 828                     i, txr->tx_avail, txr->next_tx_to_clean);
 829         }
 830         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 831         adapter->watchdog_events++;
 832
 833         ixgbe_init_locked(adapter);
 834 }
 835
 836 /*********************************************************************
 837  *  Init entry point
 838  *
 839  *  This routine is used in two ways. It is used by the stack as
 840  *  init entry point in network interface structure. It is also used
 841  *  by the driver as a hw/sw initialization routine to get to a
 842  *  consistent state.
 843  *
 844  *  return 0 on success, positive on failure
 845  **********************************************************************/
 846 #define IXGBE_MHADD_MFS_SHIFT 16
 847
 848 static void
 849 ixgbe_init_locked(struct adapter *adapter)
 850 {
 851         struct ifnet   *ifp = adapter->ifp;
 852         device_t        dev = adapter->dev;
 853         struct ixgbe_hw *hw;
 854         u32             txdctl, rxdctl, mhadd, gpie;
 855
 856         INIT_DEBUGOUT("ixgbe_init: begin");
 857
 858         hw = &adapter->hw;
 859         mtx_assert(&adapter->core_mtx, MA_OWNED);
 860
 861         ixgbe_stop(adapter);
 862
 863         /* Get the latest mac address, User can use a LAA */
 864         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
 865               IXGBE_ETH_LENGTH_OF_ADDRESS);
 866         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, 1);
 867         adapter->hw.addr_ctrl.rar_used_count = 1;
 868
 869         /* Initialize the hardware */
 870         if (ixgbe_hardware_init(adapter)) {
 871                 device_printf(dev, "Unable to initialize the hardware\n");
 872                 return;
 873         }
 874
 875 #ifndef IXGBE_VLAN_EVENTS
 876         /* With events this is done when a vlan registers */
 877         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
 878                 u32 ctrl;
 879                 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL);
 880                 ctrl |= IXGBE_VLNCTRL_VME;
 881                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
 882                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl);
 883         }
 884 #endif
 885
 886         /* Prepare transmit descriptors and buffers */
 887         if (ixgbe_setup_transmit_structures(adapter)) {
 888                 device_printf(dev,"Could not setup transmit structures\n");
 889                 ixgbe_stop(adapter);
 890                 return;
 891         }
 892
 893         ixgbe_initialize_transmit_units(adapter);
 894
 895         /* Setup Multicast table */
 896         ixgbe_set_multi(adapter);
 897
 898         /*
 899         ** If we are resetting MTU smaller than 2K
 900         ** drop to small RX buffers
 901         */
 902         if (adapter->max_frame_size <= MCLBYTES)
 903                 adapter->bigbufs = FALSE;
 904
 905         /* Prepare receive descriptors and buffers */
 906         if (ixgbe_setup_receive_structures(adapter)) {
 907                 device_printf(dev,"Could not setup receive structures\n");
 908                 ixgbe_stop(adapter);
 909                 return;
 910         }
 911
 912         /* Configure RX settings */
 913         ixgbe_initialize_receive_units(adapter);
 914
 915         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
 916         /* Enable Fan Failure Interrupt */
 917         if (adapter->hw.phy.media_type == ixgbe_media_type_copper)
 918                 gpie |= IXGBE_SDP1_GPIEN;
 919         if (adapter->msix) {
 920                 /* Enable Enhanced MSIX mode */
 921                 gpie |= IXGBE_GPIE_MSIX_MODE;
 922                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
 923                     IXGBE_GPIE_OCD;
 924         }
 925         IXGBE_WRITE_REG(&adapter->hw, IXGBE_GPIE, gpie);
 926
 927         /* Set the various hardware offload abilities */
 928         ifp->if_hwassist = 0;
 929         if (ifp->if_capenable & IFCAP_TSO4)
 930                 ifp->if_hwassist |= CSUM_TSO;
 931         else if (ifp->if_capenable & IFCAP_TXCSUM)
 932                 ifp->if_hwassist = (CSUM_TCP | CSUM_UDP);
 933
 934         /* Set MTU size */
 935         if (ifp->if_mtu > ETHERMTU) {
 936                 mhadd = IXGBE_READ_REG(&adapter->hw, IXGBE_MHADD);
 937                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
 938                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
 939                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_MHADD, mhadd);
 940         }
 941
 942         /* Now enable all the queues */
 943
 944         for (int i = 0; i < adapter->num_tx_queues; i++) {
 945                 txdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TXDCTL(i));
 946                 txdctl |= IXGBE_TXDCTL_ENABLE;
 947                 /* Set WTHRESH to 8, burst writeback */
 948                 txdctl |= (8 << 16);
 949                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TXDCTL(i), txdctl);
 950         }
 951
 952         for (int i = 0; i < adapter->num_rx_queues; i++) {
 953                 rxdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXDCTL(i));
 954                 /* PTHRESH set to 32 */
 955                 rxdctl |= 0x0020;
 956                 rxdctl |= IXGBE_RXDCTL_ENABLE;
 957                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXDCTL(i), rxdctl);
 958         }
 959
 960         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
 961
 962         /* Set up MSI/X routing */
 963         ixgbe_configure_ivars(adapter);
 964
 965         ixgbe_enable_intr(adapter);
 966
 967         /* Now inform the stack we're ready */
 968         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 969         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 970
 971         return;
 972 }
 973
 974 static void
 975 ixgbe_init(void *arg)
 976 {
 977         struct adapter *adapter = arg;
 978
 979         IXGBE_CORE_LOCK(adapter);
 980         ixgbe_init_locked(adapter);
 981         IXGBE_CORE_UNLOCK(adapter);
 982         return;
 983 }
 984
 985
 986 /*
 987 ** Legacy Deferred Interrupt Handlers
 988 */
 989
 990 static void
 991 ixgbe_handle_rx(void *context, int pending)
 992 {
 993         struct rx_ring  *rxr = context;
 994         struct adapter  *adapter = rxr->adapter;
 995         u32 loop = 0;
 996
 997         while (loop++ < MAX_INTR)
 998                 if (ixgbe_rxeof(rxr, adapter->rx_process_limit) == 0)
 999                         break;
1000 }
1001
1002 static void
1003 ixgbe_handle_tx(void *context, int pending)
1004 {
1005         struct tx_ring  *txr = context;
1006         struct adapter  *adapter = txr->adapter;
1007         struct ifnet    *ifp = adapter->ifp;
1008         u32             loop = 0;
1009
1010                 IXGBE_TX_LOCK(txr);
1011                 while (loop++ < MAX_INTR)
1012                         if (ixgbe_txeof(txr) == 0)
1013                                 break;
1014                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1015                         ixgbe_start_locked(txr, ifp);
1016                 IXGBE_TX_UNLOCK(txr);
1017 }
1018
1019
1020 /*********************************************************************
1021  *
1022  *  Legacy Interrupt Service routine
1023  *
1024  **********************************************************************/
1025
1026 static void
1027 ixgbe_legacy_irq(void *arg)
1028 {
1029         u32             reg_eicr;
1030         struct adapter  *adapter = arg;
1031         struct          tx_ring *txr = adapter->tx_rings;
1032         struct          rx_ring *rxr = adapter->rx_rings;
1033         struct ixgbe_hw *hw;
1034
1035         hw = &adapter->hw;
1036         reg_eicr = IXGBE_READ_REG(&adapter->hw, IXGBE_EICR);
1037         if (reg_eicr == 0)
1038                 return;
1039
1040         if (ixgbe_rxeof(rxr, adapter->rx_process_limit) != 0)
1041                 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1042         if (ixgbe_txeof(txr) != 0)
1043                 taskqueue_enqueue(txr->tq, &txr->tx_task);
1044
1045         /* Check for fan failure */
1046         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1047             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1048                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1049                     "REPLACE IMMEDIATELY!!\n");
1050                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS,
1051                     IXGBE_EICR_GPI_SDP1);
1052         }
1053         /* Link status change */
1054         if (reg_eicr & IXGBE_EICR_LSC)
1055                 ixgbe_update_link_status(adapter);
1056
1057         return;
1058 }
1059
1060
1061 /*********************************************************************
1062  *
1063  *  MSI TX Interrupt Service routine
1064  *
1065  **********************************************************************/
1066
1067 void
1068 ixgbe_msix_tx(void *arg)
1069 {
1070         struct tx_ring *txr = arg;
1071         struct adapter *adapter = txr->adapter;
1072         u32             loop = 0;
1073
1074         ++txr->tx_irq;
1075         IXGBE_TX_LOCK(txr);
1076         while (loop++ < MAX_INTR)
1077                 if (ixgbe_txeof(txr) == 0)
1078                         break;
1079         IXGBE_TX_UNLOCK(txr);
1080         /* Reenable this interrupt */
1081         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims);
1082
1083         return;
1084 }
1085
1086 /*********************************************************************
1087  *
1088  *  MSI RX Interrupt Service routine
1089  *
1090  **********************************************************************/
1091
1092 static void
1093 ixgbe_msix_rx(void *arg)
1094 {
1095         struct rx_ring  *rxr = arg;
1096         struct adapter  *adapter = rxr->adapter;
1097         u32             loop = 0;
1098
1099         ++rxr->rx_irq;
1100         while (loop++ < MAX_INTR)
1101                 if (ixgbe_rxeof(rxr, adapter->rx_process_limit) == 0)
1102                         break;
1103         /* Reenable this interrupt */
1104         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims);
1105         return;
1106 }
1107
1108 static void
1109 ixgbe_msix_link(void *arg)
1110 {
1111         struct adapter  *adapter = arg;
1112         struct ixgbe_hw *hw = &adapter->hw;
1113         u32             reg_eicr;
1114
1115         ++adapter->link_irq;
1116
1117         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1118
1119         if (reg_eicr & IXGBE_EICR_LSC)
1120                 ixgbe_update_link_status(adapter);
1121
1122         /* Check for fan failure */
1123         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1124             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1125                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1126                     "REPLACE IMMEDIATELY!!\n");
1127                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1128         }
1129
1130         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1131         return;
1132 }
1133
1134
1135 /*********************************************************************
1136  *
1137  *  Media Ioctl callback
1138  *
1139  *  This routine is called whenever the user queries the status of
1140  *  the interface using ifconfig.
1141  *
1142  **********************************************************************/
1143 static void
1144 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1145 {
1146         struct adapter *adapter = ifp->if_softc;
1147
1148         INIT_DEBUGOUT("ixgbe_media_status: begin");
1149         IXGBE_CORE_LOCK(adapter);
1150         ixgbe_update_link_status(adapter);
1151
1152         ifmr->ifm_status = IFM_AVALID;
1153         ifmr->ifm_active = IFM_ETHER;
1154
1155         if (!adapter->link_active) {
1156                 IXGBE_CORE_UNLOCK(adapter);
1157                 return;
1158         }
1159
1160         ifmr->ifm_status |= IFM_ACTIVE;
1161
1162         switch (adapter->link_speed) {
1163                 case IXGBE_LINK_SPEED_1GB_FULL:
1164                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1165                         break;
1166                 case IXGBE_LINK_SPEED_10GB_FULL:
1167                         ifmr->ifm_active |= ixgbe_optics | IFM_FDX;
1168                         break;
1169         }
1170
1171         IXGBE_CORE_UNLOCK(adapter);
1172
1173         return;
1174 }
1175
1176 /*********************************************************************
1177  *
1178  *  Media Ioctl callback
1179  *
1180  *  This routine is called when the user changes speed/duplex using
1181  *  media/mediopt option with ifconfig.
1182  *
1183  **********************************************************************/
1184 static int
1185 ixgbe_media_change(struct ifnet * ifp)
1186 {
1187         struct adapter *adapter = ifp->if_softc;
1188         struct ifmedia *ifm = &adapter->media;
1189
1190         INIT_DEBUGOUT("ixgbe_media_change: begin");
1191
1192         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1193                 return (EINVAL);
1194
1195         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1196         case IFM_AUTO:
1197                 adapter->hw.mac.autoneg = TRUE;
1198                 adapter->hw.phy.autoneg_advertised =
1199                     IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL;
1200                 break;
1201         default:
1202                 device_printf(adapter->dev, "Only auto media type\n");
1203                 return (EINVAL);
1204         }
1205
1206         return (0);
1207 }
1208
1209 /*********************************************************************
1210  *
1211  *  This routine maps the mbufs to tx descriptors.
1212  *    WARNING: while this code is using an MQ style infrastructure,
1213  *    it would NOT work as is with more than 1 queue.
1214  *
1215  *  return 0 on success, positive on failure
1216  **********************************************************************/
1217
1218 static int
1219 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1220 {
1221         struct adapter  *adapter = txr->adapter;
1222         u32             olinfo_status = 0, cmd_type_len = 0;
1223         u32             paylen;
1224         int             i, j, error, nsegs;
1225         int             first, last = 0;
1226         struct mbuf     *m_head;
1227         bus_dma_segment_t segs[IXGBE_MAX_SCATTER];
1228         bus_dmamap_t    map;
1229         struct ixgbe_tx_buf *txbuf, *txbuf_mapped;
1230         union ixgbe_adv_tx_desc *txd = NULL;
1231
1232         m_head = *m_headp;
1233         paylen = 0;
1234
1235         /* Basic descriptor defines */
1236         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
1237         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
1238
1239         if (m_head->m_flags & M_VLANTAG)
1240                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1241
1242         /*
1243          * Force a cleanup if number of TX descriptors
1244          * available is below the threshold. If it fails
1245          * to get above, then abort transmit.
1246          */
1247         if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) {
1248                 ixgbe_txeof(txr);
1249                 /* Make sure things have improved */
1250                 if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) {
1251                         txr->no_tx_desc_avail++;
1252                         return (ENOBUFS);
1253                 }
1254         }
1255
1256         /*
1257          * Important to capture the first descriptor
1258          * used because it will contain the index of
1259          * the one we tell the hardware to report back
1260          */
1261         first = txr->next_avail_tx_desc;
1262         txbuf = &txr->tx_buffers[first];
1263         txbuf_mapped = txbuf;
1264         map = txbuf->map;
1265
1266         /*
1267          * Map the packet for DMA.
1268          */
1269         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1270             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1271
1272         if (error == EFBIG) {
1273                 struct mbuf *m;
1274
1275                 m = m_defrag(*m_headp, M_DONTWAIT);
1276                 if (m == NULL) {
1277                         adapter->mbuf_alloc_failed++;
1278                         m_freem(*m_headp);
1279                         *m_headp = NULL;
1280                         return (ENOBUFS);
1281                 }
1282                 *m_headp = m;
1283
1284                 /* Try it again */
1285                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1286                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1287
1288                 if (error == ENOMEM) {
1289                         adapter->no_tx_dma_setup++;
1290                         return (error);
1291                 } else if (error != 0) {
1292                         adapter->no_tx_dma_setup++;
1293                         m_freem(*m_headp);
1294                         *m_headp = NULL;
1295                         return (error);
1296                 }
1297         } else if (error == ENOMEM) {
1298                 adapter->no_tx_dma_setup++;
1299                 return (error);
1300         } else if (error != 0) {
1301                 adapter->no_tx_dma_setup++;
1302                 m_freem(*m_headp);
1303                 *m_headp = NULL;
1304                 return (error);
1305         }
1306
1307         /* Make certain there are enough descriptors */
1308         if (nsegs > txr->tx_avail - 2) {
1309                 txr->no_tx_desc_avail++;
1310                 error = ENOBUFS;
1311                 goto xmit_fail;
1312         }
1313         m_head = *m_headp;
1314
1315         /*
1316         ** Set the appropriate offload context
1317         ** this becomes the first descriptor of
1318         ** a packet.
1319         */
1320         if (ixgbe_tso_setup(txr, m_head, &paylen)) {
1321                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1322                 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1323                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1324                 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1325                 ++adapter->tso_tx;
1326         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1327                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1328
1329         i = txr->next_avail_tx_desc;
1330         for (j = 0; j < nsegs; j++) {
1331                 bus_size_t seglen;
1332                 bus_addr_t segaddr;
1333
1334                 txbuf = &txr->tx_buffers[i];
1335                 txd = &txr->tx_base[i];
1336                 seglen = segs[j].ds_len;
1337                 segaddr = htole64(segs[j].ds_addr);
1338
1339                 txd->read.buffer_addr = segaddr;
1340                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1341                     cmd_type_len |seglen);
1342                 txd->read.olinfo_status = htole32(olinfo_status);
1343                 last = i; /* Next descriptor that will get completed */
1344
1345                 if (++i == adapter->num_tx_desc)
1346                         i = 0;
1347
1348                 txbuf->m_head = NULL;
1349                 /*
1350                 ** we have to do this inside the loop right now
1351                 ** because of the hardware workaround.
1352                 */
1353                 if (j == (nsegs -1)) /* Last descriptor gets EOP and RS */
1354                         txd->read.cmd_type_len |=
1355                             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1356 #ifndef NO_82598_A0_SUPPORT
1357                 if (adapter->hw.revision_id == 0)
1358                         desc_flip(txd);
1359 #endif
1360         }
1361
1362         txr->tx_avail -= nsegs;
1363         txr->next_avail_tx_desc = i;
1364
1365         txbuf->m_head = m_head;
1366         txbuf->map = map;
1367         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1368
1369         /* Set the index of the descriptor that will be marked done */
1370         txbuf = &txr->tx_buffers[first];
1371
1372         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1373             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1374         /*
1375          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1376          * hardware that this frame is available to transmit.
1377          */
1378         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1379         ++txr->tx_packets;
1380         return (0);
1381
1382 xmit_fail:
1383         bus_dmamap_unload(txr->txtag, txbuf->map);
1384         return (error);
1385
1386 }
1387
1388 static void
1389 ixgbe_set_promisc(struct adapter *adapter)
1390 {
1391
1392         u_int32_t       reg_rctl;
1393         struct ifnet   *ifp = adapter->ifp;
1394
1395         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1396
1397         if (ifp->if_flags & IFF_PROMISC) {
1398                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1399                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1400         } else if (ifp->if_flags & IFF_ALLMULTI) {
1401                 reg_rctl |= IXGBE_FCTRL_MPE;
1402                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1403                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1404         }
1405         return;
1406 }
1407
1408 static void
1409 ixgbe_disable_promisc(struct adapter * adapter)
1410 {
1411         u_int32_t       reg_rctl;
1412
1413         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1414
1415         reg_rctl &= (~IXGBE_FCTRL_UPE);
1416         reg_rctl &= (~IXGBE_FCTRL_MPE);
1417         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1418
1419         return;
1420 }
1421
1422
1423 /*********************************************************************
1424  *  Multicast Update
1425  *
1426  *  This routine is called whenever multicast address list is updated.
1427  *
1428  **********************************************************************/
1429 #define IXGBE_RAR_ENTRIES 16
1430
1431 static void
1432 ixgbe_set_multi(struct adapter *adapter)
1433 {
1434         u32     fctrl;
1435         u8      mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
1436         u8      *update_ptr;
1437         struct  ifmultiaddr *ifma;
1438         int     mcnt = 0;
1439         struct ifnet   *ifp = adapter->ifp;
1440
1441         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1442
1443         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1444         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1445         if (ifp->if_flags & IFF_PROMISC)
1446                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1447         else if (ifp->if_flags & IFF_ALLMULTI) {
1448                 fctrl |= IXGBE_FCTRL_MPE;
1449                 fctrl &= ~IXGBE_FCTRL_UPE;
1450         } else
1451                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1452
1453         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1454
1455         IF_ADDR_LOCK(ifp);
1456         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1457                 if (ifma->ifma_addr->sa_family != AF_LINK)
1458                         continue;
1459                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1460                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1461                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1462                 mcnt++;
1463         }
1464         IF_ADDR_UNLOCK(ifp);
1465
1466         update_ptr = mta;
1467         ixgbe_update_mc_addr_list(&adapter->hw,
1468             update_ptr, mcnt, ixgbe_mc_array_itr);
1469
1470         return;
1471 }
1472
1473 /*
1474  * This is an iterator function now needed by the multicast
1475  * shared code. It simply feeds the shared code routine the
1476  * addresses in the array of ixgbe_set_multi() one by one.
1477  */
1478 static u8 *
1479 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1480 {
1481         u8 *addr = *update_ptr;
1482         u8 *newptr;
1483         *vmdq = 0;
1484
1485         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1486         *update_ptr = newptr;
1487         return addr;
1488 }
1489
1490
1491 /*********************************************************************
1492  *  Timer routine
1493  *
1494  *  This routine checks for link status,updates statistics,
1495  *  and runs the watchdog timer.
1496  *
1497  **********************************************************************/
1498
1499 static void
1500 ixgbe_local_timer(void *arg)
1501 {
1502         struct adapter *adapter = arg;
1503         struct ifnet   *ifp = adapter->ifp;
1504
1505         mtx_assert(&adapter->core_mtx, MA_OWNED);
1506
1507         ixgbe_update_link_status(adapter);
1508         ixgbe_update_stats_counters(adapter);
1509         if (ixgbe_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) {
1510                 ixgbe_print_hw_stats(adapter);
1511         }
1512         /*
1513          * Each second we check the watchdog
1514          * to protect against hardware hangs.
1515          */
1516         ixgbe_watchdog(adapter);
1517
1518         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1519 }
1520
1521 static void
1522 ixgbe_update_link_status(struct adapter *adapter)
1523 {
1524         boolean_t link_up = FALSE;
1525         struct ifnet    *ifp = adapter->ifp;
1526         struct tx_ring *txr = adapter->tx_rings;
1527         device_t dev = adapter->dev;
1528
1529         ixgbe_check_link(&adapter->hw, &adapter->link_speed, &link_up, 0);
1530
1531         if (link_up){
1532                 if (adapter->link_active == FALSE) {
1533                         if (bootverbose)
1534                                 device_printf(dev,"Link is up %d Gbps %s \n",
1535                                     ((adapter->link_speed == 128)? 10:1),
1536                                     "Full Duplex");
1537                         adapter->link_active = TRUE;
1538                         if_link_state_change(ifp, LINK_STATE_UP);
1539                 }
1540         } else { /* Link down */
1541                 if (adapter->link_active == TRUE) {
1542                         if (bootverbose)
1543                                 device_printf(dev,"Link is Down\n");
1544                         if_link_state_change(ifp, LINK_STATE_DOWN);
1545                         adapter->link_active = FALSE;
1546                         for (int i = 0; i < adapter->num_tx_queues;
1547                             i++, txr++)
1548                                 txr->watchdog_timer = FALSE;
1549                 }
1550         }
1551
1552         return;
1553 }
1554
1555
1556
1557 /*********************************************************************
1558  *
1559  *  This routine disables all traffic on the adapter by issuing a
1560  *  global reset on the MAC and deallocates TX/RX buffers.
1561  *
1562  **********************************************************************/
1563
1564 static void
1565 ixgbe_stop(void *arg)
1566 {
1567         struct ifnet   *ifp;
1568         struct adapter *adapter = arg;
1569         ifp = adapter->ifp;
1570
1571         mtx_assert(&adapter->core_mtx, MA_OWNED);
1572
1573         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1574         ixgbe_disable_intr(adapter);
1575
1576         /* Tell the stack that the interface is no longer active */
1577         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1578
1579         ixgbe_reset_hw(&adapter->hw);
1580         adapter->hw.adapter_stopped = FALSE;
1581         ixgbe_stop_adapter(&adapter->hw);
1582         callout_stop(&adapter->timer);
1583
1584         /* reprogram the RAR[0] in case user changed it. */
1585         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1586
1587         return;
1588 }
1589
1590
1591 /*********************************************************************
1592  *
1593  *  Determine hardware revision.
1594  *
1595  **********************************************************************/
1596 static void
1597 ixgbe_identify_hardware(struct adapter *adapter)
1598 {
1599         device_t        dev = adapter->dev;
1600
1601         /* Save off the information about this board */
1602         adapter->hw.vendor_id = pci_get_vendor(dev);
1603         adapter->hw.device_id = pci_get_device(dev);
1604         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
1605         adapter->hw.subsystem_vendor_id =
1606             pci_read_config(dev, PCIR_SUBVEND_0, 2);
1607         adapter->hw.subsystem_device_id =
1608             pci_read_config(dev, PCIR_SUBDEV_0, 2);
1609
1610         return;
1611 }
1612
1613 /*********************************************************************
1614  *
1615  *  Setup the Legacy or MSI Interrupt handler
1616  *
1617  **********************************************************************/
1618 static int
1619 ixgbe_allocate_legacy(struct adapter *adapter)
1620 {
1621         device_t dev = adapter->dev;
1622         struct          tx_ring *txr = adapter->tx_rings;
1623         struct          rx_ring *rxr = adapter->rx_rings;
1624         int error;
1625
1626         /* Legacy RID at 0 */
1627         if (adapter->msix == 0)
1628                 adapter->rid[0] = 0;
1629
1630         /* We allocate a single interrupt resource */
1631         adapter->res[0] = bus_alloc_resource_any(dev,
1632             SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
1633         if (adapter->res[0] == NULL) {
1634                 device_printf(dev, "Unable to allocate bus resource: "
1635                     "interrupt\n");
1636                 return (ENXIO);
1637         }
1638
1639         /*
1640          * Try allocating a fast interrupt and the associated deferred
1641          * processing contexts.
1642          */
1643         TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr);
1644         TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr);
1645         txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT,
1646             taskqueue_thread_enqueue, &txr->tq);
1647         rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT,
1648             taskqueue_thread_enqueue, &rxr->tq);
1649         taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
1650             device_get_nameunit(adapter->dev));
1651         taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq",
1652             device_get_nameunit(adapter->dev));
1653         if ((error = bus_setup_intr(dev, adapter->res[0],
1654             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
1655             adapter, &adapter->tag[0])) != 0) {
1656                 device_printf(dev, "Failed to register fast interrupt "
1657                     "handler: %d\n", error);
1658                 taskqueue_free(txr->tq);
1659                 taskqueue_free(rxr->tq);
1660                 txr->tq = NULL;
1661                 rxr->tq = NULL;
1662                 return (error);
1663         }
1664
1665         return (0);
1666 }
1667
1668
1669 /*********************************************************************
1670  *
1671  *  Setup MSIX Interrupt resources and handlers
1672  *
1673  **********************************************************************/
1674 static int
1675 ixgbe_allocate_msix(struct adapter *adapter)
1676 {
1677         device_t        dev = adapter->dev;
1678         struct          tx_ring *txr = adapter->tx_rings;
1679         struct          rx_ring *rxr = adapter->rx_rings;
1680         int             error, vector = 0;
1681
1682         /* TX setup: the code is here for multi tx,
1683            there are other parts of the driver not ready for it */
1684         for (int i = 0; i < adapter->num_tx_queues; i++, vector++, txr++) {
1685                 adapter->res[vector] = bus_alloc_resource_any(dev,
1686                     SYS_RES_IRQ, &adapter->rid[vector],
1687                     RF_SHAREABLE | RF_ACTIVE);
1688                 if (!adapter->res[vector]) {
1689                         device_printf(dev,"Unable to allocate"
1690                             " bus resource: tx interrupt [%d]\n", vector);
1691                         return (ENXIO);
1692                 }
1693                 /* Set the handler function */
1694                 error = bus_setup_intr(dev, adapter->res[vector],
1695                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
1696                     ixgbe_msix_tx, txr, &adapter->tag[vector]);
1697                 if (error) {
1698                         adapter->res[vector] = NULL;
1699                         device_printf(dev, "Failed to register TX handler");
1700                         return (error);
1701                 }
1702                 txr->msix = vector;
1703                 txr->eims = IXGBE_IVAR_TX_QUEUE(vector);
1704         }
1705
1706         /* RX setup */
1707         for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rxr++) {
1708                 adapter->res[vector] = bus_alloc_resource_any(dev,
1709                     SYS_RES_IRQ, &adapter->rid[vector],
1710                     RF_SHAREABLE | RF_ACTIVE);
1711                 if (!adapter->res[vector]) {
1712                         device_printf(dev,"Unable to allocate"
1713                             " bus resource: rx interrupt [%d],"
1714                             "rid = %d\n", i, adapter->rid[vector]);
1715                         return (ENXIO);
1716                 }
1717                 /* Set the handler function */
1718                 error = bus_setup_intr(dev, adapter->res[vector],
1719                     INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_rx,
1720                     rxr, &adapter->tag[vector]);
1721                 if (error) {
1722                         adapter->res[vector] = NULL;
1723                         device_printf(dev, "Failed to register RX handler");
1724                         return (error);
1725                 }
1726                 rxr->msix = vector;
1727                 rxr->eims = IXGBE_IVAR_RX_QUEUE(vector);
1728         }
1729
1730         /* Now for Link changes */
1731         adapter->res[vector] = bus_alloc_resource_any(dev,
1732             SYS_RES_IRQ, &adapter->rid[vector], RF_SHAREABLE | RF_ACTIVE);
1733         if (!adapter->res[vector]) {
1734                 device_printf(dev,"Unable to allocate"
1735             " bus resource: Link interrupt [%d]\n", adapter->rid[vector]);
1736                 return (ENXIO);
1737         }
1738         /* Set the link handler function */
1739         error = bus_setup_intr(dev, adapter->res[vector],
1740             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_link,
1741             adapter, &adapter->tag[vector]);
1742         if (error) {
1743                 adapter->res[vector] = NULL;
1744                 device_printf(dev, "Failed to register LINK handler");
1745                 return (error);
1746         }
1747         adapter->linkvec = vector;
1748
1749         return (0);
1750 }
1751
1752
1753 /*
1754  * Setup Either MSI/X or MSI
1755  */
1756 static int
1757 ixgbe_setup_msix(struct adapter *adapter)
1758 {
1759         device_t dev = adapter->dev;
1760         int rid, want, queues, msgs;
1761
1762         /* First try MSI/X */
1763         rid = PCIR_BAR(IXGBE_MSIX_BAR);
1764         adapter->msix_mem = bus_alloc_resource_any(dev,
1765             SYS_RES_MEMORY, &rid, RF_ACTIVE);
1766         if (!adapter->msix_mem) {
1767                 /* May not be enabled */
1768                 device_printf(adapter->dev,
1769                     "Unable to map MSIX table \n");
1770                 goto msi;
1771         }
1772
1773         msgs = pci_msix_count(dev);
1774         if (msgs == 0) { /* system has msix disabled */
1775                 bus_release_resource(dev, SYS_RES_MEMORY,
1776                     PCIR_BAR(IXGBE_MSIX_BAR), adapter->msix_mem);
1777                 adapter->msix_mem = NULL;
1778                 goto msi;
1779         }
1780
1781         /* Figure out a reasonable auto config value */
1782         queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus;
1783
1784         if (ixgbe_tx_queues == 0)
1785                 ixgbe_tx_queues = queues;
1786         if (ixgbe_rx_queues == 0)
1787                 ixgbe_rx_queues = queues;
1788         want = ixgbe_tx_queues + ixgbe_rx_queues + 1;
1789         if (msgs >= want)
1790                 msgs = want;
1791         else {
1792                 device_printf(adapter->dev,
1793                     "MSIX Configuration Problem, "
1794                     "%d vectors but %d queues wanted!\n",
1795                     msgs, want);
1796                 return (ENXIO);
1797         }
1798         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
1799                 device_printf(adapter->dev,
1800                     "Using MSIX interrupts with %d vectors\n", msgs);
1801                 adapter->num_tx_queues = ixgbe_tx_queues;
1802                 adapter->num_rx_queues = ixgbe_rx_queues;
1803                 return (msgs);
1804         }
1805 msi:
1806         msgs = pci_msi_count(dev);
1807         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
1808                 device_printf(adapter->dev,"Using MSI interrupt\n");
1809         return (msgs);
1810 }
1811
1812 static int
1813 ixgbe_allocate_pci_resources(struct adapter *adapter)
1814 {
1815         int             rid;
1816         device_t        dev = adapter->dev;
1817
1818         rid = PCIR_BAR(0);
1819         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1820             &rid, RF_ACTIVE);
1821
1822         if (!(adapter->pci_mem)) {
1823                 device_printf(dev,"Unable to allocate bus resource: memory\n");
1824                 return (ENXIO);
1825         }
1826
1827         adapter->osdep.mem_bus_space_tag =
1828                 rman_get_bustag(adapter->pci_mem);
1829         adapter->osdep.mem_bus_space_handle =
1830                 rman_get_bushandle(adapter->pci_mem);
1831         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
1832
1833         /*
1834          * Init the resource arrays
1835          */
1836         for (int i = 0; i < IXGBE_MSGS; i++) {
1837                 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
1838                 adapter->tag[i] = NULL;
1839                 adapter->res[i] = NULL;
1840         }
1841
1842         /* Legacy defaults */
1843         adapter->num_tx_queues = 1;
1844         adapter->num_rx_queues = 1;
1845
1846         /* Now setup MSI or MSI/X */
1847         adapter->msix = ixgbe_setup_msix(adapter);
1848
1849         adapter->hw.back = &adapter->osdep;
1850         return (0);
1851 }
1852
1853 static void
1854 ixgbe_free_pci_resources(struct adapter * adapter)
1855 {
1856         device_t dev = adapter->dev;
1857
1858         /*
1859          * Legacy has this set to 0, but we need
1860          * to run this once, so reset it.
1861          */
1862         if (adapter->msix == 0)
1863                 adapter->msix = 1;
1864
1865         /*
1866          * First release all the interrupt resources:
1867          *      notice that since these are just kept
1868          *      in an array we can do the same logic
1869          *      whether its MSIX or just legacy.
1870          */
1871         for (int i = 0; i < adapter->msix; i++) {
1872                 if (adapter->tag[i] != NULL) {
1873                         bus_teardown_intr(dev, adapter->res[i],
1874                             adapter->tag[i]);
1875                         adapter->tag[i] = NULL;
1876                 }
1877                 if (adapter->res[i] != NULL) {
1878                         bus_release_resource(dev, SYS_RES_IRQ,
1879                             adapter->rid[i], adapter->res[i]);
1880                 }
1881         }
1882
1883         if (adapter->msix)
1884                 pci_release_msi(dev);
1885
1886         if (adapter->msix_mem != NULL)
1887                 bus_release_resource(dev, SYS_RES_MEMORY,
1888                     PCIR_BAR(IXGBE_MSIX_BAR), adapter->msix_mem);
1889
1890         if (adapter->pci_mem != NULL)
1891                 bus_release_resource(dev, SYS_RES_MEMORY,
1892                     PCIR_BAR(0), adapter->pci_mem);
1893
1894         return;
1895 }
1896
1897 /*********************************************************************
1898  *
1899  *  Initialize the hardware to a configuration as specified by the
1900  *  adapter structure. The controller is reset, the EEPROM is
1901  *  verified, the MAC address is set, then the shared initialization
1902  *  routines are called.
1903  *
1904  **********************************************************************/
1905 static int
1906 ixgbe_hardware_init(struct adapter *adapter)
1907 {
1908         device_t dev = adapter->dev;
1909         u16 csum;
1910
1911         csum = 0;
1912         /* Issue a global reset */
1913         adapter->hw.adapter_stopped = FALSE;
1914         ixgbe_stop_adapter(&adapter->hw);
1915
1916         /* Make sure we have a good EEPROM before we read from it */
1917         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
1918                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
1919                 return (EIO);
1920         }
1921
1922         /* Get Hardware Flow Control setting */
1923         adapter->hw.fc.type = ixgbe_fc_full;
1924         adapter->hw.fc.pause_time = IXGBE_FC_PAUSE;
1925         adapter->hw.fc.low_water = IXGBE_FC_LO;
1926         adapter->hw.fc.high_water = IXGBE_FC_HI;
1927         adapter->hw.fc.send_xon = TRUE;
1928
1929         if (ixgbe_init_hw(&adapter->hw)) {
1930                 device_printf(dev,"Hardware Initialization Failed");
1931                 return (EIO);
1932         }
1933
1934         return (0);
1935 }
1936
1937 /*********************************************************************
1938  *
1939  *  Setup networking device structure and register an interface.
1940  *
1941  **********************************************************************/
1942 static void
1943 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
1944 {
1945         struct ifnet   *ifp;
1946         struct ixgbe_hw *hw = &adapter->hw;
1947         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
1948
1949         ifp = adapter->ifp = if_alloc(IFT_ETHER);
1950         if (ifp == NULL)
1951                 panic("%s: can not if_alloc()\n", device_get_nameunit(dev));
1952         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1953         ifp->if_mtu = ETHERMTU;
1954         ifp->if_baudrate = 1000000000;
1955         ifp->if_init = ixgbe_init;
1956         ifp->if_softc = adapter;
1957         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1958         ifp->if_ioctl = ixgbe_ioctl;
1959         ifp->if_start = ixgbe_start;
1960         ifp->if_timer = 0;
1961         ifp->if_watchdog = NULL;
1962         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 1;
1963
1964         ether_ifattach(ifp, adapter->hw.mac.addr);
1965
1966         adapter->max_frame_size =
1967             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1968
1969         /*
1970          * Tell the upper layer(s) we support long frames.
1971          */
1972         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1973
1974         ifp->if_capabilities |= (IFCAP_HWCSUM | IFCAP_TSO4);
1975         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1976         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
1977
1978         ifp->if_capenable = ifp->if_capabilities;
1979
1980         if ((hw->device_id == IXGBE_DEV_ID_82598AT) ||
1981             (hw->device_id == IXGBE_DEV_ID_82598AT_DUAL_PORT))
1982                 ixgbe_setup_link_speed(hw, (IXGBE_LINK_SPEED_10GB_FULL |
1983                     IXGBE_LINK_SPEED_1GB_FULL), TRUE, TRUE);
1984         else
1985                 ixgbe_setup_link_speed(hw, IXGBE_LINK_SPEED_10GB_FULL,
1986                     TRUE, FALSE);
1987
1988         /*
1989          * Specify the media types supported by this adapter and register
1990          * callbacks to update media and link information
1991          */
1992         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
1993                      ixgbe_media_status);
1994         ifmedia_add(&adapter->media, IFM_ETHER | ixgbe_optics |
1995             IFM_FDX, 0, NULL);
1996         if ((hw->device_id == IXGBE_DEV_ID_82598AT) ||
1997             (hw->device_id == IXGBE_DEV_ID_82598AT_DUAL_PORT)) {
1998                 ifmedia_add(&adapter->media,
1999                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2000                 ifmedia_add(&adapter->media,
2001                     IFM_ETHER | IFM_1000_T, 0, NULL);
2002         }
2003         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2004         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2005
2006         return;
2007 }
2008
2009 /********************************************************************
2010  * Manage DMA'able memory.
2011  *******************************************************************/
2012 static void
2013 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2014 {
2015         if (error)
2016                 return;
2017         *(bus_addr_t *) arg = segs->ds_addr;
2018         return;
2019 }
2020
2021 static int
2022 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2023                 struct ixgbe_dma_alloc *dma, int mapflags)
2024 {
2025         device_t dev = adapter->dev;
2026         int             r;
2027
2028         r = bus_dma_tag_create(NULL,    /* parent */
2029                                PAGE_SIZE, 0,    /* alignment, bounds */
2030                                BUS_SPACE_MAXADDR,       /* lowaddr */
2031                                BUS_SPACE_MAXADDR,       /* highaddr */
2032                                NULL, NULL,      /* filter, filterarg */
2033                                size,    /* maxsize */
2034                                1,       /* nsegments */
2035                                size,    /* maxsegsize */
2036                                BUS_DMA_ALLOCNOW,        /* flags */
2037                                NULL,    /* lockfunc */
2038                                NULL,    /* lockfuncarg */
2039                                &dma->dma_tag);
2040         if (r != 0) {
2041                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2042                        "error %u\n", r);
2043                 goto fail_0;
2044         }
2045         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2046                              BUS_DMA_NOWAIT, &dma->dma_map);
2047         if (r != 0) {
2048                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2049                        "error %u\n", r);
2050                 goto fail_1;
2051         }
2052         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2053                             size,
2054                             ixgbe_dmamap_cb,
2055                             &dma->dma_paddr,
2056                             mapflags | BUS_DMA_NOWAIT);
2057         if (r != 0) {
2058                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2059                        "error %u\n", r);
2060                 goto fail_2;
2061         }
2062         dma->dma_size = size;
2063         return (0);
2064 fail_2:
2065         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2066 fail_1:
2067         bus_dma_tag_destroy(dma->dma_tag);
2068 fail_0:
2069         dma->dma_map = NULL;
2070         dma->dma_tag = NULL;
2071         return (r);
2072 }
2073
2074 static void
2075 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2076 {
2077         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2078             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2079         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2080         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2081         bus_dma_tag_destroy(dma->dma_tag);
2082 }
2083
2084
2085 /*********************************************************************
2086  *
2087  *  Allocate memory for the transmit and receive rings, and then
2088  *  the descriptors associated with each, called only once at attach.
2089  *
2090  **********************************************************************/
2091 static int
2092 ixgbe_allocate_queues(struct adapter *adapter)
2093 {
2094         device_t dev = adapter->dev;
2095         struct tx_ring *txr;
2096         struct rx_ring *rxr;
2097         int rsize, tsize, error = IXGBE_SUCCESS;
2098         char name_string[16];
2099         int txconf = 0, rxconf = 0;
2100
2101         /* First allocate the TX ring struct memory */
2102         if (!(adapter->tx_rings =
2103             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2104             adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2105                 device_printf(dev, "Unable to allocate TX ring memory\n");
2106                 error = ENOMEM;
2107                 goto fail;
2108         }
2109         txr = adapter->tx_rings;
2110
2111         /* Next allocate the RX */
2112         if (!(adapter->rx_rings =
2113             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2114             adapter->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2115                 device_printf(dev, "Unable to allocate RX ring memory\n");
2116                 error = ENOMEM;
2117                 goto rx_fail;
2118         }
2119         rxr = adapter->rx_rings;
2120
2121         /* For the ring itself */
2122         tsize = roundup2(adapter->num_tx_desc *
2123             sizeof(union ixgbe_adv_tx_desc), 4096);
2124
2125         /*
2126          * Now set up the TX queues, txconf is needed to handle the
2127          * possibility that things fail midcourse and we need to
2128          * undo memory gracefully
2129          */
2130         for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) {
2131                 /* Set up some basics */
2132                 txr = &adapter->tx_rings[i];
2133                 txr->adapter = adapter;
2134                 txr->me = i;
2135
2136                 /* Initialize the TX side lock */
2137                 snprintf(name_string, sizeof(name_string), "%s:tx(%d)",
2138                     device_get_nameunit(dev), txr->me);
2139                 mtx_init(&txr->tx_mtx, name_string, NULL, MTX_DEF);
2140
2141                 if (ixgbe_dma_malloc(adapter, tsize,
2142                         &txr->txdma, BUS_DMA_NOWAIT)) {
2143                         device_printf(dev,
2144                             "Unable to allocate TX Descriptor memory\n");
2145                         error = ENOMEM;
2146                         goto err_tx_desc;
2147                 }
2148                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2149                 bzero((void *)txr->tx_base, tsize);
2150
2151                 /* Now allocate transmit buffers for the ring */
2152                 if (ixgbe_allocate_transmit_buffers(txr)) {
2153                         device_printf(dev,
2154                             "Critical Failure setting up transmit buffers\n");
2155                         error = ENOMEM;
2156                         goto err_tx_desc;
2157                 }
2158
2159         }
2160
2161         /*
2162          * Next the RX queues...
2163          */
2164         rsize = roundup2(adapter->num_rx_desc *
2165             sizeof(union ixgbe_adv_rx_desc), 4096);
2166         for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) {
2167                 rxr = &adapter->rx_rings[i];
2168                 /* Set up some basics */
2169                 rxr->adapter = adapter;
2170                 rxr->me = i;
2171
2172                 /* Initialize the TX side lock */
2173                 snprintf(name_string, sizeof(name_string), "%s:rx(%d)",
2174                     device_get_nameunit(dev), rxr->me);
2175                 mtx_init(&rxr->rx_mtx, name_string, NULL, MTX_DEF);
2176
2177                 if (ixgbe_dma_malloc(adapter, rsize,
2178                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2179                         device_printf(dev,
2180                             "Unable to allocate RxDescriptor memory\n");
2181                         error = ENOMEM;
2182                         goto err_rx_desc;
2183                 }
2184                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2185                 bzero((void *)rxr->rx_base, rsize);
2186
2187                 /* Allocate receive buffers for the ring*/
2188                 if (ixgbe_allocate_receive_buffers(rxr)) {
2189                         device_printf(dev,
2190                             "Critical Failure setting up receive buffers\n");
2191                         error = ENOMEM;
2192                         goto err_rx_desc;
2193                 }
2194         }
2195
2196         return (0);
2197
2198 err_rx_desc:
2199         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2200                 ixgbe_dma_free(adapter, &rxr->rxdma);
2201 err_tx_desc:
2202         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2203                 ixgbe_dma_free(adapter, &txr->txdma);
2204         free(adapter->rx_rings, M_DEVBUF);
2205 rx_fail:
2206         free(adapter->tx_rings, M_DEVBUF);
2207 fail:
2208         return (error);
2209 }
2210
2211 /*********************************************************************
2212  *
2213  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2214  *  the information needed to transmit a packet on the wire. This is
2215  *  called only once at attach, setup is done every reset.
2216  *
2217  **********************************************************************/
2218 static int
2219 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2220 {
2221         struct adapter *adapter = txr->adapter;
2222         device_t dev = adapter->dev;
2223         struct ixgbe_tx_buf *txbuf;
2224         int error, i;
2225
2226         /*
2227          * Setup DMA descriptor areas.
2228          */
2229         if ((error = bus_dma_tag_create(NULL,           /* parent */
2230                                PAGE_SIZE, 0,            /* alignment, bounds */
2231                                BUS_SPACE_MAXADDR,       /* lowaddr */
2232                                BUS_SPACE_MAXADDR,       /* highaddr */
2233                                NULL, NULL,              /* filter, filterarg */
2234                                IXGBE_TSO_SIZE,          /* maxsize */
2235                                IXGBE_MAX_SCATTER,       /* nsegments */
2236                                PAGE_SIZE,               /* maxsegsize */
2237                                0,                       /* flags */
2238                                NULL,                    /* lockfunc */
2239                                NULL,                    /* lockfuncarg */
2240                                &txr->txtag))) {
2241                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2242                 goto fail;
2243         }
2244
2245         if (!(txr->tx_buffers =
2246             (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
2247             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2248                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2249                 error = ENOMEM;
2250                 goto fail;
2251         }
2252
2253         /* Create the descriptor buffer dma maps */
2254         txbuf = txr->tx_buffers;
2255         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2256                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2257                 if (error != 0) {
2258                         device_printf(dev, "Unable to create TX DMA map\n");
2259                         goto fail;
2260                 }
2261         }
2262
2263         return 0;
2264 fail:
2265         /* We free all, it handles case where we are in the middle */
2266         ixgbe_free_transmit_structures(adapter);
2267         return (error);
2268 }
2269
2270 /*********************************************************************
2271  *
2272  *  Initialize a transmit ring.
2273  *
2274  **********************************************************************/
2275 static void
2276 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2277 {
2278         struct adapter *adapter = txr->adapter;
2279         struct ixgbe_tx_buf *txbuf;
2280         int i;
2281
2282         /* Clear the old ring contents */
2283         bzero((void *)txr->tx_base,
2284               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2285         /* Reset indices */
2286         txr->next_avail_tx_desc = 0;
2287         txr->next_tx_to_clean = 0;
2288
2289         /* Free any existing tx buffers. */
2290         txbuf = txr->tx_buffers;
2291         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2292                 if (txbuf->m_head != NULL) {
2293                         bus_dmamap_sync(txr->txtag, txbuf->map,
2294                             BUS_DMASYNC_POSTWRITE);
2295                         bus_dmamap_unload(txr->txtag, txbuf->map);
2296                         m_freem(txbuf->m_head);
2297                         txbuf->m_head = NULL;
2298                 }
2299         }
2300
2301         /* Set number of descriptors available */
2302         txr->tx_avail = adapter->num_tx_desc;
2303
2304         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2305             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2306
2307 }
2308
2309 /*********************************************************************
2310  *
2311  *  Initialize all transmit rings.
2312  *
2313  **********************************************************************/
2314 static int
2315 ixgbe_setup_transmit_structures(struct adapter *adapter)
2316 {
2317         struct tx_ring *txr = adapter->tx_rings;
2318
2319         for (int i = 0; i < adapter->num_tx_queues; i++, txr++)
2320                 ixgbe_setup_transmit_ring(txr);
2321
2322         return (0);
2323 }
2324
2325 /*********************************************************************
2326  *
2327  *  Enable transmit unit.
2328  *
2329  **********************************************************************/
2330 static void
2331 ixgbe_initialize_transmit_units(struct adapter *adapter)
2332 {
2333         struct tx_ring  *txr = adapter->tx_rings;
2334         struct ixgbe_hw *hw = &adapter->hw;
2335
2336         /* Setup the Base and Length of the Tx Descriptor Ring */
2337
2338         for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
2339                 u64     txhwb = 0, tdba = txr->txdma.dma_paddr;
2340                 u32     txctrl;
2341
2342                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2343                        (tdba & 0x00000000ffffffffULL));
2344                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2345                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2346                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2347
2348                 /* Setup for Head WriteBack */
2349                 txhwb = (u64)vtophys(&txr->tx_hwb);
2350                 txhwb |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2351                 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i),
2352                     (txhwb & 0x00000000ffffffffULL));
2353                 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i),
2354                     (txhwb >> 32));
2355                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2356                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
2357                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2358
2359                 /* Setup the HW Tx Head and Tail descriptor pointers */
2360                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2361                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2362
2363                 /* Setup Transmit Descriptor Cmd Settings */
2364                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2365
2366                 txr->watchdog_timer = 0;
2367         }
2368
2369         return;
2370 }
2371
2372 /*********************************************************************
2373  *
2374  *  Free all transmit rings.
2375  *
2376  **********************************************************************/
2377 static void
2378 ixgbe_free_transmit_structures(struct adapter *adapter)
2379 {
2380         struct tx_ring *txr = adapter->tx_rings;
2381
2382         for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
2383                 IXGBE_TX_LOCK(txr);
2384                 ixgbe_free_transmit_buffers(txr);
2385                 ixgbe_dma_free(adapter, &txr->txdma);
2386                 IXGBE_TX_UNLOCK(txr);
2387                 IXGBE_TX_LOCK_DESTROY(txr);
2388         }
2389         free(adapter->tx_rings, M_DEVBUF);
2390 }
2391
2392 /*********************************************************************
2393  *
2394  *  Free transmit ring related data structures.
2395  *
2396  **********************************************************************/
2397 static void
2398 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2399 {
2400         struct adapter *adapter = txr->adapter;
2401         struct ixgbe_tx_buf *tx_buffer;
2402         int             i;
2403
2404         INIT_DEBUGOUT("free_transmit_ring: begin");
2405
2406         if (txr->tx_buffers == NULL)
2407                 return;
2408
2409         tx_buffer = txr->tx_buffers;
2410         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2411                 if (tx_buffer->m_head != NULL) {
2412                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2413                             BUS_DMASYNC_POSTWRITE);
2414                         bus_dmamap_unload(txr->txtag,
2415                             tx_buffer->map);
2416                         m_freem(tx_buffer->m_head);
2417                         tx_buffer->m_head = NULL;
2418                         if (tx_buffer->map != NULL) {
2419                                 bus_dmamap_destroy(txr->txtag,
2420                                     tx_buffer->map);
2421                                 tx_buffer->map = NULL;
2422                         }
2423                 } else if (tx_buffer->map != NULL) {
2424                         bus_dmamap_unload(txr->txtag,
2425                             tx_buffer->map);
2426                         bus_dmamap_destroy(txr->txtag,
2427                             tx_buffer->map);
2428                         tx_buffer->map = NULL;
2429                 }
2430         }
2431
2432         if (txr->tx_buffers != NULL) {
2433                 free(txr->tx_buffers, M_DEVBUF);
2434                 txr->tx_buffers = NULL;
2435         }
2436         if (txr->txtag != NULL) {
2437                 bus_dma_tag_destroy(txr->txtag);
2438                 txr->txtag = NULL;
2439         }
2440         return;
2441 }
2442
2443 /*********************************************************************
2444  *
2445  *  Advanced Context Descriptor setup for VLAN or CSUM
2446  *
2447  **********************************************************************/
2448
2449 static boolean_t
2450 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
2451 {
2452         struct adapter *adapter = txr->adapter;
2453         struct ixgbe_adv_tx_context_desc *TXD;
2454         struct ixgbe_tx_buf        *tx_buffer;
2455         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2456         struct ether_vlan_header *eh;
2457         struct ip *ip;
2458         struct ip6_hdr *ip6;
2459         int  ehdrlen, ip_hlen = 0;
2460         u16     etype;
2461         u8      ipproto = 0;
2462         bool    offload = TRUE;
2463         int ctxd = txr->next_avail_tx_desc;
2464 #if __FreeBSD_version < 700000
2465         struct m_tag    *mtag;
2466 #else
2467         u16 vtag = 0;
2468 #endif
2469
2470
2471         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
2472                 offload = FALSE;
2473
2474         tx_buffer = &txr->tx_buffers[ctxd];
2475         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2476
2477         /*
2478         ** In advanced descriptors the vlan tag must
2479         ** be placed into the descriptor itself.
2480         */
2481 #if __FreeBSD_version < 700000
2482         mtag = VLAN_OUTPUT_TAG(ifp, mp);
2483         if (mtag != NULL) {
2484                 vlan_macip_lens |=
2485                     htole16(VLAN_TAG_VALUE(mtag)) << IXGBE_ADVTXD_VLAN_SHIFT;
2486         } else if (offload == FALSE)
2487                 return FALSE;   /* No need for CTX */
2488 #else
2489         if (mp->m_flags & M_VLANTAG) {
2490                 vtag = htole16(mp->m_pkthdr.ether_vtag);
2491                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2492         } else if (offload == FALSE)
2493                 return FALSE;
2494 #endif
2495         /*
2496          * Determine where frame payload starts.
2497          * Jump over vlan headers if already present,
2498          * helpful for QinQ too.
2499          */
2500         eh = mtod(mp, struct ether_vlan_header *);
2501         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2502                 etype = ntohs(eh->evl_proto);
2503                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2504         } else {
2505                 etype = ntohs(eh->evl_encap_proto);
2506                 ehdrlen = ETHER_HDR_LEN;
2507         }
2508
2509         /* Set the ether header length */
2510         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2511
2512         switch (etype) {
2513                 case ETHERTYPE_IP:
2514                         ip = (struct ip *)(mp->m_data + ehdrlen);
2515                         ip_hlen = ip->ip_hl << 2;
2516                         if (mp->m_len < ehdrlen + ip_hlen)
2517                                 return FALSE; /* failure */
2518                         ipproto = ip->ip_p;
2519                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2520                         break;
2521                 case ETHERTYPE_IPV6:
2522                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
2523                         ip_hlen = sizeof(struct ip6_hdr);
2524                         if (mp->m_len < ehdrlen + ip_hlen)
2525                                 return FALSE; /* failure */
2526                         ipproto = ip6->ip6_nxt;
2527                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
2528                         break;
2529                 default:
2530                         offload = FALSE;
2531                         break;
2532         }
2533
2534         vlan_macip_lens |= ip_hlen;
2535         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2536
2537         switch (ipproto) {
2538                 case IPPROTO_TCP:
2539                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
2540                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2541                         break;
2542                 case IPPROTO_UDP:
2543                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
2544                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
2545                         break;
2546                 default:
2547                         offload = FALSE;
2548                         break;
2549         }
2550
2551         /* Now copy bits into descriptor */
2552         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
2553         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
2554         TXD->seqnum_seed = htole32(0);
2555         TXD->mss_l4len_idx = htole32(0);
2556
2557 #ifndef NO_82598_A0_SUPPORT
2558         if (adapter->hw.revision_id == 0)
2559                 desc_flip(TXD);
2560 #endif
2561
2562         tx_buffer->m_head = NULL;
2563
2564         /* We've consumed the first desc, adjust counters */
2565         if (++ctxd == adapter->num_tx_desc)
2566                 ctxd = 0;
2567         txr->next_avail_tx_desc = ctxd;
2568         --txr->tx_avail;
2569
2570         return (offload);
2571 }
2572
2573 #if __FreeBSD_version >= 700000
2574 /**********************************************************************
2575  *
2576  *  Setup work for hardware segmentation offload (TSO) on
2577  *  adapters using advanced tx descriptors
2578  *
2579  **********************************************************************/
2580 static boolean_t
2581 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
2582 {
2583         struct adapter *adapter = txr->adapter;
2584         struct ixgbe_adv_tx_context_desc *TXD;
2585         struct ixgbe_tx_buf        *tx_buffer;
2586         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2587         u32 mss_l4len_idx = 0;
2588         u16 vtag = 0;
2589         int ctxd, ehdrlen,  hdrlen, ip_hlen, tcp_hlen;
2590         struct ether_vlan_header *eh;
2591         struct ip *ip;
2592         struct tcphdr *th;
2593
2594         if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) ||
2595             (mp->m_pkthdr.len <= IXGBE_TX_BUFFER_SIZE))
2596                 return FALSE;
2597
2598         /*
2599          * Determine where frame payload starts.
2600          * Jump over vlan headers if already present
2601          */
2602         eh = mtod(mp, struct ether_vlan_header *);
2603         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
2604                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2605         else
2606                 ehdrlen = ETHER_HDR_LEN;
2607
2608         /* Ensure we have at least the IP+TCP header in the first mbuf. */
2609         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
2610                 return FALSE;
2611
2612         ctxd = txr->next_avail_tx_desc;
2613         tx_buffer = &txr->tx_buffers[ctxd];
2614         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2615
2616         ip = (struct ip *)(mp->m_data + ehdrlen);
2617         if (ip->ip_p != IPPROTO_TCP)
2618                 return FALSE;   /* 0 */
2619         ip->ip_len = 0;
2620         ip->ip_sum = 0;
2621         ip_hlen = ip->ip_hl << 2;
2622         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
2623         th->th_sum = in_pseudo(ip->ip_src.s_addr,
2624             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
2625         tcp_hlen = th->th_off << 2;
2626         hdrlen = ehdrlen + ip_hlen + tcp_hlen;
2627         /* This is used in the transmit desc in encap */
2628         *paylen = mp->m_pkthdr.len - hdrlen;
2629
2630         /* VLAN MACLEN IPLEN */
2631         if (mp->m_flags & M_VLANTAG) {
2632                 vtag = htole16(mp->m_pkthdr.ether_vtag);
2633                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2634         }
2635
2636         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2637         vlan_macip_lens |= ip_hlen;
2638         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
2639
2640         /* ADV DTYPE TUCMD */
2641         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2642         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2643         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2644         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
2645
2646
2647         /* MSS L4LEN IDX */
2648         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
2649         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
2650         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
2651
2652         TXD->seqnum_seed = htole32(0);
2653         tx_buffer->m_head = NULL;
2654
2655 #ifndef NO_82598_A0_SUPPORT
2656         if (adapter->hw.revision_id == 0)
2657                 desc_flip(TXD);
2658 #endif
2659
2660         if (++ctxd == adapter->num_tx_desc)
2661                 ctxd = 0;
2662
2663         txr->tx_avail--;
2664         txr->next_avail_tx_desc = ctxd;
2665         return TRUE;
2666 }
2667
2668 #else   /* For 6.2 RELEASE */
2669 /* This makes it easy to keep the code common */
2670 static boolean_t
2671 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
2672 {
2673         return (FALSE);
2674 }
2675 #endif
2676
2677 /**********************************************************************
2678  *
2679  *  Examine each tx_buffer in the used queue. If the hardware is done
2680  *  processing the packet then free associated resources. The
2681  *  tx_buffer is put back on the free queue.
2682  *
2683  **********************************************************************/
2684 static boolean_t
2685 ixgbe_txeof(struct tx_ring *txr)
2686 {
2687         struct adapter * adapter = txr->adapter;
2688         struct ifnet    *ifp = adapter->ifp;
2689         u32     first, last, done, num_avail;
2690         u32     cleaned = 0;
2691         struct ixgbe_tx_buf *tx_buffer;
2692         struct ixgbe_legacy_tx_desc *tx_desc;
2693
2694         mtx_assert(&txr->mtx, MA_OWNED);
2695
2696         if (txr->tx_avail == adapter->num_tx_desc)
2697                 return FALSE;
2698
2699         num_avail = txr->tx_avail;
2700         first = txr->next_tx_to_clean;
2701
2702         tx_buffer = &txr->tx_buffers[first];
2703         /* For cleanup we just use legacy struct */
2704         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
2705
2706         /* Get the HWB */
2707         rmb();
2708         done = txr->tx_hwb;
2709
2710         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2711             BUS_DMASYNC_POSTREAD);
2712
2713         while (TRUE) {
2714                 /* We clean the range til last head write back */
2715                 while (first != done) {
2716                         tx_desc->upper.data = 0;
2717                         tx_desc->lower.data = 0;
2718                         tx_desc->buffer_addr = 0;
2719                         num_avail++; cleaned++;
2720
2721                         if (tx_buffer->m_head) {
2722                                 ifp->if_opackets++;
2723                                 bus_dmamap_sync(txr->txtag,
2724                                     tx_buffer->map,
2725                                     BUS_DMASYNC_POSTWRITE);
2726                                 bus_dmamap_unload(txr->txtag,
2727                                     tx_buffer->map);
2728                                 m_freem(tx_buffer->m_head);
2729                                 tx_buffer->m_head = NULL;
2730                                 tx_buffer->map = NULL;
2731                         }
2732
2733                         if (++first == adapter->num_tx_desc)
2734                                 first = 0;
2735
2736                         tx_buffer = &txr->tx_buffers[first];
2737                         tx_desc =
2738                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
2739                 }
2740                 /* See if there is more work now */
2741                 last = done;
2742                 rmb();
2743                 done = txr->tx_hwb;
2744                 if (last == done)
2745                         break;
2746         }
2747         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2748             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2749
2750         txr->next_tx_to_clean = first;
2751
2752         /*
2753          * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that
2754          * it is OK to send packets. If there are no pending descriptors,
2755          * clear the timeout. Otherwise, if some descriptors have been freed,
2756          * restart the timeout.
2757          */
2758         if (num_avail > IXGBE_TX_CLEANUP_THRESHOLD) {
2759                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2760                 /* If all are clean turn off the timer */
2761                 if (num_avail == adapter->num_tx_desc) {
2762                         txr->watchdog_timer = 0;
2763                         txr->tx_avail = num_avail;
2764                         return FALSE;
2765                 }
2766         }
2767
2768         /* Some were cleaned, so reset timer */
2769         if (cleaned)
2770                 txr->watchdog_timer = IXGBE_TX_TIMEOUT;
2771         txr->tx_avail = num_avail;
2772         return TRUE;
2773 }
2774
2775 /*********************************************************************
2776  *
2777  *  Get a buffer from system mbuf buffer pool.
2778  *
2779  **********************************************************************/
2780 static int
2781 ixgbe_get_buf(struct rx_ring *rxr, int i)
2782 {
2783         struct adapter  *adapter = rxr->adapter;
2784         struct mbuf     *mp;
2785         bus_dmamap_t    map;
2786         int             nsegs, error, old, s = 0;
2787         int             size = MCLBYTES;
2788
2789
2790         bus_dma_segment_t       segs[1];
2791         struct ixgbe_rx_buf     *rxbuf;
2792
2793         /* Are we going to Jumbo clusters? */
2794         if (adapter->bigbufs) {
2795                 size = MJUMPAGESIZE;
2796                 s = 1;
2797         };
2798
2799         mp = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, size);
2800         if (mp == NULL) {
2801                 adapter->mbuf_alloc_failed++;
2802                 return (ENOBUFS);
2803         }
2804
2805         mp->m_len = mp->m_pkthdr.len = size;
2806
2807         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
2808                 m_adj(mp, ETHER_ALIGN);
2809
2810         /*
2811          * Using memory from the mbuf cluster pool, invoke the bus_dma
2812          * machinery to arrange the memory mapping.
2813          */
2814         error = bus_dmamap_load_mbuf_sg(rxr->rxtag[s], rxr->spare_map[s],
2815             mp, segs, &nsegs, BUS_DMA_NOWAIT);
2816         if (error) {
2817                 m_free(mp);
2818                 return (error);
2819         }
2820
2821         /* Now check our target buffer for existing mapping */
2822         rxbuf = &rxr->rx_buffers[i];
2823         old = rxbuf->bigbuf;
2824         if (rxbuf->m_head != NULL)
2825                 bus_dmamap_unload(rxr->rxtag[old], rxbuf->map[old]);
2826
2827         map = rxbuf->map[old];
2828         rxbuf->map[s] = rxr->spare_map[s];
2829         rxr->spare_map[old] = map;
2830         bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s], BUS_DMASYNC_PREREAD);
2831         rxbuf->m_head = mp;
2832         rxbuf->bigbuf = s;
2833
2834         rxr->rx_base[i].read.pkt_addr = htole64(segs[0].ds_addr);
2835
2836 #ifndef NO_82598_A0_SUPPORT
2837         /* A0 needs to One's Compliment descriptors */
2838         if (adapter->hw.revision_id == 0) {
2839                 struct dhack {u32 a1; u32 a2; u32 b1; u32 b2;};
2840                 struct dhack *d;
2841
2842                 d = (struct dhack *)&rxr->rx_base[i];
2843                 d->a1 = ~(d->a1);
2844                 d->a2 = ~(d->a2);
2845         }
2846 #endif
2847
2848         return (0);
2849 }
2850
2851 /*********************************************************************
2852  *
2853  *  Allocate memory for rx_buffer structures. Since we use one
2854  *  rx_buffer per received packet, the maximum number of rx_buffer's
2855  *  that we'll need is equal to the number of receive descriptors
2856  *  that we've allocated.
2857  *
2858  **********************************************************************/
2859 static int
2860 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
2861 {
2862         struct  adapter         *adapter = rxr->adapter;
2863         device_t                dev = adapter->dev;
2864         struct ixgbe_rx_buf     *rxbuf;
2865         int                     i, bsize, error;
2866
2867         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
2868         if (!(rxr->rx_buffers =
2869             (struct ixgbe_rx_buf *) malloc(bsize,
2870             M_DEVBUF, M_NOWAIT | M_ZERO))) {
2871                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
2872                 error = ENOMEM;
2873                 goto fail;
2874         }
2875
2876         /* First make the small (2K) tag/map */
2877         if ((error = bus_dma_tag_create(NULL,           /* parent */
2878                                    PAGE_SIZE, 0,        /* alignment, bounds */
2879                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2880                                    BUS_SPACE_MAXADDR,   /* highaddr */
2881                                    NULL, NULL,          /* filter, filterarg */
2882                                    MCLBYTES,            /* maxsize */
2883                                    1,                   /* nsegments */
2884                                    MCLBYTES,            /* maxsegsize */
2885                                    0,                   /* flags */
2886                                    NULL,                /* lockfunc */
2887                                    NULL,                /* lockfuncarg */
2888                                    &rxr->rxtag[0]))) {
2889                 device_printf(dev, "Unable to create RX Small DMA tag\n");
2890                 goto fail;
2891         }
2892
2893         /* Next make the large (4K) tag/map */
2894         if ((error = bus_dma_tag_create(NULL,           /* parent */
2895                                    PAGE_SIZE, 0,        /* alignment, bounds */
2896                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2897                                    BUS_SPACE_MAXADDR,   /* highaddr */
2898                                    NULL, NULL,          /* filter, filterarg */
2899                                    MJUMPAGESIZE,        /* maxsize */
2900                                    1,                   /* nsegments */
2901                                    MJUMPAGESIZE,        /* maxsegsize */
2902                                    0,                   /* flags */
2903                                    NULL,                /* lockfunc */
2904                                    NULL,                /* lockfuncarg */
2905                                    &rxr->rxtag[1]))) {
2906                 device_printf(dev, "Unable to create RX Large DMA tag\n");
2907                 goto fail;
2908         }
2909
2910         /* Create the spare maps (used by getbuf) */
2911         error = bus_dmamap_create(rxr->rxtag[0], BUS_DMA_NOWAIT,
2912              &rxr->spare_map[0]);
2913         error = bus_dmamap_create(rxr->rxtag[1], BUS_DMA_NOWAIT,
2914              &rxr->spare_map[1]);
2915         if (error) {
2916                 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
2917                     __func__, error);
2918                 goto fail;
2919         }
2920
2921         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
2922                 rxbuf = &rxr->rx_buffers[i];
2923                 error = bus_dmamap_create(rxr->rxtag[0],
2924                     BUS_DMA_NOWAIT, &rxbuf->map[0]);
2925                 if (error) {
2926                         device_printf(dev, "Unable to create Small RX DMA map\n");
2927                         goto fail;
2928                 }
2929                 error = bus_dmamap_create(rxr->rxtag[1],
2930                     BUS_DMA_NOWAIT, &rxbuf->map[1]);
2931                 if (error) {
2932                         device_printf(dev, "Unable to create Large RX DMA map\n");
2933                         goto fail;
2934                 }
2935         }
2936
2937         return (0);
2938
2939 fail:
2940         /* Frees all, but can handle partial completion */
2941         ixgbe_free_receive_structures(adapter);
2942         return (error);
2943 }
2944
2945 /*********************************************************************
2946  *
2947  *  Initialize a receive ring and its buffers.
2948  *
2949  **********************************************************************/
2950 static int
2951 ixgbe_setup_receive_ring(struct rx_ring *rxr)
2952 {
2953         struct  adapter         *adapter;
2954         device_t                dev;
2955         struct ixgbe_rx_buf     *rxbuf;
2956         struct lro_ctrl         *lro = &rxr->lro;
2957         int                     j, rsize, s = 0;
2958
2959         adapter = rxr->adapter;
2960         dev = adapter->dev;
2961         rsize = roundup2(adapter->num_rx_desc *
2962             sizeof(union ixgbe_adv_rx_desc), 4096);
2963         /* Clear the ring contents */
2964         bzero((void *)rxr->rx_base, rsize);
2965
2966         /*
2967         ** Free current RX buffers: the size buffer
2968         ** that is loaded is indicated by the buffer
2969         ** bigbuf value.
2970         */
2971         for (int i = 0; i < adapter->num_rx_desc; i++) {
2972                 rxbuf = &rxr->rx_buffers[i];
2973                 s = rxbuf->bigbuf;
2974                 if (rxbuf->m_head != NULL) {
2975                         bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s],
2976                             BUS_DMASYNC_POSTREAD);
2977                         bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]);
2978                         m_freem(rxbuf->m_head);
2979                         rxbuf->m_head = NULL;
2980                 }
2981         }
2982
2983         for (j = 0; j < adapter->num_rx_desc; j++) {
2984                 if (ixgbe_get_buf(rxr, j) == ENOBUFS) {
2985                         rxr->rx_buffers[j].m_head = NULL;
2986                         rxr->rx_base[j].read.pkt_addr = 0;
2987                         /* If we fail some may have change size */
2988                         s = adapter->bigbufs;
2989                         goto fail;
2990                 }
2991         }
2992
2993         /* Setup our descriptor indices */
2994         rxr->next_to_check = 0;
2995         rxr->last_cleaned = 0;
2996
2997         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
2998             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2999
3000         /* Now set up the LRO interface */
3001         if (ixgbe_enable_lro) {
3002                 int err = tcp_lro_init(lro);
3003                 if (err) {
3004                         device_printf(dev,"LRO Initialization failed!\n");
3005                         goto fail;
3006                 }
3007                 device_printf(dev,"RX LRO Initialized\n");
3008                 lro->ifp = adapter->ifp;
3009         }
3010
3011
3012         return (0);
3013 fail:
3014         /*
3015          * We need to clean up any buffers allocated so far
3016          * 'j' is the failing index, decrement it to get the
3017          * last success.
3018          */
3019         for (--j; j < 0; j--) {
3020                 rxbuf = &rxr->rx_buffers[j];
3021                 if (rxbuf->m_head != NULL) {
3022                         bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s],
3023                             BUS_DMASYNC_POSTREAD);
3024                         bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]);
3025                         m_freem(rxbuf->m_head);
3026                         rxbuf->m_head = NULL;
3027                 }
3028         }
3029         return (ENOBUFS);
3030 }
3031
3032 /*********************************************************************
3033  *
3034  *  Initialize all receive rings.
3035  *
3036  **********************************************************************/
3037 static int
3038 ixgbe_setup_receive_structures(struct adapter *adapter)
3039 {
3040         struct rx_ring *rxr = adapter->rx_rings;
3041         int i, j, s;
3042
3043         for (i = 0; i < adapter->num_rx_queues; i++, rxr++)
3044                 if (ixgbe_setup_receive_ring(rxr))
3045                         goto fail;
3046
3047         return (0);
3048 fail:
3049         /*
3050          * Free RX buffers allocated so far, we will only handle
3051          * the rings that completed, the failing case will have
3052          * cleaned up for itself. The value of 'i' will be the
3053          * failed ring so we must pre-decrement it.
3054          */
3055         rxr = adapter->rx_rings;
3056         for (--i; i > 0; i--, rxr++) {
3057                 for (j = 0; j < adapter->num_rx_desc; j++) {
3058                         struct ixgbe_rx_buf *rxbuf;
3059                         rxbuf = &rxr->rx_buffers[j];
3060                         s = rxbuf->bigbuf;
3061                         if (rxbuf->m_head != NULL) {
3062                                 bus_dmamap_sync(rxr->rxtag[s], rxbuf->map[s],
3063                                   BUS_DMASYNC_POSTREAD);
3064                                 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]);
3065                                 m_freem(rxbuf->m_head);
3066                                 rxbuf->m_head = NULL;
3067                         }
3068                 }
3069         }
3070
3071         return (ENOBUFS);
3072 }
3073
3074 /*********************************************************************
3075  *
3076  *  Enable receive unit.
3077  *
3078  **********************************************************************/
3079 static void
3080 ixgbe_initialize_receive_units(struct adapter *adapter)
3081 {
3082         struct  rx_ring *rxr = adapter->rx_rings;
3083         struct ifnet   *ifp = adapter->ifp;
3084         u32             rxctrl, fctrl, srrctl, rxcsum;
3085         u32             mrqc, hlreg, linkvec;
3086         u32             random[10];
3087         int             i,j;
3088         union {
3089                 u8      c[128];
3090                 u32     i[32];
3091         } reta;
3092
3093
3094         /*
3095          * Make sure receives are disabled while
3096          * setting up the descriptor ring
3097          */
3098         rxctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCTRL);
3099         IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXCTRL,
3100             rxctrl & ~IXGBE_RXCTRL_RXEN);
3101
3102         /* Enable broadcasts */
3103         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
3104         fctrl |= IXGBE_FCTRL_BAM;
3105         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
3106
3107         hlreg = IXGBE_READ_REG(&adapter->hw, IXGBE_HLREG0);
3108         if (ifp->if_mtu > ETHERMTU)
3109                 hlreg |= IXGBE_HLREG0_JUMBOEN;
3110         else
3111                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
3112         IXGBE_WRITE_REG(&adapter->hw, IXGBE_HLREG0, hlreg);
3113
3114         srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(0));
3115         srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
3116         srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
3117         if (adapter->bigbufs)
3118                 srrctl |= 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3119         else
3120                 srrctl |= 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3121         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
3122         IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(0), srrctl);
3123
3124         /* Set Queue moderation rate */
3125         for (i = 0; i < IXGBE_MSGS; i++)
3126                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(i), DEFAULT_ITR);
3127
3128         /* Set Link moderation lower */
3129         linkvec = adapter->num_tx_queues + adapter->num_rx_queues;
3130         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(linkvec), LINK_ITR);
3131
3132         for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
3133                 u64 rdba = rxr->rxdma.dma_paddr;
3134                 /* Setup the Base and Length of the Rx Descriptor Ring */
3135                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDBAL(i),
3136                                (rdba & 0x00000000ffffffffULL));
3137                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDBAH(i), (rdba >> 32));
3138                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDLEN(i),
3139                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
3140
3141                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
3142                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDH(i), 0);
3143                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(i),
3144                     adapter->num_rx_desc - 1);
3145         }
3146
3147         rxcsum = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCSUM);
3148
3149         if (adapter->num_rx_queues > 1) {
3150                 /* set up random bits */
3151                 arc4rand(&random, sizeof(random), 0);
3152
3153                 /* Create reta data */
3154                 for (i = 0; i < 128; )
3155                         for (j = 0; j < adapter->num_rx_queues &&
3156                             i < 128; j++, i++)
3157                                 reta.c[i] = j;
3158
3159                 /* Set up the redirection table */
3160                 for (i = 0; i < 32; i++)
3161                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_RETA(i), reta.i[i]);
3162
3163                 /* Now fill our hash function seeds */
3164                 for (int i = 0; i < 10; i++)
3165                         IXGBE_WRITE_REG_ARRAY(&adapter->hw,
3166                             IXGBE_RSSRK(0), i, random[i]);
3167
3168                 mrqc = IXGBE_MRQC_RSSEN
3169                     /* Perform hash on these packet types */
3170                     | IXGBE_MRQC_RSS_FIELD_IPV4
3171                     | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
3172                     | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
3173                     | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
3174                     | IXGBE_MRQC_RSS_FIELD_IPV6_EX
3175                     | IXGBE_MRQC_RSS_FIELD_IPV6
3176                     | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
3177                     | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
3178                     | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3179                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_MRQC, mrqc);
3180
3181                 /* RSS and RX IPP Checksum are mutually exclusive */
3182                 rxcsum |= IXGBE_RXCSUM_PCSD;
3183         }
3184
3185         if (ifp->if_capenable & IFCAP_RXCSUM)
3186                 rxcsum |= IXGBE_RXCSUM_PCSD;
3187
3188         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
3189                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
3190
3191         IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXCSUM, rxcsum);
3192
3193         /* Enable Receive engine */
3194         rxctrl |= (IXGBE_RXCTRL_RXEN | IXGBE_RXCTRL_DMBYPS);
3195         IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXCTRL, rxctrl);
3196
3197         return;
3198 }
3199
3200 /*********************************************************************
3201  *
3202  *  Free all receive rings.
3203  *
3204  **********************************************************************/
3205 static void
3206 ixgbe_free_receive_structures(struct adapter *adapter)
3207 {
3208         struct rx_ring *rxr = adapter->rx_rings;
3209
3210         for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
3211                 struct lro_ctrl         *lro = &rxr->lro;
3212                 ixgbe_free_receive_buffers(rxr);
3213                 /* Free LRO memory */
3214                 tcp_lro_free(lro);
3215                 /* Free the ring memory as well */
3216                 ixgbe_dma_free(adapter, &rxr->rxdma);
3217         }
3218
3219         free(adapter->rx_rings, M_DEVBUF);
3220 }
3221
3222 /*********************************************************************
3223  *
3224  *  Free receive ring data structures
3225  *
3226  **********************************************************************/
3227 void
3228 ixgbe_free_receive_buffers(struct rx_ring *rxr)
3229 {
3230         struct adapter          *adapter = NULL;
3231         struct ixgbe_rx_buf     *rxbuf = NULL;
3232
3233         INIT_DEBUGOUT("free_receive_buffers: begin");
3234         adapter = rxr->adapter;
3235         if (rxr->rx_buffers != NULL) {
3236                 rxbuf = &rxr->rx_buffers[0];
3237                 for (int i = 0; i < adapter->num_rx_desc; i++) {
3238                         int s = rxbuf->bigbuf;
3239                         if (rxbuf->map != NULL) {
3240                                 bus_dmamap_unload(rxr->rxtag[s], rxbuf->map[s]);
3241                                 bus_dmamap_destroy(rxr->rxtag[s], rxbuf->map[s]);
3242                         }
3243                         if (rxbuf->m_head != NULL) {
3244                                 m_freem(rxbuf->m_head);
3245                         }
3246                         rxbuf->m_head = NULL;
3247                         ++rxbuf;
3248                 }
3249         }
3250         if (rxr->rx_buffers != NULL) {
3251                 free(rxr->rx_buffers, M_DEVBUF);
3252                 rxr->rx_buffers = NULL;
3253         }
3254         for (int s = 0; s < 2; s++) {
3255                 if (rxr->rxtag[s] != NULL) {
3256                         bus_dma_tag_destroy(rxr->rxtag[s]);
3257                         rxr->rxtag[s] = NULL;
3258                 }
3259         }
3260         return;
3261 }
3262
3263 /*********************************************************************
3264  *
3265  *  This routine executes in interrupt context. It replenishes
3266  *  the mbufs in the descriptor and sends data which has been
3267  *  dma'ed into host memory to upper layer.
3268  *
3269  *  We loop at most count times if count is > 0, or until done if
3270  *  count < 0.
3271  *
3272  *********************************************************************/
3273 static bool
3274 ixgbe_rxeof(struct rx_ring *rxr, int count)
3275 {
3276         struct adapter          *adapter = rxr->adapter;
3277         struct ifnet            *ifp = adapter->ifp;
3278         struct lro_ctrl         *lro = &rxr->lro;
3279         struct lro_entry        *queued;
3280         struct mbuf             *mp;
3281         int                     len, i, eop = 0;
3282         u8                      accept_frame = 0;
3283         u32             staterr;
3284         union ixgbe_adv_rx_desc *cur;
3285
3286
3287         IXGBE_RX_LOCK(rxr);
3288         i = rxr->next_to_check;
3289         cur = &rxr->rx_base[i];
3290         staterr = cur->wb.upper.status_error;
3291
3292         if (!(staterr & IXGBE_RXD_STAT_DD)) {
3293                 IXGBE_RX_UNLOCK(rxr);
3294                 return FALSE;
3295         }
3296
3297         while ((staterr & IXGBE_RXD_STAT_DD) && (count != 0) &&
3298             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3299                 struct mbuf *m = NULL;
3300                 int s;
3301
3302                 mp = rxr->rx_buffers[i].m_head;
3303                 s = rxr->rx_buffers[i].bigbuf;
3304                 bus_dmamap_sync(rxr->rxtag[s], rxr->rx_buffers[i].map[s],
3305                                 BUS_DMASYNC_POSTREAD);
3306                 accept_frame = 1;
3307                 if (staterr & IXGBE_RXD_STAT_EOP) {
3308                         count--;
3309                         eop = 1;
3310                 } else {
3311                         eop = 0;
3312                 }
3313                 len = cur->wb.upper.length;
3314
3315                 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)
3316                         accept_frame = 0;
3317
3318                 if (accept_frame) {
3319                         /* Get a fresh buffer first */
3320                         if (ixgbe_get_buf(rxr, i) != 0) {
3321                                 ifp->if_iqdrops++;
3322                                 goto discard;
3323                         }
3324
3325                         /* Assign correct length to the current fragment */
3326                         mp->m_len = len;
3327
3328                         if (rxr->fmp == NULL) {
3329                                 mp->m_pkthdr.len = len;
3330                                 rxr->fmp = mp; /* Store the first mbuf */
3331                                 rxr->lmp = mp;
3332                         } else {
3333                                 /* Chain mbuf's together */
3334                                 mp->m_flags &= ~M_PKTHDR;
3335                                 rxr->lmp->m_next = mp;
3336                                 rxr->lmp = rxr->lmp->m_next;
3337                                 rxr->fmp->m_pkthdr.len += len;
3338                         }
3339
3340                         if (eop) {
3341                                 rxr->fmp->m_pkthdr.rcvif = ifp;
3342                                 ifp->if_ipackets++;
3343                                 rxr->packet_count++;
3344                                 rxr->byte_count += rxr->fmp->m_pkthdr.len;
3345
3346                                 ixgbe_rx_checksum(adapter,
3347                                     staterr, rxr->fmp);
3348
3349                                 if (staterr & IXGBE_RXD_STAT_VP) {
3350 #if __FreeBSD_version < 700000
3351                                         VLAN_INPUT_TAG_NEW(ifp, rxr->fmp,
3352                                             (le16toh(cur->wb.upper.vlan) &
3353                                             IXGBE_RX_DESC_SPECIAL_VLAN_MASK));
3354 #else
3355                                         rxr->fmp->m_pkthdr.ether_vtag =
3356                                             le16toh(cur->wb.upper.vlan);
3357                                         rxr->fmp->m_flags |= M_VLANTAG;
3358 #endif
3359                                 }
3360                                 m = rxr->fmp;
3361                                 rxr->fmp = NULL;
3362                                 rxr->lmp = NULL;
3363                         }
3364                 } else {
3365                         ifp->if_ierrors++;
3366 discard:
3367                         /* Reuse loaded DMA map and just update mbuf chain */
3368                         mp = rxr->rx_buffers[i].m_head;
3369                         mp->m_len = mp->m_pkthdr.len =
3370                             (rxr->rx_buffers[i].bigbuf ? MJUMPAGESIZE:MCLBYTES);
3371                         mp->m_data = mp->m_ext.ext_buf;
3372                         mp->m_next = NULL;
3373                         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3374                                 m_adj(mp, ETHER_ALIGN);
3375                         if (rxr->fmp != NULL) {
3376                                 m_freem(rxr->fmp);
3377                                 rxr->fmp = NULL;
3378                                 rxr->lmp = NULL;
3379                         }
3380                         m = NULL;
3381                 }
3382
3383                 /* Zero out the receive descriptors status  */
3384                 cur->wb.upper.status_error = 0;
3385                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3386                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3387
3388                 rxr->last_cleaned = i; /* for updating tail */
3389
3390                 if (++i == adapter->num_rx_desc)
3391                         i = 0;
3392
3393                 /* Now send up to the stack */
3394                 if (m != NULL) {
3395                         rxr->next_to_check = i;
3396                         /* Use LRO if possible */
3397                         if ((!lro->lro_cnt) || (tcp_lro_rx(lro, m, 0))) {
3398                                 IXGBE_RX_UNLOCK(rxr);
3399                                 (*ifp->if_input)(ifp, m);
3400                                 IXGBE_RX_LOCK(rxr);
3401                                 i = rxr->next_to_check;
3402                         }
3403                 }
3404                 /* Get next descriptor */
3405                 cur = &rxr->rx_base[i];
3406                 staterr = cur->wb.upper.status_error;
3407         }
3408         rxr->next_to_check = i;
3409
3410         /* Advance the IXGB's Receive Queue "Tail Pointer" */
3411         IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), rxr->last_cleaned);
3412         IXGBE_RX_UNLOCK(rxr);
3413
3414         /*
3415         ** Flush any outstanding LRO work
3416         ** this may call into the stack and
3417         ** must not hold a driver lock.
3418         */
3419         while(!SLIST_EMPTY(&lro->lro_active)) {
3420                 queued = SLIST_FIRST(&lro->lro_active);
3421                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
3422                 tcp_lro_flush(lro, queued);
3423         }
3424
3425         if (!(staterr & IXGBE_RXD_STAT_DD))
3426                 return FALSE;
3427
3428         return TRUE;
3429 }
3430
3431 /*********************************************************************
3432  *
3433  *  Verify that the hardware indicated that the checksum is valid.
3434  *  Inform the stack about the status of checksum so that stack
3435  *  doesn't spend time verifying the checksum.
3436  *
3437  *********************************************************************/
3438 static void
3439 ixgbe_rx_checksum(struct adapter *adapter,
3440     u32 staterr, struct mbuf * mp)
3441 {
3442         struct ifnet    *ifp = adapter->ifp;
3443         u16 status = (u16) staterr;
3444         u8  errors = (u8) (staterr >> 24);
3445
3446         /* Not offloading */
3447         if ((ifp->if_capenable & IFCAP_RXCSUM) == 0) {
3448                 mp->m_pkthdr.csum_flags = 0;
3449                 return;
3450         }
3451
3452         if (status & IXGBE_RXD_STAT_IPCS) {
3453                 /* Did it pass? */
3454                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
3455                         /* IP Checksum Good */
3456                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3457                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3458
3459                 } else
3460                         mp->m_pkthdr.csum_flags = 0;
3461         }
3462         if (status & IXGBE_RXD_STAT_L4CS) {
3463                 /* Did it pass? */
3464                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
3465                         mp->m_pkthdr.csum_flags |=
3466                                 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3467                         mp->m_pkthdr.csum_data = htons(0xffff);
3468                 }
3469         }
3470         return;
3471 }
3472
3473 #ifdef IXGBE_VLAN_EVENTS
3474 /*
3475  * This routine is run via an vlan
3476  * config EVENT
3477  */
3478 static void
3479 ixgbe_register_vlan(void *unused, struct ifnet *ifp, u16 vtag)
3480 {
3481         struct adapter  *adapter = ifp->if_softc;
3482         u32             ctrl;
3483
3484         ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL);
3485         ctrl |= IXGBE_VLNCTRL_VME | IXGBE_VLNCTRL_VFE;
3486         ctrl &= ~IXGBE_VLNCTRL_CFIEN;
3487         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl);
3488
3489         /* Make entry in the hardware filter table */
3490         ixgbe_set_vfta(&adapter->hw, vtag, 0, TRUE);
3491 }
3492
3493 /*
3494  * This routine is run via an vlan
3495  * unconfig EVENT
3496  */
3497 static void
3498 ixgbe_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag)
3499 {
3500         struct adapter  *adapter = ifp->if_softc;
3501
3502         /* Remove entry in the hardware filter table */
3503         ixgbe_set_vfta(&adapter->hw, vtag, 0, FALSE);
3504
3505         /* Have all vlans unregistered? */
3506         if (adapter->ifp->if_vlantrunk == NULL) {
3507                 u32 ctrl;
3508                 /* Turn off the filter table */
3509                 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL);
3510                 ctrl &= ~IXGBE_VLNCTRL_VME;
3511                 ctrl &=  ~IXGBE_VLNCTRL_VFE;
3512                 ctrl |= IXGBE_VLNCTRL_CFIEN;
3513                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl);
3514         }
3515 }
3516 #endif /* IXGBE_VLAN_EVENTS */
3517
3518 static void
3519 ixgbe_enable_intr(struct adapter *adapter)
3520 {
3521         struct ixgbe_hw *hw = &adapter->hw;
3522         u32 mask = IXGBE_EIMS_ENABLE_MASK;
3523
3524         /* Enable Fan Failure detection */
3525         if (hw->phy.media_type == ixgbe_media_type_copper)
3526                     mask |= IXGBE_EIMS_GPI_SDP1;
3527         /* With RSS we use auto clear */
3528         if (adapter->msix_mem) {
3529                 /* Dont autoclear Link */
3530                 mask &= ~IXGBE_EIMS_OTHER;
3531                 mask &= ~IXGBE_EIMS_LSC;
3532                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC,
3533                     adapter->eims_mask | mask);
3534         }
3535
3536         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3537         IXGBE_WRITE_FLUSH(hw);
3538
3539         return;
3540 }
3541
3542 static void
3543 ixgbe_disable_intr(struct adapter *adapter)
3544 {
3545         if (adapter->msix_mem)
3546                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
3547         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
3548         IXGBE_WRITE_FLUSH(&adapter->hw);
3549         return;
3550 }
3551
3552 u16
3553 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
3554 {
3555         u16 value;
3556
3557         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
3558             reg, 2);
3559
3560         return (value);
3561 }
3562
3563 static void
3564 ixgbe_set_ivar(struct adapter *adapter, u16 entry, u8 vector)
3565 {
3566         u32 ivar, index;
3567
3568         vector |= IXGBE_IVAR_ALLOC_VAL;
3569         index = (entry >> 2) & 0x1F;
3570         ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR(index));
3571         ivar &= ~(0xFF << (8 * (entry & 0x3)));
3572         ivar |= (vector << (8 * (entry & 0x3)));
3573         IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
3574 }
3575
3576 static void
3577 ixgbe_configure_ivars(struct adapter *adapter)
3578 {
3579         struct  tx_ring *txr = adapter->tx_rings;
3580         struct  rx_ring *rxr = adapter->rx_rings;
3581
3582         for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
3583                 ixgbe_set_ivar(adapter, IXGBE_IVAR_RX_QUEUE(i), rxr->msix);
3584                 adapter->eims_mask |= rxr->eims;
3585         }
3586
3587         for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
3588                 ixgbe_set_ivar(adapter, IXGBE_IVAR_TX_QUEUE(i), txr->msix);
3589                 adapter->eims_mask |= txr->eims;
3590         }
3591
3592         /* For the Link interrupt */
3593         ixgbe_set_ivar(adapter, IXGBE_IVAR_OTHER_CAUSES_INDEX,
3594             adapter->linkvec);
3595         adapter->eims_mask |= IXGBE_IVAR_OTHER_CAUSES_INDEX;
3596 }
3597
3598 /**********************************************************************
3599  *
3600  *  Update the board statistics counters.
3601  *
3602  **********************************************************************/
3603 static void
3604 ixgbe_update_stats_counters(struct adapter *adapter)
3605 {
3606         struct ifnet   *ifp = adapter->ifp;;
3607         struct ixgbe_hw *hw = &adapter->hw;
3608         u32  missed_rx = 0, bprc, lxon, lxoff, total;
3609
3610         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
3611
3612         for (int i = 0; i < 8; i++) {
3613                 int mp;
3614                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
3615                 missed_rx += mp;
3616                 adapter->stats.mpc[i] += mp;
3617                 adapter->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
3618         }
3619
3620         /* Hardware workaround, gprc counts missed packets */
3621         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
3622         adapter->stats.gprc -= missed_rx;
3623
3624         adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
3625         adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
3626         adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
3627
3628         /*
3629          * Workaround: mprc hardware is incorrectly counting
3630          * broadcasts, so for now we subtract those.
3631          */
3632         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
3633         adapter->stats.bprc += bprc;
3634         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
3635         adapter->stats.mprc -= bprc;
3636
3637         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
3638         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
3639         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
3640         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
3641         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
3642         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
3643         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
3644         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
3645
3646         adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
3647         adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
3648
3649         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
3650         adapter->stats.lxontxc += lxon;
3651         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
3652         adapter->stats.lxofftxc += lxoff;
3653         total = lxon + lxoff;
3654
3655         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
3656         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
3657         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
3658         adapter->stats.gptc -= total;
3659         adapter->stats.mptc -= total;
3660         adapter->stats.ptc64 -= total;
3661         adapter->stats.gotc -= total * ETHER_MIN_LEN;
3662
3663         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
3664         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
3665         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
3666         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
3667         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
3668         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
3669         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
3670         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
3671         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
3672         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
3673
3674
3675         /* Fill out the OS statistics structure */
3676         ifp->if_ipackets = adapter->stats.gprc;
3677         ifp->if_opackets = adapter->stats.gptc;
3678         ifp->if_ibytes = adapter->stats.gorc;
3679         ifp->if_obytes = adapter->stats.gotc;
3680         ifp->if_imcasts = adapter->stats.mprc;
3681         ifp->if_collisions = 0;
3682
3683         /* Rx Errors */
3684         ifp->if_ierrors = missed_rx + adapter->stats.crcerrs +
3685                 adapter->stats.rlec;
3686 }
3687
3688
3689 /**********************************************************************
3690  *
3691  *  This routine is called only when ixgbe_display_debug_stats is enabled.
3692  *  This routine provides a way to take a look at important statistics
3693  *  maintained by the driver and hardware.
3694  *
3695  **********************************************************************/
3696 static void
3697 ixgbe_print_hw_stats(struct adapter * adapter)
3698 {
3699         device_t dev = adapter->dev;
3700
3701
3702         device_printf(dev,"Std Mbuf Failed = %lu\n",
3703                adapter->mbuf_alloc_failed);
3704         device_printf(dev,"Std Cluster Failed = %lu\n",
3705                adapter->mbuf_cluster_failed);
3706
3707         device_printf(dev,"Missed Packets = %llu\n",
3708                (long long)adapter->stats.mpc[0]);
3709         device_printf(dev,"Receive length errors = %llu\n",
3710                ((long long)adapter->stats.roc +
3711                (long long)adapter->stats.ruc));
3712         device_printf(dev,"Crc errors = %llu\n",
3713                (long long)adapter->stats.crcerrs);
3714         device_printf(dev,"Driver dropped packets = %lu\n",
3715                adapter->dropped_pkts);
3716         device_printf(dev, "watchdog timeouts = %ld\n",
3717                adapter->watchdog_events);
3718
3719         device_printf(dev,"XON Rcvd = %llu\n",
3720                (long long)adapter->stats.lxonrxc);
3721         device_printf(dev,"XON Xmtd = %llu\n",
3722                (long long)adapter->stats.lxontxc);
3723         device_printf(dev,"XOFF Rcvd = %llu\n",
3724                (long long)adapter->stats.lxoffrxc);
3725         device_printf(dev,"XOFF Xmtd = %llu\n",
3726                (long long)adapter->stats.lxofftxc);
3727
3728         device_printf(dev,"Total Packets Rcvd = %llu\n",
3729                (long long)adapter->stats.tpr);
3730         device_printf(dev,"Good Packets Rcvd = %llu\n",
3731                (long long)adapter->stats.gprc);
3732         device_printf(dev,"Good Packets Xmtd = %llu\n",
3733                (long long)adapter->stats.gptc);
3734         device_printf(dev,"TSO Transmissions = %lu\n",
3735                adapter->tso_tx);
3736
3737         return;
3738 }
3739
3740 /**********************************************************************
3741  *
3742  *  This routine is called only when em_display_debug_stats is enabled.
3743  *  This routine provides a way to take a look at important statistics
3744  *  maintained by the driver and hardware.
3745  *
3746  **********************************************************************/
3747 static void
3748 ixgbe_print_debug_info(struct adapter *adapter)
3749 {
3750         device_t dev = adapter->dev;
3751         struct rx_ring *rxr = adapter->rx_rings;
3752         struct tx_ring *txr = adapter->tx_rings;
3753         struct ixgbe_hw *hw = &adapter->hw;
3754
3755         device_printf(dev,"Error Byte Count = %u \n",
3756             IXGBE_READ_REG(hw, IXGBE_ERRBC));
3757
3758         for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
3759                 struct lro_ctrl         *lro = &rxr->lro;
3760                 device_printf(dev,"Queue[%d]: rdh = %d, hw rdt = %d\n",
3761                     i, IXGBE_READ_REG(hw, IXGBE_RDH(i)),
3762                     IXGBE_READ_REG(hw, IXGBE_RDT(i)));
3763                 device_printf(dev,"RX(%d) Packets Received: %lu\n",
3764                     rxr->me, (long)rxr->packet_count);
3765                 device_printf(dev,"RX(%d) Bytes Received: %lu\n",
3766                     rxr->me, (long)rxr->byte_count);
3767                 device_printf(dev,"RX(%d) IRQ Handled: %lu\n",
3768                     rxr->me, (long)rxr->rx_irq);
3769                 device_printf(dev,"RX(%d) LRO Queued= %d\n",
3770                     rxr->me, lro->lro_queued);
3771                 device_printf(dev,"RX(%d) LRO Flushed= %d\n",
3772                     rxr->me, lro->lro_flushed);
3773         }
3774
3775         for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
3776                 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i,
3777                     IXGBE_READ_REG(hw, IXGBE_TDH(i)),
3778                     IXGBE_READ_REG(hw, IXGBE_TDT(i)));
3779                 device_printf(dev,"TX(%d) Packets Sent: %lu\n",
3780                     txr->me, (long)txr->tx_packets);
3781                 device_printf(dev,"TX(%d) IRQ Handled: %lu\n",
3782                     txr->me, (long)txr->tx_irq);
3783                 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n",
3784                     txr->me, (long)txr->no_tx_desc_avail);
3785         }
3786
3787         device_printf(dev,"Link IRQ Handled: %lu\n",
3788             (long)adapter->link_irq);
3789         return;
3790 }
3791
3792 static int
3793 ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS)
3794 {
3795         int             error;
3796         int             result;
3797         struct adapter *adapter;
3798
3799         result = -1;
3800         error = sysctl_handle_int(oidp, &result, 0, req);
3801
3802         if (error || !req->newptr)
3803                 return (error);
3804
3805         if (result == 1) {
3806                 adapter = (struct adapter *) arg1;
3807                 ixgbe_print_hw_stats(adapter);
3808         }
3809         return error;
3810 }
3811
3812 static int
3813 ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS)
3814 {
3815         int error, result;
3816         struct adapter *adapter;
3817
3818         result = -1;
3819         error = sysctl_handle_int(oidp, &result, 0, req);
3820
3821         if (error || !req->newptr)
3822                 return (error);
3823
3824         if (result == 1) {
3825                 adapter = (struct adapter *) arg1;
3826                 ixgbe_print_debug_info(adapter);
3827         }
3828         return error;
3829 }
3830
3831 /*
3832 ** Set flow control using sysctl:
3833 ** Flow control values:
3834 **      0 - off
3835 **      1 - rx pause
3836 **      2 - tx pause
3837 **      3 - full
3838 */
3839 static int
3840 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
3841 {
3842         int error;
3843         struct adapter *adapter;
3844
3845         error = sysctl_handle_int(oidp, &ixgbe_flow_control, 0, req);
3846
3847         if (error)
3848                 return (error);
3849
3850         adapter = (struct adapter *) arg1;
3851         switch (ixgbe_flow_control) {
3852                 case ixgbe_fc_rx_pause:
3853                 case ixgbe_fc_tx_pause:
3854                 case ixgbe_fc_full:
3855                         adapter->hw.fc.type = ixgbe_flow_control;
3856                         break;
3857                 case ixgbe_fc_none:
3858                 default:
3859                         adapter->hw.fc.type = ixgbe_fc_none;
3860         }
3861
3862         ixgbe_setup_fc(&adapter->hw, 0);
3863         return error;
3864 }
3865
3866 static void
3867 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
3868         const char *description, int *limit, int value)
3869 {
3870         *limit = value;
3871         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
3872             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
3873             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
3874 }
3875
3876 #ifndef NO_82598_A0_SUPPORT
3877 /*
3878  * A0 Workaround: invert descriptor for hardware
3879  */
3880 void
3881 desc_flip(void *desc)
3882 {
3883         struct dhack {u32 a1; u32 a2; u32 b1; u32 b2;};
3884         struct dhack *d;
3885
3886         d = (struct dhack *)desc;
3887         d->a1 = ~(d->a1);
3888         d->a2 = ~(d->a2);
3889         d->b1 = ~(d->b1);
3890         d->b2 = ~(d->b2);
3891         d->b2 &= 0xFFFFFFF0;
3892         d->b1 &= ~IXGBE_ADVTXD_DCMD_RS;
3893 }
3894 #endif
3895
3896
3897