sys/dev/e1000/if_igb.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2010, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #ifdef HAVE_KERNEL_OPTION_HEADERS
  37 #include "opt_device_polling.h"
  38 #include "opt_inet.h"
  39 #include "opt_altq.h"
  40 #endif
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #if __FreeBSD_version >= 800000
  45 #include <sys/buf_ring.h>
  46 #endif
  47 #include <sys/bus.h>
  48 #include <sys/endian.h>
  49 #include <sys/kernel.h>
  50 #include <sys/kthread.h>
  51 #include <sys/malloc.h>
  52 #include <sys/mbuf.h>
  53 #include <sys/module.h>
  54 #include <sys/rman.h>
  55 #include <sys/socket.h>
  56 #include <sys/sockio.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/taskqueue.h>
  59 #include <sys/eventhandler.h>
  60 #include <sys/pcpu.h>
  61 #include <sys/smp.h>
  62 #include <machine/smp.h>
  63 #include <machine/bus.h>
  64 #include <machine/resource.h>
  65
  66 #include <net/bpf.h>
  67 #include <net/ethernet.h>
  68 #include <net/if.h>
  69 #include <net/if_arp.h>
  70 #include <net/if_dl.h>
  71 #include <net/if_media.h>
  72
  73 #include <net/if_types.h>
  74 #include <net/if_vlan_var.h>
  75
  76 #include <netinet/in_systm.h>
  77 #include <netinet/in.h>
  78 #include <netinet/if_ether.h>
  79 #include <netinet/ip.h>
  80 #include <netinet/ip6.h>
  81 #include <netinet/tcp.h>
  82 #include <netinet/tcp_lro.h>
  83 #include <netinet/udp.h>
  84
  85 #include <machine/in_cksum.h>
  86 #include <dev/led/led.h>
  87 #include <dev/pci/pcivar.h>
  88 #include <dev/pci/pcireg.h>
  89
  90 #include "e1000_api.h"
  91 #include "e1000_82575.h"
  92 #include "if_igb.h"
  93
  94 /*********************************************************************
  95  *  Set this to one to display debug statistics
  96  *********************************************************************/
  97 int     igb_display_debug_stats = 0;
  98
  99 /*********************************************************************
 100  *  Driver version:
 101  *********************************************************************/
 102 char igb_driver_version[] = "version - 1.9.6";
 103
 104
 105 /*********************************************************************
 106  *  PCI Device ID Table
 107  *
 108  *  Used by probe to select devices to load on
 109  *  Last field stores an index into e1000_strings
 110  *  Last entry must be all 0s
 111  *
 112  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 113  *********************************************************************/
 114
 115 static igb_vendor_info_t igb_vendor_info_array[] =
 116 {
 117         { 0x8086, E1000_DEV_ID_82575EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 118         { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
 119                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 120         { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
 121                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 122         { 0x8086, E1000_DEV_ID_82576,           PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82576_NS,        PCI_ANY_ID, PCI_ANY_ID, 0},
 124         { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82576_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
 126         { 0x8086, E1000_DEV_ID_82576_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
 127         { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
 128                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 129         { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
 130                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 131         { 0x8086, E1000_DEV_ID_82580_COPPER,    PCI_ANY_ID, PCI_ANY_ID, 0},
 132         { 0x8086, E1000_DEV_ID_82580_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82580_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82580_SGMII,     PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL,
 136                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 137         /* required last entry */
 138         { 0, 0, 0, 0, 0}
 139 };
 140
 141 /*********************************************************************
 142  *  Table of branding strings for all supported NICs.
 143  *********************************************************************/
 144
 145 static char *igb_strings[] = {
 146         "Intel(R) PRO/1000 Network Connection"
 147 };
 148
 149 /*********************************************************************
 150  *  Function prototypes
 151  *********************************************************************/
 152 static int      igb_probe(device_t);
 153 static int      igb_attach(device_t);
 154 static int      igb_detach(device_t);
 155 static int      igb_shutdown(device_t);
 156 static int      igb_suspend(device_t);
 157 static int      igb_resume(device_t);
 158 static void     igb_start(struct ifnet *);
 159 static void     igb_start_locked(struct tx_ring *, struct ifnet *ifp);
 160 #if __FreeBSD_version >= 800000
 161 static int      igb_mq_start(struct ifnet *, struct mbuf *);
 162 static int      igb_mq_start_locked(struct ifnet *,
 163                     struct tx_ring *, struct mbuf *);
 164 static void     igb_qflush(struct ifnet *);
 165 #endif
 166 static int      igb_ioctl(struct ifnet *, u_long, caddr_t);
 167 static void     igb_init(void *);
 168 static void     igb_init_locked(struct adapter *);
 169 static void     igb_stop(void *);
 170 static void     igb_media_status(struct ifnet *, struct ifmediareq *);
 171 static int      igb_media_change(struct ifnet *);
 172 static void     igb_identify_hardware(struct adapter *);
 173 static int      igb_allocate_pci_resources(struct adapter *);
 174 static int      igb_allocate_msix(struct adapter *);
 175 static int      igb_allocate_legacy(struct adapter *);
 176 static int      igb_setup_msix(struct adapter *);
 177 static void     igb_free_pci_resources(struct adapter *);
 178 static void     igb_local_timer(void *);
 179 static void     igb_reset(struct adapter *);
 180 static void     igb_setup_interface(device_t, struct adapter *);
 181 static int      igb_allocate_queues(struct adapter *);
 182 static void     igb_configure_queues(struct adapter *);
 183
 184 static int      igb_allocate_transmit_buffers(struct tx_ring *);
 185 static void     igb_setup_transmit_structures(struct adapter *);
 186 static void     igb_setup_transmit_ring(struct tx_ring *);
 187 static void     igb_initialize_transmit_units(struct adapter *);
 188 static void     igb_free_transmit_structures(struct adapter *);
 189 static void     igb_free_transmit_buffers(struct tx_ring *);
 190
 191 static int      igb_allocate_receive_buffers(struct rx_ring *);
 192 static int      igb_setup_receive_structures(struct adapter *);
 193 static int      igb_setup_receive_ring(struct rx_ring *);
 194 static void     igb_initialize_receive_units(struct adapter *);
 195 static void     igb_free_receive_structures(struct adapter *);
 196 static void     igb_free_receive_buffers(struct rx_ring *);
 197 static void     igb_free_receive_ring(struct rx_ring *);
 198
 199 static void     igb_enable_intr(struct adapter *);
 200 static void     igb_disable_intr(struct adapter *);
 201 static void     igb_update_stats_counters(struct adapter *);
 202 static bool     igb_txeof(struct tx_ring *);
 203
 204 static __inline void igb_rx_discard(struct rx_ring *, int);
 205 static __inline void igb_rx_input(struct rx_ring *,
 206                     struct ifnet *, struct mbuf *, u32);
 207
 208 static bool     igb_rxeof(struct igb_queue *, int, int *);
 209 static void     igb_rx_checksum(u32, struct mbuf *, u32);
 210 static int      igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 211 static bool     igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 212 static void     igb_set_promisc(struct adapter *);
 213 static void     igb_disable_promisc(struct adapter *);
 214 static void     igb_set_multi(struct adapter *);
 215 static void     igb_update_link_status(struct adapter *);
 216 static void     igb_refresh_mbufs(struct rx_ring *, int);
 217
 218 static void     igb_register_vlan(void *, struct ifnet *, u16);
 219 static void     igb_unregister_vlan(void *, struct ifnet *, u16);
 220 static void     igb_setup_vlan_hw_support(struct adapter *);
 221
 222 static int      igb_xmit(struct tx_ring *, struct mbuf **);
 223 static int      igb_dma_malloc(struct adapter *, bus_size_t,
 224                     struct igb_dma_alloc *, int);
 225 static void     igb_dma_free(struct adapter *, struct igb_dma_alloc *);
 226 static int      igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
 227 static void     igb_print_nvm_info(struct adapter *);
 228 static int      igb_is_valid_ether_addr(u8 *);
 229 static void     igb_add_hw_stats(struct adapter *adapter);
 230 /* Management and WOL Support */
 231 static void     igb_init_manageability(struct adapter *);
 232 static void     igb_release_manageability(struct adapter *);
 233 static void     igb_get_hw_control(struct adapter *);
 234 static void     igb_release_hw_control(struct adapter *);
 235 static void     igb_enable_wakeup(device_t);
 236 static void     igb_led_func(void *, int);
 237
 238 static int      igb_irq_fast(void *);
 239 static void     igb_add_rx_process_limit(struct adapter *, const char *,
 240                     const char *, int *, int);
 241 static void     igb_handle_que(void *context, int pending);
 242 static void     igb_handle_link(void *context, int pending);
 243
 244 /* These are MSIX only irq handlers */
 245 static void     igb_msix_que(void *);
 246 static void     igb_msix_link(void *);
 247
 248 #ifdef DEVICE_POLLING
 249 static poll_handler_t igb_poll;
 250 #endif /* POLLING */
 251
 252 /*********************************************************************
 253  *  FreeBSD Device Interface Entry Points
 254  *********************************************************************/
 255
 256 static device_method_t igb_methods[] = {
 257         /* Device interface */
 258         DEVMETHOD(device_probe, igb_probe),
 259         DEVMETHOD(device_attach, igb_attach),
 260         DEVMETHOD(device_detach, igb_detach),
 261         DEVMETHOD(device_shutdown, igb_shutdown),
 262         DEVMETHOD(device_suspend, igb_suspend),
 263         DEVMETHOD(device_resume, igb_resume),
 264         {0, 0}
 265 };
 266
 267 static driver_t igb_driver = {
 268         "igb", igb_methods, sizeof(struct adapter),
 269 };
 270
 271 static devclass_t igb_devclass;
 272 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
 273 MODULE_DEPEND(igb, pci, 1, 1, 1);
 274 MODULE_DEPEND(igb, ether, 1, 1, 1);
 275
 276 /*********************************************************************
 277  *  Tunable default values.
 278  *********************************************************************/
 279
 280 /* Descriptor defaults */
 281 static int igb_rxd = IGB_DEFAULT_RXD;
 282 static int igb_txd = IGB_DEFAULT_TXD;
 283 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
 284 TUNABLE_INT("hw.igb.txd", &igb_txd);
 285
 286 /*
 287 ** AIM: Adaptive Interrupt Moderation
 288 ** which means that the interrupt rate
 289 ** is varied over time based on the
 290 ** traffic for that interrupt vector
 291 */
 292 static int igb_enable_aim = TRUE;
 293 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
 294
 295 /*
 296  * MSIX should be the default for best performance,
 297  * but this allows it to be forced off for testing.
 298  */
 299 static int igb_enable_msix = 1;
 300 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
 301
 302 /*
 303  * Header split has seemed to be beneficial in
 304  * many circumstances tested, however there have
 305  * been some stability issues, so the default is
 306  * off.
 307  */
 308 static bool igb_header_split = FALSE;
 309 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
 310
 311 /*
 312 ** This will autoconfigure based on
 313 ** the number of CPUs if left at 0.
 314 */
 315 static int igb_num_queues = 0;
 316 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
 317
 318 /* How many packets rxeof tries to clean at a time */
 319 static int igb_rx_process_limit = 100;
 320 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
 321
 322 /* Flow control setting - default to FULL */
 323 static int igb_fc_setting = e1000_fc_full;
 324 TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
 325
 326 /*
 327 ** Shadow VFTA table, this is needed because
 328 ** the real filter table gets cleared during
 329 ** a soft reset and the driver needs to be able
 330 ** to repopulate it.
 331 */
 332 static u32 igb_shadow_vfta[IGB_VFTA_SIZE];
 333
 334
 335 /*********************************************************************
 336  *  Device identification routine
 337  *
 338  *  igb_probe determines if the driver should be loaded on
 339  *  adapter based on PCI vendor/device id of the adapter.
 340  *
 341  *  return BUS_PROBE_DEFAULT on success, positive on failure
 342  *********************************************************************/
 343
 344 static int
 345 igb_probe(device_t dev)
 346 {
 347         char            adapter_name[60];
 348         uint16_t        pci_vendor_id = 0;
 349         uint16_t        pci_device_id = 0;
 350         uint16_t        pci_subvendor_id = 0;
 351         uint16_t        pci_subdevice_id = 0;
 352         igb_vendor_info_t *ent;
 353
 354         INIT_DEBUGOUT("igb_probe: begin");
 355
 356         pci_vendor_id = pci_get_vendor(dev);
 357         if (pci_vendor_id != IGB_VENDOR_ID)
 358                 return (ENXIO);
 359
 360         pci_device_id = pci_get_device(dev);
 361         pci_subvendor_id = pci_get_subvendor(dev);
 362         pci_subdevice_id = pci_get_subdevice(dev);
 363
 364         ent = igb_vendor_info_array;
 365         while (ent->vendor_id != 0) {
 366                 if ((pci_vendor_id == ent->vendor_id) &&
 367                     (pci_device_id == ent->device_id) &&
 368
 369                     ((pci_subvendor_id == ent->subvendor_id) ||
 370                     (ent->subvendor_id == PCI_ANY_ID)) &&
 371
 372                     ((pci_subdevice_id == ent->subdevice_id) ||
 373                     (ent->subdevice_id == PCI_ANY_ID))) {
 374                         sprintf(adapter_name, "%s %s",
 375                                 igb_strings[ent->index],
 376                                 igb_driver_version);
 377                         device_set_desc_copy(dev, adapter_name);
 378                         return (BUS_PROBE_DEFAULT);
 379                 }
 380                 ent++;
 381         }
 382
 383         return (ENXIO);
 384 }
 385
 386 /*********************************************************************
 387  *  Device initialization routine
 388  *
 389  *  The attach entry point is called when the driver is being loaded.
 390  *  This routine identifies the type of hardware, allocates all resources
 391  *  and initializes the hardware.
 392  *
 393  *  return 0 on success, positive on failure
 394  *********************************************************************/
 395
 396 static int
 397 igb_attach(device_t dev)
 398 {
 399         struct adapter  *adapter;
 400         int             error = 0;
 401         u16             eeprom_data;
 402
 403         INIT_DEBUGOUT("igb_attach: begin");
 404
 405         adapter = device_get_softc(dev);
 406         adapter->dev = adapter->osdep.dev = dev;
 407         IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 408
 409         /* SYSCTL stuff */
 410         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 411             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 412             OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 413             igb_sysctl_nvm_info, "I", "NVM Information");
 414
 415         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
 416             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
 417             OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW,
 418             &igb_fc_setting, 0, "Flow Control");
 419
 420         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 421             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 422             OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 423             &igb_enable_aim, 1, "Interrupt Moderation");
 424
 425         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 426
 427         /* Determine hardware and mac info */
 428         igb_identify_hardware(adapter);
 429
 430         /* Setup PCI resources */
 431         if (igb_allocate_pci_resources(adapter)) {
 432                 device_printf(dev, "Allocation of PCI resources failed\n");
 433                 error = ENXIO;
 434                 goto err_pci;
 435         }
 436
 437         /* Do Shared Code initialization */
 438         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 439                 device_printf(dev, "Setup of Shared code failed\n");
 440                 error = ENXIO;
 441                 goto err_pci;
 442         }
 443
 444         e1000_get_bus_info(&adapter->hw);
 445
 446         /* Sysctls for limiting the amount of work done in the taskqueue */
 447         igb_add_rx_process_limit(adapter, "rx_processing_limit",
 448             "max number of rx packets to process", &adapter->rx_process_limit,
 449             igb_rx_process_limit);
 450
 451         /*
 452          * Validate number of transmit and receive descriptors. It
 453          * must not exceed hardware maximum, and must be multiple
 454          * of E1000_DBA_ALIGN.
 455          */
 456         if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
 457             (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
 458                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 459                     IGB_DEFAULT_TXD, igb_txd);
 460                 adapter->num_tx_desc = IGB_DEFAULT_TXD;
 461         } else
 462                 adapter->num_tx_desc = igb_txd;
 463         if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
 464             (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
 465                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 466                     IGB_DEFAULT_RXD, igb_rxd);
 467                 adapter->num_rx_desc = IGB_DEFAULT_RXD;
 468         } else
 469                 adapter->num_rx_desc = igb_rxd;
 470
 471         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 472         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 473         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 474
 475         /* Copper options */
 476         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 477                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 478                 adapter->hw.phy.disable_polarity_correction = FALSE;
 479                 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
 480         }
 481
 482         /*
 483          * Set the frame limits assuming
 484          * standard ethernet sized frames.
 485          */
 486         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 487         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 488
 489         /*
 490         ** Allocate and Setup Queues
 491         */
 492         if (igb_allocate_queues(adapter)) {
 493                 error = ENOMEM;
 494                 goto err_pci;
 495         }
 496
 497         /*
 498         ** Start from a known state, this is
 499         ** important in reading the nvm and
 500         ** mac from that.
 501         */
 502         e1000_reset_hw(&adapter->hw);
 503
 504         /* Make sure we have a good EEPROM before we read from it */
 505         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 506                 /*
 507                 ** Some PCI-E parts fail the first check due to
 508                 ** the link being in sleep state, call it again,
 509                 ** if it fails a second time its a real issue.
 510                 */
 511                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 512                         device_printf(dev,
 513                             "The EEPROM Checksum Is Not Valid\n");
 514                         error = EIO;
 515                         goto err_late;
 516                 }
 517         }
 518
 519         /*
 520         ** Copy the permanent MAC address out of the EEPROM
 521         */
 522         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 523                 device_printf(dev, "EEPROM read error while reading MAC"
 524                     " address\n");
 525                 error = EIO;
 526                 goto err_late;
 527         }
 528         /* Check its sanity */
 529         if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
 530                 device_printf(dev, "Invalid MAC address\n");
 531                 error = EIO;
 532                 goto err_late;
 533         }
 534
 535         /*
 536         ** Configure Interrupts
 537         */
 538         if ((adapter->msix > 1) && (igb_enable_msix))
 539                 error = igb_allocate_msix(adapter);
 540         else /* MSI or Legacy */
 541                 error = igb_allocate_legacy(adapter);
 542         if (error)
 543                 goto err_late;
 544
 545         /* Setup OS specific network interface */
 546         igb_setup_interface(dev, adapter);
 547
 548         /* Now get a good starting state */
 549         igb_reset(adapter);
 550
 551         /* Initialize statistics */
 552         igb_update_stats_counters(adapter);
 553
 554         adapter->hw.mac.get_link_status = 1;
 555         igb_update_link_status(adapter);
 556
 557         /* Indicate SOL/IDER usage */
 558         if (e1000_check_reset_block(&adapter->hw))
 559                 device_printf(dev,
 560                     "PHY reset is blocked due to SOL/IDER session.\n");
 561
 562         /* Determine if we have to control management hardware */
 563         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
 564
 565         /*
 566          * Setup Wake-on-Lan
 567          */
 568         /* APME bit in EEPROM is mapped to WUC.APME */
 569         eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
 570         if (eeprom_data)
 571                 adapter->wol = E1000_WUFC_MAG;
 572
 573         /* Register for VLAN events */
 574         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 575              igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 576         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 577              igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 578
 579         igb_add_hw_stats(adapter);
 580
 581         /* Tell the stack that the interface is not active */
 582         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 583
 584         adapter->led_dev = led_create(igb_led_func, adapter,
 585             device_get_nameunit(dev));
 586
 587         INIT_DEBUGOUT("igb_attach: end");
 588
 589         return (0);
 590
 591 err_late:
 592         igb_free_transmit_structures(adapter);
 593         igb_free_receive_structures(adapter);
 594         igb_release_hw_control(adapter);
 595 err_pci:
 596         igb_free_pci_resources(adapter);
 597         IGB_CORE_LOCK_DESTROY(adapter);
 598
 599         return (error);
 600 }
 601
 602 /*********************************************************************
 603  *  Device removal routine
 604  *
 605  *  The detach entry point is called when the driver is being removed.
 606  *  This routine stops the adapter and deallocates all the resources
 607  *  that were allocated for driver operation.
 608  *
 609  *  return 0 on success, positive on failure
 610  *********************************************************************/
 611
 612 static int
 613 igb_detach(device_t dev)
 614 {
 615         struct adapter  *adapter = device_get_softc(dev);
 616         struct ifnet    *ifp = adapter->ifp;
 617
 618         INIT_DEBUGOUT("igb_detach: begin");
 619
 620         /* Make sure VLANS are not using driver */
 621         if (adapter->ifp->if_vlantrunk != NULL) {
 622                 device_printf(dev,"Vlan in use, detach first\n");
 623                 return (EBUSY);
 624         }
 625
 626         if (adapter->led_dev != NULL)
 627                 led_destroy(adapter->led_dev);
 628
 629 #ifdef DEVICE_POLLING
 630         if (ifp->if_capenable & IFCAP_POLLING)
 631                 ether_poll_deregister(ifp);
 632 #endif
 633
 634         IGB_CORE_LOCK(adapter);
 635         adapter->in_detach = 1;
 636         igb_stop(adapter);
 637         IGB_CORE_UNLOCK(adapter);
 638
 639         e1000_phy_hw_reset(&adapter->hw);
 640
 641         /* Give control back to firmware */
 642         igb_release_manageability(adapter);
 643         igb_release_hw_control(adapter);
 644
 645         if (adapter->wol) {
 646                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 647                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 648                 igb_enable_wakeup(dev);
 649         }
 650
 651         /* Unregister VLAN events */
 652         if (adapter->vlan_attach != NULL)
 653                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 654         if (adapter->vlan_detach != NULL)
 655                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 656
 657         ether_ifdetach(adapter->ifp);
 658
 659         callout_drain(&adapter->timer);
 660
 661         igb_free_pci_resources(adapter);
 662         bus_generic_detach(dev);
 663         if_free(ifp);
 664
 665         igb_free_transmit_structures(adapter);
 666         igb_free_receive_structures(adapter);
 667
 668         IGB_CORE_LOCK_DESTROY(adapter);
 669
 670         return (0);
 671 }
 672
 673 /*********************************************************************
 674  *
 675  *  Shutdown entry point
 676  *
 677  **********************************************************************/
 678
 679 static int
 680 igb_shutdown(device_t dev)
 681 {
 682         return igb_suspend(dev);
 683 }
 684
 685 /*
 686  * Suspend/resume device methods.
 687  */
 688 static int
 689 igb_suspend(device_t dev)
 690 {
 691         struct adapter *adapter = device_get_softc(dev);
 692
 693         IGB_CORE_LOCK(adapter);
 694
 695         igb_stop(adapter);
 696
 697         igb_release_manageability(adapter);
 698         igb_release_hw_control(adapter);
 699
 700         if (adapter->wol) {
 701                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 702                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 703                 igb_enable_wakeup(dev);
 704         }
 705
 706         IGB_CORE_UNLOCK(adapter);
 707
 708         return bus_generic_suspend(dev);
 709 }
 710
 711 static int
 712 igb_resume(device_t dev)
 713 {
 714         struct adapter *adapter = device_get_softc(dev);
 715         struct ifnet *ifp = adapter->ifp;
 716
 717         IGB_CORE_LOCK(adapter);
 718         igb_init_locked(adapter);
 719         igb_init_manageability(adapter);
 720
 721         if ((ifp->if_flags & IFF_UP) &&
 722             (ifp->if_drv_flags & IFF_DRV_RUNNING))
 723                 igb_start(ifp);
 724
 725         IGB_CORE_UNLOCK(adapter);
 726
 727         return bus_generic_resume(dev);
 728 }
 729
 730
 731 /*********************************************************************
 732  *  Transmit entry point
 733  *
 734  *  igb_start is called by the stack to initiate a transmit.
 735  *  The driver will remain in this routine as long as there are
 736  *  packets to transmit and transmit resources are available.
 737  *  In case resources are not available stack is notified and
 738  *  the packet is requeued.
 739  **********************************************************************/
 740
 741 static void
 742 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
 743 {
 744         struct adapter  *adapter = ifp->if_softc;
 745         struct mbuf     *m_head;
 746
 747         IGB_TX_LOCK_ASSERT(txr);
 748
 749         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 750             IFF_DRV_RUNNING)
 751                 return;
 752         if (!adapter->link_active)
 753                 return;
 754
 755         /* Call cleanup if number of TX descriptors low */
 756         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
 757                 igb_txeof(txr);
 758
 759         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 760                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
 761                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 762                         break;
 763                 }
 764                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 765                 if (m_head == NULL)
 766                         break;
 767                 /*
 768                  *  Encapsulation can modify our pointer, and or make it
 769                  *  NULL on failure.  In that event, we can't requeue.
 770                  */
 771                 if (igb_xmit(txr, &m_head)) {
 772                         if (m_head == NULL)
 773                                 break;
 774                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 775                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 776                         break;
 777                 }
 778
 779                 /* Send a copy of the frame to the BPF listener */
 780                 ETHER_BPF_MTAP(ifp, m_head);
 781
 782                 /* Set watchdog on */
 783                 txr->watchdog_time = ticks;
 784                 txr->watchdog_check = TRUE;
 785         }
 786 }
 787
 788 /*
 789  * Legacy TX driver routine, called from the
 790  * stack, always uses tx[0], and spins for it.
 791  * Should not be used with multiqueue tx
 792  */
 793 static void
 794 igb_start(struct ifnet *ifp)
 795 {
 796         struct adapter  *adapter = ifp->if_softc;
 797         struct tx_ring  *txr = adapter->tx_rings;
 798
 799         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 800                 IGB_TX_LOCK(txr);
 801                 igb_start_locked(txr, ifp);
 802                 IGB_TX_UNLOCK(txr);
 803         }
 804         return;
 805 }
 806
 807 #if __FreeBSD_version >= 800000
 808 /*
 809 ** Multiqueue Transmit driver
 810 **
 811 */
 812 static int
 813 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
 814 {
 815         struct adapter          *adapter = ifp->if_softc;
 816         struct igb_queue        *que;
 817         struct tx_ring          *txr;
 818         int                     i = 0, err = 0;
 819
 820         /* Which queue to use */
 821         if ((m->m_flags & M_FLOWID) != 0)
 822                 i = m->m_pkthdr.flowid % adapter->num_queues;
 823
 824         txr = &adapter->tx_rings[i];
 825         que = &adapter->queues[i];
 826
 827         if (IGB_TX_TRYLOCK(txr)) {
 828                 err = igb_mq_start_locked(ifp, txr, m);
 829                 IGB_TX_UNLOCK(txr);
 830         } else {
 831                 err = drbr_enqueue(ifp, txr->br, m);
 832                 taskqueue_enqueue(que->tq, &que->que_task);
 833         }
 834
 835         return (err);
 836 }
 837
 838 static int
 839 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 840 {
 841         struct adapter  *adapter = txr->adapter;
 842         struct mbuf     *next;
 843         int             err = 0, enq;
 844
 845         IGB_TX_LOCK_ASSERT(txr);
 846
 847         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 848             IFF_DRV_RUNNING || adapter->link_active == 0) {
 849                 if (m != NULL)
 850                         err = drbr_enqueue(ifp, txr->br, m);
 851                 return (err);
 852         }
 853
 854         /* Call cleanup if number of TX descriptors low */
 855         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
 856                 igb_txeof(txr);
 857
 858         enq = 0;
 859         if (m == NULL) {
 860                 next = drbr_dequeue(ifp, txr->br);
 861         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 862                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 863                         return (err);
 864                 next = drbr_dequeue(ifp, txr->br);
 865         } else
 866                 next = m;
 867
 868         /* Process the queue */
 869         while (next != NULL) {
 870                 if ((err = igb_xmit(txr, &next)) != 0) {
 871                         if (next != NULL)
 872                                 err = drbr_enqueue(ifp, txr->br, next);
 873                         break;
 874                 }
 875                 enq++;
 876                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 877                 ETHER_BPF_MTAP(ifp, next);
 878                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 879                         break;
 880                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
 881                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 882                         break;
 883                 }
 884                 next = drbr_dequeue(ifp, txr->br);
 885         }
 886         if (enq > 0) {
 887                 /* Set the watchdog */
 888                 txr->watchdog_check = TRUE;
 889                 txr->watchdog_time = ticks;
 890         }
 891         return (err);
 892 }
 893
 894 /*
 895 ** Flush all ring buffers
 896 */
 897 static void
 898 igb_qflush(struct ifnet *ifp)
 899 {
 900         struct adapter  *adapter = ifp->if_softc;
 901         struct tx_ring  *txr = adapter->tx_rings;
 902         struct mbuf     *m;
 903
 904         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 905                 IGB_TX_LOCK(txr);
 906                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 907                         m_freem(m);
 908                 IGB_TX_UNLOCK(txr);
 909         }
 910         if_qflush(ifp);
 911 }
 912 #endif /* __FreeBSD_version >= 800000 */
 913
 914 /*********************************************************************
 915  *  Ioctl entry point
 916  *
 917  *  igb_ioctl is called when the user wants to configure the
 918  *  interface.
 919  *
 920  *  return 0 on success, positive on failure
 921  **********************************************************************/
 922
 923 static int
 924 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 925 {
 926         struct adapter  *adapter = ifp->if_softc;
 927         struct ifreq *ifr = (struct ifreq *)data;
 928 #ifdef INET
 929         struct ifaddr *ifa = (struct ifaddr *)data;
 930 #endif
 931         int error = 0;
 932
 933         if (adapter->in_detach)
 934                 return (error);
 935
 936         switch (command) {
 937         case SIOCSIFADDR:
 938 #ifdef INET
 939                 if (ifa->ifa_addr->sa_family == AF_INET) {
 940                         /*
 941                          * XXX
 942                          * Since resetting hardware takes a very long time
 943                          * and results in link renegotiation we only
 944                          * initialize the hardware only when it is absolutely
 945                          * required.
 946                          */
 947                         ifp->if_flags |= IFF_UP;
 948                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 949                                 IGB_CORE_LOCK(adapter);
 950                                 igb_init_locked(adapter);
 951                                 IGB_CORE_UNLOCK(adapter);
 952                         }
 953                         if (!(ifp->if_flags & IFF_NOARP))
 954                                 arp_ifinit(ifp, ifa);
 955                 } else
 956 #endif
 957                         error = ether_ioctl(ifp, command, data);
 958                 break;
 959         case SIOCSIFMTU:
 960             {
 961                 int max_frame_size;
 962
 963                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 964
 965                 IGB_CORE_LOCK(adapter);
 966                 max_frame_size = 9234;
 967                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
 968                     ETHER_CRC_LEN) {
 969                         IGB_CORE_UNLOCK(adapter);
 970                         error = EINVAL;
 971                         break;
 972                 }
 973
 974                 ifp->if_mtu = ifr->ifr_mtu;
 975                 adapter->max_frame_size =
 976                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 977                 igb_init_locked(adapter);
 978                 IGB_CORE_UNLOCK(adapter);
 979                 break;
 980             }
 981         case SIOCSIFFLAGS:
 982                 IOCTL_DEBUGOUT("ioctl rcv'd:\
 983                     SIOCSIFFLAGS (Set Interface Flags)");
 984                 IGB_CORE_LOCK(adapter);
 985                 if (ifp->if_flags & IFF_UP) {
 986                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 987                                 if ((ifp->if_flags ^ adapter->if_flags) &
 988                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 989                                         igb_disable_promisc(adapter);
 990                                         igb_set_promisc(adapter);
 991                                 }
 992                         } else
 993                                 igb_init_locked(adapter);
 994                 } else
 995                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 996                                 igb_stop(adapter);
 997                 adapter->if_flags = ifp->if_flags;
 998                 IGB_CORE_UNLOCK(adapter);
 999                 break;
1000         case SIOCADDMULTI:
1001         case SIOCDELMULTI:
1002                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1003                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1004                         IGB_CORE_LOCK(adapter);
1005                         igb_disable_intr(adapter);
1006                         igb_set_multi(adapter);
1007 #ifdef DEVICE_POLLING
1008                         if (!(ifp->if_capenable & IFCAP_POLLING))
1009 #endif
1010                                 igb_enable_intr(adapter);
1011                         IGB_CORE_UNLOCK(adapter);
1012                 }
1013                 break;
1014         case SIOCSIFMEDIA:
1015                 /* Check SOL/IDER usage */
1016                 IGB_CORE_LOCK(adapter);
1017                 if (e1000_check_reset_block(&adapter->hw)) {
1018                         IGB_CORE_UNLOCK(adapter);
1019                         device_printf(adapter->dev, "Media change is"
1020                             " blocked due to SOL/IDER session.\n");
1021                         break;
1022                 }
1023                 IGB_CORE_UNLOCK(adapter);
1024         case SIOCGIFMEDIA:
1025                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1026                     SIOCxIFMEDIA (Get/Set Interface Media)");
1027                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1028                 break;
1029         case SIOCSIFCAP:
1030             {
1031                 int mask, reinit;
1032
1033                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1034                 reinit = 0;
1035                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1036 #ifdef DEVICE_POLLING
1037                 if (mask & IFCAP_POLLING) {
1038                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1039                                 error = ether_poll_register(igb_poll, ifp);
1040                                 if (error)
1041                                         return (error);
1042                                 IGB_CORE_LOCK(adapter);
1043                                 igb_disable_intr(adapter);
1044                                 ifp->if_capenable |= IFCAP_POLLING;
1045                                 IGB_CORE_UNLOCK(adapter);
1046                         } else {
1047                                 error = ether_poll_deregister(ifp);
1048                                 /* Enable interrupt even in error case */
1049                                 IGB_CORE_LOCK(adapter);
1050                                 igb_enable_intr(adapter);
1051                                 ifp->if_capenable &= ~IFCAP_POLLING;
1052                                 IGB_CORE_UNLOCK(adapter);
1053                         }
1054                 }
1055 #endif
1056                 if (mask & IFCAP_HWCSUM) {
1057                         ifp->if_capenable ^= IFCAP_HWCSUM;
1058                         reinit = 1;
1059                 }
1060                 if (mask & IFCAP_TSO4) {
1061                         ifp->if_capenable ^= IFCAP_TSO4;
1062                         reinit = 1;
1063                 }
1064                 if (mask & IFCAP_VLAN_HWTAGGING) {
1065                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1066                         reinit = 1;
1067                 }
1068                 if (mask & IFCAP_VLAN_HWFILTER) {
1069                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1070                         reinit = 1;
1071                 }
1072                 if (mask & IFCAP_LRO) {
1073                         ifp->if_capenable ^= IFCAP_LRO;
1074                         reinit = 1;
1075                 }
1076                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1077                         igb_init(adapter);
1078                 VLAN_CAPABILITIES(ifp);
1079                 break;
1080             }
1081
1082         default:
1083                 error = ether_ioctl(ifp, command, data);
1084                 break;
1085         }
1086
1087         return (error);
1088 }
1089
1090
1091 /*********************************************************************
1092  *  Init entry point
1093  *
1094  *  This routine is used in two ways. It is used by the stack as
1095  *  init entry point in network interface structure. It is also used
1096  *  by the driver as a hw/sw initialization routine to get to a
1097  *  consistent state.
1098  *
1099  *  return 0 on success, positive on failure
1100  **********************************************************************/
1101
1102 static void
1103 igb_init_locked(struct adapter *adapter)
1104 {
1105         struct ifnet    *ifp = adapter->ifp;
1106         device_t        dev = adapter->dev;
1107
1108         INIT_DEBUGOUT("igb_init: begin");
1109
1110         IGB_CORE_LOCK_ASSERT(adapter);
1111
1112         igb_disable_intr(adapter);
1113         callout_stop(&adapter->timer);
1114
1115         /* Get the latest mac address, User can use a LAA */
1116         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1117               ETHER_ADDR_LEN);
1118
1119         /* Put the address into the Receive Address Array */
1120         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1121
1122         igb_reset(adapter);
1123         igb_update_link_status(adapter);
1124
1125         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1126
1127         /* Use real VLAN Filter support? */
1128         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1129                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1130                         /* Use real VLAN Filter support */
1131                         igb_setup_vlan_hw_support(adapter);
1132                 else {
1133                         u32 ctrl;
1134                         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1135                         ctrl |= E1000_CTRL_VME;
1136                         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1137                 }
1138         }
1139
1140         /* Set hardware offload abilities */
1141         ifp->if_hwassist = 0;
1142         if (ifp->if_capenable & IFCAP_TXCSUM) {
1143                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1144 #if __FreeBSD_version >= 800000
1145                 if (adapter->hw.mac.type == e1000_82576)
1146                         ifp->if_hwassist |= CSUM_SCTP;
1147 #endif
1148         }
1149
1150         if (ifp->if_capenable & IFCAP_TSO4)
1151                 ifp->if_hwassist |= CSUM_TSO;
1152
1153         /* Configure for OS presence */
1154         igb_init_manageability(adapter);
1155
1156         /* Prepare transmit descriptors and buffers */
1157         igb_setup_transmit_structures(adapter);
1158         igb_initialize_transmit_units(adapter);
1159
1160         /* Setup Multicast table */
1161         igb_set_multi(adapter);
1162
1163         /*
1164         ** Figure out the desired mbuf pool
1165         ** for doing jumbo/packetsplit
1166         */
1167         if (ifp->if_mtu > ETHERMTU)
1168                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1169         else
1170                 adapter->rx_mbuf_sz = MCLBYTES;
1171
1172         /* Prepare receive descriptors and buffers */
1173         if (igb_setup_receive_structures(adapter)) {
1174                 device_printf(dev, "Could not setup receive structures\n");
1175                 return;
1176         }
1177         igb_initialize_receive_units(adapter);
1178
1179         /* Don't lose promiscuous settings */
1180         igb_set_promisc(adapter);
1181
1182         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1183         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1184
1185         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1186         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1187
1188         if (adapter->msix > 1) /* Set up queue routing */
1189                 igb_configure_queues(adapter);
1190
1191         /* Set up VLAN tag offload and filter */
1192         igb_setup_vlan_hw_support(adapter);
1193
1194         /* this clears any pending interrupts */
1195         E1000_READ_REG(&adapter->hw, E1000_ICR);
1196 #ifdef DEVICE_POLLING
1197         /*
1198          * Only enable interrupts if we are not polling, make sure
1199          * they are off otherwise.
1200          */
1201         if (ifp->if_capenable & IFCAP_POLLING)
1202                 igb_disable_intr(adapter);
1203         else
1204 #endif /* DEVICE_POLLING */
1205         {
1206         igb_enable_intr(adapter);
1207         E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1208         }
1209
1210         /* Don't reset the phy next time init gets called */
1211         adapter->hw.phy.reset_disable = TRUE;
1212 }
1213
1214 static void
1215 igb_init(void *arg)
1216 {
1217         struct adapter *adapter = arg;
1218
1219         IGB_CORE_LOCK(adapter);
1220         igb_init_locked(adapter);
1221         IGB_CORE_UNLOCK(adapter);
1222 }
1223
1224
1225 static void
1226 igb_handle_que(void *context, int pending)
1227 {
1228         struct igb_queue *que = context;
1229         struct adapter *adapter = que->adapter;
1230         struct tx_ring *txr = que->txr;
1231         struct ifnet    *ifp = adapter->ifp;
1232
1233         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1234                 bool    more;
1235
1236                 more = igb_rxeof(que, -1, NULL);
1237
1238                 IGB_TX_LOCK(txr);
1239                 if (igb_txeof(txr))
1240                         more = TRUE;
1241 #if __FreeBSD_version >= 800000
1242                 if (!drbr_empty(ifp, txr->br))
1243                         igb_mq_start_locked(ifp, txr, NULL);
1244 #else
1245                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1246                         igb_start_locked(txr, ifp);
1247 #endif
1248                 IGB_TX_UNLOCK(txr);
1249                 if (more) {
1250                         taskqueue_enqueue(que->tq, &que->que_task);
1251                         return;
1252                 }
1253         }
1254
1255 #ifdef DEVICE_POLLING
1256         if (ifp->if_capenable & IFCAP_POLLING)
1257                 return;
1258 #endif
1259         /* Reenable this interrupt */
1260         if (que->eims)
1261                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1262         else
1263                 igb_enable_intr(adapter);
1264 }
1265
1266 /* Deal with link in a sleepable context */
1267 static void
1268 igb_handle_link(void *context, int pending)
1269 {
1270         struct adapter *adapter = context;
1271
1272         adapter->hw.mac.get_link_status = 1;
1273         igb_update_link_status(adapter);
1274 }
1275
1276 /*********************************************************************
1277  *
1278  *  MSI/Legacy Deferred
1279  *  Interrupt Service routine
1280  *
1281  *********************************************************************/
1282 static int
1283 igb_irq_fast(void *arg)
1284 {
1285         struct adapter          *adapter = arg;
1286         struct igb_queue        *que = adapter->queues;
1287         u32                     reg_icr;
1288
1289
1290         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1291
1292         /* Hot eject?  */
1293         if (reg_icr == 0xffffffff)
1294                 return FILTER_STRAY;
1295
1296         /* Definitely not our interrupt.  */
1297         if (reg_icr == 0x0)
1298                 return FILTER_STRAY;
1299
1300         if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1301                 return FILTER_STRAY;
1302
1303         /*
1304          * Mask interrupts until the taskqueue is finished running.  This is
1305          * cheap, just assume that it is needed.  This also works around the
1306          * MSI message reordering errata on certain systems.
1307          */
1308         igb_disable_intr(adapter);
1309         taskqueue_enqueue(que->tq, &que->que_task);
1310
1311         /* Link status change */
1312         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1313                 taskqueue_enqueue(que->tq, &adapter->link_task);
1314
1315         if (reg_icr & E1000_ICR_RXO)
1316                 adapter->rx_overruns++;
1317         return FILTER_HANDLED;
1318 }
1319
1320 #ifdef DEVICE_POLLING
1321 /*********************************************************************
1322  *
1323  *  Legacy polling routine : if using this code you MUST be sure that
1324  *  multiqueue is not defined, ie, set igb_num_queues to 1.
1325  *
1326  *********************************************************************/
1327 #if __FreeBSD_version >= 800000
1328 #define POLL_RETURN_COUNT(a) (a)
1329 static int
1330 #else
1331 #define POLL_RETURN_COUNT(a)
1332 static void
1333 #endif
1334 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1335 {
1336         struct adapter          *adapter = ifp->if_softc;
1337         struct igb_queue        *que = adapter->queues;
1338         struct tx_ring          *txr = adapter->tx_rings;
1339         u32                     reg_icr, rx_done = 0;
1340         u32                     loop = IGB_MAX_LOOP;
1341         bool                    more;
1342
1343         IGB_CORE_LOCK(adapter);
1344         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1345                 IGB_CORE_UNLOCK(adapter);
1346                 return POLL_RETURN_COUNT(rx_done);
1347         }
1348
1349         if (cmd == POLL_AND_CHECK_STATUS) {
1350                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1351                 /* Link status change */
1352                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1353                         igb_handle_link(adapter, 0);
1354
1355                 if (reg_icr & E1000_ICR_RXO)
1356                         adapter->rx_overruns++;
1357         }
1358         IGB_CORE_UNLOCK(adapter);
1359
1360         igb_rxeof(que, count, &rx_done);
1361
1362         IGB_TX_LOCK(txr);
1363         do {
1364                 more = igb_txeof(txr);
1365         } while (loop-- && more);
1366 #if __FreeBSD_version >= 800000
1367         if (!drbr_empty(ifp, txr->br))
1368                 igb_mq_start_locked(ifp, txr, NULL);
1369 #else
1370         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1371                 igb_start_locked(txr, ifp);
1372 #endif
1373         IGB_TX_UNLOCK(txr);
1374         return POLL_RETURN_COUNT(rx_done);
1375 }
1376 #endif /* DEVICE_POLLING */
1377
1378 /*********************************************************************
1379  *
1380  *  MSIX TX Interrupt Service routine
1381  *
1382  **********************************************************************/
1383 static void
1384 igb_msix_que(void *arg)
1385 {
1386         struct igb_queue *que = arg;
1387         struct adapter *adapter = que->adapter;
1388         struct tx_ring *txr = que->txr;
1389         struct rx_ring *rxr = que->rxr;
1390         u32             newitr = 0;
1391         bool            more_tx, more_rx;
1392
1393         E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1394         ++que->irqs;
1395
1396         IGB_TX_LOCK(txr);
1397         more_tx = igb_txeof(txr);
1398         IGB_TX_UNLOCK(txr);
1399
1400         more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
1401
1402         if (igb_enable_aim == FALSE)
1403                 goto no_calc;
1404         /*
1405         ** Do Adaptive Interrupt Moderation:
1406         **  - Write out last calculated setting
1407         **  - Calculate based on average size over
1408         **    the last interval.
1409         */
1410         if (que->eitr_setting)
1411                 E1000_WRITE_REG(&adapter->hw,
1412                     E1000_EITR(que->msix), que->eitr_setting);
1413
1414         que->eitr_setting = 0;
1415
1416         /* Idle, do nothing */
1417         if ((txr->bytes == 0) && (rxr->bytes == 0))
1418                 goto no_calc;
1419
1420         /* Used half Default if sub-gig */
1421         if (adapter->link_speed != 1000)
1422                 newitr = IGB_DEFAULT_ITR / 2;
1423         else {
1424                 if ((txr->bytes) && (txr->packets))
1425                         newitr = txr->bytes/txr->packets;
1426                 if ((rxr->bytes) && (rxr->packets))
1427                         newitr = max(newitr,
1428                             (rxr->bytes / rxr->packets));
1429                 newitr += 24; /* account for hardware frame, crc */
1430                 /* set an upper boundary */
1431                 newitr = min(newitr, 3000);
1432                 /* Be nice to the mid range */
1433                 if ((newitr > 300) && (newitr < 1200))
1434                         newitr = (newitr / 3);
1435                 else
1436                         newitr = (newitr / 2);
1437         }
1438         newitr &= 0x7FFC;  /* Mask invalid bits */
1439         if (adapter->hw.mac.type == e1000_82575)
1440                 newitr |= newitr << 16;
1441         else
1442                 newitr |= E1000_EITR_CNT_IGNR;
1443
1444         /* save for next interrupt */
1445         que->eitr_setting = newitr;
1446
1447         /* Reset state */
1448         txr->bytes = 0;
1449         txr->packets = 0;
1450         rxr->bytes = 0;
1451         rxr->packets = 0;
1452
1453 no_calc:
1454         /* Schedule a clean task if needed*/
1455         if (more_tx || more_rx)
1456                 taskqueue_enqueue(que->tq, &que->que_task);
1457         else
1458                 /* Reenable this interrupt */
1459                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1460         return;
1461 }
1462
1463
1464 /*********************************************************************
1465  *
1466  *  MSIX Link Interrupt Service routine
1467  *
1468  **********************************************************************/
1469
1470 static void
1471 igb_msix_link(void *arg)
1472 {
1473         struct adapter  *adapter = arg;
1474         u32             icr;
1475
1476         ++adapter->link_irq;
1477         icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1478         if (!(icr & E1000_ICR_LSC))
1479                 goto spurious;
1480         igb_handle_link(adapter, 0);
1481
1482 spurious:
1483         /* Rearm */
1484         E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1485         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1486         return;
1487 }
1488
1489
1490 /*********************************************************************
1491  *
1492  *  Media Ioctl callback
1493  *
1494  *  This routine is called whenever the user queries the status of
1495  *  the interface using ifconfig.
1496  *
1497  **********************************************************************/
1498 static void
1499 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1500 {
1501         struct adapter *adapter = ifp->if_softc;
1502         u_char fiber_type = IFM_1000_SX;
1503
1504         INIT_DEBUGOUT("igb_media_status: begin");
1505
1506         IGB_CORE_LOCK(adapter);
1507         igb_update_link_status(adapter);
1508
1509         ifmr->ifm_status = IFM_AVALID;
1510         ifmr->ifm_active = IFM_ETHER;
1511
1512         if (!adapter->link_active) {
1513                 IGB_CORE_UNLOCK(adapter);
1514                 return;
1515         }
1516
1517         ifmr->ifm_status |= IFM_ACTIVE;
1518
1519         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1520             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes))
1521                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1522         else {
1523                 switch (adapter->link_speed) {
1524                 case 10:
1525                         ifmr->ifm_active |= IFM_10_T;
1526                         break;
1527                 case 100:
1528                         ifmr->ifm_active |= IFM_100_TX;
1529                         break;
1530                 case 1000:
1531                         ifmr->ifm_active |= IFM_1000_T;
1532                         break;
1533                 }
1534                 if (adapter->link_duplex == FULL_DUPLEX)
1535                         ifmr->ifm_active |= IFM_FDX;
1536                 else
1537                         ifmr->ifm_active |= IFM_HDX;
1538         }
1539         IGB_CORE_UNLOCK(adapter);
1540 }
1541
1542 /*********************************************************************
1543  *
1544  *  Media Ioctl callback
1545  *
1546  *  This routine is called when the user changes speed/duplex using
1547  *  media/mediopt option with ifconfig.
1548  *
1549  **********************************************************************/
1550 static int
1551 igb_media_change(struct ifnet *ifp)
1552 {
1553         struct adapter *adapter = ifp->if_softc;
1554         struct ifmedia  *ifm = &adapter->media;
1555
1556         INIT_DEBUGOUT("igb_media_change: begin");
1557
1558         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1559                 return (EINVAL);
1560
1561         IGB_CORE_LOCK(adapter);
1562         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1563         case IFM_AUTO:
1564                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1565                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1566                 break;
1567         case IFM_1000_LX:
1568         case IFM_1000_SX:
1569         case IFM_1000_T:
1570                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1571                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1572                 break;
1573         case IFM_100_TX:
1574                 adapter->hw.mac.autoneg = FALSE;
1575                 adapter->hw.phy.autoneg_advertised = 0;
1576                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1577                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1578                 else
1579                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1580                 break;
1581         case IFM_10_T:
1582                 adapter->hw.mac.autoneg = FALSE;
1583                 adapter->hw.phy.autoneg_advertised = 0;
1584                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1585                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1586                 else
1587                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1588                 break;
1589         default:
1590                 device_printf(adapter->dev, "Unsupported media type\n");
1591         }
1592
1593         /* As the speed/duplex settings my have changed we need to
1594          * reset the PHY.
1595          */
1596         adapter->hw.phy.reset_disable = FALSE;
1597
1598         igb_init_locked(adapter);
1599         IGB_CORE_UNLOCK(adapter);
1600
1601         return (0);
1602 }
1603
1604
1605 /*********************************************************************
1606  *
1607  *  This routine maps the mbufs to Advanced TX descriptors.
1608  *  used by the 82575 adapter.
1609  *
1610  **********************************************************************/
1611
1612 static int
1613 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1614 {
1615         struct adapter          *adapter = txr->adapter;
1616         bus_dma_segment_t       segs[IGB_MAX_SCATTER];
1617         bus_dmamap_t            map;
1618         struct igb_tx_buffer    *tx_buffer, *tx_buffer_mapped;
1619         union e1000_adv_tx_desc *txd = NULL;
1620         struct mbuf             *m_head;
1621         u32                     olinfo_status = 0, cmd_type_len = 0;
1622         int                     nsegs, i, j, error, first, last = 0;
1623         u32                     hdrlen = 0;
1624
1625         m_head = *m_headp;
1626
1627
1628         /* Set basic descriptor constants */
1629         cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
1630         cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
1631         if (m_head->m_flags & M_VLANTAG)
1632                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1633
1634         /*
1635          * Force a cleanup if number of TX descriptors
1636          * available hits the threshold
1637          */
1638         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) {
1639                 igb_txeof(txr);
1640                 /* Now do we at least have a minimal? */
1641                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
1642                         txr->no_desc_avail++;
1643                         return (ENOBUFS);
1644                 }
1645         }
1646
1647         /*
1648          * Map the packet for DMA.
1649          *
1650          * Capture the first descriptor index,
1651          * this descriptor will have the index
1652          * of the EOP which is the only one that
1653          * now gets a DONE bit writeback.
1654          */
1655         first = txr->next_avail_desc;
1656         tx_buffer = &txr->tx_buffers[first];
1657         tx_buffer_mapped = tx_buffer;
1658         map = tx_buffer->map;
1659
1660         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1661             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1662
1663         if (error == EFBIG) {
1664                 struct mbuf *m;
1665
1666                 m = m_defrag(*m_headp, M_DONTWAIT);
1667                 if (m == NULL) {
1668                         adapter->mbuf_defrag_failed++;
1669                         m_freem(*m_headp);
1670                         *m_headp = NULL;
1671                         return (ENOBUFS);
1672                 }
1673                 *m_headp = m;
1674
1675                 /* Try it again */
1676                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1677                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1678
1679                 if (error == ENOMEM) {
1680                         adapter->no_tx_dma_setup++;
1681                         return (error);
1682                 } else if (error != 0) {
1683                         adapter->no_tx_dma_setup++;
1684                         m_freem(*m_headp);
1685                         *m_headp = NULL;
1686                         return (error);
1687                 }
1688         } else if (error == ENOMEM) {
1689                 adapter->no_tx_dma_setup++;
1690                 return (error);
1691         } else if (error != 0) {
1692                 adapter->no_tx_dma_setup++;
1693                 m_freem(*m_headp);
1694                 *m_headp = NULL;
1695                 return (error);
1696         }
1697
1698         /* Check again to be sure we have enough descriptors */
1699         if (nsegs > (txr->tx_avail - 2)) {
1700                 txr->no_desc_avail++;
1701                 bus_dmamap_unload(txr->txtag, map);
1702                 return (ENOBUFS);
1703         }
1704         m_head = *m_headp;
1705
1706         /*
1707          * Set up the context descriptor:
1708          * used when any hardware offload is done.
1709          * This includes CSUM, VLAN, and TSO. It
1710          * will use the first descriptor.
1711          */
1712         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1713                 if (igb_tso_setup(txr, m_head, &hdrlen)) {
1714                         cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1715                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
1716                         olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1717                 } else
1718                         return (ENXIO);
1719         } else if (igb_tx_ctx_setup(txr, m_head))
1720                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1721
1722         /* Calculate payload length */
1723         olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
1724             << E1000_ADVTXD_PAYLEN_SHIFT);
1725
1726         /* 82575 needs the queue index added */
1727         if (adapter->hw.mac.type == e1000_82575)
1728                 olinfo_status |= txr->me << 4;
1729
1730         /* Set up our transmit descriptors */
1731         i = txr->next_avail_desc;
1732         for (j = 0; j < nsegs; j++) {
1733                 bus_size_t seg_len;
1734                 bus_addr_t seg_addr;
1735
1736                 tx_buffer = &txr->tx_buffers[i];
1737                 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
1738                 seg_addr = segs[j].ds_addr;
1739                 seg_len  = segs[j].ds_len;
1740
1741                 txd->read.buffer_addr = htole64(seg_addr);
1742                 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len);
1743                 txd->read.olinfo_status = htole32(olinfo_status);
1744                 last = i;
1745                 if (++i == adapter->num_tx_desc)
1746                         i = 0;
1747                 tx_buffer->m_head = NULL;
1748                 tx_buffer->next_eop = -1;
1749         }
1750
1751         txr->next_avail_desc = i;
1752         txr->tx_avail -= nsegs;
1753
1754         tx_buffer->m_head = m_head;
1755         tx_buffer_mapped->map = tx_buffer->map;
1756         tx_buffer->map = map;
1757         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1758
1759         /*
1760          * Last Descriptor of Packet
1761          * needs End Of Packet (EOP)
1762          * and Report Status (RS)
1763          */
1764         txd->read.cmd_type_len |=
1765             htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS);
1766         /*
1767          * Keep track in the first buffer which
1768          * descriptor will be written back
1769          */
1770         tx_buffer = &txr->tx_buffers[first];
1771         tx_buffer->next_eop = last;
1772         txr->watchdog_time = ticks;
1773
1774         /*
1775          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
1776          * that this frame is available to transmit.
1777          */
1778         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1779             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1780         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1781         ++txr->tx_packets;
1782
1783         return (0);
1784
1785 }
1786
1787 static void
1788 igb_set_promisc(struct adapter *adapter)
1789 {
1790         struct ifnet    *ifp = adapter->ifp;
1791         uint32_t        reg_rctl;
1792
1793         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1794
1795         if (ifp->if_flags & IFF_PROMISC) {
1796                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1797                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1798         } else if (ifp->if_flags & IFF_ALLMULTI) {
1799                 reg_rctl |= E1000_RCTL_MPE;
1800                 reg_rctl &= ~E1000_RCTL_UPE;
1801                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1802         }
1803 }
1804
1805 static void
1806 igb_disable_promisc(struct adapter *adapter)
1807 {
1808         uint32_t        reg_rctl;
1809
1810         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1811
1812         reg_rctl &=  (~E1000_RCTL_UPE);
1813         reg_rctl &=  (~E1000_RCTL_MPE);
1814         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1815 }
1816
1817
1818 /*********************************************************************
1819  *  Multicast Update
1820  *
1821  *  This routine is called whenever multicast address list is updated.
1822  *
1823  **********************************************************************/
1824
1825 static void
1826 igb_set_multi(struct adapter *adapter)
1827 {
1828         struct ifnet    *ifp = adapter->ifp;
1829         struct ifmultiaddr *ifma;
1830         u32 reg_rctl = 0;
1831         u8  mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN];
1832
1833         int mcnt = 0;
1834
1835         IOCTL_DEBUGOUT("igb_set_multi: begin");
1836
1837 #if __FreeBSD_version < 800000
1838         IF_ADDR_LOCK(ifp);
1839 #else
1840         if_maddr_rlock(ifp);
1841 #endif
1842         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1843                 if (ifma->ifma_addr->sa_family != AF_LINK)
1844                         continue;
1845
1846                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1847                         break;
1848
1849                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1850                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
1851                 mcnt++;
1852         }
1853 #if __FreeBSD_version < 800000
1854         IF_ADDR_UNLOCK(ifp);
1855 #else
1856         if_maddr_runlock(ifp);
1857 #endif
1858
1859         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1860                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1861                 reg_rctl |= E1000_RCTL_MPE;
1862                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1863         } else
1864                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
1865 }
1866
1867
1868 /*********************************************************************
1869  *  Timer routine:
1870  *      This routine checks for link status,
1871  *      updates statistics, and does the watchdog.
1872  *
1873  **********************************************************************/
1874
1875 static void
1876 igb_local_timer(void *arg)
1877 {
1878         struct adapter          *adapter = arg;
1879         device_t                dev = adapter->dev;
1880         struct tx_ring          *txr = adapter->tx_rings;
1881
1882
1883         IGB_CORE_LOCK_ASSERT(adapter);
1884
1885         igb_update_link_status(adapter);
1886         igb_update_stats_counters(adapter);
1887
1888         /*
1889         ** Watchdog: check for time since any descriptor was cleaned
1890         */
1891         for (int i = 0; i < adapter->num_queues; i++, txr++) {
1892                 if (txr->watchdog_check == FALSE)
1893                         continue;
1894                 if ((ticks - txr->watchdog_time) > IGB_WATCHDOG)
1895                         goto timeout;
1896         }
1897
1898         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1899         return;
1900
1901 timeout:
1902         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1903         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1904             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
1905             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
1906         device_printf(dev,"TX(%d) desc avail = %d,"
1907             "Next TX to Clean = %d\n",
1908             txr->me, txr->tx_avail, txr->next_to_clean);
1909         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1910         adapter->watchdog_events++;
1911         igb_init_locked(adapter);
1912 }
1913
1914 static void
1915 igb_update_link_status(struct adapter *adapter)
1916 {
1917         struct e1000_hw *hw = &adapter->hw;
1918         struct ifnet *ifp = adapter->ifp;
1919         device_t dev = adapter->dev;
1920         struct tx_ring *txr = adapter->tx_rings;
1921         u32 link_check = 0;
1922
1923         /* Get the cached link value or read for real */
1924         switch (hw->phy.media_type) {
1925         case e1000_media_type_copper:
1926                 if (hw->mac.get_link_status) {
1927                         /* Do the work to read phy */
1928                         e1000_check_for_link(hw);
1929                         link_check = !hw->mac.get_link_status;
1930                 } else
1931                         link_check = TRUE;
1932                 break;
1933         case e1000_media_type_fiber:
1934                 e1000_check_for_link(hw);
1935                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1936                                  E1000_STATUS_LU);
1937                 break;
1938         case e1000_media_type_internal_serdes:
1939                 e1000_check_for_link(hw);
1940                 link_check = adapter->hw.mac.serdes_has_link;
1941                 break;
1942         default:
1943         case e1000_media_type_unknown:
1944                 break;
1945         }
1946
1947         /* Now we check if a transition has happened */
1948         if (link_check && (adapter->link_active == 0)) {
1949                 e1000_get_speed_and_duplex(&adapter->hw,
1950                     &adapter->link_speed, &adapter->link_duplex);
1951                 if (bootverbose)
1952                         device_printf(dev, "Link is up %d Mbps %s\n",
1953                             adapter->link_speed,
1954                             ((adapter->link_duplex == FULL_DUPLEX) ?
1955                             "Full Duplex" : "Half Duplex"));
1956                 adapter->link_active = 1;
1957                 ifp->if_baudrate = adapter->link_speed * 1000000;
1958                 /* This can sleep */
1959                 if_link_state_change(ifp, LINK_STATE_UP);
1960         } else if (!link_check && (adapter->link_active == 1)) {
1961                 ifp->if_baudrate = adapter->link_speed = 0;
1962                 adapter->link_duplex = 0;
1963                 if (bootverbose)
1964                         device_printf(dev, "Link is Down\n");
1965                 adapter->link_active = 0;
1966                 /* This can sleep */
1967                 if_link_state_change(ifp, LINK_STATE_DOWN);
1968                 /* Turn off watchdogs */
1969                 for (int i = 0; i < adapter->num_queues; i++, txr++)
1970                         txr->watchdog_check = FALSE;
1971         }
1972 }
1973
1974 /*********************************************************************
1975  *
1976  *  This routine disables all traffic on the adapter by issuing a
1977  *  global reset on the MAC and deallocates TX/RX buffers.
1978  *
1979  **********************************************************************/
1980
1981 static void
1982 igb_stop(void *arg)
1983 {
1984         struct adapter  *adapter = arg;
1985         struct ifnet    *ifp = adapter->ifp;
1986         struct tx_ring *txr = adapter->tx_rings;
1987
1988         IGB_CORE_LOCK_ASSERT(adapter);
1989
1990         INIT_DEBUGOUT("igb_stop: begin");
1991
1992         igb_disable_intr(adapter);
1993
1994         callout_stop(&adapter->timer);
1995
1996         /* Tell the stack that the interface is no longer active */
1997         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1998
1999         /* Unarm watchdog timer. */
2000         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2001                 IGB_TX_LOCK(txr);
2002                 txr->watchdog_check = FALSE;
2003                 IGB_TX_UNLOCK(txr);
2004         }
2005
2006         e1000_reset_hw(&adapter->hw);
2007         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2008
2009         e1000_led_off(&adapter->hw);
2010         e1000_cleanup_led(&adapter->hw);
2011 }
2012
2013
2014 /*********************************************************************
2015  *
2016  *  Determine hardware revision.
2017  *
2018  **********************************************************************/
2019 static void
2020 igb_identify_hardware(struct adapter *adapter)
2021 {
2022         device_t dev = adapter->dev;
2023
2024         /* Make sure our PCI config space has the necessary stuff set */
2025         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2026         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2027             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2028                 device_printf(dev, "Memory Access and/or Bus Master bits "
2029                     "were not set!\n");
2030                 adapter->hw.bus.pci_cmd_word |=
2031                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2032                 pci_write_config(dev, PCIR_COMMAND,
2033                     adapter->hw.bus.pci_cmd_word, 2);
2034         }
2035
2036         /* Save off the information about this board */
2037         adapter->hw.vendor_id = pci_get_vendor(dev);
2038         adapter->hw.device_id = pci_get_device(dev);
2039         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2040         adapter->hw.subsystem_vendor_id =
2041             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2042         adapter->hw.subsystem_device_id =
2043             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2044
2045         /* Do Shared Code Init and Setup */
2046         if (e1000_set_mac_type(&adapter->hw)) {
2047                 device_printf(dev, "Setup init failure\n");
2048                 return;
2049         }
2050 }
2051
2052 static int
2053 igb_allocate_pci_resources(struct adapter *adapter)
2054 {
2055         device_t        dev = adapter->dev;
2056         int             rid;
2057
2058         rid = PCIR_BAR(0);
2059         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2060             &rid, RF_ACTIVE);
2061         if (adapter->pci_mem == NULL) {
2062                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2063                 return (ENXIO);
2064         }
2065         adapter->osdep.mem_bus_space_tag =
2066             rman_get_bustag(adapter->pci_mem);
2067         adapter->osdep.mem_bus_space_handle =
2068             rman_get_bushandle(adapter->pci_mem);
2069         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2070
2071         adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2072
2073         /* This will setup either MSI/X or MSI */
2074         adapter->msix = igb_setup_msix(adapter);
2075         adapter->hw.back = &adapter->osdep;
2076
2077         return (0);
2078 }
2079
2080 /*********************************************************************
2081  *
2082  *  Setup the Legacy or MSI Interrupt handler
2083  *
2084  **********************************************************************/
2085 static int
2086 igb_allocate_legacy(struct adapter *adapter)
2087 {
2088         device_t                dev = adapter->dev;
2089         struct igb_queue        *que = adapter->queues;
2090         int                     error, rid = 0;
2091
2092         /* Turn off all interrupts */
2093         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2094
2095         /* MSI RID is 1 */
2096         if (adapter->msix == 1)
2097                 rid = 1;
2098
2099         /* We allocate a single interrupt resource */
2100         adapter->res = bus_alloc_resource_any(dev,
2101             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2102         if (adapter->res == NULL) {
2103                 device_printf(dev, "Unable to allocate bus resource: "
2104                     "interrupt\n");
2105                 return (ENXIO);
2106         }
2107
2108         /*
2109          * Try allocating a fast interrupt and the associated deferred
2110          * processing contexts.
2111          */
2112         TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2113         /* Make tasklet for deferred link handling */
2114         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2115         que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2116             taskqueue_thread_enqueue, &que->tq);
2117         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
2118             device_get_nameunit(adapter->dev));
2119         if ((error = bus_setup_intr(dev, adapter->res,
2120             INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2121             adapter, &adapter->tag)) != 0) {
2122                 device_printf(dev, "Failed to register fast interrupt "
2123                             "handler: %d\n", error);
2124                 taskqueue_free(que->tq);
2125                 que->tq = NULL;
2126                 return (error);
2127         }
2128
2129         return (0);
2130 }
2131
2132
2133 /*********************************************************************
2134  *
2135  *  Setup the MSIX Queue Interrupt handlers:
2136  *
2137  **********************************************************************/
2138 static int
2139 igb_allocate_msix(struct adapter *adapter)
2140 {
2141         device_t                dev = adapter->dev;
2142         struct igb_queue        *que = adapter->queues;
2143         int                     error, rid, vector = 0;
2144
2145
2146         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2147                 rid = vector +1;
2148                 que->res = bus_alloc_resource_any(dev,
2149                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2150                 if (que->res == NULL) {
2151                         device_printf(dev,
2152                             "Unable to allocate bus resource: "
2153                             "MSIX Queue Interrupt\n");
2154                         return (ENXIO);
2155                 }
2156                 error = bus_setup_intr(dev, que->res,
2157                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2158                     igb_msix_que, que, &que->tag);
2159                 if (error) {
2160                         que->res = NULL;
2161                         device_printf(dev, "Failed to register Queue handler");
2162                         return (error);
2163                 }
2164 #if __FreeBSD_version >= 800504
2165                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2166 #endif
2167                 que->msix = vector;
2168                 if (adapter->hw.mac.type == e1000_82575)
2169                         que->eims = E1000_EICR_TX_QUEUE0 << i;
2170                 else
2171                         que->eims = 1 << vector;
2172                 /*
2173                 ** Bind the msix vector, and thus the
2174                 ** rings to the corresponding cpu.
2175                 */
2176                 if (adapter->num_queues > 1)
2177                         bus_bind_intr(dev, que->res, i);
2178                 /* Make tasklet for deferred handling */
2179                 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2180                 que->tq = taskqueue_create_fast("igb_que", M_NOWAIT,
2181                     taskqueue_thread_enqueue, &que->tq);
2182                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2183                     device_get_nameunit(adapter->dev));
2184         }
2185
2186         /* And Link */
2187         rid = vector + 1;
2188         adapter->res = bus_alloc_resource_any(dev,
2189             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2190         if (adapter->res == NULL) {
2191                 device_printf(dev,
2192                     "Unable to allocate bus resource: "
2193                     "MSIX Link Interrupt\n");
2194                 return (ENXIO);
2195         }
2196         if ((error = bus_setup_intr(dev, adapter->res,
2197             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2198             igb_msix_link, adapter, &adapter->tag)) != 0) {
2199                 device_printf(dev, "Failed to register Link handler");
2200                 return (error);
2201         }
2202 #if __FreeBSD_version >= 800504
2203         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2204 #endif
2205         adapter->linkvec = vector;
2206
2207         return (0);
2208 }
2209
2210
2211 static void
2212 igb_configure_queues(struct adapter *adapter)
2213 {
2214         struct  e1000_hw        *hw = &adapter->hw;
2215         struct  igb_queue       *que;
2216         u32                     tmp, ivar = 0;
2217         u32                     newitr = IGB_DEFAULT_ITR;
2218
2219         /* First turn on RSS capability */
2220         if (adapter->hw.mac.type > e1000_82575)
2221                 E1000_WRITE_REG(hw, E1000_GPIE,
2222                     E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2223                     E1000_GPIE_PBA | E1000_GPIE_NSICR);
2224
2225         /* Turn on MSIX */
2226         switch (adapter->hw.mac.type) {
2227         case e1000_82580:
2228                 /* RX entries */
2229                 for (int i = 0; i < adapter->num_queues; i++) {
2230                         u32 index = i >> 1;
2231                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2232                         que = &adapter->queues[i];
2233                         if (i & 1) {
2234                                 ivar &= 0xFF00FFFF;
2235                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2236                         } else {
2237                                 ivar &= 0xFFFFFF00;
2238                                 ivar |= que->msix | E1000_IVAR_VALID;
2239                         }
2240                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2241                 }
2242                 /* TX entries */
2243                 for (int i = 0; i < adapter->num_queues; i++) {
2244                         u32 index = i >> 1;
2245                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2246                         que = &adapter->queues[i];
2247                         if (i & 1) {
2248                                 ivar &= 0x00FFFFFF;
2249                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2250                         } else {
2251                                 ivar &= 0xFFFF00FF;
2252                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2253                         }
2254                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2255                         adapter->eims_mask |= que->eims;
2256                 }
2257
2258                 /* And for the link interrupt */
2259                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2260                 adapter->link_mask = 1 << adapter->linkvec;
2261                 adapter->eims_mask |= adapter->link_mask;
2262                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2263                 break;
2264         case e1000_82576:
2265                 /* RX entries */
2266                 for (int i = 0; i < adapter->num_queues; i++) {
2267                         u32 index = i & 0x7; /* Each IVAR has two entries */
2268                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2269                         que = &adapter->queues[i];
2270                         if (i < 8) {
2271                                 ivar &= 0xFFFFFF00;
2272                                 ivar |= que->msix | E1000_IVAR_VALID;
2273                         } else {
2274                                 ivar &= 0xFF00FFFF;
2275                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2276                         }
2277                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2278                         adapter->eims_mask |= que->eims;
2279                 }
2280                 /* TX entries */
2281                 for (int i = 0; i < adapter->num_queues; i++) {
2282                         u32 index = i & 0x7; /* Each IVAR has two entries */
2283                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2284                         que = &adapter->queues[i];
2285                         if (i < 8) {
2286                                 ivar &= 0xFFFF00FF;
2287                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2288                         } else {
2289                                 ivar &= 0x00FFFFFF;
2290                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2291                         }
2292                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2293                         adapter->eims_mask |= que->eims;
2294                 }
2295
2296                 /* And for the link interrupt */
2297                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2298                 adapter->link_mask = 1 << adapter->linkvec;
2299                 adapter->eims_mask |= adapter->link_mask;
2300                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2301                 break;
2302
2303         case e1000_82575:
2304                 /* enable MSI-X support*/
2305                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2306                 tmp |= E1000_CTRL_EXT_PBA_CLR;
2307                 /* Auto-Mask interrupts upon ICR read. */
2308                 tmp |= E1000_CTRL_EXT_EIAME;
2309                 tmp |= E1000_CTRL_EXT_IRCA;
2310                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2311
2312                 /* Queues */
2313                 for (int i = 0; i < adapter->num_queues; i++) {
2314                         que = &adapter->queues[i];
2315                         tmp = E1000_EICR_RX_QUEUE0 << i;
2316                         tmp |= E1000_EICR_TX_QUEUE0 << i;
2317                         que->eims = tmp;
2318                         E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2319                             i, que->eims);
2320                         adapter->eims_mask |= que->eims;
2321                 }
2322
2323                 /* Link */
2324                 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2325                     E1000_EIMS_OTHER);
2326                 adapter->link_mask |= E1000_EIMS_OTHER;
2327                 adapter->eims_mask |= adapter->link_mask;
2328         default:
2329                 break;
2330         }
2331
2332         /* Set the starting interrupt rate */
2333         if (hw->mac.type == e1000_82575)
2334                 newitr |= newitr << 16;
2335         else
2336                 newitr |= E1000_EITR_CNT_IGNR;
2337
2338         for (int i = 0; i < adapter->num_queues; i++) {
2339                 que = &adapter->queues[i];
2340                 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2341         }
2342
2343         return;
2344 }
2345
2346
2347 static void
2348 igb_free_pci_resources(struct adapter *adapter)
2349 {
2350         struct          igb_queue *que = adapter->queues;
2351         device_t        dev = adapter->dev;
2352         int             rid;
2353
2354         /*
2355         ** There is a slight possibility of a failure mode
2356         ** in attach that will result in entering this function
2357         ** before interrupt resources have been initialized, and
2358         ** in that case we do not want to execute the loops below
2359         ** We can detect this reliably by the state of the adapter
2360         ** res pointer.
2361         */
2362         if (adapter->res == NULL)
2363                 goto mem;
2364
2365         /*
2366          * First release all the interrupt resources:
2367          */
2368         for (int i = 0; i < adapter->num_queues; i++, que++) {
2369                 rid = que->msix + 1;
2370                 if (que->tag != NULL) {
2371                         bus_teardown_intr(dev, que->res, que->tag);
2372                         que->tag = NULL;
2373                 }
2374                 if (que->res != NULL)
2375                         bus_release_resource(dev,
2376                             SYS_RES_IRQ, rid, que->res);
2377         }
2378
2379         /* Clean the Legacy or Link interrupt last */
2380         if (adapter->linkvec) /* we are doing MSIX */
2381                 rid = adapter->linkvec + 1;
2382         else
2383                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2384
2385         if (adapter->tag != NULL) {
2386                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2387                 adapter->tag = NULL;
2388         }
2389         if (adapter->res != NULL)
2390                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2391
2392 mem:
2393         if (adapter->msix)
2394                 pci_release_msi(dev);
2395
2396         if (adapter->msix_mem != NULL)
2397                 bus_release_resource(dev, SYS_RES_MEMORY,
2398                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2399
2400         if (adapter->pci_mem != NULL)
2401                 bus_release_resource(dev, SYS_RES_MEMORY,
2402                     PCIR_BAR(0), adapter->pci_mem);
2403
2404 }
2405
2406 /*
2407  * Setup Either MSI/X or MSI
2408  */
2409 static int
2410 igb_setup_msix(struct adapter *adapter)
2411 {
2412         device_t dev = adapter->dev;
2413         int rid, want, queues, msgs;
2414
2415         /* tuneable override */
2416         if (igb_enable_msix == 0)
2417                 goto msi;
2418
2419         /* First try MSI/X */
2420         rid = PCIR_BAR(IGB_MSIX_BAR);
2421         adapter->msix_mem = bus_alloc_resource_any(dev,
2422             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2423         if (!adapter->msix_mem) {
2424                 /* May not be enabled */
2425                 device_printf(adapter->dev,
2426                     "Unable to map MSIX table \n");
2427                 goto msi;
2428         }
2429
2430         msgs = pci_msix_count(dev);
2431         if (msgs == 0) { /* system has msix disabled */
2432                 bus_release_resource(dev, SYS_RES_MEMORY,
2433                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2434                 adapter->msix_mem = NULL;
2435                 goto msi;
2436         }
2437
2438         /* Figure out a reasonable auto config value */
2439         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2440
2441         /* Manual override */
2442         if (igb_num_queues != 0)
2443                 queues = igb_num_queues;
2444
2445         /* Can have max of 4 queues on 82575 */
2446         if ((adapter->hw.mac.type == e1000_82575) && (queues > 4))
2447                 queues = 4;
2448
2449         /*
2450         ** One vector (RX/TX pair) per queue
2451         ** plus an additional for Link interrupt
2452         */
2453         want = queues + 1;
2454         if (msgs >= want)
2455                 msgs = want;
2456         else {
2457                 device_printf(adapter->dev,
2458                     "MSIX Configuration Problem, "
2459                     "%d vectors configured, but %d queues wanted!\n",
2460                     msgs, want);
2461                 return (ENXIO);
2462         }
2463         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2464                 device_printf(adapter->dev,
2465                     "Using MSIX interrupts with %d vectors\n", msgs);
2466                 adapter->num_queues = queues;
2467                 return (msgs);
2468         }
2469 msi:
2470         msgs = pci_msi_count(dev);
2471         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2472                 device_printf(adapter->dev,"Using MSI interrupt\n");
2473         return (msgs);
2474 }
2475
2476 /*********************************************************************
2477  *
2478  *  Set up an fresh starting state
2479  *
2480  **********************************************************************/
2481 static void
2482 igb_reset(struct adapter *adapter)
2483 {
2484         device_t        dev = adapter->dev;
2485         struct e1000_hw *hw = &adapter->hw;
2486         struct e1000_fc_info *fc = &hw->fc;
2487         struct ifnet    *ifp = adapter->ifp;
2488         u32             pba = 0;
2489         u16             hwm;
2490
2491         INIT_DEBUGOUT("igb_reset: begin");
2492
2493         /* Let the firmware know the OS is in control */
2494         igb_get_hw_control(adapter);
2495
2496         /*
2497          * Packet Buffer Allocation (PBA)
2498          * Writing PBA sets the receive portion of the buffer
2499          * the remainder is used for the transmit buffer.
2500          */
2501         switch (hw->mac.type) {
2502         case e1000_82575:
2503                 pba = E1000_PBA_32K;
2504                 break;
2505         case e1000_82576:
2506                 pba = E1000_PBA_64K;
2507                 break;
2508         case e1000_82580:
2509                 pba = E1000_PBA_35K;
2510         default:
2511                 break;
2512         }
2513
2514         /* Special needs in case of Jumbo frames */
2515         if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
2516                 u32 tx_space, min_tx, min_rx;
2517                 pba = E1000_READ_REG(hw, E1000_PBA);
2518                 tx_space = pba >> 16;
2519                 pba &= 0xffff;
2520                 min_tx = (adapter->max_frame_size +
2521                     sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
2522                 min_tx = roundup2(min_tx, 1024);
2523                 min_tx >>= 10;
2524                 min_rx = adapter->max_frame_size;
2525                 min_rx = roundup2(min_rx, 1024);
2526                 min_rx >>= 10;
2527                 if (tx_space < min_tx &&
2528                     ((min_tx - tx_space) < pba)) {
2529                         pba = pba - (min_tx - tx_space);
2530                         /*
2531                          * if short on rx space, rx wins
2532                          * and must trump tx adjustment
2533                          */
2534                         if (pba < min_rx)
2535                                 pba = min_rx;
2536                 }
2537                 E1000_WRITE_REG(hw, E1000_PBA, pba);
2538         }
2539
2540         INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
2541
2542         /*
2543          * These parameters control the automatic generation (Tx) and
2544          * response (Rx) to Ethernet PAUSE frames.
2545          * - High water mark should allow for at least two frames to be
2546          *   received after sending an XOFF.
2547          * - Low water mark works best when it is very near the high water mark.
2548          *   This allows the receiver to restart by sending XON when it has
2549          *   drained a bit.
2550          */
2551         hwm = min(((pba << 10) * 9 / 10),
2552             ((pba << 10) - 2 * adapter->max_frame_size));
2553
2554         if (hw->mac.type < e1000_82576) {
2555                 fc->high_water = hwm & 0xFFF8;  /* 8-byte granularity */
2556                 fc->low_water = fc->high_water - 8;
2557         } else {
2558                 fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
2559                 fc->low_water = fc->high_water - 16;
2560         }
2561
2562         fc->pause_time = IGB_FC_PAUSE_TIME;
2563         fc->send_xon = TRUE;
2564
2565         /* Set Flow control, use the tunable location if sane */
2566         if ((igb_fc_setting >= 0) || (igb_fc_setting < 4))
2567                 fc->requested_mode = igb_fc_setting;
2568         else
2569                 fc->requested_mode = e1000_fc_none;
2570
2571         fc->current_mode = fc->requested_mode;
2572
2573         /* Issue a global reset */
2574         e1000_reset_hw(hw);
2575         E1000_WRITE_REG(hw, E1000_WUC, 0);
2576
2577         if (e1000_init_hw(hw) < 0)
2578                 device_printf(dev, "Hardware Initialization Failed\n");
2579
2580         if (hw->mac.type == e1000_82580) {
2581                 u32 reg;
2582
2583                 hwm = (pba << 10) - (2 * adapter->max_frame_size);
2584                 /*
2585                  * 0x80000000 - enable DMA COAL
2586                  * 0x10000000 - use L0s as low power
2587                  * 0x20000000 - use L1 as low power
2588                  * X << 16 - exit dma coal when rx data exceeds X kB
2589                  * Y - upper limit to stay in dma coal in units of 32usecs
2590                  */
2591                 E1000_WRITE_REG(hw, E1000_DMACR,
2592                     0xA0000006 | ((hwm << 6) & 0x00FF0000));
2593
2594                 /* set hwm to PBA -  2 * max frame size */
2595                 E1000_WRITE_REG(hw, E1000_FCRTC, hwm);
2596                 /*
2597                  * This sets the time to wait before requesting transition to
2598                  * low power state to number of usecs needed to receive 1 512
2599                  * byte frame at gigabit line rate
2600                  */
2601                 E1000_WRITE_REG(hw, E1000_DMCTLX, 4);
2602
2603                 /* free space in tx packet buffer to wake from DMA coal */
2604                 E1000_WRITE_REG(hw, E1000_DMCTXTH,
2605                     (20480 - (2 * adapter->max_frame_size)) >> 6);
2606
2607                 /* make low power state decision controlled by DMA coal */
2608                 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2609                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
2610                     reg | E1000_PCIEMISC_LX_DECISION);
2611         }
2612
2613         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
2614         e1000_get_phy_info(hw);
2615         e1000_check_for_link(hw);
2616         return;
2617 }
2618
2619 /*********************************************************************
2620  *
2621  *  Setup networking device structure and register an interface.
2622  *
2623  **********************************************************************/
2624 static void
2625 igb_setup_interface(device_t dev, struct adapter *adapter)
2626 {
2627         struct ifnet   *ifp;
2628
2629         INIT_DEBUGOUT("igb_setup_interface: begin");
2630
2631         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2632         if (ifp == NULL)
2633                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
2634         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2635         ifp->if_mtu = ETHERMTU;
2636         ifp->if_init =  igb_init;
2637         ifp->if_softc = adapter;
2638         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2639         ifp->if_ioctl = igb_ioctl;
2640         ifp->if_start = igb_start;
2641 #if __FreeBSD_version >= 800000
2642         ifp->if_transmit = igb_mq_start;
2643         ifp->if_qflush = igb_qflush;
2644 #endif
2645         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2646         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2647         IFQ_SET_READY(&ifp->if_snd);
2648
2649         ether_ifattach(ifp, adapter->hw.mac.addr);
2650
2651         ifp->if_capabilities = ifp->if_capenable = 0;
2652
2653         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU;
2654         ifp->if_capabilities |= IFCAP_TSO4;
2655         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2656         if (igb_header_split)
2657                 ifp->if_capabilities |= IFCAP_LRO;
2658
2659         ifp->if_capenable = ifp->if_capabilities;
2660 #ifdef DEVICE_POLLING
2661         ifp->if_capabilities |= IFCAP_POLLING;
2662 #endif
2663
2664         /*
2665          * Tell the upper layer(s) we
2666          * support full VLAN capability.
2667          */
2668         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2669         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2670         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2671
2672         /*
2673         ** Dont turn this on by default, if vlans are
2674         ** created on another pseudo device (eg. lagg)
2675         ** then vlan events are not passed thru, breaking
2676         ** operation, but with HW FILTER off it works. If
2677         ** using vlans directly on the em driver you can
2678         ** enable this and get full hardware tag filtering.
2679         */
2680         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2681
2682         /*
2683          * Specify the media types supported by this adapter and register
2684          * callbacks to update media and link information
2685          */
2686         ifmedia_init(&adapter->media, IFM_IMASK,
2687             igb_media_change, igb_media_status);
2688         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2689             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2690                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
2691                             0, NULL);
2692                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2693         } else {
2694                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2695                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2696                             0, NULL);
2697                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2698                             0, NULL);
2699                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2700                             0, NULL);
2701                 if (adapter->hw.phy.type != e1000_phy_ife) {
2702                         ifmedia_add(&adapter->media,
2703                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2704                         ifmedia_add(&adapter->media,
2705                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2706                 }
2707         }
2708         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2709         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2710 }
2711
2712
2713 /*
2714  * Manage DMA'able memory.
2715  */
2716 static void
2717 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2718 {
2719         if (error)
2720                 return;
2721         *(bus_addr_t *) arg = segs[0].ds_addr;
2722 }
2723
2724 static int
2725 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
2726         struct igb_dma_alloc *dma, int mapflags)
2727 {
2728         int error;
2729
2730         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2731                                 IGB_DBA_ALIGN, 0,       /* alignment, bounds */
2732                                 BUS_SPACE_MAXADDR,      /* lowaddr */
2733                                 BUS_SPACE_MAXADDR,      /* highaddr */
2734                                 NULL, NULL,             /* filter, filterarg */
2735                                 size,                   /* maxsize */
2736                                 1,                      /* nsegments */
2737                                 size,                   /* maxsegsize */
2738                                 0,                      /* flags */
2739                                 NULL,                   /* lockfunc */
2740                                 NULL,                   /* lockarg */
2741                                 &dma->dma_tag);
2742         if (error) {
2743                 device_printf(adapter->dev,
2744                     "%s: bus_dma_tag_create failed: %d\n",
2745                     __func__, error);
2746                 goto fail_0;
2747         }
2748
2749         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2750             BUS_DMA_NOWAIT, &dma->dma_map);
2751         if (error) {
2752                 device_printf(adapter->dev,
2753                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2754                     __func__, (uintmax_t)size, error);
2755                 goto fail_2;
2756         }
2757
2758         dma->dma_paddr = 0;
2759         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2760             size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2761         if (error || dma->dma_paddr == 0) {
2762                 device_printf(adapter->dev,
2763                     "%s: bus_dmamap_load failed: %d\n",
2764                     __func__, error);
2765                 goto fail_3;
2766         }
2767
2768         return (0);
2769
2770 fail_3:
2771         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2772 fail_2:
2773         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2774         bus_dma_tag_destroy(dma->dma_tag);
2775 fail_0:
2776         dma->dma_map = NULL;
2777         dma->dma_tag = NULL;
2778
2779         return (error);
2780 }
2781
2782 static void
2783 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
2784 {
2785         if (dma->dma_tag == NULL)
2786                 return;
2787         if (dma->dma_map != NULL) {
2788                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2789                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2790                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2791                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2792                 dma->dma_map = NULL;
2793         }
2794         bus_dma_tag_destroy(dma->dma_tag);
2795         dma->dma_tag = NULL;
2796 }
2797
2798
2799 /*********************************************************************
2800  *
2801  *  Allocate memory for the transmit and receive rings, and then
2802  *  the descriptors associated with each, called only once at attach.
2803  *
2804  **********************************************************************/
2805 static int
2806 igb_allocate_queues(struct adapter *adapter)
2807 {
2808         device_t dev = adapter->dev;
2809         struct igb_queue        *que = NULL;
2810         struct tx_ring          *txr = NULL;
2811         struct rx_ring          *rxr = NULL;
2812         int rsize, tsize, error = E1000_SUCCESS;
2813         int txconf = 0, rxconf = 0;
2814
2815         /* First allocate the top level queue structs */
2816         if (!(adapter->queues =
2817             (struct igb_queue *) malloc(sizeof(struct igb_queue) *
2818             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2819                 device_printf(dev, "Unable to allocate queue memory\n");
2820                 error = ENOMEM;
2821                 goto fail;
2822         }
2823
2824         /* Next allocate the TX ring struct memory */
2825         if (!(adapter->tx_rings =
2826             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2827             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2828                 device_printf(dev, "Unable to allocate TX ring memory\n");
2829                 error = ENOMEM;
2830                 goto tx_fail;
2831         }
2832
2833         /* Now allocate the RX */
2834         if (!(adapter->rx_rings =
2835             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2836             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2837                 device_printf(dev, "Unable to allocate RX ring memory\n");
2838                 error = ENOMEM;
2839                 goto rx_fail;
2840         }
2841
2842         tsize = roundup2(adapter->num_tx_desc *
2843             sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
2844         /*
2845          * Now set up the TX queues, txconf is needed to handle the
2846          * possibility that things fail midcourse and we need to
2847          * undo memory gracefully
2848          */
2849         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2850                 /* Set up some basics */
2851                 txr = &adapter->tx_rings[i];
2852                 txr->adapter = adapter;
2853                 txr->me = i;
2854
2855                 /* Initialize the TX lock */
2856                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2857                     device_get_nameunit(dev), txr->me);
2858                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2859
2860                 if (igb_dma_malloc(adapter, tsize,
2861                         &txr->txdma, BUS_DMA_NOWAIT)) {
2862                         device_printf(dev,
2863                             "Unable to allocate TX Descriptor memory\n");
2864                         error = ENOMEM;
2865                         goto err_tx_desc;
2866                 }
2867                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
2868                 bzero((void *)txr->tx_base, tsize);
2869
2870                 /* Now allocate transmit buffers for the ring */
2871                 if (igb_allocate_transmit_buffers(txr)) {
2872                         device_printf(dev,
2873                             "Critical Failure setting up transmit buffers\n");
2874                         error = ENOMEM;
2875                         goto err_tx_desc;
2876                 }
2877 #if __FreeBSD_version >= 800000
2878                 /* Allocate a buf ring */
2879                 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF,
2880                     M_WAITOK, &txr->tx_mtx);
2881 #endif
2882         }
2883
2884         /*
2885          * Next the RX queues...
2886          */
2887         rsize = roundup2(adapter->num_rx_desc *
2888             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
2889         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2890                 rxr = &adapter->rx_rings[i];
2891                 rxr->adapter = adapter;
2892                 rxr->me = i;
2893
2894                 /* Initialize the RX lock */
2895                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2896                     device_get_nameunit(dev), txr->me);
2897                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2898
2899                 if (igb_dma_malloc(adapter, rsize,
2900                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2901                         device_printf(dev,
2902                             "Unable to allocate RxDescriptor memory\n");
2903                         error = ENOMEM;
2904                         goto err_rx_desc;
2905                 }
2906                 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2907                 bzero((void *)rxr->rx_base, rsize);
2908
2909                 /* Allocate receive buffers for the ring*/
2910                 if (igb_allocate_receive_buffers(rxr)) {
2911                         device_printf(dev,
2912                             "Critical Failure setting up receive buffers\n");
2913                         error = ENOMEM;
2914                         goto err_rx_desc;
2915                 }
2916         }
2917
2918         /*
2919         ** Finally set up the queue holding structs
2920         */
2921         for (int i = 0; i < adapter->num_queues; i++) {
2922                 que = &adapter->queues[i];
2923                 que->adapter = adapter;
2924                 que->txr = &adapter->tx_rings[i];
2925                 que->rxr = &adapter->rx_rings[i];
2926         }
2927
2928         return (0);
2929
2930 err_rx_desc:
2931         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2932                 igb_dma_free(adapter, &rxr->rxdma);
2933 err_tx_desc:
2934         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2935                 igb_dma_free(adapter, &txr->txdma);
2936         free(adapter->rx_rings, M_DEVBUF);
2937 rx_fail:
2938 #if __FreeBSD_version >= 800000
2939         buf_ring_free(txr->br, M_DEVBUF);
2940 #endif
2941         free(adapter->tx_rings, M_DEVBUF);
2942 tx_fail:
2943         free(adapter->queues, M_DEVBUF);
2944 fail:
2945         return (error);
2946 }
2947
2948 /*********************************************************************
2949  *
2950  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2951  *  the information needed to transmit a packet on the wire. This is
2952  *  called only once at attach, setup is done every reset.
2953  *
2954  **********************************************************************/
2955 static int
2956 igb_allocate_transmit_buffers(struct tx_ring *txr)
2957 {
2958         struct adapter *adapter = txr->adapter;
2959         device_t dev = adapter->dev;
2960         struct igb_tx_buffer *txbuf;
2961         int error, i;
2962
2963         /*
2964          * Setup DMA descriptor areas.
2965          */
2966         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
2967                                1, 0,                    /* alignment, bounds */
2968                                BUS_SPACE_MAXADDR,       /* lowaddr */
2969                                BUS_SPACE_MAXADDR,       /* highaddr */
2970                                NULL, NULL,              /* filter, filterarg */
2971                                IGB_TSO_SIZE,            /* maxsize */
2972                                IGB_MAX_SCATTER,         /* nsegments */
2973                                PAGE_SIZE,               /* maxsegsize */
2974                                0,                       /* flags */
2975                                NULL,                    /* lockfunc */
2976                                NULL,                    /* lockfuncarg */
2977                                &txr->txtag))) {
2978                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2979                 goto fail;
2980         }
2981
2982         if (!(txr->tx_buffers =
2983             (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) *
2984             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2985                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2986                 error = ENOMEM;
2987                 goto fail;
2988         }
2989
2990         /* Create the descriptor buffer dma maps */
2991         txbuf = txr->tx_buffers;
2992         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2993                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2994                 if (error != 0) {
2995                         device_printf(dev, "Unable to create TX DMA map\n");
2996                         goto fail;
2997                 }
2998         }
2999
3000         return 0;
3001 fail:
3002         /* We free all, it handles case where we are in the middle */
3003         igb_free_transmit_structures(adapter);
3004         return (error);
3005 }
3006
3007 /*********************************************************************
3008  *
3009  *  Initialize a transmit ring.
3010  *
3011  **********************************************************************/
3012 static void
3013 igb_setup_transmit_ring(struct tx_ring *txr)
3014 {
3015         struct adapter *adapter = txr->adapter;
3016         struct igb_tx_buffer *txbuf;
3017         int i;
3018
3019         /* Clear the old descriptor contents */
3020         IGB_TX_LOCK(txr);
3021         bzero((void *)txr->tx_base,
3022               (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3023         /* Reset indices */
3024         txr->next_avail_desc = 0;
3025         txr->next_to_clean = 0;
3026
3027         /* Free any existing tx buffers. */
3028         txbuf = txr->tx_buffers;
3029         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3030                 if (txbuf->m_head != NULL) {
3031                         bus_dmamap_sync(txr->txtag, txbuf->map,
3032                             BUS_DMASYNC_POSTWRITE);
3033                         bus_dmamap_unload(txr->txtag, txbuf->map);
3034                         m_freem(txbuf->m_head);
3035                         txbuf->m_head = NULL;
3036                 }
3037                 /* clear the watch index */
3038                 txbuf->next_eop = -1;
3039         }
3040
3041         /* Set number of descriptors available */
3042         txr->tx_avail = adapter->num_tx_desc;
3043
3044         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3045             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3046         IGB_TX_UNLOCK(txr);
3047 }
3048
3049 /*********************************************************************
3050  *
3051  *  Initialize all transmit rings.
3052  *
3053  **********************************************************************/
3054 static void
3055 igb_setup_transmit_structures(struct adapter *adapter)
3056 {
3057         struct tx_ring *txr = adapter->tx_rings;
3058
3059         for (int i = 0; i < adapter->num_queues; i++, txr++)
3060                 igb_setup_transmit_ring(txr);
3061
3062         return;
3063 }
3064
3065 /*********************************************************************
3066  *
3067  *  Enable transmit unit.
3068  *
3069  **********************************************************************/
3070 static void
3071 igb_initialize_transmit_units(struct adapter *adapter)
3072 {
3073         struct tx_ring  *txr = adapter->tx_rings;
3074         struct e1000_hw *hw = &adapter->hw;
3075         u32             tctl, txdctl;
3076
3077          INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3078
3079         /* Setup the Tx Descriptor Rings */
3080         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3081                 u64 bus_addr = txr->txdma.dma_paddr;
3082
3083                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3084                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3085                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3086                     (uint32_t)(bus_addr >> 32));
3087                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3088                     (uint32_t)bus_addr);
3089
3090                 /* Setup the HW Tx Head and Tail descriptor pointers */
3091                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3092                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3093
3094                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3095                     E1000_READ_REG(hw, E1000_TDBAL(i)),
3096                     E1000_READ_REG(hw, E1000_TDLEN(i)));
3097
3098                 txr->watchdog_check = FALSE;
3099
3100                 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i));
3101                 txdctl |= IGB_TX_PTHRESH;
3102                 txdctl |= IGB_TX_HTHRESH << 8;
3103                 txdctl |= IGB_TX_WTHRESH << 16;
3104                 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3105                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3106         }
3107
3108         /* Program the Transmit Control Register */
3109         tctl = E1000_READ_REG(hw, E1000_TCTL);
3110         tctl &= ~E1000_TCTL_CT;
3111         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3112                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3113
3114         e1000_config_collision_dist(hw);
3115
3116         /* This write will effectively turn on the transmit unit. */
3117         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3118 }
3119
3120 /*********************************************************************
3121  *
3122  *  Free all transmit rings.
3123  *
3124  **********************************************************************/
3125 static void
3126 igb_free_transmit_structures(struct adapter *adapter)
3127 {
3128         struct tx_ring *txr = adapter->tx_rings;
3129
3130         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3131                 IGB_TX_LOCK(txr);
3132                 igb_free_transmit_buffers(txr);
3133                 igb_dma_free(adapter, &txr->txdma);
3134                 IGB_TX_UNLOCK(txr);
3135                 IGB_TX_LOCK_DESTROY(txr);
3136         }
3137         free(adapter->tx_rings, M_DEVBUF);
3138 }
3139
3140 /*********************************************************************
3141  *
3142  *  Free transmit ring related data structures.
3143  *
3144  **********************************************************************/
3145 static void
3146 igb_free_transmit_buffers(struct tx_ring *txr)
3147 {
3148         struct adapter *adapter = txr->adapter;
3149         struct igb_tx_buffer *tx_buffer;
3150         int             i;
3151
3152         INIT_DEBUGOUT("free_transmit_ring: begin");
3153
3154         if (txr->tx_buffers == NULL)
3155                 return;
3156
3157         tx_buffer = txr->tx_buffers;
3158         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3159                 if (tx_buffer->m_head != NULL) {
3160                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3161                             BUS_DMASYNC_POSTWRITE);
3162                         bus_dmamap_unload(txr->txtag,
3163                             tx_buffer->map);
3164                         m_freem(tx_buffer->m_head);
3165                         tx_buffer->m_head = NULL;
3166                         if (tx_buffer->map != NULL) {
3167                                 bus_dmamap_destroy(txr->txtag,
3168                                     tx_buffer->map);
3169                                 tx_buffer->map = NULL;
3170                         }
3171                 } else if (tx_buffer->map != NULL) {
3172                         bus_dmamap_unload(txr->txtag,
3173                             tx_buffer->map);
3174                         bus_dmamap_destroy(txr->txtag,
3175                             tx_buffer->map);
3176                         tx_buffer->map = NULL;
3177                 }
3178         }
3179 #if __FreeBSD_version >= 800000
3180         if (txr->br != NULL)
3181                 buf_ring_free(txr->br, M_DEVBUF);
3182 #endif
3183         if (txr->tx_buffers != NULL) {
3184                 free(txr->tx_buffers, M_DEVBUF);
3185                 txr->tx_buffers = NULL;
3186         }
3187         if (txr->txtag != NULL) {
3188                 bus_dma_tag_destroy(txr->txtag);
3189                 txr->txtag = NULL;
3190         }
3191         return;
3192 }
3193
3194 /**********************************************************************
3195  *
3196  *  Setup work for hardware segmentation offload (TSO)
3197  *
3198  **********************************************************************/
3199 static boolean_t
3200 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
3201 {
3202         struct adapter *adapter = txr->adapter;
3203         struct e1000_adv_tx_context_desc *TXD;
3204         struct igb_tx_buffer        *tx_buffer;
3205         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3206         u32 mss_l4len_idx = 0;
3207         u16 vtag = 0;
3208         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3209         struct ether_vlan_header *eh;
3210         struct ip *ip;
3211         struct tcphdr *th;
3212
3213
3214         /*
3215          * Determine where frame payload starts.
3216          * Jump over vlan headers if already present
3217          */
3218         eh = mtod(mp, struct ether_vlan_header *);
3219         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3220                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3221         else
3222                 ehdrlen = ETHER_HDR_LEN;
3223
3224         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3225         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3226                 return FALSE;
3227
3228         /* Only supports IPV4 for now */
3229         ctxd = txr->next_avail_desc;
3230         tx_buffer = &txr->tx_buffers[ctxd];
3231         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3232
3233         ip = (struct ip *)(mp->m_data + ehdrlen);
3234         if (ip->ip_p != IPPROTO_TCP)
3235                 return FALSE;   /* 0 */
3236         ip->ip_sum = 0;
3237         ip_hlen = ip->ip_hl << 2;
3238         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3239         th->th_sum = in_pseudo(ip->ip_src.s_addr,
3240             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3241         tcp_hlen = th->th_off << 2;
3242         /*
3243          * Calculate header length, this is used
3244          * in the transmit desc in igb_xmit
3245          */
3246         *hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3247
3248         /* VLAN MACLEN IPLEN */
3249         if (mp->m_flags & M_VLANTAG) {
3250                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3251                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3252         }
3253
3254         vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT);
3255         vlan_macip_lens |= ip_hlen;
3256         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3257
3258         /* ADV DTYPE TUCMD */
3259         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3260         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3261         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3262         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3263
3264         /* MSS L4LEN IDX */
3265         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3266         mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3267         /* 82575 needs the queue index added */
3268         if (adapter->hw.mac.type == e1000_82575)
3269                 mss_l4len_idx |= txr->me << 4;
3270         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3271
3272         TXD->seqnum_seed = htole32(0);
3273         tx_buffer->m_head = NULL;
3274         tx_buffer->next_eop = -1;
3275
3276         if (++ctxd == adapter->num_tx_desc)
3277                 ctxd = 0;
3278
3279         txr->tx_avail--;
3280         txr->next_avail_desc = ctxd;
3281         return TRUE;
3282 }
3283
3284
3285 /*********************************************************************
3286  *
3287  *  Context Descriptor setup for VLAN or CSUM
3288  *
3289  **********************************************************************/
3290
3291 static bool
3292 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3293 {
3294         struct adapter *adapter = txr->adapter;
3295         struct e1000_adv_tx_context_desc *TXD;
3296         struct igb_tx_buffer        *tx_buffer;
3297         u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx;
3298         struct ether_vlan_header *eh;
3299         struct ip *ip = NULL;
3300         struct ip6_hdr *ip6;
3301         int  ehdrlen, ctxd, ip_hlen = 0;
3302         u16     etype, vtag = 0;
3303         u8      ipproto = 0;
3304         bool    offload = TRUE;
3305
3306         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3307                 offload = FALSE;
3308
3309         vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0;
3310         ctxd = txr->next_avail_desc;
3311         tx_buffer = &txr->tx_buffers[ctxd];
3312         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3313
3314         /*
3315         ** In advanced descriptors the vlan tag must
3316         ** be placed into the context descriptor, thus
3317         ** we need to be here just for that setup.
3318         */
3319         if (mp->m_flags & M_VLANTAG) {
3320                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3321                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3322         } else if (offload == FALSE)
3323                 return FALSE;
3324
3325         /*
3326          * Determine where frame payload starts.
3327          * Jump over vlan headers if already present,
3328          * helpful for QinQ too.
3329          */
3330         eh = mtod(mp, struct ether_vlan_header *);
3331         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3332                 etype = ntohs(eh->evl_proto);
3333                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3334         } else {
3335                 etype = ntohs(eh->evl_encap_proto);
3336                 ehdrlen = ETHER_HDR_LEN;
3337         }
3338
3339         /* Set the ether header length */
3340         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3341
3342         switch (etype) {
3343                 case ETHERTYPE_IP:
3344                         ip = (struct ip *)(mp->m_data + ehdrlen);
3345                         ip_hlen = ip->ip_hl << 2;
3346                         if (mp->m_len < ehdrlen + ip_hlen) {
3347                                 offload = FALSE;
3348                                 break;
3349                         }
3350                         ipproto = ip->ip_p;
3351                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3352                         break;
3353                 case ETHERTYPE_IPV6:
3354                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3355                         ip_hlen = sizeof(struct ip6_hdr);
3356                         if (mp->m_len < ehdrlen + ip_hlen)
3357                                 return (FALSE);
3358                         ipproto = ip6->ip6_nxt;
3359                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3360                         break;
3361                 default:
3362                         offload = FALSE;
3363                         break;
3364         }
3365
3366         vlan_macip_lens |= ip_hlen;
3367         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3368
3369         switch (ipproto) {
3370                 case IPPROTO_TCP:
3371                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3372                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3373                         break;
3374                 case IPPROTO_UDP:
3375                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3376                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3377                         break;
3378 #if __FreeBSD_version >= 800000
3379                 case IPPROTO_SCTP:
3380                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3381                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3382                         break;
3383 #endif
3384                 default:
3385                         offload = FALSE;
3386                         break;
3387         }
3388
3389         /* 82575 needs the queue index added */
3390         if (adapter->hw.mac.type == e1000_82575)
3391                 mss_l4len_idx = txr->me << 4;
3392
3393         /* Now copy bits into descriptor */
3394         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3395         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3396         TXD->seqnum_seed = htole32(0);
3397         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3398
3399         tx_buffer->m_head = NULL;
3400         tx_buffer->next_eop = -1;
3401
3402         /* We've consumed the first desc, adjust counters */
3403         if (++ctxd == adapter->num_tx_desc)
3404                 ctxd = 0;
3405         txr->next_avail_desc = ctxd;
3406         --txr->tx_avail;
3407
3408         return (offload);
3409 }
3410
3411
3412 /**********************************************************************
3413  *
3414  *  Examine each tx_buffer in the used queue. If the hardware is done
3415  *  processing the packet then free associated resources. The
3416  *  tx_buffer is put back on the free queue.
3417  *
3418  *  TRUE return means there's work in the ring to clean, FALSE its empty.
3419  **********************************************************************/
3420 static bool
3421 igb_txeof(struct tx_ring *txr)
3422 {
3423         struct adapter  *adapter = txr->adapter;
3424         int first, last, done;
3425         struct igb_tx_buffer *tx_buffer;
3426         struct e1000_tx_desc   *tx_desc, *eop_desc;
3427         struct ifnet   *ifp = adapter->ifp;
3428
3429         IGB_TX_LOCK_ASSERT(txr);
3430
3431         if (txr->tx_avail == adapter->num_tx_desc)
3432                 return FALSE;
3433
3434         first = txr->next_to_clean;
3435         tx_desc = &txr->tx_base[first];
3436         tx_buffer = &txr->tx_buffers[first];
3437         last = tx_buffer->next_eop;
3438         eop_desc = &txr->tx_base[last];
3439
3440         /*
3441          * What this does is get the index of the
3442          * first descriptor AFTER the EOP of the
3443          * first packet, that way we can do the
3444          * simple comparison on the inner while loop.
3445          */
3446         if (++last == adapter->num_tx_desc)
3447                 last = 0;
3448         done = last;
3449
3450         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3451             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3452
3453         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3454                 /* We clean the range of the packet */
3455                 while (first != done) {
3456                         tx_desc->upper.data = 0;
3457                         tx_desc->lower.data = 0;
3458                         tx_desc->buffer_addr = 0;
3459                         ++txr->tx_avail;
3460
3461                         if (tx_buffer->m_head) {
3462                                 txr->bytes +=
3463                                     tx_buffer->m_head->m_pkthdr.len;
3464                                 bus_dmamap_sync(txr->txtag,
3465                                     tx_buffer->map,
3466                                     BUS_DMASYNC_POSTWRITE);
3467                                 bus_dmamap_unload(txr->txtag,
3468                                     tx_buffer->map);
3469
3470                                 m_freem(tx_buffer->m_head);
3471                                 tx_buffer->m_head = NULL;
3472                         }
3473                         tx_buffer->next_eop = -1;
3474                         txr->watchdog_time = ticks;
3475
3476                         if (++first == adapter->num_tx_desc)
3477                                 first = 0;
3478
3479                         tx_buffer = &txr->tx_buffers[first];
3480                         tx_desc = &txr->tx_base[first];
3481                 }
3482                 ++txr->packets;
3483                 ++ifp->if_opackets;
3484                 /* See if we can continue to the next packet */
3485                 last = tx_buffer->next_eop;
3486                 if (last != -1) {
3487                         eop_desc = &txr->tx_base[last];
3488                         /* Get new done point */
3489                         if (++last == adapter->num_tx_desc) last = 0;
3490                         done = last;
3491                 } else
3492                         break;
3493         }
3494         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3495             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3496
3497         txr->next_to_clean = first;
3498
3499         /*
3500          * If we have enough room, clear IFF_DRV_OACTIVE
3501          * to tell the stack that it is OK to send packets.
3502          */
3503         if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) {
3504                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3505                 /* All clean, turn off the watchdog */
3506                 if (txr->tx_avail == adapter->num_tx_desc) {
3507                         txr->watchdog_check = FALSE;
3508                         return FALSE;
3509                 }
3510         }
3511
3512         return (TRUE);
3513 }
3514
3515
3516 /*********************************************************************
3517  *
3518  *  Refresh mbuf buffers for RX descriptor rings
3519  *   - now keeps its own state so discards due to resource
3520  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3521  *     it just returns, keeping its placeholder, thus it can simply
3522  *     be recalled to try again.
3523  *
3524  **********************************************************************/
3525 static void
3526 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
3527 {
3528         struct adapter          *adapter = rxr->adapter;
3529         bus_dma_segment_t       hseg[1];
3530         bus_dma_segment_t       pseg[1];
3531         struct igb_rx_buf       *rxbuf;
3532         struct mbuf             *mh, *mp;
3533         int                     i, nsegs, error, cleaned;
3534
3535         i = rxr->next_to_refresh;
3536         cleaned = -1; /* Signify no completions */
3537         while (i != limit) {
3538                 rxbuf = &rxr->rx_buffers[i];
3539                 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) {
3540                         mh = m_gethdr(M_DONTWAIT, MT_DATA);
3541                         if (mh == NULL)
3542                                 goto update;
3543                         mh->m_pkthdr.len = mh->m_len = MHLEN;
3544                         mh->m_len = MHLEN;
3545                         mh->m_flags |= M_PKTHDR;
3546                         m_adj(mh, ETHER_ALIGN);
3547                         /* Get the memory mapping */
3548                         error = bus_dmamap_load_mbuf_sg(rxr->htag,
3549                             rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
3550                         if (error != 0) {
3551                                 printf("GET BUF: dmamap load"
3552                                     " failure - %d\n", error);
3553                                 m_free(mh);
3554                                 goto update;
3555                         }
3556                         rxbuf->m_head = mh;
3557                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3558                             BUS_DMASYNC_PREREAD);
3559                         rxr->rx_base[i].read.hdr_addr =
3560                             htole64(hseg[0].ds_addr);
3561                 }
3562
3563                 if (rxbuf->m_pack == NULL) {
3564                         mp = m_getjcl(M_DONTWAIT, MT_DATA,
3565                             M_PKTHDR, adapter->rx_mbuf_sz);
3566                         if (mp == NULL)
3567                                 goto update;
3568                         mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3569                         /* Get the memory mapping */
3570                         error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3571                             rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
3572                         if (error != 0) {
3573                                 printf("GET BUF: dmamap load"
3574                                     " failure - %d\n", error);
3575                                 m_free(mp);
3576                                 goto update;
3577                         }
3578                         rxbuf->m_pack = mp;
3579                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3580                             BUS_DMASYNC_PREREAD);
3581                         rxr->rx_base[i].read.pkt_addr =
3582                             htole64(pseg[0].ds_addr);
3583                 }
3584
3585                 cleaned = i;
3586                 /* Calculate next index */
3587                 if (++i == adapter->num_rx_desc)
3588                         i = 0;
3589                 /* This is the work marker for refresh */
3590                 rxr->next_to_refresh = i;
3591         }
3592 update:
3593         if (cleaned != -1) /* If we refreshed some, bump tail */
3594                 E1000_WRITE_REG(&adapter->hw,
3595                     E1000_RDT(rxr->me), cleaned);
3596         return;
3597 }
3598
3599
3600 /*********************************************************************
3601  *
3602  *  Allocate memory for rx_buffer structures. Since we use one
3603  *  rx_buffer per received packet, the maximum number of rx_buffer's
3604  *  that we'll need is equal to the number of receive descriptors
3605  *  that we've allocated.
3606  *
3607  **********************************************************************/
3608 static int
3609 igb_allocate_receive_buffers(struct rx_ring *rxr)
3610 {
3611         struct  adapter         *adapter = rxr->adapter;
3612         device_t                dev = adapter->dev;
3613         struct igb_rx_buf       *rxbuf;
3614         int                     i, bsize, error;
3615
3616         bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
3617         if (!(rxr->rx_buffers =
3618             (struct igb_rx_buf *) malloc(bsize,
3619             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3620                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3621                 error = ENOMEM;
3622                 goto fail;
3623         }
3624
3625         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3626                                    1, 0,                /* alignment, bounds */
3627                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3628                                    BUS_SPACE_MAXADDR,   /* highaddr */
3629                                    NULL, NULL,          /* filter, filterarg */
3630                                    MSIZE,               /* maxsize */
3631                                    1,                   /* nsegments */
3632                                    MSIZE,               /* maxsegsize */
3633                                    0,                   /* flags */
3634                                    NULL,                /* lockfunc */
3635                                    NULL,                /* lockfuncarg */
3636                                    &rxr->htag))) {
3637                 device_printf(dev, "Unable to create RX DMA tag\n");
3638                 goto fail;
3639         }
3640
3641         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3642                                    1, 0,                /* alignment, bounds */
3643                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3644                                    BUS_SPACE_MAXADDR,   /* highaddr */
3645                                    NULL, NULL,          /* filter, filterarg */
3646                                    MJUMPAGESIZE,        /* maxsize */
3647                                    1,                   /* nsegments */
3648                                    MJUMPAGESIZE,        /* maxsegsize */
3649                                    0,                   /* flags */
3650                                    NULL,                /* lockfunc */
3651                                    NULL,                /* lockfuncarg */
3652                                    &rxr->ptag))) {
3653                 device_printf(dev, "Unable to create RX payload DMA tag\n");
3654                 goto fail;
3655         }
3656
3657         for (i = 0; i < adapter->num_rx_desc; i++) {
3658                 rxbuf = &rxr->rx_buffers[i];
3659                 error = bus_dmamap_create(rxr->htag,
3660                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3661                 if (error) {
3662                         device_printf(dev,
3663                             "Unable to create RX head DMA maps\n");
3664                         goto fail;
3665                 }
3666                 error = bus_dmamap_create(rxr->ptag,
3667                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3668                 if (error) {
3669                         device_printf(dev,
3670                             "Unable to create RX packet DMA maps\n");
3671                         goto fail;
3672                 }
3673         }
3674
3675         return (0);
3676
3677 fail:
3678         /* Frees all, but can handle partial completion */
3679         igb_free_receive_structures(adapter);
3680         return (error);
3681 }
3682
3683
3684 static void
3685 igb_free_receive_ring(struct rx_ring *rxr)
3686 {
3687         struct  adapter         *adapter;
3688         struct igb_rx_buf       *rxbuf;
3689         int i;
3690
3691         adapter = rxr->adapter;
3692         for (i = 0; i < adapter->num_rx_desc; i++) {
3693                 rxbuf = &rxr->rx_buffers[i];
3694                 if (rxbuf->m_head != NULL) {
3695                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3696                             BUS_DMASYNC_POSTREAD);
3697                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3698                         rxbuf->m_head->m_flags |= M_PKTHDR;
3699                         m_freem(rxbuf->m_head);
3700                 }
3701                 if (rxbuf->m_pack != NULL) {
3702                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3703                             BUS_DMASYNC_POSTREAD);
3704                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3705                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3706                         m_freem(rxbuf->m_pack);
3707                 }
3708                 rxbuf->m_head = NULL;
3709                 rxbuf->m_pack = NULL;
3710         }
3711 }
3712
3713
3714 /*********************************************************************
3715  *
3716  *  Initialize a receive ring and its buffers.
3717  *
3718  **********************************************************************/
3719 static int
3720 igb_setup_receive_ring(struct rx_ring *rxr)
3721 {
3722         struct  adapter         *adapter;
3723         struct  ifnet           *ifp;
3724         device_t                dev;
3725         struct igb_rx_buf       *rxbuf;
3726         bus_dma_segment_t       pseg[1], hseg[1];
3727         struct lro_ctrl         *lro = &rxr->lro;
3728         int                     rsize, nsegs, error = 0;
3729
3730         adapter = rxr->adapter;
3731         dev = adapter->dev;
3732         ifp = adapter->ifp;
3733
3734         /* Clear the ring contents */
3735         IGB_RX_LOCK(rxr);
3736         rsize = roundup2(adapter->num_rx_desc *
3737             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3738         bzero((void *)rxr->rx_base, rsize);
3739
3740         /*
3741         ** Free current RX buffer structures and their mbufs
3742         */
3743         igb_free_receive_ring(rxr);
3744
3745         /* Configure for header split? */
3746         if (igb_header_split)
3747                 rxr->hdr_split = TRUE;
3748
3749         /* Now replenish the ring mbufs */
3750         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3751                 struct mbuf     *mh, *mp;
3752
3753                 rxbuf = &rxr->rx_buffers[j];
3754                 if (rxr->hdr_split == FALSE)
3755                         goto skip_head;
3756
3757                 /* First the header */
3758                 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA);
3759                 if (rxbuf->m_head == NULL) {
3760                         error = ENOBUFS;
3761                         goto fail;
3762                 }
3763                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3764                 mh = rxbuf->m_head;
3765                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3766                 mh->m_flags |= M_PKTHDR;
3767                 /* Get the memory mapping */
3768                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
3769                     rxbuf->hmap, rxbuf->m_head, hseg,
3770                     &nsegs, BUS_DMA_NOWAIT);
3771                 if (error != 0) /* Nothing elegant to do here */
3772                         goto fail;
3773                 bus_dmamap_sync(rxr->htag,
3774                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3775                 /* Update descriptor */
3776                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3777
3778 skip_head:
3779                 /* Now the payload cluster */
3780                 rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA,
3781                     M_PKTHDR, adapter->rx_mbuf_sz);
3782                 if (rxbuf->m_pack == NULL) {
3783                         error = ENOBUFS;
3784                         goto fail;
3785                 }
3786                 mp = rxbuf->m_pack;
3787                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3788                 /* Get the memory mapping */
3789                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
3790                     rxbuf->pmap, mp, pseg,
3791                     &nsegs, BUS_DMA_NOWAIT);
3792                 if (error != 0)
3793                         goto fail;
3794                 bus_dmamap_sync(rxr->ptag,
3795                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3796                 /* Update descriptor */
3797                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3798         }
3799
3800         /* Setup our descriptor indices */
3801         rxr->next_to_check = 0;
3802         rxr->next_to_refresh = 0;
3803         rxr->lro_enabled = FALSE;
3804         rxr->rx_split_packets = 0;
3805         rxr->rx_bytes = 0;
3806
3807         rxr->fmp = NULL;
3808         rxr->lmp = NULL;
3809         rxr->discard = FALSE;
3810
3811         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3812             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3813
3814         /*
3815         ** Now set up the LRO interface, we
3816         ** also only do head split when LRO
3817         ** is enabled, since so often they
3818         ** are undesireable in similar setups.
3819         */
3820         if (ifp->if_capenable & IFCAP_LRO) {
3821                 int err = tcp_lro_init(lro);
3822                 if (err) {
3823                         device_printf(dev, "LRO Initialization failed!\n");
3824                         goto fail;
3825                 }
3826                 INIT_DEBUGOUT("RX LRO Initialized\n");
3827                 rxr->lro_enabled = TRUE;
3828                 lro->ifp = adapter->ifp;
3829         }
3830
3831         IGB_RX_UNLOCK(rxr);
3832         return (0);
3833
3834 fail:
3835         igb_free_receive_ring(rxr);
3836         IGB_RX_UNLOCK(rxr);
3837         return (error);
3838 }
3839
3840 /*********************************************************************
3841  *
3842  *  Initialize all receive rings.
3843  *
3844  **********************************************************************/
3845 static int
3846 igb_setup_receive_structures(struct adapter *adapter)
3847 {
3848         struct rx_ring *rxr = adapter->rx_rings;
3849         int i;
3850
3851         for (i = 0; i < adapter->num_queues; i++, rxr++)
3852                 if (igb_setup_receive_ring(rxr))
3853                         goto fail;
3854
3855         return (0);
3856 fail:
3857         /*
3858          * Free RX buffers allocated so far, we will only handle
3859          * the rings that completed, the failing case will have
3860          * cleaned up for itself. 'i' is the endpoint.
3861          */
3862         for (int j = 0; j > i; ++j) {
3863                 rxr = &adapter->rx_rings[i];
3864                 igb_free_receive_ring(rxr);
3865         }
3866
3867         return (ENOBUFS);
3868 }
3869
3870 /*********************************************************************
3871  *
3872  *  Enable receive unit.
3873  *
3874  **********************************************************************/
3875 static void
3876 igb_initialize_receive_units(struct adapter *adapter)
3877 {
3878         struct rx_ring  *rxr = adapter->rx_rings;
3879         struct ifnet    *ifp = adapter->ifp;
3880         struct e1000_hw *hw = &adapter->hw;
3881         u32             rctl, rxcsum, psize, srrctl = 0;
3882
3883         INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
3884
3885         /*
3886          * Make sure receives are disabled while setting
3887          * up the descriptor ring
3888          */
3889         rctl = E1000_READ_REG(hw, E1000_RCTL);
3890         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3891
3892         /*
3893         ** Set up for header split
3894         */
3895         if (rxr->hdr_split) {
3896                 /* Use a standard mbuf for the header */
3897                 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3898                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3899         } else
3900                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3901
3902         /*
3903         ** Set up for jumbo frames
3904         */
3905         if (ifp->if_mtu > ETHERMTU) {
3906                 rctl |= E1000_RCTL_LPE;
3907                 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3908                 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
3909
3910                 /* Set maximum packet len */
3911                 psize = adapter->max_frame_size;
3912                 /* are we on a vlan? */
3913                 if (adapter->ifp->if_vlantrunk != NULL)
3914                         psize += VLAN_TAG_SIZE;
3915                 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
3916         } else {
3917                 rctl &= ~E1000_RCTL_LPE;
3918                 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3919                 rctl |= E1000_RCTL_SZ_2048;
3920         }
3921
3922         /* Setup the Base and Length of the Rx Descriptor Rings */
3923         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3924                 u64 bus_addr = rxr->rxdma.dma_paddr;
3925                 u32 rxdctl;
3926
3927                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
3928                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3929                 E1000_WRITE_REG(hw, E1000_RDBAH(i),
3930                     (uint32_t)(bus_addr >> 32));
3931                 E1000_WRITE_REG(hw, E1000_RDBAL(i),
3932                     (uint32_t)bus_addr);
3933                 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
3934                 /* Enable this Queue */
3935                 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
3936                 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3937                 rxdctl &= 0xFFF00000;
3938                 rxdctl |= IGB_RX_PTHRESH;
3939                 rxdctl |= IGB_RX_HTHRESH << 8;
3940                 rxdctl |= IGB_RX_WTHRESH << 16;
3941                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
3942         }
3943
3944         /*
3945         ** Setup for RX MultiQueue
3946         */
3947         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3948         if (adapter->num_queues >1) {
3949                 u32 random[10], mrqc, shift = 0;
3950                 union igb_reta {
3951                         u32 dword;
3952                         u8  bytes[4];
3953                 } reta;
3954
3955                 arc4rand(&random, sizeof(random), 0);
3956                 if (adapter->hw.mac.type == e1000_82575)
3957                         shift = 6;
3958                 /* Warning FM follows */
3959                 for (int i = 0; i < 128; i++) {
3960                         reta.bytes[i & 3] =
3961                             (i % adapter->num_queues) << shift;
3962                         if ((i & 3) == 3)
3963                                 E1000_WRITE_REG(hw,
3964                                     E1000_RETA(i >> 2), reta.dword);
3965                 }
3966                 /* Now fill in hash table */
3967                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3968                 for (int i = 0; i < 10; i++)
3969                         E1000_WRITE_REG_ARRAY(hw,
3970                             E1000_RSSRK(0), i, random[i]);
3971
3972                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
3973                     E1000_MRQC_RSS_FIELD_IPV4_TCP);
3974                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
3975                     E1000_MRQC_RSS_FIELD_IPV6_TCP);
3976                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
3977                     E1000_MRQC_RSS_FIELD_IPV6_UDP);
3978                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
3979                     E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
3980
3981                 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
3982
3983                 /*
3984                 ** NOTE: Receive Full-Packet Checksum Offload
3985                 ** is mutually exclusive with Multiqueue. However
3986                 ** this is not the same as TCP/IP checksums which
3987                 ** still work.
3988                 */
3989                 rxcsum |= E1000_RXCSUM_PCSD;
3990 #if __FreeBSD_version >= 800000
3991                 /* For SCTP Offload */
3992                 if ((hw->mac.type == e1000_82576)
3993                     && (ifp->if_capenable & IFCAP_RXCSUM))
3994                         rxcsum |= E1000_RXCSUM_CRCOFL;
3995 #endif
3996         } else {
3997                 /* Non RSS setup */
3998                 if (ifp->if_capenable & IFCAP_RXCSUM) {
3999                         rxcsum |= E1000_RXCSUM_IPPCSE;
4000 #if __FreeBSD_version >= 800000
4001                         if (adapter->hw.mac.type == e1000_82576)
4002                                 rxcsum |= E1000_RXCSUM_CRCOFL;
4003 #endif
4004                 } else
4005                         rxcsum &= ~E1000_RXCSUM_TUOFL;
4006         }
4007         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4008
4009         /* Setup the Receive Control Register */
4010         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4011         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4012                    E1000_RCTL_RDMTS_HALF |
4013                    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4014         /* Strip CRC bytes. */
4015         rctl |= E1000_RCTL_SECRC;
4016         /* Make sure VLAN Filters are off */
4017         rctl &= ~E1000_RCTL_VFE;
4018         /* Don't store bad packets */
4019         rctl &= ~E1000_RCTL_SBP;
4020
4021         /* Enable Receives */
4022         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4023
4024         /*
4025          * Setup the HW Rx Head and Tail Descriptor Pointers
4026          *   - needs to be after enable
4027          */
4028         for (int i = 0; i < adapter->num_queues; i++) {
4029                 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4030                 E1000_WRITE_REG(hw, E1000_RDT(i),
4031                      adapter->num_rx_desc - 1);
4032         }
4033         return;
4034 }
4035
4036 /*********************************************************************
4037  *
4038  *  Free receive rings.
4039  *
4040  **********************************************************************/
4041 static void
4042 igb_free_receive_structures(struct adapter *adapter)
4043 {
4044         struct rx_ring *rxr = adapter->rx_rings;
4045
4046         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4047                 struct lro_ctrl *lro = &rxr->lro;
4048                 igb_free_receive_buffers(rxr);
4049                 tcp_lro_free(lro);
4050                 igb_dma_free(adapter, &rxr->rxdma);
4051         }
4052
4053         free(adapter->rx_rings, M_DEVBUF);
4054 }
4055
4056 /*********************************************************************
4057  *
4058  *  Free receive ring data structures.
4059  *
4060  **********************************************************************/
4061 static void
4062 igb_free_receive_buffers(struct rx_ring *rxr)
4063 {
4064         struct adapter          *adapter = rxr->adapter;
4065         struct igb_rx_buf       *rxbuf;
4066         int i;
4067
4068         INIT_DEBUGOUT("free_receive_structures: begin");
4069
4070         /* Cleanup any existing buffers */
4071         if (rxr->rx_buffers != NULL) {
4072                 for (i = 0; i < adapter->num_rx_desc; i++) {
4073                         rxbuf = &rxr->rx_buffers[i];
4074                         if (rxbuf->m_head != NULL) {
4075                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4076                                     BUS_DMASYNC_POSTREAD);
4077                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4078                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4079                                 m_freem(rxbuf->m_head);
4080                         }
4081                         if (rxbuf->m_pack != NULL) {
4082                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4083                                     BUS_DMASYNC_POSTREAD);
4084                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4085                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4086                                 m_freem(rxbuf->m_pack);
4087                         }
4088                         rxbuf->m_head = NULL;
4089                         rxbuf->m_pack = NULL;
4090                         if (rxbuf->hmap != NULL) {
4091                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4092                                 rxbuf->hmap = NULL;
4093                         }
4094                         if (rxbuf->pmap != NULL) {
4095                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4096                                 rxbuf->pmap = NULL;
4097                         }
4098                 }
4099                 if (rxr->rx_buffers != NULL) {
4100                         free(rxr->rx_buffers, M_DEVBUF);
4101                         rxr->rx_buffers = NULL;
4102                 }
4103         }
4104
4105         if (rxr->htag != NULL) {
4106                 bus_dma_tag_destroy(rxr->htag);
4107                 rxr->htag = NULL;
4108         }
4109         if (rxr->ptag != NULL) {
4110                 bus_dma_tag_destroy(rxr->ptag);
4111                 rxr->ptag = NULL;
4112         }
4113 }
4114
4115 static __inline void
4116 igb_rx_discard(struct rx_ring *rxr, int i)
4117 {
4118         struct adapter          *adapter = rxr->adapter;
4119         struct igb_rx_buf       *rbuf;
4120         struct mbuf             *mh, *mp;
4121
4122         rbuf = &rxr->rx_buffers[i];
4123         if (rxr->fmp != NULL) {
4124                 rxr->fmp->m_flags |= M_PKTHDR;
4125                 m_freem(rxr->fmp);
4126                 rxr->fmp = NULL;
4127                 rxr->lmp = NULL;
4128         }
4129
4130         mh = rbuf->m_head;
4131         mp = rbuf->m_pack;
4132
4133         /* Reuse loaded DMA map and just update mbuf chain */
4134         mh->m_len = MHLEN;
4135         mh->m_flags |= M_PKTHDR;
4136         mh->m_next = NULL;
4137
4138         mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz;
4139         mp->m_data = mp->m_ext.ext_buf;
4140         mp->m_next = NULL;
4141         return;
4142 }
4143
4144 static __inline void
4145 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4146 {
4147
4148         /*
4149          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4150          * should be computed by hardware. Also it should not have VLAN tag in
4151          * ethernet header.
4152          */
4153         if (rxr->lro_enabled &&
4154             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4155             (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4156             (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4157             (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4158             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4159             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4160                 /*
4161                  * Send to the stack if:
4162                  **  - LRO not enabled, or
4163                  **  - no LRO resources, or
4164                  **  - lro enqueue fails
4165                  */
4166                 if (rxr->lro.lro_cnt != 0)
4167                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4168                                 return;
4169         }
4170         IGB_RX_UNLOCK(rxr);
4171         (*ifp->if_input)(ifp, m);
4172         IGB_RX_LOCK(rxr);
4173 }
4174
4175 /*********************************************************************
4176  *
4177  *  This routine executes in interrupt context. It replenishes
4178  *  the mbufs in the descriptor and sends data which has been
4179  *  dma'ed into host memory to upper layer.
4180  *
4181  *  We loop at most count times if count is > 0, or until done if
4182  *  count < 0.
4183  *
4184  *  Return TRUE if more to clean, FALSE otherwise
4185  *********************************************************************/
4186 static bool
4187 igb_rxeof(struct igb_queue *que, int count, int *done)
4188 {
4189         struct adapter          *adapter = que->adapter;
4190         struct rx_ring          *rxr = que->rxr;
4191         struct ifnet            *ifp = adapter->ifp;
4192         struct lro_ctrl         *lro = &rxr->lro;
4193         struct lro_entry        *queued;
4194         int                     i, processed = 0, rxdone = 0;
4195         u32                     ptype, staterr = 0;
4196         union e1000_adv_rx_desc *cur;
4197
4198         IGB_RX_LOCK(rxr);
4199         /* Sync the ring. */
4200         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4201             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4202
4203         /* Main clean loop */
4204         for (i = rxr->next_to_check; count != 0;) {
4205                 struct mbuf             *sendmp, *mh, *mp;
4206                 struct igb_rx_buf       *rxbuf;
4207                 u16                     hlen, plen, hdr, vtag;
4208                 bool                    eop = FALSE;
4209
4210                 cur = &rxr->rx_base[i];
4211                 staterr = le32toh(cur->wb.upper.status_error);
4212                 if ((staterr & E1000_RXD_STAT_DD) == 0)
4213                         break;
4214                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4215                         break;
4216                 count--;
4217                 sendmp = mh = mp = NULL;
4218                 cur->wb.upper.status_error = 0;
4219                 rxbuf = &rxr->rx_buffers[i];
4220                 plen = le16toh(cur->wb.upper.length);
4221                 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
4222                 vtag = le16toh(cur->wb.upper.vlan);
4223                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4224                 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
4225
4226                 /* Make sure all segments of a bad packet are discarded */
4227                 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) ||
4228                     (rxr->discard)) {
4229                         ifp->if_ierrors++;
4230                         ++rxr->rx_discarded;
4231                         if (!eop) /* Catch subsequent segs */
4232                                 rxr->discard = TRUE;
4233                         else
4234                                 rxr->discard = FALSE;
4235                         igb_rx_discard(rxr, i);
4236                         goto next_desc;
4237                 }
4238
4239                 /*
4240                 ** The way the hardware is configured to
4241                 ** split, it will ONLY use the header buffer
4242                 ** when header split is enabled, otherwise we
4243                 ** get normal behavior, ie, both header and
4244                 ** payload are DMA'd into the payload buffer.
4245                 **
4246                 ** The fmp test is to catch the case where a
4247                 ** packet spans multiple descriptors, in that
4248                 ** case only the first header is valid.
4249                 */
4250                 if (rxr->hdr_split && rxr->fmp == NULL) {
4251                         hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4252                             E1000_RXDADV_HDRBUFLEN_SHIFT;
4253                         if (hlen > IGB_HDR_BUF)
4254                                 hlen = IGB_HDR_BUF;
4255                         /* Handle the header mbuf */
4256                         mh = rxr->rx_buffers[i].m_head;
4257                         mh->m_len = hlen;
4258                         /* clear buf info for refresh */
4259                         rxbuf->m_head = NULL;
4260                         /*
4261                         ** Get the payload length, this
4262                         ** could be zero if its a small
4263                         ** packet.
4264                         */
4265                         if (plen > 0) {
4266                                 mp = rxr->rx_buffers[i].m_pack;
4267                                 mp->m_len = plen;
4268                                 mh->m_next = mp;
4269                                 /* clear buf info for refresh */
4270                                 rxbuf->m_pack = NULL;
4271                                 rxr->rx_split_packets++;
4272                         }
4273                 } else {
4274                         /*
4275                         ** Either no header split, or a
4276                         ** secondary piece of a fragmented
4277                         ** split packet.
4278                         */
4279                         mh = rxr->rx_buffers[i].m_pack;
4280                         mh->m_len = plen;
4281                         /* clear buf info for refresh */
4282                         rxbuf->m_pack = NULL;
4283                 }
4284
4285                 ++processed; /* So we know when to refresh */
4286
4287                 /* Initial frame - setup */
4288                 if (rxr->fmp == NULL) {
4289                         mh->m_pkthdr.len = mh->m_len;
4290                         /* Store the first mbuf */
4291                         rxr->fmp = mh;
4292                         rxr->lmp = mh;
4293                         if (mp != NULL) {
4294                                 /* Add payload if split */
4295                                 mh->m_pkthdr.len += mp->m_len;
4296                                 rxr->lmp = mh->m_next;
4297                         }
4298                 } else {
4299                         /* Chain mbuf's together */
4300                         rxr->lmp->m_next = mh;
4301                         rxr->lmp = rxr->lmp->m_next;
4302                         rxr->fmp->m_pkthdr.len += mh->m_len;
4303                 }
4304
4305                 if (eop) {
4306                         rxr->fmp->m_pkthdr.rcvif = ifp;
4307                         ifp->if_ipackets++;
4308                         rxr->rx_packets++;
4309                         /* capture data for AIM */
4310                         rxr->packets++;
4311                         rxr->bytes += rxr->fmp->m_pkthdr.len;
4312                         rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4313
4314                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4315                                 igb_rx_checksum(staterr, rxr->fmp, ptype);
4316
4317                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4318                             (staterr & E1000_RXD_STAT_VP) != 0) {
4319                                 rxr->fmp->m_pkthdr.ether_vtag = vtag;
4320                                 rxr->fmp->m_flags |= M_VLANTAG;
4321                         }
4322 #if __FreeBSD_version >= 800000
4323                         rxr->fmp->m_pkthdr.flowid = que->msix;
4324                         rxr->fmp->m_flags |= M_FLOWID;
4325 #endif
4326                         sendmp = rxr->fmp;
4327                         /* Make sure to set M_PKTHDR. */
4328                         sendmp->m_flags |= M_PKTHDR;
4329                         rxr->fmp = NULL;
4330                         rxr->lmp = NULL;
4331                 }
4332
4333 next_desc:
4334                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4335                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4336
4337                 /* Advance our pointers to the next descriptor. */
4338                 if (++i == adapter->num_rx_desc)
4339                         i = 0;
4340                 /*
4341                 ** Send to the stack or LRO
4342                 */
4343                 if (sendmp != NULL) {
4344                         rxr->next_to_check = i;
4345                         igb_rx_input(rxr, ifp, sendmp, ptype);
4346                         i = rxr->next_to_check;
4347                         rxdone++;
4348                 }
4349
4350                 /* Every 8 descriptors we go to refresh mbufs */
4351                 if (processed == 8) {
4352                         igb_refresh_mbufs(rxr, i);
4353                         processed = 0;
4354                 }
4355         }
4356
4357         /* Catch any remainders */
4358         if (processed != 0) {
4359                 igb_refresh_mbufs(rxr, i);
4360                 processed = 0;
4361         }
4362
4363         rxr->next_to_check = i;
4364
4365         /*
4366          * Flush any outstanding LRO work
4367          */
4368         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4369                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4370                 tcp_lro_flush(lro, queued);
4371         }
4372
4373         IGB_RX_UNLOCK(rxr);
4374
4375         if (done != NULL)
4376                 *done = rxdone;
4377
4378         /*
4379         ** We still have cleaning to do?
4380         ** Schedule another interrupt if so.
4381         */
4382         if ((staterr & E1000_RXD_STAT_DD) != 0)
4383                 return (TRUE);
4384
4385         return (FALSE);
4386 }
4387
4388 /*********************************************************************
4389  *
4390  *  Verify that the hardware indicated that the checksum is valid.
4391  *  Inform the stack about the status of checksum so that stack
4392  *  doesn't spend time verifying the checksum.
4393  *
4394  *********************************************************************/
4395 static void
4396 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
4397 {
4398         u16 status = (u16)staterr;
4399         u8  errors = (u8) (staterr >> 24);
4400         int sctp;
4401
4402         /* Ignore Checksum bit is set */
4403         if (status & E1000_RXD_STAT_IXSM) {
4404                 mp->m_pkthdr.csum_flags = 0;
4405                 return;
4406         }
4407
4408         if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4409             (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
4410                 sctp = 1;
4411         else
4412                 sctp = 0;
4413         if (status & E1000_RXD_STAT_IPCS) {
4414                 /* Did it pass? */
4415                 if (!(errors & E1000_RXD_ERR_IPE)) {
4416                         /* IP Checksum Good */
4417                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4418                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4419                 } else
4420                         mp->m_pkthdr.csum_flags = 0;
4421         }
4422
4423         if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
4424                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4425 #if __FreeBSD_version >= 800000
4426                 if (sctp) /* reassign */
4427                         type = CSUM_SCTP_VALID;
4428 #endif
4429                 /* Did it pass? */
4430                 if (!(errors & E1000_RXD_ERR_TCPE)) {
4431                         mp->m_pkthdr.csum_flags |= type;
4432                         if (sctp == 0)
4433                                 mp->m_pkthdr.csum_data = htons(0xffff);
4434                 }
4435         }
4436         return;
4437 }
4438
4439 /*
4440  * This routine is run via an vlan
4441  * config EVENT
4442  */
4443 static void
4444 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4445 {
4446         struct adapter  *adapter = ifp->if_softc;
4447         u32             index, bit;
4448
4449         if (ifp->if_softc !=  arg)   /* Not our event */
4450                 return;
4451
4452         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4453                 return;
4454
4455         index = (vtag >> 5) & 0x7F;
4456         bit = vtag & 0x1F;
4457         igb_shadow_vfta[index] |= (1 << bit);
4458         ++adapter->num_vlans;
4459         /* Re-init to load the changes */
4460         igb_init(adapter);
4461 }
4462
4463 /*
4464  * This routine is run via an vlan
4465  * unconfig EVENT
4466  */
4467 static void
4468 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4469 {
4470         struct adapter  *adapter = ifp->if_softc;
4471         u32             index, bit;
4472
4473         if (ifp->if_softc !=  arg)
4474                 return;
4475
4476         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4477                 return;
4478
4479         index = (vtag >> 5) & 0x7F;
4480         bit = vtag & 0x1F;
4481         igb_shadow_vfta[index] &= ~(1 << bit);
4482         --adapter->num_vlans;
4483         /* Re-init to load the changes */
4484         igb_init(adapter);
4485 }
4486
4487 static void
4488 igb_setup_vlan_hw_support(struct adapter *adapter)
4489 {
4490         struct e1000_hw *hw = &adapter->hw;
4491         u32             reg;
4492
4493         /*
4494         ** We get here thru init_locked, meaning
4495         ** a soft reset, this has already cleared
4496         ** the VFTA and other state, so if there
4497         ** have been no vlan's registered do nothing.
4498         */
4499         if (adapter->num_vlans == 0)
4500                 return;
4501
4502         /*
4503         ** A soft reset zero's out the VFTA, so
4504         ** we need to repopulate it now.
4505         */
4506         for (int i = 0; i < IGB_VFTA_SIZE; i++)
4507                 if (igb_shadow_vfta[i] != 0)
4508                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4509                             i, igb_shadow_vfta[i]);
4510
4511         reg = E1000_READ_REG(hw, E1000_CTRL);
4512         reg |= E1000_CTRL_VME;
4513         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4514
4515         /* Enable the Filter Table */
4516         reg = E1000_READ_REG(hw, E1000_RCTL);
4517         reg &= ~E1000_RCTL_CFIEN;
4518         reg |= E1000_RCTL_VFE;
4519         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4520
4521         /* Update the frame size */
4522         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4523             adapter->max_frame_size + VLAN_TAG_SIZE);
4524 }
4525
4526 static void
4527 igb_enable_intr(struct adapter *adapter)
4528 {
4529         /* With RSS set up what to auto clear */
4530         if (adapter->msix_mem) {
4531                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC,
4532                     adapter->eims_mask);
4533                 E1000_WRITE_REG(&adapter->hw, E1000_EIAM,
4534                     adapter->eims_mask);
4535                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS,
4536                     adapter->eims_mask);
4537                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4538                     E1000_IMS_LSC);
4539         } else {
4540                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4541                     IMS_ENABLE_MASK);
4542         }
4543         E1000_WRITE_FLUSH(&adapter->hw);
4544
4545         return;
4546 }
4547
4548 static void
4549 igb_disable_intr(struct adapter *adapter)
4550 {
4551         if (adapter->msix_mem) {
4552                 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
4553                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
4554         }
4555         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
4556         E1000_WRITE_FLUSH(&adapter->hw);
4557         return;
4558 }
4559
4560 /*
4561  * Bit of a misnomer, what this really means is
4562  * to enable OS management of the system... aka
4563  * to disable special hardware management features
4564  */
4565 static void
4566 igb_init_manageability(struct adapter *adapter)
4567 {
4568         if (adapter->has_manage) {
4569                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4570                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4571
4572                 /* disable hardware interception of ARP */
4573                 manc &= ~(E1000_MANC_ARP_EN);
4574
4575                 /* enable receiving management packets to the host */
4576                 manc |= E1000_MANC_EN_MNG2HOST;
4577                 manc2h |= 1 << 5;  /* Mng Port 623 */
4578                 manc2h |= 1 << 6;  /* Mng Port 664 */
4579                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4580                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4581         }
4582 }
4583
4584 /*
4585  * Give control back to hardware management
4586  * controller if there is one.
4587  */
4588 static void
4589 igb_release_manageability(struct adapter *adapter)
4590 {
4591         if (adapter->has_manage) {
4592                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4593
4594                 /* re-enable hardware interception of ARP */
4595                 manc |= E1000_MANC_ARP_EN;
4596                 manc &= ~E1000_MANC_EN_MNG2HOST;
4597
4598                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4599         }
4600 }
4601
4602 /*
4603  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
4604  * For ASF and Pass Through versions of f/w this means that
4605  * the driver is loaded.
4606  *
4607  */
4608 static void
4609 igb_get_hw_control(struct adapter *adapter)
4610 {
4611         u32 ctrl_ext;
4612
4613         /* Let firmware know the driver has taken over */
4614         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4615         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4616             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4617 }
4618
4619 /*
4620  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
4621  * For ASF and Pass Through versions of f/w this means that the
4622  * driver is no longer loaded.
4623  *
4624  */
4625 static void
4626 igb_release_hw_control(struct adapter *adapter)
4627 {
4628         u32 ctrl_ext;
4629
4630         /* Let firmware taken over control of h/w */
4631         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4632         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4633             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4634 }
4635
4636 static int
4637 igb_is_valid_ether_addr(uint8_t *addr)
4638 {
4639         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4640
4641         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4642                 return (FALSE);
4643         }
4644
4645         return (TRUE);
4646 }
4647
4648
4649 /*
4650  * Enable PCI Wake On Lan capability
4651  */
4652 static void
4653 igb_enable_wakeup(device_t dev)
4654 {
4655         u16     cap, status;
4656         u8      id;
4657
4658         /* First find the capabilities pointer*/
4659         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4660         /* Read the PM Capabilities */
4661         id = pci_read_config(dev, cap, 1);
4662         if (id != PCIY_PMG)     /* Something wrong */
4663                 return;
4664         /* OK, we have the power capabilities, so
4665            now get the status register */
4666         cap += PCIR_POWER_STATUS;
4667         status = pci_read_config(dev, cap, 2);
4668         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4669         pci_write_config(dev, cap, status, 2);
4670         return;
4671 }
4672
4673 static void
4674 igb_led_func(void *arg, int onoff)
4675 {
4676         struct adapter  *adapter = arg;
4677
4678         IGB_CORE_LOCK(adapter);
4679         if (onoff) {
4680                 e1000_setup_led(&adapter->hw);
4681                 e1000_led_on(&adapter->hw);
4682         } else {
4683                 e1000_led_off(&adapter->hw);
4684                 e1000_cleanup_led(&adapter->hw);
4685         }
4686         IGB_CORE_UNLOCK(adapter);
4687 }
4688
4689 /**********************************************************************
4690  *
4691  *  Update the board statistics counters.
4692  *
4693  **********************************************************************/
4694 static void
4695 igb_update_stats_counters(struct adapter *adapter)
4696 {
4697         struct ifnet   *ifp;
4698
4699         if (adapter->hw.phy.media_type == e1000_media_type_copper ||
4700            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4701                 adapter->stats.symerrs +=
4702                     E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4703                 adapter->stats.sec +=
4704                     E1000_READ_REG(&adapter->hw, E1000_SEC);
4705         }
4706         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4707         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4708         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4709         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4710
4711         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4712         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4713         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4714         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4715         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4716         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4717         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4718         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4719         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4720         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4721         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4722         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4723         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4724         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4725         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4726         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4727         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4728         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4729         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4730         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4731
4732         /* For the 64-bit byte counters the low dword must be read first. */
4733         /* Both registers clear on the read of the high dword */
4734
4735         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
4736           ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
4737         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
4738           ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32) ;
4739
4740         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4741         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4742         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4743         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4744         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4745
4746         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4747         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4748
4749         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
4750         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
4751         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
4752         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
4753         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
4754         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
4755         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
4756         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
4757         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
4758         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
4759
4760         /* Interrupt Counts */
4761
4762         adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC);
4763         adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC);
4764         adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC);
4765         adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC);
4766         adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC);
4767         adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC);
4768         adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC);
4769         adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC);
4770         adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC);
4771
4772         /* Host to Card Statistics */
4773
4774         adapter->stats.cbtmpc += E1000_READ_REG(&adapter->hw, E1000_CBTMPC);
4775         adapter->stats.htdpmc += E1000_READ_REG(&adapter->hw, E1000_HTDPMC);
4776         adapter->stats.cbrdpc += E1000_READ_REG(&adapter->hw, E1000_CBRDPC);
4777         adapter->stats.cbrmpc += E1000_READ_REG(&adapter->hw, E1000_CBRMPC);
4778         adapter->stats.rpthc += E1000_READ_REG(&adapter->hw, E1000_RPTHC);
4779         adapter->stats.hgptc += E1000_READ_REG(&adapter->hw, E1000_HGPTC);
4780         adapter->stats.htcbdpc += E1000_READ_REG(&adapter->hw, E1000_HTCBDPC);
4781         adapter->stats.hgorc += (E1000_READ_REG(&adapter->hw, E1000_HGORCL) +
4782                                  ((u64)E1000_READ_REG(&adapter->hw,
4783                                                       E1000_HGORCH) << 32));
4784
4785         adapter->stats.hgotc += (E1000_READ_REG(&adapter->hw, E1000_HGOTCL) +
4786                                  ((u64)E1000_READ_REG(&adapter->hw,
4787                                                       E1000_HGOTCH) << 32));
4788         adapter->stats.lenerrs += E1000_READ_REG(&adapter->hw, E1000_LENERRS);
4789         adapter->stats.scvpc += E1000_READ_REG(&adapter->hw, E1000_SCVPC);
4790         adapter->stats.hrmpc += E1000_READ_REG(&adapter->hw, E1000_HRMPC);
4791
4792         adapter->stats.algnerrc +=
4793                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
4794         adapter->stats.rxerrc +=
4795                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
4796         adapter->stats.tncrs +=
4797                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
4798         adapter->stats.cexterr +=
4799                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
4800         adapter->stats.tsctc +=
4801                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
4802         adapter->stats.tsctfc +=
4803                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
4804         ifp = adapter->ifp;
4805
4806         ifp->if_collisions = adapter->stats.colc;
4807
4808         /* Rx Errors */
4809         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
4810             adapter->stats.crcerrs + adapter->stats.algnerrc +
4811             adapter->stats.ruc + adapter->stats.roc +
4812             adapter->stats.mpc + adapter->stats.cexterr;
4813
4814         /* Tx Errors */
4815         ifp->if_oerrors = adapter->stats.ecol +
4816             adapter->stats.latecol + adapter->watchdog_events;
4817 }
4818
4819
4820 /** igb_sysctl_tdh_handler - Handler function
4821  *  Retrieves the TDH value from the hardware
4822  */
4823 static int
4824 igb_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
4825 {
4826         int error;
4827
4828         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
4829         if (!txr) return 0;
4830
4831         unsigned val = E1000_READ_REG(&txr->adapter->hw, E1000_TDH(txr->me));
4832         error = sysctl_handle_int(oidp, &val, 0, req);
4833         if (error || !req->newptr)
4834                 return error;
4835         return 0;
4836 }
4837
4838 /** igb_sysctl_tdt_handler - Handler function
4839  *  Retrieves the TDT value from the hardware
4840  */
4841 static int
4842 igb_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
4843 {
4844         int error;
4845
4846         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
4847         if (!txr) return 0;
4848
4849         unsigned val = E1000_READ_REG(&txr->adapter->hw, E1000_TDT(txr->me));
4850         error = sysctl_handle_int(oidp, &val, 0, req);
4851         if (error || !req->newptr)
4852                 return error;
4853         return 0;
4854 }
4855
4856 /** igb_sysctl_rdh_handler - Handler function
4857  *  Retrieves the RDH value from the hardware
4858  */
4859 static int
4860 igb_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
4861 {
4862         int error;
4863
4864         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
4865         if (!rxr) return 0;
4866
4867         unsigned val = E1000_READ_REG(&rxr->adapter->hw, E1000_RDH(rxr->me));
4868         error = sysctl_handle_int(oidp, &val, 0, req);
4869         if (error || !req->newptr)
4870                 return error;
4871         return 0;
4872 }
4873
4874 /** igb_sysctl_rdt_handler - Handler function
4875  *  Retrieves the RDT value from the hardware
4876  */
4877 static int
4878 igb_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
4879 {
4880         int error;
4881
4882         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
4883         if (!rxr) return 0;
4884
4885         unsigned val = E1000_READ_REG(&rxr->adapter->hw, E1000_RDT(rxr->me));
4886         error = sysctl_handle_int(oidp, &val, 0, req);
4887         if (error || !req->newptr)
4888                 return error;
4889         return 0;
4890 }
4891
4892 /*
4893  * Add sysctl variables, one per statistic, to the system.
4894  */
4895 static void
4896 igb_add_hw_stats(struct adapter *adapter)
4897 {
4898
4899         device_t dev = adapter->dev;
4900
4901         struct tx_ring *txr = adapter->tx_rings;
4902         struct rx_ring *rxr = adapter->rx_rings;
4903
4904         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
4905         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
4906         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
4907         struct e1000_hw_stats *stats = &adapter->stats;
4908
4909         struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
4910         struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
4911
4912 #define QUEUE_NAME_LEN 32
4913         char namebuf[QUEUE_NAME_LEN];
4914
4915         /* Driver Statistics */
4916         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "link_irq",
4917                         CTLFLAG_RD, &adapter->link_irq, 0,
4918                         "Link MSIX IRQ Handled");
4919         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
4920                         CTLFLAG_RD, &adapter->dropped_pkts,
4921                         "Driver dropped packets");
4922         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
4923                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
4924                         "Driver tx dma failure in xmit");
4925
4926         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control",
4927                         CTLFLAG_RD, &adapter->device_control,
4928                         "Device Control Register");
4929         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control",
4930                         CTLFLAG_RD, &adapter->rx_control,
4931                         "Receiver Control Register");
4932         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask",
4933                         CTLFLAG_RD, &adapter->int_mask,
4934                         "Interrupt Mask");
4935         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask",
4936                         CTLFLAG_RD, &adapter->eint_mask,
4937                         "Extended Interrupt Mask");
4938         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc",
4939                         CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
4940                         "Transmit Buffer Packet Allocation");
4941         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc",
4942                         CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
4943                         "Receive Buffer Packet Allocation");
4944         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
4945                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
4946                         "Flow Control High Watermark");
4947         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
4948                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
4949                         "Flow Control Low Watermark");
4950
4951         for (int i = 0; i < adapter->num_queues; i++, txr++) {
4952                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
4953                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
4954                                             CTLFLAG_RD, NULL, "Queue Name");
4955                 queue_list = SYSCTL_CHILDREN(queue_node);
4956
4957                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
4958                                 CTLFLAG_RD, txr, sizeof(txr),
4959                                 igb_sysctl_tdh_handler, "IU",
4960                                 "Transmit Descriptor Head");
4961                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
4962                                 CTLFLAG_RD, txr, sizeof(txr),
4963                                 igb_sysctl_tdt_handler, "IU",
4964                                 "Transmit Descriptor Tail");
4965                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
4966                                 CTLFLAG_RD, &txr->no_desc_avail,
4967                                 "Queue No Descriptor Available");
4968                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets",
4969                                 CTLFLAG_RD, &txr->tx_packets,
4970                                 "Queue Packets Transmitted");
4971         }
4972
4973         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4974                 struct lro_ctrl *lro = &rxr->lro;
4975
4976                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
4977                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
4978                                             CTLFLAG_RD, NULL, "Queue Name");
4979                 queue_list = SYSCTL_CHILDREN(queue_node);
4980
4981                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
4982                                 CTLFLAG_RD, rxr, sizeof(rxr),
4983                                 igb_sysctl_rdh_handler, "IU",
4984                                 "Receive Descriptor Head");
4985                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
4986                                 CTLFLAG_RD, rxr, sizeof(rxr),
4987                                 igb_sysctl_rdt_handler, "IU",
4988                                 "Receive Descriptor Tail");
4989                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
4990                                 CTLFLAG_RD, &rxr->rx_packets,
4991                                 "Queue Packets Received");
4992                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
4993                                 CTLFLAG_RD, &rxr->rx_bytes,
4994                                 "Queue Bytes Received");
4995                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
4996                                 CTLFLAG_RD, &lro->lro_queued, 0,
4997                                 "LRO Queued");
4998                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
4999                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5000                                 "LRO Flushed");
5001         }
5002
5003         /* MAC stats get the own sub node */
5004
5005         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5006                                     CTLFLAG_RD, NULL, "MAC Statistics");
5007         stat_list = SYSCTL_CHILDREN(stat_node);
5008
5009         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5010                         CTLFLAG_RD, &stats->ecol,
5011                         "Excessive collisions");
5012         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll",
5013                         CTLFLAG_RD, &stats->scc,
5014                         "Single collisions");
5015         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5016                         CTLFLAG_RD, &stats->mcc,
5017                         "Multiple collisions");
5018         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll",
5019                         CTLFLAG_RD, &stats->latecol,
5020                         "Late collisions");
5021         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count",
5022                         CTLFLAG_RD, &stats->colc,
5023                         "Collision Count");
5024         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5025                         CTLFLAG_RD, &adapter->stats.symerrs,
5026                         "Symbol Errors");
5027         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5028                         CTLFLAG_RD, &adapter->stats.sec,
5029                         "Sequence Errors");
5030         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
5031                         CTLFLAG_RD, &adapter->stats.dc,
5032                         "Defer Count");
5033         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5034                         CTLFLAG_RD, &adapter->stats.mpc,
5035                         "Missed Packets");
5036         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5037                         CTLFLAG_RD, &adapter->stats.rnbc,
5038                         "Receive No Buffers");
5039         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5040                         CTLFLAG_RD, &adapter->stats.ruc,
5041                         "Receive Undersize");
5042         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5043                         CTLFLAG_RD, &adapter->stats.rfc,
5044                         "Fragmented Packets Received ");
5045         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5046                         CTLFLAG_RD, &adapter->stats.roc,
5047                         "Oversized Packets Received");
5048         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5049                         CTLFLAG_RD, &adapter->stats.rjc,
5050                         "Recevied Jabber");
5051
5052         /* RLEC is inaccurate on some hardware, calculate our own. */
5053 /*      SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_len_errs", */
5054 /*                      CTLFLAG_RD, adapter->stats.roc + adapter->stats.ruc, */
5055 /*                      "Receive Length Errors"); */
5056
5057         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5058                         CTLFLAG_RD, &adapter->stats.rxerrc,
5059                         "Receive Errors");
5060         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5061                         CTLFLAG_RD, &adapter->stats.crcerrs,
5062                         "CRC errors");
5063         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5064                         CTLFLAG_RD, &adapter->stats.algnerrc,
5065                         "Alignment Errors");
5066         /* On 82575 these are collision counts */
5067         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5068                         CTLFLAG_RD, &adapter->stats.cexterr,
5069                         "Collision/Carrier extension errors");
5070         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_overruns",
5071                         CTLFLAG_RD, &adapter->rx_overruns,
5072                         "RX overruns");
5073         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "watchdog_timeouts",
5074                         CTLFLAG_RD, &adapter->watchdog_events,
5075                         "Watchdog timeouts");
5076         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5077                         CTLFLAG_RD, &adapter->stats.xonrxc,
5078                         "XON Received");
5079         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5080                         CTLFLAG_RD, &adapter->stats.xontxc,
5081                         "XON Transmitted");
5082         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5083                         CTLFLAG_RD, &adapter->stats.xoffrxc,
5084                         "XOFF Received");
5085         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5086                         CTLFLAG_RD, &adapter->stats.xofftxc,
5087                         "XOFF Transmitted");
5088         /* Packet Reception Stats */
5089         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5090                         CTLFLAG_RD, &adapter->stats.tpr,
5091                         "Total Packets Received ");
5092         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5093                         CTLFLAG_RD, &adapter->stats.gprc,
5094                         "Good Packets Received");
5095         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5096                         CTLFLAG_RD, &adapter->stats.bprc,
5097                         "Broadcast Packets Received");
5098         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5099                         CTLFLAG_RD, &adapter->stats.mprc,
5100                         "Multicast Packets Received");
5101         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5102                         CTLFLAG_RD, &adapter->stats.prc64,
5103                         "64 byte frames received ");
5104         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5105                         CTLFLAG_RD, &adapter->stats.prc127,
5106                         "65-127 byte frames received");
5107         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5108                         CTLFLAG_RD, &adapter->stats.prc255,
5109                         "128-255 byte frames received");
5110         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5111                         CTLFLAG_RD, &adapter->stats.prc511,
5112                         "256-511 byte frames received");
5113         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5114                         CTLFLAG_RD, &adapter->stats.prc1023,
5115                         "512-1023 byte frames received");
5116         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5117                         CTLFLAG_RD, &adapter->stats.prc1522,
5118                         "1023-1522 byte frames received");
5119         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5120                         CTLFLAG_RD, &adapter->stats.gorc,
5121                         "Good Octets Received");
5122
5123         /* Packet Transmission Stats */
5124         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octest_txd",
5125                         CTLFLAG_RD, &adapter->stats.gotc,
5126                         "Good Octest Transmitted");
5127         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5128                         CTLFLAG_RD, &adapter->stats.tpt,
5129                         "Total Packets Transmitted");
5130         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5131                         CTLFLAG_RD, &adapter->stats.gptc,
5132                         "Good Packets Transmitted");
5133         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5134                         CTLFLAG_RD, &adapter->stats.bptc,
5135                         "Broadcast Packets Transmitted");
5136         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5137                         CTLFLAG_RD, &adapter->stats.mptc,
5138                         "Multicast Packets Transmitted");
5139         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5140                         CTLFLAG_RD, &adapter->stats.ptc64,
5141                         "64 byte frames transmitted ");
5142         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5143                         CTLFLAG_RD, &adapter->stats.ptc127,
5144                         "65-127 byte frames transmitted");
5145         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5146                         CTLFLAG_RD, &adapter->stats.ptc255,
5147                         "128-255 byte frames transmitted");
5148         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5149                         CTLFLAG_RD, &adapter->stats.ptc511,
5150                         "256-511 byte frames transmitted");
5151         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5152                         CTLFLAG_RD, &adapter->stats.ptc1023,
5153                         "512-1023 byte frames transmitted");
5154         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5155                         CTLFLAG_RD, &adapter->stats.ptc1522,
5156                         "1024-1522 byte frames transmitted");
5157         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5158                         CTLFLAG_RD, &adapter->stats.tsctc,
5159                         "TSO Contexts Transmitted");
5160         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5161                         CTLFLAG_RD, &adapter->stats.tsctfc,
5162                         "TSO Contexts Failed");
5163
5164
5165         /* Interrupt Stats */
5166
5167         int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5168                                     CTLFLAG_RD, NULL, "Interrupt Statistics");
5169         int_list = SYSCTL_CHILDREN(int_node);
5170
5171         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
5172                         CTLFLAG_RD, &adapter->stats.iac,
5173                         "Interrupt Assertion Count");
5174
5175         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5176                         CTLFLAG_RD, &adapter->stats.icrxptc,
5177                         "Interrupt Cause Rx Pkt Timer Expire Count");
5178
5179         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5180                         CTLFLAG_RD, &adapter->stats.icrxatc,
5181                         "Interrupt Cause Rx Abs Timer Expire Count");
5182
5183         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5184                         CTLFLAG_RD, &adapter->stats.ictxptc,
5185                         "Interrupt Cause Tx Pkt Timer Expire Count");
5186
5187         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5188                         CTLFLAG_RD, &adapter->stats.ictxatc,
5189                         "Interrupt Cause Tx Abs Timer Expire Count");
5190
5191         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5192                         CTLFLAG_RD, &adapter->stats.ictxqec,
5193                         "Interrupt Cause Tx Queue Empty Count");
5194
5195         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5196                         CTLFLAG_RD, &adapter->stats.ictxqmtc,
5197                         "Interrupt Cause Tx Queue Min Thresh Count");
5198
5199         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5200                         CTLFLAG_RD, &adapter->stats.icrxdmtc,
5201                         "Interrupt Cause Rx Desc Min Thresh Count");
5202
5203         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5204                         CTLFLAG_RD, &adapter->stats.icrxoc,
5205                         "Interrupt Cause Receiver Overrun Count");
5206
5207         /* Host to Card Stats */
5208
5209         host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host",
5210                                     CTLFLAG_RD, NULL,
5211                                     "Host to Card Statistics");
5212
5213         host_list = SYSCTL_CHILDREN(host_node);
5214
5215         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
5216                         CTLFLAG_RD, &adapter->stats.cbtmpc,
5217                         "Circuit Breaker Tx Packet Count");
5218
5219         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
5220                         CTLFLAG_RD, &adapter->stats.htdpmc,
5221                         "Host Transmit Discarded Packets");
5222
5223         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
5224                         CTLFLAG_RD, &adapter->stats.rpthc,
5225                         "Rx Packets To Host");
5226
5227         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
5228                         CTLFLAG_RD, &adapter->stats.cbrmpc,
5229                         "Circuit Breaker Rx Packet Count");
5230
5231         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
5232                         CTLFLAG_RD, &adapter->stats.cbrdpc,
5233                         "Circuit Breaker Rx Dropped Count");
5234
5235         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
5236                         CTLFLAG_RD, &adapter->stats.hgptc,
5237                         "Host Good Packets Tx Count");
5238
5239         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
5240                         CTLFLAG_RD, &adapter->stats.htcbdpc,
5241                         "Host Tx Circuit Breaker Dropped Count");
5242
5243         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
5244                         CTLFLAG_RD, &adapter->stats.hgorc,
5245                         "Host Good Octets Received Count");
5246
5247         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
5248                         CTLFLAG_RD, &adapter->stats.hgotc,
5249                         "Host Good Octets Transmit Count");
5250
5251         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
5252                         CTLFLAG_RD, &adapter->stats.lenerrs,
5253                         "Length Errors");
5254
5255         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
5256                         CTLFLAG_RD, &adapter->stats.scvpc,
5257                         "SerDes/SGMII Code Violation Pkt Count");
5258
5259         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
5260                         CTLFLAG_RD, &adapter->stats.hrmpc,
5261                         "Header Redirection Missed Packet Count");
5262
5263
5264 }
5265 /**********************************************************************
5266  *
5267  *  This routine provides a way to dump out the adapter eeprom,
5268  *  often a useful debug/service tool. This only dumps the first
5269  *  32 words, stuff that matters is in that extent.
5270  *
5271  **********************************************************************/
5272 static int
5273 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5274 {
5275         struct adapter *adapter;
5276         int error;
5277         int result;
5278
5279         result = -1;
5280         error = sysctl_handle_int(oidp, &result, 0, req);
5281
5282         if (error || !req->newptr)
5283                 return (error);
5284
5285         /*
5286          * This value will cause a hex dump of the
5287          * first 32 16-bit words of the EEPROM to
5288          * the screen.
5289          */
5290         if (result == 1) {
5291                 adapter = (struct adapter *)arg1;
5292                 igb_print_nvm_info(adapter);
5293         }
5294
5295         return (error);
5296 }
5297
5298 static void
5299 igb_print_nvm_info(struct adapter *adapter)
5300 {
5301         u16     eeprom_data;
5302         int     i, j, row = 0;
5303
5304         /* Its a bit crude, but it gets the job done */
5305         printf("\nInterface EEPROM Dump:\n");
5306         printf("Offset\n0x0000  ");
5307         for (i = 0, j = 0; i < 32; i++, j++) {
5308                 if (j == 8) { /* Make the offset block */
5309                         j = 0; ++row;
5310                         printf("\n0x00%x0  ",row);
5311                 }
5312                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5313                 printf("%04x ", eeprom_data);
5314         }
5315         printf("\n");
5316 }
5317
5318 static void
5319 igb_add_rx_process_limit(struct adapter *adapter, const char *name,
5320         const char *description, int *limit, int value)
5321 {
5322         *limit = value;
5323         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5324             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5325             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5326 }