sys/dev/e1000/if_em.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2010, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #endif
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #if __FreeBSD_version >= 800000
  43 #include <sys/buf_ring.h>
  44 #endif
  45 #include <sys/bus.h>
  46 #include <sys/endian.h>
  47 #include <sys/kernel.h>
  48 #include <sys/kthread.h>
  49 #include <sys/malloc.h>
  50 #include <sys/mbuf.h>
  51 #include <sys/module.h>
  52 #include <sys/rman.h>
  53 #include <sys/socket.h>
  54 #include <sys/sockio.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/taskqueue.h>
  57 #include <sys/eventhandler.h>
  58 #include <machine/bus.h>
  59 #include <machine/resource.h>
  60
  61 #include <net/bpf.h>
  62 #include <net/ethernet.h>
  63 #include <net/if.h>
  64 #include <net/if_arp.h>
  65 #include <net/if_dl.h>
  66 #include <net/if_media.h>
  67
  68 #include <net/if_types.h>
  69 #include <net/if_vlan_var.h>
  70
  71 #include <netinet/in_systm.h>
  72 #include <netinet/in.h>
  73 #include <netinet/if_ether.h>
  74 #include <netinet/ip.h>
  75 #include <netinet/ip6.h>
  76 #include <netinet/tcp.h>
  77 #include <netinet/udp.h>
  78
  79 #include <machine/in_cksum.h>
  80 #include <dev/led/led.h>
  81 #include <dev/pci/pcivar.h>
  82 #include <dev/pci/pcireg.h>
  83
  84 #include "e1000_api.h"
  85 #include "e1000_82571.h"
  86 #include "if_em.h"
  87
  88 /*********************************************************************
  89  *  Set this to one to display debug statistics
  90  *********************************************************************/
  91 int     em_display_debug_stats = 0;
  92
  93 /*********************************************************************
  94  *  Driver version:
  95  *********************************************************************/
  96 char em_driver_version[] = "7.0.5";
  97
  98
  99 /*********************************************************************
 100  *  PCI Device ID Table
 101  *
 102  *  Used by probe to select devices to load on
 103  *  Last field stores an index into e1000_strings
 104  *  Last entry must be all 0s
 105  *
 106  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 107  *********************************************************************/
 108
 109 static em_vendor_info_t em_vendor_info_array[] =
 110 {
 111         /* Intel(R) PRO/1000 Network Connection */
 112         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 113         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 114         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
 116                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
 118                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 119         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 120                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
 122                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
 124                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
 126                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 127         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 128         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 129         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 131
 132         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
 136         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 137                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 138         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 139                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 141                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 142         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 143                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 144         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_ICH8_82567V_3,   PCI_ANY_ID, PCI_ANY_ID, 0},
 152         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 153         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 154         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 155         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 157         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 158         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 159         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 160         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 162         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
 163         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 164         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 165         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 166         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 167         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 168         { 0x8086, E1000_DEV_ID_PCH_M_HV_LM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 169         { 0x8086, E1000_DEV_ID_PCH_M_HV_LC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 170         { 0x8086, E1000_DEV_ID_PCH_D_HV_DM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 171         { 0x8086, E1000_DEV_ID_PCH_D_HV_DC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 172         /* required last entry */
 173         { 0, 0, 0, 0, 0}
 174 };
 175
 176 /*********************************************************************
 177  *  Table of branding strings for all supported NICs.
 178  *********************************************************************/
 179
 180 static char *em_strings[] = {
 181         "Intel(R) PRO/1000 Network Connection"
 182 };
 183
 184 /*********************************************************************
 185  *  Function prototypes
 186  *********************************************************************/
 187 static int      em_probe(device_t);
 188 static int      em_attach(device_t);
 189 static int      em_detach(device_t);
 190 static int      em_shutdown(device_t);
 191 static int      em_suspend(device_t);
 192 static int      em_resume(device_t);
 193 static void     em_start(struct ifnet *);
 194 static void     em_start_locked(struct ifnet *, struct tx_ring *);
 195 #ifdef EM_MULTIQUEUE
 196 static int      em_mq_start(struct ifnet *, struct mbuf *);
 197 static int      em_mq_start_locked(struct ifnet *,
 198                     struct tx_ring *, struct mbuf *);
 199 static void     em_qflush(struct ifnet *);
 200 #endif
 201 static int      em_ioctl(struct ifnet *, u_long, caddr_t);
 202 static void     em_init(void *);
 203 static void     em_init_locked(struct adapter *);
 204 static void     em_stop(void *);
 205 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 206 static int      em_media_change(struct ifnet *);
 207 static void     em_identify_hardware(struct adapter *);
 208 static int      em_allocate_pci_resources(struct adapter *);
 209 static int      em_allocate_legacy(struct adapter *);
 210 static int      em_allocate_msix(struct adapter *);
 211 static int      em_allocate_queues(struct adapter *);
 212 static int      em_setup_msix(struct adapter *);
 213 static void     em_free_pci_resources(struct adapter *);
 214 static void     em_local_timer(void *);
 215 static void     em_reset(struct adapter *);
 216 static void     em_setup_interface(device_t, struct adapter *);
 217
 218 static void     em_setup_transmit_structures(struct adapter *);
 219 static void     em_initialize_transmit_unit(struct adapter *);
 220 static int      em_allocate_transmit_buffers(struct tx_ring *);
 221 static void     em_free_transmit_structures(struct adapter *);
 222 static void     em_free_transmit_buffers(struct tx_ring *);
 223
 224 static int      em_setup_receive_structures(struct adapter *);
 225 static int      em_allocate_receive_buffers(struct rx_ring *);
 226 static void     em_initialize_receive_unit(struct adapter *);
 227 static void     em_free_receive_structures(struct adapter *);
 228 static void     em_free_receive_buffers(struct rx_ring *);
 229
 230 static void     em_enable_intr(struct adapter *);
 231 static void     em_disable_intr(struct adapter *);
 232 static void     em_update_stats_counters(struct adapter *);
 233 static bool     em_txeof(struct tx_ring *);
 234 static int      em_rxeof(struct rx_ring *, int);
 235 #ifndef __NO_STRICT_ALIGNMENT
 236 static int      em_fixup_rx(struct rx_ring *);
 237 #endif
 238 static void     em_receive_checksum(struct e1000_rx_desc *, struct mbuf *);
 239 static void     em_transmit_checksum_setup(struct tx_ring *, struct mbuf *,
 240                     u32 *, u32 *);
 241 static bool     em_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 242 static void     em_set_promisc(struct adapter *);
 243 static void     em_disable_promisc(struct adapter *);
 244 static void     em_set_multi(struct adapter *);
 245 static void     em_print_hw_stats(struct adapter *);
 246 static void     em_update_link_status(struct adapter *);
 247 static void     em_refresh_mbufs(struct rx_ring *, int);
 248 static void     em_register_vlan(void *, struct ifnet *, u16);
 249 static void     em_unregister_vlan(void *, struct ifnet *, u16);
 250 static void     em_setup_vlan_hw_support(struct adapter *);
 251 static int      em_xmit(struct tx_ring *, struct mbuf **);
 252 static int      em_dma_malloc(struct adapter *, bus_size_t,
 253                     struct em_dma_alloc *, int);
 254 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 255 static void     em_print_debug_info(struct adapter *);
 256 static void     em_print_nvm_info(struct adapter *);
 257 static int      em_is_valid_ether_addr(u8 *);
 258 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 259 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 260 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 261 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 262                     const char *, struct em_int_delay_info *, int, int);
 263 /* Management and WOL Support */
 264 static void     em_init_manageability(struct adapter *);
 265 static void     em_release_manageability(struct adapter *);
 266 static void     em_get_hw_control(struct adapter *);
 267 static void     em_release_hw_control(struct adapter *);
 268 static void     em_get_wakeup(device_t);
 269 static void     em_enable_wakeup(device_t);
 270 static int      em_enable_phy_wakeup(struct adapter *);
 271 static void     em_led_func(void *, int);
 272
 273 static int      em_irq_fast(void *);
 274
 275 /* MSIX handlers */
 276 static void     em_msix_tx(void *);
 277 static void     em_msix_rx(void *);
 278 static void     em_msix_link(void *);
 279 static void     em_handle_tx(void *context, int pending);
 280 static void     em_handle_rx(void *context, int pending);
 281 static void     em_handle_link(void *context, int pending);
 282
 283 static void     em_add_rx_process_limit(struct adapter *, const char *,
 284                     const char *, int *, int);
 285
 286 #ifdef DEVICE_POLLING
 287 static poll_handler_t em_poll;
 288 #endif /* POLLING */
 289
 290 /*********************************************************************
 291  *  FreeBSD Device Interface Entry Points
 292  *********************************************************************/
 293
 294 static device_method_t em_methods[] = {
 295         /* Device interface */
 296         DEVMETHOD(device_probe, em_probe),
 297         DEVMETHOD(device_attach, em_attach),
 298         DEVMETHOD(device_detach, em_detach),
 299         DEVMETHOD(device_shutdown, em_shutdown),
 300         DEVMETHOD(device_suspend, em_suspend),
 301         DEVMETHOD(device_resume, em_resume),
 302         {0, 0}
 303 };
 304
 305 static driver_t em_driver = {
 306         "em", em_methods, sizeof(struct adapter),
 307 };
 308
 309 devclass_t em_devclass;
 310 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 311 MODULE_DEPEND(em, pci, 1, 1, 1);
 312 MODULE_DEPEND(em, ether, 1, 1, 1);
 313
 314 /*********************************************************************
 315  *  Tunable default values.
 316  *********************************************************************/
 317
 318 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 319 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 320 #define M_TSO_LEN                       66
 321
 322 /* Allow common code without TSO */
 323 #ifndef CSUM_TSO
 324 #define CSUM_TSO        0
 325 #endif
 326
 327 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 328 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 329 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 330 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 331
 332 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 333 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 334 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 335 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 336
 337 static int em_rxd = EM_DEFAULT_RXD;
 338 static int em_txd = EM_DEFAULT_TXD;
 339 TUNABLE_INT("hw.em.rxd", &em_rxd);
 340 TUNABLE_INT("hw.em.txd", &em_txd);
 341
 342 static int em_smart_pwr_down = FALSE;
 343 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 344
 345 /* Controls whether promiscuous also shows bad packets */
 346 static int em_debug_sbp = FALSE;
 347 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 348
 349 /* Local controls for MSI/MSIX */
 350 #ifdef EM_MULTIQUEUE
 351 static int em_enable_msix = TRUE;
 352 static int em_msix_queues = 2; /* for 82574, can be 1 or 2 */
 353 #else
 354 static int em_enable_msix = FALSE;
 355 static int em_msix_queues = 0; /* disable */
 356 #endif
 357 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix);
 358 TUNABLE_INT("hw.em.msix_queues", &em_msix_queues);
 359
 360 /* How many packets rxeof tries to clean at a time */
 361 static int em_rx_process_limit = 100;
 362 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
 363
 364 /* Flow control setting - default to FULL */
 365 static int em_fc_setting = e1000_fc_full;
 366 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting);
 367
 368 /*
 369 ** Shadow VFTA table, this is needed because
 370 ** the real vlan filter table gets cleared during
 371 ** a soft reset and the driver needs to be able
 372 ** to repopulate it.
 373 */
 374 static u32 em_shadow_vfta[EM_VFTA_SIZE];
 375
 376 /* Global used in WOL setup with multiport cards */
 377 static int global_quad_port_a = 0;
 378
 379 /*********************************************************************
 380  *  Device identification routine
 381  *
 382  *  em_probe determines if the driver should be loaded on
 383  *  adapter based on PCI vendor/device id of the adapter.
 384  *
 385  *  return BUS_PROBE_DEFAULT on success, positive on failure
 386  *********************************************************************/
 387
 388 static int
 389 em_probe(device_t dev)
 390 {
 391         char            adapter_name[60];
 392         u16             pci_vendor_id = 0;
 393         u16             pci_device_id = 0;
 394         u16             pci_subvendor_id = 0;
 395         u16             pci_subdevice_id = 0;
 396         em_vendor_info_t *ent;
 397
 398         INIT_DEBUGOUT("em_probe: begin");
 399
 400         pci_vendor_id = pci_get_vendor(dev);
 401         if (pci_vendor_id != EM_VENDOR_ID)
 402                 return (ENXIO);
 403
 404         pci_device_id = pci_get_device(dev);
 405         pci_subvendor_id = pci_get_subvendor(dev);
 406         pci_subdevice_id = pci_get_subdevice(dev);
 407
 408         ent = em_vendor_info_array;
 409         while (ent->vendor_id != 0) {
 410                 if ((pci_vendor_id == ent->vendor_id) &&
 411                     (pci_device_id == ent->device_id) &&
 412
 413                     ((pci_subvendor_id == ent->subvendor_id) ||
 414                     (ent->subvendor_id == PCI_ANY_ID)) &&
 415
 416                     ((pci_subdevice_id == ent->subdevice_id) ||
 417                     (ent->subdevice_id == PCI_ANY_ID))) {
 418                         sprintf(adapter_name, "%s %s",
 419                                 em_strings[ent->index],
 420                                 em_driver_version);
 421                         device_set_desc_copy(dev, adapter_name);
 422                         return (BUS_PROBE_DEFAULT);
 423                 }
 424                 ent++;
 425         }
 426
 427         return (ENXIO);
 428 }
 429
 430 /*********************************************************************
 431  *  Device initialization routine
 432  *
 433  *  The attach entry point is called when the driver is being loaded.
 434  *  This routine identifies the type of hardware, allocates all resources
 435  *  and initializes the hardware.
 436  *
 437  *  return 0 on success, positive on failure
 438  *********************************************************************/
 439
 440 static int
 441 em_attach(device_t dev)
 442 {
 443         struct adapter  *adapter;
 444         int             error = 0;
 445
 446         INIT_DEBUGOUT("em_attach: begin");
 447
 448         adapter = device_get_softc(dev);
 449         adapter->dev = adapter->osdep.dev = dev;
 450         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 451
 452         /* SYSCTL stuff */
 453         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 454             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 455             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 456             em_sysctl_debug_info, "I", "Debug Information");
 457
 458         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 459             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 460             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 461             em_sysctl_stats, "I", "Statistics");
 462
 463         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 464
 465         /* Determine hardware and mac info */
 466         em_identify_hardware(adapter);
 467
 468         /* Setup PCI resources */
 469         if (em_allocate_pci_resources(adapter)) {
 470                 device_printf(dev, "Allocation of PCI resources failed\n");
 471                 error = ENXIO;
 472                 goto err_pci;
 473         }
 474
 475         /*
 476         ** For ICH8 and family we need to
 477         ** map the flash memory, and this
 478         ** must happen after the MAC is
 479         ** identified
 480         */
 481         if ((adapter->hw.mac.type == e1000_ich8lan) ||
 482             (adapter->hw.mac.type == e1000_pchlan) ||
 483             (adapter->hw.mac.type == e1000_ich9lan) ||
 484             (adapter->hw.mac.type == e1000_ich10lan)) {
 485                 int rid = EM_BAR_TYPE_FLASH;
 486                 adapter->flash = bus_alloc_resource_any(dev,
 487                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 488                 if (adapter->flash == NULL) {
 489                         device_printf(dev, "Mapping of Flash failed\n");
 490                         error = ENXIO;
 491                         goto err_pci;
 492                 }
 493                 /* This is used in the shared code */
 494                 adapter->hw.flash_address = (u8 *)adapter->flash;
 495                 adapter->osdep.flash_bus_space_tag =
 496                     rman_get_bustag(adapter->flash);
 497                 adapter->osdep.flash_bus_space_handle =
 498                     rman_get_bushandle(adapter->flash);
 499         }
 500
 501         /* Do Shared Code initialization */
 502         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 503                 device_printf(dev, "Setup of Shared code failed\n");
 504                 error = ENXIO;
 505                 goto err_pci;
 506         }
 507
 508         e1000_get_bus_info(&adapter->hw);
 509
 510         /* Set up some sysctls for the tunable interrupt delays */
 511         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 512             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 513             E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
 514         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 515             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 516             E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
 517         em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 518             "receive interrupt delay limit in usecs",
 519             &adapter->rx_abs_int_delay,
 520             E1000_REGISTER(&adapter->hw, E1000_RADV),
 521             em_rx_abs_int_delay_dflt);
 522         em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 523             "transmit interrupt delay limit in usecs",
 524             &adapter->tx_abs_int_delay,
 525             E1000_REGISTER(&adapter->hw, E1000_TADV),
 526             em_tx_abs_int_delay_dflt);
 527
 528         /* Sysctls for limiting the amount of work done in the taskqueue */
 529         em_add_rx_process_limit(adapter, "rx_processing_limit",
 530             "max number of rx packets to process", &adapter->rx_process_limit,
 531             em_rx_process_limit);
 532
 533         /*
 534          * Validate number of transmit and receive descriptors. It
 535          * must not exceed hardware maximum, and must be multiple
 536          * of E1000_DBA_ALIGN.
 537          */
 538         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 539             (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) {
 540                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 541                     EM_DEFAULT_TXD, em_txd);
 542                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 543         } else
 544                 adapter->num_tx_desc = em_txd;
 545
 546         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 547             (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) {
 548                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 549                     EM_DEFAULT_RXD, em_rxd);
 550                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 551         } else
 552                 adapter->num_rx_desc = em_rxd;
 553
 554         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 555         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 556         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 557
 558         /* Copper options */
 559         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 560                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 561                 adapter->hw.phy.disable_polarity_correction = FALSE;
 562                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 563         }
 564
 565         /*
 566          * Set the frame limits assuming
 567          * standard ethernet sized frames.
 568          */
 569         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 570         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 571
 572         /*
 573          * This controls when hardware reports transmit completion
 574          * status.
 575          */
 576         adapter->hw.mac.report_tx_early = 1;
 577
 578         /*
 579         ** Get queue/ring memory
 580         */
 581         if (em_allocate_queues(adapter)) {
 582                 error = ENOMEM;
 583                 goto err_pci;
 584         }
 585
 586         /*
 587         ** Start from a known state, this is
 588         ** important in reading the nvm and
 589         ** mac from that.
 590         */
 591         e1000_reset_hw(&adapter->hw);
 592
 593         /* Make sure we have a good EEPROM before we read from it */
 594         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 595                 /*
 596                 ** Some PCI-E parts fail the first check due to
 597                 ** the link being in sleep state, call it again,
 598                 ** if it fails a second time its a real issue.
 599                 */
 600                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 601                         device_printf(dev,
 602                             "The EEPROM Checksum Is Not Valid\n");
 603                         error = EIO;
 604                         goto err_late;
 605                 }
 606         }
 607
 608         /* Copy the permanent MAC address out of the EEPROM */
 609         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 610                 device_printf(dev, "EEPROM read error while reading MAC"
 611                     " address\n");
 612                 error = EIO;
 613                 goto err_late;
 614         }
 615
 616         if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
 617                 device_printf(dev, "Invalid MAC address\n");
 618                 error = EIO;
 619                 goto err_late;
 620         }
 621
 622         /*
 623         **  Do interrupt configuration
 624         */
 625         if (adapter->msix > 1) /* Do MSIX */
 626                 error = em_allocate_msix(adapter);
 627         else  /* MSI or Legacy */
 628                 error = em_allocate_legacy(adapter);
 629         if (error)
 630                 goto err_late;
 631
 632         /*
 633          * Get Wake-on-Lan and Management info for later use
 634          */
 635         em_get_wakeup(dev);
 636
 637         /* Setup OS specific network interface */
 638         em_setup_interface(dev, adapter);
 639
 640         em_reset(adapter);
 641
 642         /* Initialize statistics */
 643         em_update_stats_counters(adapter);
 644
 645         adapter->hw.mac.get_link_status = 1;
 646         em_update_link_status(adapter);
 647
 648         /* Indicate SOL/IDER usage */
 649         if (e1000_check_reset_block(&adapter->hw))
 650                 device_printf(dev,
 651                     "PHY reset is blocked due to SOL/IDER session.\n");
 652
 653         /* Register for VLAN events */
 654         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 655             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 656         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 657             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 658
 659         /* Non-AMT based hardware can now take control from firmware */
 660         if (adapter->has_manage && !adapter->has_amt)
 661                 em_get_hw_control(adapter);
 662
 663         /* Tell the stack that the interface is not active */
 664         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 665
 666         adapter->led_dev = led_create(em_led_func, adapter,
 667             device_get_nameunit(dev));
 668
 669         INIT_DEBUGOUT("em_attach: end");
 670
 671         return (0);
 672
 673 err_late:
 674         em_free_transmit_structures(adapter);
 675         em_free_receive_structures(adapter);
 676         em_release_hw_control(adapter);
 677 err_pci:
 678         em_free_pci_resources(adapter);
 679         EM_CORE_LOCK_DESTROY(adapter);
 680
 681         return (error);
 682 }
 683
 684 /*********************************************************************
 685  *  Device removal routine
 686  *
 687  *  The detach entry point is called when the driver is being removed.
 688  *  This routine stops the adapter and deallocates all the resources
 689  *  that were allocated for driver operation.
 690  *
 691  *  return 0 on success, positive on failure
 692  *********************************************************************/
 693
 694 static int
 695 em_detach(device_t dev)
 696 {
 697         struct adapter  *adapter = device_get_softc(dev);
 698         struct ifnet    *ifp = adapter->ifp;
 699
 700         INIT_DEBUGOUT("em_detach: begin");
 701
 702         /* Make sure VLANS are not using driver */
 703         if (adapter->ifp->if_vlantrunk != NULL) {
 704                 device_printf(dev,"Vlan in use, detach first\n");
 705                 return (EBUSY);
 706         }
 707
 708 #ifdef DEVICE_POLLING
 709         if (ifp->if_capenable & IFCAP_POLLING)
 710                 ether_poll_deregister(ifp);
 711 #endif
 712
 713         EM_CORE_LOCK(adapter);
 714         adapter->in_detach = 1;
 715         em_stop(adapter);
 716         EM_CORE_UNLOCK(adapter);
 717         EM_CORE_LOCK_DESTROY(adapter);
 718
 719         e1000_phy_hw_reset(&adapter->hw);
 720
 721         em_release_manageability(adapter);
 722         em_release_hw_control(adapter);
 723
 724         /* Unregister VLAN events */
 725         if (adapter->vlan_attach != NULL)
 726                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 727         if (adapter->vlan_detach != NULL)
 728                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 729
 730         ether_ifdetach(adapter->ifp);
 731         callout_drain(&adapter->timer);
 732
 733         em_free_pci_resources(adapter);
 734         bus_generic_detach(dev);
 735         if_free(ifp);
 736
 737         em_free_transmit_structures(adapter);
 738         em_free_receive_structures(adapter);
 739
 740         em_release_hw_control(adapter);
 741
 742         return (0);
 743 }
 744
 745 /*********************************************************************
 746  *
 747  *  Shutdown entry point
 748  *
 749  **********************************************************************/
 750
 751 static int
 752 em_shutdown(device_t dev)
 753 {
 754         return em_suspend(dev);
 755 }
 756
 757 /*
 758  * Suspend/resume device methods.
 759  */
 760 static int
 761 em_suspend(device_t dev)
 762 {
 763         struct adapter *adapter = device_get_softc(dev);
 764
 765         EM_CORE_LOCK(adapter);
 766
 767         em_release_manageability(adapter);
 768         em_release_hw_control(adapter);
 769         em_enable_wakeup(dev);
 770
 771         EM_CORE_UNLOCK(adapter);
 772
 773         return bus_generic_suspend(dev);
 774 }
 775
 776 static int
 777 em_resume(device_t dev)
 778 {
 779         struct adapter *adapter = device_get_softc(dev);
 780         struct ifnet *ifp = adapter->ifp;
 781
 782         if (adapter->led_dev != NULL)
 783                 led_destroy(adapter->led_dev);
 784
 785         EM_CORE_LOCK(adapter);
 786         em_init_locked(adapter);
 787         em_init_manageability(adapter);
 788         EM_CORE_UNLOCK(adapter);
 789         em_start(ifp);
 790
 791         return bus_generic_resume(dev);
 792 }
 793
 794
 795 /*********************************************************************
 796  *  Transmit entry point
 797  *
 798  *  em_start is called by the stack to initiate a transmit.
 799  *  The driver will remain in this routine as long as there are
 800  *  packets to transmit and transmit resources are available.
 801  *  In case resources are not available stack is notified and
 802  *  the packet is requeued.
 803  **********************************************************************/
 804
 805 #ifdef EM_MULTIQUEUE
 806 static int
 807 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 808 {
 809         struct adapter  *adapter = txr->adapter;
 810         struct mbuf     *next;
 811         int             err = 0, enq = 0;
 812
 813         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 814             IFF_DRV_RUNNING || adapter->link_active == 0) {
 815                 if (m != NULL)
 816                         err = drbr_enqueue(ifp, txr->br, m);
 817                 return (err);
 818         }
 819
 820         /* Call cleanup if number of TX descriptors low */
 821         if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
 822                 em_txeof(txr);
 823
 824         enq = 0;
 825         if (m == NULL) {
 826                 next = drbr_dequeue(ifp, txr->br);
 827         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 828                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 829                         return (err);
 830                 next = drbr_dequeue(ifp, txr->br);
 831         } else
 832                 next = m;
 833
 834         /* Process the queue */
 835         while (next != NULL) {
 836                 if ((err = em_xmit(txr, &next)) != 0) {
 837                         if (next != NULL)
 838                                 err = drbr_enqueue(ifp, txr->br, next);
 839                         break;
 840                 }
 841                 enq++;
 842                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 843                 ETHER_BPF_MTAP(ifp, next);
 844                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 845                         break;
 846                 if (txr->tx_avail < EM_MAX_SCATTER) {
 847                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 848                         break;
 849                 }
 850                 next = drbr_dequeue(ifp, txr->br);
 851         }
 852
 853         if (enq > 0) {
 854                 /* Set the watchdog */
 855                 txr->watchdog_check = TRUE;
 856                 txr->watchdog_time = ticks;
 857         }
 858         return (err);
 859 }
 860
 861 /*
 862 ** Multiqueue capable stack interface, this is not
 863 ** yet truely multiqueue, but that is coming...
 864 */
 865 static int
 866 em_mq_start(struct ifnet *ifp, struct mbuf *m)
 867 {
 868         struct adapter  *adapter = ifp->if_softc;
 869         struct tx_ring  *txr;
 870         int             i, error = 0;
 871
 872         /* Which queue to use */
 873         if ((m->m_flags & M_FLOWID) != 0)
 874                 i = m->m_pkthdr.flowid % adapter->num_queues;
 875         else
 876                 i = curcpu % adapter->num_queues;
 877
 878         txr = &adapter->tx_rings[i];
 879
 880         if (EM_TX_TRYLOCK(txr)) {
 881                 error = em_mq_start_locked(ifp, txr, m);
 882                 EM_TX_UNLOCK(txr);
 883         } else
 884                 error = drbr_enqueue(ifp, txr->br, m);
 885
 886         return (error);
 887 }
 888
 889 /*
 890 ** Flush all ring buffers
 891 */
 892 static void
 893 em_qflush(struct ifnet *ifp)
 894 {
 895         struct adapter  *adapter = ifp->if_softc;
 896         struct tx_ring  *txr = adapter->tx_rings;
 897         struct mbuf     *m;
 898
 899         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 900                 EM_TX_LOCK(txr);
 901                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 902                         m_freem(m);
 903                 EM_TX_UNLOCK(txr);
 904         }
 905         if_qflush(ifp);
 906 }
 907
 908 #endif /* EM_MULTIQUEUE */
 909
 910 static void
 911 em_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 912 {
 913         struct adapter  *adapter = ifp->if_softc;
 914         struct mbuf     *m_head;
 915
 916         EM_TX_LOCK_ASSERT(txr);
 917
 918         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 919             IFF_DRV_RUNNING)
 920                 return;
 921
 922         if (!adapter->link_active)
 923                 return;
 924
 925         /* Call cleanup if number of TX descriptors low */
 926         if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
 927                 em_txeof(txr);
 928
 929         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 930                 if (txr->tx_avail < EM_MAX_SCATTER) {
 931                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 932                         break;
 933                 }
 934                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 935                 if (m_head == NULL)
 936                         break;
 937                 /*
 938                  *  Encapsulation can modify our pointer, and or make it
 939                  *  NULL on failure.  In that event, we can't requeue.
 940                  */
 941                 if (em_xmit(txr, &m_head)) {
 942                         if (m_head == NULL)
 943                                 break;
 944                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 945                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 946                         break;
 947                 }
 948
 949                 /* Send a copy of the frame to the BPF listener */
 950                 ETHER_BPF_MTAP(ifp, m_head);
 951
 952                 /* Set timeout in case hardware has problems transmitting. */
 953                 txr->watchdog_time = ticks;
 954                 txr->watchdog_check = TRUE;
 955         }
 956
 957         return;
 958 }
 959
 960 static void
 961 em_start(struct ifnet *ifp)
 962 {
 963         struct adapter  *adapter = ifp->if_softc;
 964         struct tx_ring  *txr = adapter->tx_rings;
 965
 966         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 967                 EM_TX_LOCK(txr);
 968                 em_start_locked(ifp, txr);
 969                 EM_TX_UNLOCK(txr);
 970         }
 971         return;
 972 }
 973
 974 /*********************************************************************
 975  *  Ioctl entry point
 976  *
 977  *  em_ioctl is called when the user wants to configure the
 978  *  interface.
 979  *
 980  *  return 0 on success, positive on failure
 981  **********************************************************************/
 982
 983 static int
 984 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 985 {
 986         struct adapter  *adapter = ifp->if_softc;
 987         struct ifreq *ifr = (struct ifreq *)data;
 988 #ifdef INET
 989         struct ifaddr *ifa = (struct ifaddr *)data;
 990 #endif
 991         int error = 0;
 992
 993         if (adapter->in_detach)
 994                 return (error);
 995
 996         switch (command) {
 997         case SIOCSIFADDR:
 998 #ifdef INET
 999                 if (ifa->ifa_addr->sa_family == AF_INET) {
1000                         /*
1001                          * XXX
1002                          * Since resetting hardware takes a very long time
1003                          * and results in link renegotiation we only
1004                          * initialize the hardware only when it is absolutely
1005                          * required.
1006                          */
1007                         ifp->if_flags |= IFF_UP;
1008                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1009                                 EM_CORE_LOCK(adapter);
1010                                 em_init_locked(adapter);
1011                                 EM_CORE_UNLOCK(adapter);
1012                         }
1013                         arp_ifinit(ifp, ifa);
1014                 } else
1015 #endif
1016                         error = ether_ioctl(ifp, command, data);
1017                 break;
1018         case SIOCSIFMTU:
1019             {
1020                 int max_frame_size;
1021
1022                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1023
1024                 EM_CORE_LOCK(adapter);
1025                 switch (adapter->hw.mac.type) {
1026                 case e1000_82571:
1027                 case e1000_82572:
1028                 case e1000_ich9lan:
1029                 case e1000_ich10lan:
1030                 case e1000_82574:
1031                 case e1000_80003es2lan: /* 9K Jumbo Frame size */
1032                         max_frame_size = 9234;
1033                         break;
1034                 case e1000_pchlan:
1035                         max_frame_size = 4096;
1036                         break;
1037                         /* Adapters that do not support jumbo frames */
1038                 case e1000_82583:
1039                 case e1000_ich8lan:
1040                         max_frame_size = ETHER_MAX_LEN;
1041                         break;
1042                 default:
1043                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1044                 }
1045                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1046                     ETHER_CRC_LEN) {
1047                         EM_CORE_UNLOCK(adapter);
1048                         error = EINVAL;
1049                         break;
1050                 }
1051
1052                 ifp->if_mtu = ifr->ifr_mtu;
1053                 adapter->max_frame_size =
1054                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1055                 em_init_locked(adapter);
1056                 EM_CORE_UNLOCK(adapter);
1057                 break;
1058             }
1059         case SIOCSIFFLAGS:
1060                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1061                     SIOCSIFFLAGS (Set Interface Flags)");
1062                 EM_CORE_LOCK(adapter);
1063                 if (ifp->if_flags & IFF_UP) {
1064                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1065                                 if ((ifp->if_flags ^ adapter->if_flags) &
1066                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1067                                         em_disable_promisc(adapter);
1068                                         em_set_promisc(adapter);
1069                                 }
1070                         } else
1071                                 em_init_locked(adapter);
1072                 } else
1073                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1074                                 em_stop(adapter);
1075                 adapter->if_flags = ifp->if_flags;
1076                 EM_CORE_UNLOCK(adapter);
1077                 break;
1078         case SIOCADDMULTI:
1079         case SIOCDELMULTI:
1080                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1081                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1082                         EM_CORE_LOCK(adapter);
1083                         em_disable_intr(adapter);
1084                         em_set_multi(adapter);
1085 #ifdef DEVICE_POLLING
1086                         if (!(ifp->if_capenable & IFCAP_POLLING))
1087 #endif
1088                                 em_enable_intr(adapter);
1089                         EM_CORE_UNLOCK(adapter);
1090                 }
1091                 break;
1092         case SIOCSIFMEDIA:
1093                 /* Check SOL/IDER usage */
1094                 EM_CORE_LOCK(adapter);
1095                 if (e1000_check_reset_block(&adapter->hw)) {
1096                         EM_CORE_UNLOCK(adapter);
1097                         device_printf(adapter->dev, "Media change is"
1098                             " blocked due to SOL/IDER session.\n");
1099                         break;
1100                 }
1101                 EM_CORE_UNLOCK(adapter);
1102         case SIOCGIFMEDIA:
1103                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1104                     SIOCxIFMEDIA (Get/Set Interface Media)");
1105                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1106                 break;
1107         case SIOCSIFCAP:
1108             {
1109                 int mask, reinit;
1110
1111                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1112                 reinit = 0;
1113                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1114 #ifdef DEVICE_POLLING
1115                 if (mask & IFCAP_POLLING) {
1116                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1117                                 error = ether_poll_register(em_poll, ifp);
1118                                 if (error)
1119                                         return (error);
1120                                 EM_CORE_LOCK(adapter);
1121                                 em_disable_intr(adapter);
1122                                 ifp->if_capenable |= IFCAP_POLLING;
1123                                 EM_CORE_UNLOCK(adapter);
1124                         } else {
1125                                 error = ether_poll_deregister(ifp);
1126                                 /* Enable interrupt even in error case */
1127                                 EM_CORE_LOCK(adapter);
1128                                 em_enable_intr(adapter);
1129                                 ifp->if_capenable &= ~IFCAP_POLLING;
1130                                 EM_CORE_UNLOCK(adapter);
1131                         }
1132                 }
1133 #endif
1134                 if (mask & IFCAP_HWCSUM) {
1135                         ifp->if_capenable ^= IFCAP_HWCSUM;
1136                         reinit = 1;
1137                 }
1138                 if (mask & IFCAP_TSO4) {
1139                         ifp->if_capenable ^= IFCAP_TSO4;
1140                         reinit = 1;
1141                 }
1142                 if (mask & IFCAP_VLAN_HWTAGGING) {
1143                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1144                         reinit = 1;
1145                 }
1146                 if (mask & IFCAP_VLAN_HWFILTER) {
1147                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1148                         reinit = 1;
1149                 }
1150                 if ((mask & IFCAP_WOL) &&
1151                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
1152                         if (mask & IFCAP_WOL_MCAST)
1153                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1154                         if (mask & IFCAP_WOL_MAGIC)
1155                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1156                 }
1157                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1158                         em_init(adapter);
1159                 VLAN_CAPABILITIES(ifp);
1160                 break;
1161             }
1162
1163         default:
1164                 error = ether_ioctl(ifp, command, data);
1165                 break;
1166         }
1167
1168         return (error);
1169 }
1170
1171
1172 /*********************************************************************
1173  *  Init entry point
1174  *
1175  *  This routine is used in two ways. It is used by the stack as
1176  *  init entry point in network interface structure. It is also used
1177  *  by the driver as a hw/sw initialization routine to get to a
1178  *  consistent state.
1179  *
1180  *  return 0 on success, positive on failure
1181  **********************************************************************/
1182
1183 static void
1184 em_init_locked(struct adapter *adapter)
1185 {
1186         struct ifnet    *ifp = adapter->ifp;
1187         device_t        dev = adapter->dev;
1188         u32             pba;
1189
1190         INIT_DEBUGOUT("em_init: begin");
1191
1192         EM_CORE_LOCK_ASSERT(adapter);
1193
1194         em_disable_intr(adapter);
1195         callout_stop(&adapter->timer);
1196
1197         /*
1198          * Packet Buffer Allocation (PBA)
1199          * Writing PBA sets the receive portion of the buffer
1200          * the remainder is used for the transmit buffer.
1201          */
1202         switch (adapter->hw.mac.type) {
1203         /* Total Packet Buffer on these is 48K */
1204         case e1000_82571:
1205         case e1000_82572:
1206         case e1000_80003es2lan:
1207                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1208                 break;
1209         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1210                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1211                 break;
1212         case e1000_82574:
1213         case e1000_82583:
1214                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1215                 break;
1216         case e1000_ich9lan:
1217         case e1000_ich10lan:
1218         case e1000_pchlan:
1219                 pba = E1000_PBA_10K;
1220                 break;
1221         case e1000_ich8lan:
1222                 pba = E1000_PBA_8K;
1223                 break;
1224         default:
1225                 if (adapter->max_frame_size > 8192)
1226                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1227                 else
1228                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1229         }
1230
1231         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1232         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1233
1234         /* Get the latest mac address, User can use a LAA */
1235         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1236               ETHER_ADDR_LEN);
1237
1238         /* Put the address into the Receive Address Array */
1239         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1240
1241         /*
1242          * With the 82571 adapter, RAR[0] may be overwritten
1243          * when the other port is reset, we make a duplicate
1244          * in RAR[14] for that eventuality, this assures
1245          * the interface continues to function.
1246          */
1247         if (adapter->hw.mac.type == e1000_82571) {
1248                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1249                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1250                     E1000_RAR_ENTRIES - 1);
1251         }
1252
1253         /* Initialize the hardware */
1254         em_reset(adapter);
1255         em_update_link_status(adapter);
1256
1257         /* Setup VLAN support, basic and offload if available */
1258         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1259
1260         /* Use real VLAN Filter support? */
1261         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1262                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1263                         /* Use real VLAN Filter support */
1264                         em_setup_vlan_hw_support(adapter);
1265                 else {
1266                         u32 ctrl;
1267                         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1268                         ctrl |= E1000_CTRL_VME;
1269                         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1270                 }
1271         }
1272
1273         /* Set hardware offload abilities */
1274         ifp->if_hwassist = 0;
1275         if (ifp->if_capenable & IFCAP_TXCSUM)
1276                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1277         if (ifp->if_capenable & IFCAP_TSO4)
1278                 ifp->if_hwassist |= CSUM_TSO;
1279
1280         /* Configure for OS presence */
1281         em_init_manageability(adapter);
1282
1283         /* Prepare transmit descriptors and buffers */
1284         em_setup_transmit_structures(adapter);
1285         em_initialize_transmit_unit(adapter);
1286
1287         /* Setup Multicast table */
1288         em_set_multi(adapter);
1289
1290         /* Prepare receive descriptors and buffers */
1291         if (em_setup_receive_structures(adapter)) {
1292                 device_printf(dev, "Could not setup receive structures\n");
1293                 em_stop(adapter);
1294                 return;
1295         }
1296         em_initialize_receive_unit(adapter);
1297
1298         /* Don't lose promiscuous settings */
1299         em_set_promisc(adapter);
1300
1301         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1302         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1303
1304         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1305         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1306
1307         /* MSI/X configuration for 82574 */
1308         if (adapter->hw.mac.type == e1000_82574) {
1309                 int tmp;
1310                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1311                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1312                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1313                 /* Set the IVAR - interrupt vector routing. */
1314                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
1315         }
1316
1317 #ifdef DEVICE_POLLING
1318         /*
1319          * Only enable interrupts if we are not polling, make sure
1320          * they are off otherwise.
1321          */
1322         if (ifp->if_capenable & IFCAP_POLLING)
1323                 em_disable_intr(adapter);
1324         else
1325 #endif /* DEVICE_POLLING */
1326                 em_enable_intr(adapter);
1327
1328         /* AMT based hardware can now take control from firmware */
1329         if (adapter->has_manage && adapter->has_amt)
1330                 em_get_hw_control(adapter);
1331
1332         /* Don't reset the phy next time init gets called */
1333         adapter->hw.phy.reset_disable = TRUE;
1334 }
1335
1336 static void
1337 em_init(void *arg)
1338 {
1339         struct adapter *adapter = arg;
1340
1341         EM_CORE_LOCK(adapter);
1342         em_init_locked(adapter);
1343         EM_CORE_UNLOCK(adapter);
1344 }
1345
1346
1347 #ifdef DEVICE_POLLING
1348 /*********************************************************************
1349  *
1350  *  Legacy polling routine: note this only works with single queue
1351  *
1352  *********************************************************************/
1353 static int
1354 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1355 {
1356         struct adapter *adapter = ifp->if_softc;
1357         struct tx_ring  *txr = adapter->tx_rings;
1358         struct rx_ring  *rxr = adapter->rx_rings;
1359         u32             reg_icr, rx_done = 0;
1360
1361         EM_CORE_LOCK(adapter);
1362         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1363                 EM_CORE_UNLOCK(adapter);
1364                 return (rx_done);
1365         }
1366
1367         if (cmd == POLL_AND_CHECK_STATUS) {
1368                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1369                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1370                         callout_stop(&adapter->timer);
1371                         adapter->hw.mac.get_link_status = 1;
1372                         em_update_link_status(adapter);
1373                         callout_reset(&adapter->timer, hz,
1374                             em_local_timer, adapter);
1375                 }
1376         }
1377         EM_CORE_UNLOCK(adapter);
1378
1379         rx_done = em_rxeof(rxr, count);
1380
1381         EM_TX_LOCK(txr);
1382         em_txeof(txr);
1383 #ifdef EM_MULTIQUEUE
1384         if (!drbr_empty(ifp, txr->br))
1385                 em_mq_start_locked(ifp, txr, NULL);
1386 #else
1387         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1388                 em_start_locked(ifp, txr);
1389 #endif
1390         EM_TX_UNLOCK(txr);
1391
1392         return (rx_done);
1393 }
1394 #endif /* DEVICE_POLLING */
1395
1396
1397 /*********************************************************************
1398  *
1399  *  Fast Legacy/MSI Combined Interrupt Service routine
1400  *
1401  *********************************************************************/
1402 static int
1403 em_irq_fast(void *arg)
1404 {
1405         struct adapter  *adapter = arg;
1406         struct ifnet    *ifp;
1407         u32             reg_icr;
1408
1409         ifp = adapter->ifp;
1410
1411         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1412
1413         /* Hot eject?  */
1414         if (reg_icr == 0xffffffff)
1415                 return FILTER_STRAY;
1416
1417         /* Definitely not our interrupt.  */
1418         if (reg_icr == 0x0)
1419                 return FILTER_STRAY;
1420
1421         /*
1422          * Starting with the 82571 chip, bit 31 should be used to
1423          * determine whether the interrupt belongs to us.
1424          */
1425         if (adapter->hw.mac.type >= e1000_82571 &&
1426             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1427                 return FILTER_STRAY;
1428
1429         em_disable_intr(adapter);
1430         taskqueue_enqueue(adapter->tq, &adapter->que_task);
1431
1432         /* Link status change */
1433         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1434                 adapter->hw.mac.get_link_status = 1;
1435                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1436         }
1437
1438         if (reg_icr & E1000_ICR_RXO)
1439                 adapter->rx_overruns++;
1440         return FILTER_HANDLED;
1441 }
1442
1443 /* Combined RX/TX handler, used by Legacy and MSI */
1444 static void
1445 em_handle_que(void *context, int pending)
1446 {
1447         struct adapter  *adapter = context;
1448         struct ifnet    *ifp = adapter->ifp;
1449         struct tx_ring  *txr = adapter->tx_rings;
1450         struct rx_ring  *rxr = adapter->rx_rings;
1451         bool            more_rx;
1452
1453
1454         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1455                 more_rx = em_rxeof(rxr, adapter->rx_process_limit);
1456
1457                 EM_TX_LOCK(txr);
1458                 em_txeof(txr);
1459 #ifdef EM_MULTIQUEUE
1460                 if (!drbr_empty(ifp, txr->br))
1461                         em_mq_start_locked(ifp, txr, NULL);
1462 #else
1463                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1464                         em_start_locked(ifp, txr);
1465 #endif
1466                 EM_TX_UNLOCK(txr);
1467                 if (more_rx) {
1468                         taskqueue_enqueue(adapter->tq, &adapter->que_task);
1469                         return;
1470                 }
1471         }
1472
1473         em_enable_intr(adapter);
1474         return;
1475 }
1476
1477
1478 /*********************************************************************
1479  *
1480  *  MSIX Interrupt Service Routines
1481  *
1482  **********************************************************************/
1483 static void
1484 em_msix_tx(void *arg)
1485 {
1486         struct tx_ring *txr = arg;
1487         struct adapter *adapter = txr->adapter;
1488         bool            more;
1489
1490         ++txr->tx_irq;
1491         EM_TX_LOCK(txr);
1492         more = em_txeof(txr);
1493         EM_TX_UNLOCK(txr);
1494         if (more)
1495                 taskqueue_enqueue(txr->tq, &txr->tx_task);
1496         else
1497                 /* Reenable this interrupt */
1498                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1499         return;
1500 }
1501
1502 /*********************************************************************
1503  *
1504  *  MSIX RX Interrupt Service routine
1505  *
1506  **********************************************************************/
1507
1508 static void
1509 em_msix_rx(void *arg)
1510 {
1511         struct rx_ring  *rxr = arg;
1512         struct adapter  *adapter = rxr->adapter;
1513         bool            more;
1514
1515         ++rxr->rx_irq;
1516         more = em_rxeof(rxr, adapter->rx_process_limit);
1517         if (more)
1518                 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1519         else
1520                 /* Reenable this interrupt */
1521                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1522         return;
1523 }
1524
1525 /*********************************************************************
1526  *
1527  *  MSIX Link Fast Interrupt Service routine
1528  *
1529  **********************************************************************/
1530 static void
1531 em_msix_link(void *arg)
1532 {
1533         struct adapter  *adapter = arg;
1534         u32             reg_icr;
1535
1536         ++adapter->link_irq;
1537         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1538
1539         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1540                 adapter->hw.mac.get_link_status = 1;
1541                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1542         } else
1543                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1544                     EM_MSIX_LINK | E1000_IMS_LSC);
1545         return;
1546 }
1547
1548 static void
1549 em_handle_rx(void *context, int pending)
1550 {
1551         struct rx_ring  *rxr = context;
1552         struct adapter  *adapter = rxr->adapter;
1553         bool            more;
1554
1555         more = em_rxeof(rxr, adapter->rx_process_limit);
1556         if (more)
1557                 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1558         else
1559                 /* Reenable this interrupt */
1560                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1561 }
1562
1563 static void
1564 em_handle_tx(void *context, int pending)
1565 {
1566         struct tx_ring  *txr = context;
1567         struct adapter  *adapter = txr->adapter;
1568         struct ifnet    *ifp = adapter->ifp;
1569
1570         if (!EM_TX_TRYLOCK(txr))
1571                 return;
1572
1573         em_txeof(txr);
1574
1575 #ifdef EM_MULTIQUEUE
1576         if (!drbr_empty(ifp, txr->br))
1577                 em_mq_start_locked(ifp, txr, NULL);
1578 #else
1579         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1580                 em_start_locked(ifp, txr);
1581 #endif
1582         E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1583         EM_TX_UNLOCK(txr);
1584 }
1585
1586 static void
1587 em_handle_link(void *context, int pending)
1588 {
1589         struct adapter  *adapter = context;
1590         struct ifnet *ifp = adapter->ifp;
1591
1592         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1593                 return;
1594
1595         EM_CORE_LOCK(adapter);
1596         callout_stop(&adapter->timer);
1597         em_update_link_status(adapter);
1598         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1599         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1600             EM_MSIX_LINK | E1000_IMS_LSC);
1601         EM_CORE_UNLOCK(adapter);
1602 }
1603
1604
1605 /*********************************************************************
1606  *
1607  *  Media Ioctl callback
1608  *
1609  *  This routine is called whenever the user queries the status of
1610  *  the interface using ifconfig.
1611  *
1612  **********************************************************************/
1613 static void
1614 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1615 {
1616         struct adapter *adapter = ifp->if_softc;
1617         u_char fiber_type = IFM_1000_SX;
1618
1619         INIT_DEBUGOUT("em_media_status: begin");
1620
1621         EM_CORE_LOCK(adapter);
1622         em_update_link_status(adapter);
1623
1624         ifmr->ifm_status = IFM_AVALID;
1625         ifmr->ifm_active = IFM_ETHER;
1626
1627         if (!adapter->link_active) {
1628                 EM_CORE_UNLOCK(adapter);
1629                 return;
1630         }
1631
1632         ifmr->ifm_status |= IFM_ACTIVE;
1633
1634         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1635             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1636                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1637         } else {
1638                 switch (adapter->link_speed) {
1639                 case 10:
1640                         ifmr->ifm_active |= IFM_10_T;
1641                         break;
1642                 case 100:
1643                         ifmr->ifm_active |= IFM_100_TX;
1644                         break;
1645                 case 1000:
1646                         ifmr->ifm_active |= IFM_1000_T;
1647                         break;
1648                 }
1649                 if (adapter->link_duplex == FULL_DUPLEX)
1650                         ifmr->ifm_active |= IFM_FDX;
1651                 else
1652                         ifmr->ifm_active |= IFM_HDX;
1653         }
1654         EM_CORE_UNLOCK(adapter);
1655 }
1656
1657 /*********************************************************************
1658  *
1659  *  Media Ioctl callback
1660  *
1661  *  This routine is called when the user changes speed/duplex using
1662  *  media/mediopt option with ifconfig.
1663  *
1664  **********************************************************************/
1665 static int
1666 em_media_change(struct ifnet *ifp)
1667 {
1668         struct adapter *adapter = ifp->if_softc;
1669         struct ifmedia  *ifm = &adapter->media;
1670
1671         INIT_DEBUGOUT("em_media_change: begin");
1672
1673         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1674                 return (EINVAL);
1675
1676         EM_CORE_LOCK(adapter);
1677         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1678         case IFM_AUTO:
1679                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1680                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1681                 break;
1682         case IFM_1000_LX:
1683         case IFM_1000_SX:
1684         case IFM_1000_T:
1685                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1686                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1687                 break;
1688         case IFM_100_TX:
1689                 adapter->hw.mac.autoneg = FALSE;
1690                 adapter->hw.phy.autoneg_advertised = 0;
1691                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1692                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1693                 else
1694                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1695                 break;
1696         case IFM_10_T:
1697                 adapter->hw.mac.autoneg = FALSE;
1698                 adapter->hw.phy.autoneg_advertised = 0;
1699                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1700                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1701                 else
1702                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1703                 break;
1704         default:
1705                 device_printf(adapter->dev, "Unsupported media type\n");
1706         }
1707
1708         /* As the speed/duplex settings my have changed we need to
1709          * reset the PHY.
1710          */
1711         adapter->hw.phy.reset_disable = FALSE;
1712
1713         em_init_locked(adapter);
1714         EM_CORE_UNLOCK(adapter);
1715
1716         return (0);
1717 }
1718
1719 /*********************************************************************
1720  *
1721  *  This routine maps the mbufs to tx descriptors.
1722  *
1723  *  return 0 on success, positive on failure
1724  **********************************************************************/
1725
1726 static int
1727 em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1728 {
1729         struct adapter          *adapter = txr->adapter;
1730         bus_dma_segment_t       segs[EM_MAX_SCATTER];
1731         bus_dmamap_t            map;
1732         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
1733         struct e1000_tx_desc    *ctxd = NULL;
1734         struct mbuf             *m_head;
1735         u32                     txd_upper, txd_lower, txd_used, txd_saved;
1736         int                     nsegs, i, j, first, last = 0;
1737         int                     error, do_tso, tso_desc = 0;
1738
1739         m_head = *m_headp;
1740         txd_upper = txd_lower = txd_used = txd_saved = 0;
1741         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1742
1743         /*
1744          * TSO workaround:
1745          *  If an mbuf is only header we need
1746          *     to pull 4 bytes of data into it.
1747          */
1748         if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
1749                 m_head = m_pullup(m_head, M_TSO_LEN + 4);
1750                 *m_headp = m_head;
1751                 if (m_head == NULL)
1752                         return (ENOBUFS);
1753         }
1754
1755         /*
1756          * Map the packet for DMA
1757          *
1758          * Capture the first descriptor index,
1759          * this descriptor will have the index
1760          * of the EOP which is the only one that
1761          * now gets a DONE bit writeback.
1762          */
1763         first = txr->next_avail_desc;
1764         tx_buffer = &txr->tx_buffers[first];
1765         tx_buffer_mapped = tx_buffer;
1766         map = tx_buffer->map;
1767
1768         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1769             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1770
1771         /*
1772          * There are two types of errors we can (try) to handle:
1773          * - EFBIG means the mbuf chain was too long and bus_dma ran
1774          *   out of segments.  Defragment the mbuf chain and try again.
1775          * - ENOMEM means bus_dma could not obtain enough bounce buffers
1776          *   at this point in time.  Defer sending and try again later.
1777          * All other errors, in particular EINVAL, are fatal and prevent the
1778          * mbuf chain from ever going through.  Drop it and report error.
1779          */
1780         if (error == EFBIG) {
1781                 struct mbuf *m;
1782
1783                 m = m_defrag(*m_headp, M_DONTWAIT);
1784                 if (m == NULL) {
1785                         adapter->mbuf_alloc_failed++;
1786                         m_freem(*m_headp);
1787                         *m_headp = NULL;
1788                         return (ENOBUFS);
1789                 }
1790                 *m_headp = m;
1791
1792                 /* Try it again */
1793                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1794                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1795
1796                 if (error) {
1797                         adapter->no_tx_dma_setup++;
1798                         m_freem(*m_headp);
1799                         *m_headp = NULL;
1800                         return (error);
1801                 }
1802         } else if (error != 0) {
1803                 adapter->no_tx_dma_setup++;
1804                 return (error);
1805         }
1806
1807         /*
1808          * TSO Hardware workaround, if this packet is not
1809          * TSO, and is only a single descriptor long, and
1810          * it follows a TSO burst, then we need to add a
1811          * sentinel descriptor to prevent premature writeback.
1812          */
1813         if ((do_tso == 0) && (txr->tx_tso == TRUE)) {
1814                 if (nsegs == 1)
1815                         tso_desc = TRUE;
1816                 txr->tx_tso = FALSE;
1817         }
1818
1819         if (nsegs > (txr->tx_avail - 2)) {
1820                 txr->no_desc_avail++;
1821                 bus_dmamap_unload(txr->txtag, map);
1822                 return (ENOBUFS);
1823         }
1824         m_head = *m_headp;
1825
1826         /* Do hardware assists */
1827 #if __FreeBSD_version >= 700000
1828         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1829                 error = em_tso_setup(txr, m_head, &txd_upper, &txd_lower);
1830                 if (error != TRUE)
1831                         return (ENXIO); /* something foobar */
1832                 /* we need to make a final sentinel transmit desc */
1833                 tso_desc = TRUE;
1834         } else
1835 #endif
1836         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
1837                 em_transmit_checksum_setup(txr,  m_head,
1838                     &txd_upper, &txd_lower);
1839
1840         i = txr->next_avail_desc;
1841
1842         /* Set up our transmit descriptors */
1843         for (j = 0; j < nsegs; j++) {
1844                 bus_size_t seg_len;
1845                 bus_addr_t seg_addr;
1846
1847                 tx_buffer = &txr->tx_buffers[i];
1848                 ctxd = &txr->tx_base[i];
1849                 seg_addr = segs[j].ds_addr;
1850                 seg_len  = segs[j].ds_len;
1851                 /*
1852                 ** TSO Workaround:
1853                 ** If this is the last descriptor, we want to
1854                 ** split it so we have a small final sentinel
1855                 */
1856                 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
1857                         seg_len -= 4;
1858                         ctxd->buffer_addr = htole64(seg_addr);
1859                         ctxd->lower.data = htole32(
1860                         adapter->txd_cmd | txd_lower | seg_len);
1861                         ctxd->upper.data =
1862                             htole32(txd_upper);
1863                         if (++i == adapter->num_tx_desc)
1864                                 i = 0;
1865                         /* Now make the sentinel */
1866                         ++txd_used; /* using an extra txd */
1867                         ctxd = &txr->tx_base[i];
1868                         tx_buffer = &txr->tx_buffers[i];
1869                         ctxd->buffer_addr =
1870                             htole64(seg_addr + seg_len);
1871                         ctxd->lower.data = htole32(
1872                         adapter->txd_cmd | txd_lower | 4);
1873                         ctxd->upper.data =
1874                             htole32(txd_upper);
1875                         last = i;
1876                         if (++i == adapter->num_tx_desc)
1877                                 i = 0;
1878                 } else {
1879                         ctxd->buffer_addr = htole64(seg_addr);
1880                         ctxd->lower.data = htole32(
1881                         adapter->txd_cmd | txd_lower | seg_len);
1882                         ctxd->upper.data =
1883                             htole32(txd_upper);
1884                         last = i;
1885                         if (++i == adapter->num_tx_desc)
1886                                 i = 0;
1887                 }
1888                 tx_buffer->m_head = NULL;
1889                 tx_buffer->next_eop = -1;
1890         }
1891
1892         txr->next_avail_desc = i;
1893         txr->tx_avail -= nsegs;
1894         if (tso_desc) /* TSO used an extra for sentinel */
1895                 txr->tx_avail -= txd_used;
1896
1897         if (m_head->m_flags & M_VLANTAG) {
1898                 /* Set the vlan id. */
1899                 ctxd->upper.fields.special =
1900                     htole16(m_head->m_pkthdr.ether_vtag);
1901                 /* Tell hardware to add tag */
1902                 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
1903         }
1904
1905         tx_buffer->m_head = m_head;
1906         tx_buffer_mapped->map = tx_buffer->map;
1907         tx_buffer->map = map;
1908         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1909
1910         /*
1911          * Last Descriptor of Packet
1912          * needs End Of Packet (EOP)
1913          * and Report Status (RS)
1914          */
1915         ctxd->lower.data |=
1916             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1917         /*
1918          * Keep track in the first buffer which
1919          * descriptor will be written back
1920          */
1921         tx_buffer = &txr->tx_buffers[first];
1922         tx_buffer->next_eop = last;
1923
1924         /*
1925          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
1926          * that this frame is available to transmit.
1927          */
1928         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1929             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1930         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1931
1932         return (0);
1933 }
1934
1935 static void
1936 em_set_promisc(struct adapter *adapter)
1937 {
1938         struct ifnet    *ifp = adapter->ifp;
1939         u32             reg_rctl;
1940
1941         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1942
1943         if (ifp->if_flags & IFF_PROMISC) {
1944                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1945                 /* Turn this on if you want to see bad packets */
1946                 if (em_debug_sbp)
1947                         reg_rctl |= E1000_RCTL_SBP;
1948                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1949         } else if (ifp->if_flags & IFF_ALLMULTI) {
1950                 reg_rctl |= E1000_RCTL_MPE;
1951                 reg_rctl &= ~E1000_RCTL_UPE;
1952                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1953         }
1954 }
1955
1956 static void
1957 em_disable_promisc(struct adapter *adapter)
1958 {
1959         u32     reg_rctl;
1960
1961         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1962
1963         reg_rctl &=  (~E1000_RCTL_UPE);
1964         reg_rctl &=  (~E1000_RCTL_MPE);
1965         reg_rctl &=  (~E1000_RCTL_SBP);
1966         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1967 }
1968
1969
1970 /*********************************************************************
1971  *  Multicast Update
1972  *
1973  *  This routine is called whenever multicast address list is updated.
1974  *
1975  **********************************************************************/
1976
1977 static void
1978 em_set_multi(struct adapter *adapter)
1979 {
1980         struct ifnet    *ifp = adapter->ifp;
1981         struct ifmultiaddr *ifma;
1982         u32 reg_rctl = 0;
1983         u8  *mta; /* Multicast array memory */
1984         int mcnt = 0;
1985
1986         IOCTL_DEBUGOUT("em_set_multi: begin");
1987
1988         if (adapter->hw.mac.type == e1000_82542 &&
1989             adapter->hw.revision_id == E1000_REVISION_2) {
1990                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1991                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1992                         e1000_pci_clear_mwi(&adapter->hw);
1993                 reg_rctl |= E1000_RCTL_RST;
1994                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1995                 msec_delay(5);
1996         }
1997
1998         /* Allocate temporary memory to setup array */
1999         mta = malloc(sizeof(u8) *
2000             (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
2001             M_DEVBUF, M_NOWAIT | M_ZERO);
2002         if (mta == NULL)
2003                 panic("em_set_multi memory failure\n");
2004
2005 #if __FreeBSD_version < 800000
2006         IF_ADDR_LOCK(ifp);
2007 #else
2008         if_maddr_rlock(ifp);
2009 #endif
2010         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2011                 if (ifma->ifma_addr->sa_family != AF_LINK)
2012                         continue;
2013
2014                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2015                         break;
2016
2017                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2018                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2019                 mcnt++;
2020         }
2021 #if __FreeBSD_version < 800000
2022         IF_ADDR_UNLOCK(ifp);
2023 #else
2024         if_maddr_runlock(ifp);
2025 #endif
2026         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2027                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2028                 reg_rctl |= E1000_RCTL_MPE;
2029                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2030         } else
2031                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2032
2033         if (adapter->hw.mac.type == e1000_82542 &&
2034             adapter->hw.revision_id == E1000_REVISION_2) {
2035                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2036                 reg_rctl &= ~E1000_RCTL_RST;
2037                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2038                 msec_delay(5);
2039                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2040                         e1000_pci_set_mwi(&adapter->hw);
2041         }
2042         free(mta, M_DEVBUF);
2043 }
2044
2045
2046 /*********************************************************************
2047  *  Timer routine
2048  *
2049  *  This routine checks for link status and updates statistics.
2050  *
2051  **********************************************************************/
2052
2053 static void
2054 em_local_timer(void *arg)
2055 {
2056         struct adapter  *adapter = arg;
2057         struct ifnet    *ifp = adapter->ifp;
2058         struct tx_ring  *txr = adapter->tx_rings;
2059
2060         EM_CORE_LOCK_ASSERT(adapter);
2061
2062         em_update_link_status(adapter);
2063         em_update_stats_counters(adapter);
2064
2065         /* Reset LAA into RAR[0] on 82571 */
2066         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2067                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2068
2069         if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
2070                 em_print_hw_stats(adapter);
2071
2072         /*
2073         ** Check for time since any descriptor was cleaned
2074         */
2075         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2076                 EM_TX_LOCK(txr);
2077                 if (txr->watchdog_check == FALSE) {
2078                         EM_TX_UNLOCK(txr);
2079                         continue;
2080                 }
2081                 if ((ticks - txr->watchdog_time) > EM_WATCHDOG)
2082                         goto hung;
2083                 EM_TX_UNLOCK(txr);
2084         }
2085
2086         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2087         return;
2088 hung:
2089         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2090         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2091         adapter->watchdog_events++;
2092         EM_TX_UNLOCK(txr);
2093         em_init_locked(adapter);
2094 }
2095
2096
2097 static void
2098 em_update_link_status(struct adapter *adapter)
2099 {
2100         struct e1000_hw *hw = &adapter->hw;
2101         struct ifnet *ifp = adapter->ifp;
2102         device_t dev = adapter->dev;
2103         u32 link_check = 0;
2104
2105         /* Get the cached link value or read phy for real */
2106         switch (hw->phy.media_type) {
2107         case e1000_media_type_copper:
2108                 if (hw->mac.get_link_status) {
2109                         /* Do the work to read phy */
2110                         e1000_check_for_link(hw);
2111                         link_check = !hw->mac.get_link_status;
2112                         if (link_check) /* ESB2 fix */
2113                                 e1000_cfg_on_link_up(hw);
2114                 } else
2115                         link_check = TRUE;
2116                 break;
2117         case e1000_media_type_fiber:
2118                 e1000_check_for_link(hw);
2119                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2120                                  E1000_STATUS_LU);
2121                 break;
2122         case e1000_media_type_internal_serdes:
2123                 e1000_check_for_link(hw);
2124                 link_check = adapter->hw.mac.serdes_has_link;
2125                 break;
2126         default:
2127         case e1000_media_type_unknown:
2128                 break;
2129         }
2130
2131         /* Now check for a transition */
2132         if (link_check && (adapter->link_active == 0)) {
2133                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2134                     &adapter->link_duplex);
2135                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2136                 if ((adapter->link_speed != SPEED_1000) &&
2137                     ((hw->mac.type == e1000_82571) ||
2138                     (hw->mac.type == e1000_82572))) {
2139                         int tarc0;
2140                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2141                         tarc0 &= ~SPEED_MODE_BIT;
2142                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2143                 }
2144                 if (bootverbose)
2145                         device_printf(dev, "Link is up %d Mbps %s\n",
2146                             adapter->link_speed,
2147                             ((adapter->link_duplex == FULL_DUPLEX) ?
2148                             "Full Duplex" : "Half Duplex"));
2149                 adapter->link_active = 1;
2150                 adapter->smartspeed = 0;
2151                 ifp->if_baudrate = adapter->link_speed * 1000000;
2152                 if_link_state_change(ifp, LINK_STATE_UP);
2153         } else if (!link_check && (adapter->link_active == 1)) {
2154                 ifp->if_baudrate = adapter->link_speed = 0;
2155                 adapter->link_duplex = 0;
2156                 if (bootverbose)
2157                         device_printf(dev, "Link is Down\n");
2158                 adapter->link_active = 0;
2159                 /* Link down, disable watchdog */
2160                 // JFV change later
2161                 //adapter->watchdog_check = FALSE;
2162                 if_link_state_change(ifp, LINK_STATE_DOWN);
2163         }
2164 }
2165
2166 /*********************************************************************
2167  *
2168  *  This routine disables all traffic on the adapter by issuing a
2169  *  global reset on the MAC and deallocates TX/RX buffers.
2170  *
2171  *  This routine should always be called with BOTH the CORE
2172  *  and TX locks.
2173  **********************************************************************/
2174
2175 static void
2176 em_stop(void *arg)
2177 {
2178         struct adapter  *adapter = arg;
2179         struct ifnet    *ifp = adapter->ifp;
2180         struct tx_ring  *txr = adapter->tx_rings;
2181
2182         EM_CORE_LOCK_ASSERT(adapter);
2183
2184         INIT_DEBUGOUT("em_stop: begin");
2185
2186         em_disable_intr(adapter);
2187         callout_stop(&adapter->timer);
2188
2189         /* Tell the stack that the interface is no longer active */
2190         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2191
2192         /* Unarm watchdog timer. */
2193         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2194                 EM_TX_LOCK(txr);
2195                 txr->watchdog_check = FALSE;
2196                 EM_TX_UNLOCK(txr);
2197         }
2198
2199         e1000_reset_hw(&adapter->hw);
2200         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2201
2202         e1000_led_off(&adapter->hw);
2203         e1000_cleanup_led(&adapter->hw);
2204 }
2205
2206
2207 /*********************************************************************
2208  *
2209  *  Determine hardware revision.
2210  *
2211  **********************************************************************/
2212 static void
2213 em_identify_hardware(struct adapter *adapter)
2214 {
2215         device_t dev = adapter->dev;
2216
2217         /* Make sure our PCI config space has the necessary stuff set */
2218         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2219         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2220             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2221                 device_printf(dev, "Memory Access and/or Bus Master bits "
2222                     "were not set!\n");
2223                 adapter->hw.bus.pci_cmd_word |=
2224                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2225                 pci_write_config(dev, PCIR_COMMAND,
2226                     adapter->hw.bus.pci_cmd_word, 2);
2227         }
2228
2229         /* Save off the information about this board */
2230         adapter->hw.vendor_id = pci_get_vendor(dev);
2231         adapter->hw.device_id = pci_get_device(dev);
2232         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2233         adapter->hw.subsystem_vendor_id =
2234             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2235         adapter->hw.subsystem_device_id =
2236             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2237
2238         /* Do Shared Code Init and Setup */
2239         if (e1000_set_mac_type(&adapter->hw)) {
2240                 device_printf(dev, "Setup init failure\n");
2241                 return;
2242         }
2243 }
2244
2245 static int
2246 em_allocate_pci_resources(struct adapter *adapter)
2247 {
2248         device_t        dev = adapter->dev;
2249         int             rid;
2250
2251         rid = PCIR_BAR(0);
2252         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2253             &rid, RF_ACTIVE);
2254         if (adapter->memory == NULL) {
2255                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2256                 return (ENXIO);
2257         }
2258         adapter->osdep.mem_bus_space_tag =
2259             rman_get_bustag(adapter->memory);
2260         adapter->osdep.mem_bus_space_handle =
2261             rman_get_bushandle(adapter->memory);
2262         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2263
2264         /* Default to a single queue */
2265         adapter->num_queues = 1;
2266
2267         /*
2268          * Setup MSI/X or MSI if PCI Express
2269          */
2270         adapter->msix = em_setup_msix(adapter);
2271
2272         adapter->hw.back = &adapter->osdep;
2273
2274         return (0);
2275 }
2276
2277 /*********************************************************************
2278  *
2279  *  Setup the Legacy or MSI Interrupt handler
2280  *
2281  **********************************************************************/
2282 int
2283 em_allocate_legacy(struct adapter *adapter)
2284 {
2285         device_t dev = adapter->dev;
2286         int error, rid = 0;
2287
2288         /* Manually turn off all interrupts */
2289         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2290
2291         if (adapter->msix == 1) /* using MSI */
2292                 rid = 1;
2293         /* We allocate a single interrupt resource */
2294         adapter->res = bus_alloc_resource_any(dev,
2295             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2296         if (adapter->res == NULL) {
2297                 device_printf(dev, "Unable to allocate bus resource: "
2298                     "interrupt\n");
2299                 return (ENXIO);
2300         }
2301
2302         /*
2303          * Allocate a fast interrupt and the associated
2304          * deferred processing contexts.
2305          */
2306         TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter);
2307         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2308         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2309             taskqueue_thread_enqueue, &adapter->tq);
2310         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2311             device_get_nameunit(adapter->dev));
2312         if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET,
2313             em_irq_fast, NULL, adapter, &adapter->tag)) != 0) {
2314                 device_printf(dev, "Failed to register fast interrupt "
2315                             "handler: %d\n", error);
2316                 taskqueue_free(adapter->tq);
2317                 adapter->tq = NULL;
2318                 return (error);
2319         }
2320
2321         return (0);
2322 }
2323
2324 /*********************************************************************
2325  *
2326  *  Setup the MSIX Interrupt handlers
2327  *   This is not really Multiqueue, rather
2328  *   its just multiple interrupt vectors.
2329  *
2330  **********************************************************************/
2331 int
2332 em_allocate_msix(struct adapter *adapter)
2333 {
2334         device_t        dev = adapter->dev;
2335         struct          tx_ring *txr = adapter->tx_rings;
2336         struct          rx_ring *rxr = adapter->rx_rings;
2337         int             error, rid, vector = 0;
2338
2339
2340         /* Make sure all interrupts are disabled */
2341         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2342
2343         /* First set up ring resources */
2344         for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
2345
2346                 /* RX ring */
2347                 rid = vector + 1;
2348
2349                 rxr->res = bus_alloc_resource_any(dev,
2350                     SYS_RES_IRQ, &rid, RF_ACTIVE);
2351                 if (rxr->res == NULL) {
2352                         device_printf(dev,
2353                             "Unable to allocate bus resource: "
2354                             "RX MSIX Interrupt %d\n", i);
2355                         return (ENXIO);
2356                 }
2357                 if ((error = bus_setup_intr(dev, rxr->res,
2358                     INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx,
2359                     rxr, &rxr->tag)) != 0) {
2360                         device_printf(dev, "Failed to register RX handler");
2361                         return (error);
2362                 }
2363                 rxr->msix = vector++; /* NOTE increment vector for TX */
2364                 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr);
2365                 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT,
2366                     taskqueue_thread_enqueue, &rxr->tq);
2367                 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq",
2368                     device_get_nameunit(adapter->dev));
2369                 /*
2370                 ** Set the bit to enable interrupt
2371                 ** in E1000_IMS -- bits 20 and 21
2372                 ** are for RX0 and RX1, note this has
2373                 ** NOTHING to do with the MSIX vector
2374                 */
2375                 rxr->ims = 1 << (20 + i);
2376                 adapter->ivars |= (8 | rxr->msix) << (i * 4);
2377
2378                 /* TX ring */
2379                 rid = vector + 1;
2380                 txr->res = bus_alloc_resource_any(dev,
2381                     SYS_RES_IRQ, &rid, RF_ACTIVE);
2382                 if (txr->res == NULL) {
2383                         device_printf(dev,
2384                             "Unable to allocate bus resource: "
2385                             "TX MSIX Interrupt %d\n", i);
2386                         return (ENXIO);
2387                 }
2388                 if ((error = bus_setup_intr(dev, txr->res,
2389                     INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx,
2390                     txr, &txr->tag)) != 0) {
2391                         device_printf(dev, "Failed to register TX handler");
2392                         return (error);
2393                 }
2394                 txr->msix = vector++; /* Increment vector for next pass */
2395                 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2396                 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2397                     taskqueue_thread_enqueue, &txr->tq);
2398                 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2399                     device_get_nameunit(adapter->dev));
2400                 /*
2401                 ** Set the bit to enable interrupt
2402                 ** in E1000_IMS -- bits 22 and 23
2403                 ** are for TX0 and TX1, note this has
2404                 ** NOTHING to do with the MSIX vector
2405                 */
2406                 txr->ims = 1 << (22 + i);
2407                 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4));
2408         }
2409
2410         /* Link interrupt */
2411         ++rid;
2412         adapter->res = bus_alloc_resource_any(dev,
2413             SYS_RES_IRQ, &rid, RF_ACTIVE);
2414         if (!adapter->res) {
2415                 device_printf(dev,"Unable to allocate "
2416                     "bus resource: Link interrupt [%d]\n", rid);
2417                 return (ENXIO);
2418         }
2419         /* Set the link handler function */
2420         error = bus_setup_intr(dev, adapter->res,
2421             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2422             em_msix_link, adapter, &adapter->tag);
2423         if (error) {
2424                 adapter->res = NULL;
2425                 device_printf(dev, "Failed to register LINK handler");
2426                 return (error);
2427         }
2428         adapter->linkvec = vector;
2429         adapter->ivars |=  (8 | vector) << 16;
2430         adapter->ivars |= 0x80000000;
2431         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2432         adapter->tq = taskqueue_create_fast("em_link", M_NOWAIT,
2433             taskqueue_thread_enqueue, &adapter->tq);
2434         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
2435             device_get_nameunit(adapter->dev));
2436
2437         return (0);
2438 }
2439
2440
2441 static void
2442 em_free_pci_resources(struct adapter *adapter)
2443 {
2444         device_t        dev = adapter->dev;
2445         struct tx_ring  *txr;
2446         struct rx_ring  *rxr;
2447         int             rid;
2448
2449
2450         /*
2451         ** Release all the queue interrupt resources:
2452         */
2453         for (int i = 0; i < adapter->num_queues; i++) {
2454                 txr = &adapter->tx_rings[i];
2455                 rxr = &adapter->rx_rings[i];
2456                 rid = txr->msix +1;
2457                 if (txr->tag != NULL) {
2458                         bus_teardown_intr(dev, txr->res, txr->tag);
2459                         txr->tag = NULL;
2460                 }
2461                 if (txr->res != NULL)
2462                         bus_release_resource(dev, SYS_RES_IRQ,
2463                             rid, txr->res);
2464                 rid = rxr->msix +1;
2465                 if (rxr->tag != NULL) {
2466                         bus_teardown_intr(dev, rxr->res, rxr->tag);
2467                         rxr->tag = NULL;
2468                 }
2469                 if (rxr->res != NULL)
2470                         bus_release_resource(dev, SYS_RES_IRQ,
2471                             rid, rxr->res);
2472         }
2473
2474         if (adapter->linkvec) /* we are doing MSIX */
2475                 rid = adapter->linkvec + 1;
2476         else
2477                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2478
2479         if (adapter->tag != NULL) {
2480                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2481                 adapter->tag = NULL;
2482         }
2483
2484         if (adapter->res != NULL)
2485                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2486
2487
2488         if (adapter->msix)
2489                 pci_release_msi(dev);
2490
2491         if (adapter->msix_mem != NULL)
2492                 bus_release_resource(dev, SYS_RES_MEMORY,
2493                     PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2494
2495         if (adapter->memory != NULL)
2496                 bus_release_resource(dev, SYS_RES_MEMORY,
2497                     PCIR_BAR(0), adapter->memory);
2498
2499         if (adapter->flash != NULL)
2500                 bus_release_resource(dev, SYS_RES_MEMORY,
2501                     EM_FLASH, adapter->flash);
2502 }
2503
2504 /*
2505  * Setup MSI or MSI/X
2506  */
2507 static int
2508 em_setup_msix(struct adapter *adapter)
2509 {
2510         device_t dev = adapter->dev;
2511         int val = 0;
2512
2513
2514         /* Setup MSI/X for Hartwell */
2515         if ((adapter->hw.mac.type == e1000_82574) &&
2516             (em_enable_msix == TRUE)) {
2517                 /* Map the MSIX BAR */
2518                 int rid = PCIR_BAR(EM_MSIX_BAR);
2519                 adapter->msix_mem = bus_alloc_resource_any(dev,
2520                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2521                 if (!adapter->msix_mem) {
2522                         /* May not be enabled */
2523                         device_printf(adapter->dev,
2524                             "Unable to map MSIX table \n");
2525                         goto msi;
2526                 }
2527                 val = pci_msix_count(dev);
2528                 if (val != 5) {
2529                         bus_release_resource(dev, SYS_RES_MEMORY,
2530                             PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2531                         adapter->msix_mem = NULL;
2532                         device_printf(adapter->dev,
2533                             "MSIX vectors wrong, using MSI \n");
2534                         goto msi;
2535                 }
2536                 if (em_msix_queues == 2) {
2537                         val = 5;
2538                         adapter->num_queues = 2;
2539                 } else {
2540                         val = 3;
2541                         adapter->num_queues = 1;
2542                 }
2543                 if (pci_alloc_msix(dev, &val) == 0) {
2544                         device_printf(adapter->dev,
2545                             "Using MSIX interrupts "
2546                             "with %d vectors\n", val);
2547                 }
2548
2549                 return (val);
2550         }
2551 msi:
2552         val = pci_msi_count(dev);
2553         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2554                 adapter->msix = 1;
2555                 device_printf(adapter->dev,"Using MSI interrupt\n");
2556                 return (val);
2557         }
2558         /* Should only happen due to manual invention */
2559         device_printf(adapter->dev,"Setup MSIX failure\n");
2560         return (0);
2561 }
2562
2563
2564 /*********************************************************************
2565  *
2566  *  Initialize the hardware to a configuration
2567  *  as specified by the adapter structure.
2568  *
2569  **********************************************************************/
2570 static void
2571 em_reset(struct adapter *adapter)
2572 {
2573         device_t        dev = adapter->dev;
2574         struct e1000_hw *hw = &adapter->hw;
2575         u16             rx_buffer_size;
2576
2577         INIT_DEBUGOUT("em_reset: begin");
2578
2579         /* Set up smart power down as default off on newer adapters. */
2580         if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
2581             hw->mac.type == e1000_82572)) {
2582                 u16 phy_tmp = 0;
2583
2584                 /* Speed up time to link by disabling smart power down. */
2585                 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2586                 phy_tmp &= ~IGP02E1000_PM_SPD;
2587                 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2588         }
2589
2590         /*
2591          * These parameters control the automatic generation (Tx) and
2592          * response (Rx) to Ethernet PAUSE frames.
2593          * - High water mark should allow for at least two frames to be
2594          *   received after sending an XOFF.
2595          * - Low water mark works best when it is very near the high water mark.
2596          *   This allows the receiver to restart by sending XON when it has
2597          *   drained a bit. Here we use an arbitary value of 1500 which will
2598          *   restart after one full frame is pulled from the buffer. There
2599          *   could be several smaller frames in the buffer and if so they will
2600          *   not trigger the XON until their total number reduces the buffer
2601          *   by 1500.
2602          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2603          */
2604         rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 );
2605
2606         hw->fc.high_water = rx_buffer_size -
2607             roundup2(adapter->max_frame_size, 1024);
2608         hw->fc.low_water = hw->fc.high_water - 1500;
2609
2610         if (hw->mac.type == e1000_80003es2lan)
2611                 hw->fc.pause_time = 0xFFFF;
2612         else
2613                 hw->fc.pause_time = EM_FC_PAUSE_TIME;
2614
2615         hw->fc.send_xon = TRUE;
2616
2617         /* Set Flow control, use the tunable location if sane */
2618         if ((em_fc_setting >= 0) || (em_fc_setting < 4))
2619                 hw->fc.requested_mode = em_fc_setting;
2620         else
2621                 hw->fc.requested_mode = e1000_fc_none;
2622
2623         /* Override - workaround for PCHLAN issue */
2624         if (hw->mac.type == e1000_pchlan)
2625                 hw->fc.requested_mode = e1000_fc_rx_pause;
2626
2627         /* Issue a global reset */
2628         e1000_reset_hw(hw);
2629         E1000_WRITE_REG(hw, E1000_WUC, 0);
2630
2631         if (e1000_init_hw(hw) < 0) {
2632                 device_printf(dev, "Hardware Initialization Failed\n");
2633                 return;
2634         }
2635
2636         E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
2637         e1000_get_phy_info(hw);
2638         e1000_check_for_link(hw);
2639         return;
2640 }
2641
2642 /*********************************************************************
2643  *
2644  *  Setup networking device structure and register an interface.
2645  *
2646  **********************************************************************/
2647 static void
2648 em_setup_interface(device_t dev, struct adapter *adapter)
2649 {
2650         struct ifnet   *ifp;
2651
2652         INIT_DEBUGOUT("em_setup_interface: begin");
2653
2654         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2655         if (ifp == NULL)
2656                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
2657         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2658         ifp->if_mtu = ETHERMTU;
2659         ifp->if_init =  em_init;
2660         ifp->if_softc = adapter;
2661         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2662         ifp->if_ioctl = em_ioctl;
2663         ifp->if_start = em_start;
2664         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2665         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2666         IFQ_SET_READY(&ifp->if_snd);
2667
2668         ether_ifattach(ifp, adapter->hw.mac.addr);
2669
2670         ifp->if_capabilities = ifp->if_capenable = 0;
2671
2672 #ifdef EM_MULTIQUEUE
2673         /* Multiqueue tx functions */
2674         ifp->if_transmit = em_mq_start;
2675         ifp->if_qflush = em_qflush;
2676 #endif
2677
2678         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2679         ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2680
2681         /* Enable TSO by default, can disable with ifconfig */
2682         ifp->if_capabilities |= IFCAP_TSO4;
2683         ifp->if_capenable |= IFCAP_TSO4;
2684
2685         /*
2686          * Tell the upper layer(s) we
2687          * support full VLAN capability
2688          */
2689         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2690         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2691         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2692
2693         /*
2694         ** Dont turn this on by default, if vlans are
2695         ** created on another pseudo device (eg. lagg)
2696         ** then vlan events are not passed thru, breaking
2697         ** operation, but with HW FILTER off it works. If
2698         ** using vlans directly on the em driver you can
2699         ** enable this and get full hardware tag filtering.
2700         */
2701         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2702
2703 #ifdef DEVICE_POLLING
2704         ifp->if_capabilities |= IFCAP_POLLING;
2705 #endif
2706
2707         /* Enable only WOL MAGIC by default */
2708         if (adapter->wol) {
2709                 ifp->if_capabilities |= IFCAP_WOL;
2710                 ifp->if_capenable |= IFCAP_WOL_MAGIC;
2711         }
2712
2713         /*
2714          * Specify the media types supported by this adapter and register
2715          * callbacks to update media and link information
2716          */
2717         ifmedia_init(&adapter->media, IFM_IMASK,
2718             em_media_change, em_media_status);
2719         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2720             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2721                 u_char fiber_type = IFM_1000_SX;        /* default type */
2722
2723                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2724                             0, NULL);
2725                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2726         } else {
2727                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2728                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2729                             0, NULL);
2730                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2731                             0, NULL);
2732                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2733                             0, NULL);
2734                 if (adapter->hw.phy.type != e1000_phy_ife) {
2735                         ifmedia_add(&adapter->media,
2736                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2737                         ifmedia_add(&adapter->media,
2738                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2739                 }
2740         }
2741         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2742         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2743 }
2744
2745
2746 /*
2747  * Manage DMA'able memory.
2748  */
2749 static void
2750 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2751 {
2752         if (error)
2753                 return;
2754         *(bus_addr_t *) arg = segs[0].ds_addr;
2755 }
2756
2757 static int
2758 em_dma_malloc(struct adapter *adapter, bus_size_t size,
2759         struct em_dma_alloc *dma, int mapflags)
2760 {
2761         int error;
2762
2763         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2764                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
2765                                 BUS_SPACE_MAXADDR,      /* lowaddr */
2766                                 BUS_SPACE_MAXADDR,      /* highaddr */
2767                                 NULL, NULL,             /* filter, filterarg */
2768                                 size,                   /* maxsize */
2769                                 1,                      /* nsegments */
2770                                 size,                   /* maxsegsize */
2771                                 0,                      /* flags */
2772                                 NULL,                   /* lockfunc */
2773                                 NULL,                   /* lockarg */
2774                                 &dma->dma_tag);
2775         if (error) {
2776                 device_printf(adapter->dev,
2777                     "%s: bus_dma_tag_create failed: %d\n",
2778                     __func__, error);
2779                 goto fail_0;
2780         }
2781
2782         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2783             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
2784         if (error) {
2785                 device_printf(adapter->dev,
2786                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2787                     __func__, (uintmax_t)size, error);
2788                 goto fail_2;
2789         }
2790
2791         dma->dma_paddr = 0;
2792         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2793             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2794         if (error || dma->dma_paddr == 0) {
2795                 device_printf(adapter->dev,
2796                     "%s: bus_dmamap_load failed: %d\n",
2797                     __func__, error);
2798                 goto fail_3;
2799         }
2800
2801         return (0);
2802
2803 fail_3:
2804         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2805 fail_2:
2806         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2807         bus_dma_tag_destroy(dma->dma_tag);
2808 fail_0:
2809         dma->dma_map = NULL;
2810         dma->dma_tag = NULL;
2811
2812         return (error);
2813 }
2814
2815 static void
2816 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2817 {
2818         if (dma->dma_tag == NULL)
2819                 return;
2820         if (dma->dma_map != NULL) {
2821                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2822                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2823                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2824                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2825                 dma->dma_map = NULL;
2826         }
2827         bus_dma_tag_destroy(dma->dma_tag);
2828         dma->dma_tag = NULL;
2829 }
2830
2831
2832 /*********************************************************************
2833  *
2834  *  Allocate memory for the transmit and receive rings, and then
2835  *  the descriptors associated with each, called only once at attach.
2836  *
2837  **********************************************************************/
2838 static int
2839 em_allocate_queues(struct adapter *adapter)
2840 {
2841         device_t                dev = adapter->dev;
2842         struct tx_ring          *txr = NULL;
2843         struct rx_ring          *rxr = NULL;
2844         int rsize, tsize, error = E1000_SUCCESS;
2845         int txconf = 0, rxconf = 0;
2846
2847
2848         /* Allocate the TX ring struct memory */
2849         if (!(adapter->tx_rings =
2850             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2851             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2852                 device_printf(dev, "Unable to allocate TX ring memory\n");
2853                 error = ENOMEM;
2854                 goto fail;
2855         }
2856
2857         /* Now allocate the RX */
2858         if (!(adapter->rx_rings =
2859             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2860             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2861                 device_printf(dev, "Unable to allocate RX ring memory\n");
2862                 error = ENOMEM;
2863                 goto rx_fail;
2864         }
2865
2866         tsize = roundup2(adapter->num_tx_desc *
2867             sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
2868         /*
2869          * Now set up the TX queues, txconf is needed to handle the
2870          * possibility that things fail midcourse and we need to
2871          * undo memory gracefully
2872          */
2873         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2874                 /* Set up some basics */
2875                 txr = &adapter->tx_rings[i];
2876                 txr->adapter = adapter;
2877                 txr->me = i;
2878
2879                 /* Initialize the TX lock */
2880                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2881                     device_get_nameunit(dev), txr->me);
2882                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2883
2884                 if (em_dma_malloc(adapter, tsize,
2885                         &txr->txdma, BUS_DMA_NOWAIT)) {
2886                         device_printf(dev,
2887                             "Unable to allocate TX Descriptor memory\n");
2888                         error = ENOMEM;
2889                         goto err_tx_desc;
2890                 }
2891                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
2892                 bzero((void *)txr->tx_base, tsize);
2893
2894                 if (em_allocate_transmit_buffers(txr)) {
2895                         device_printf(dev,
2896                             "Critical Failure setting up transmit buffers\n");
2897                         error = ENOMEM;
2898                         goto err_tx_desc;
2899                 }
2900 #if __FreeBSD_version >= 800000
2901                 /* Allocate a buf ring */
2902                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
2903                     M_WAITOK, &txr->tx_mtx);
2904 #endif
2905         }
2906
2907         /*
2908          * Next the RX queues...
2909          */
2910         rsize = roundup2(adapter->num_rx_desc *
2911             sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
2912         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2913                 rxr = &adapter->rx_rings[i];
2914                 rxr->adapter = adapter;
2915                 rxr->me = i;
2916
2917                 /* Initialize the RX lock */
2918                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2919                     device_get_nameunit(dev), txr->me);
2920                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2921
2922                 if (em_dma_malloc(adapter, rsize,
2923                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2924                         device_printf(dev,
2925                             "Unable to allocate RxDescriptor memory\n");
2926                         error = ENOMEM;
2927                         goto err_rx_desc;
2928                 }
2929                 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr;
2930                 bzero((void *)rxr->rx_base, rsize);
2931
2932                 /* Allocate receive buffers for the ring*/
2933                 if (em_allocate_receive_buffers(rxr)) {
2934                         device_printf(dev,
2935                             "Critical Failure setting up receive buffers\n");
2936                         error = ENOMEM;
2937                         goto err_rx_desc;
2938                 }
2939         }
2940
2941         return (0);
2942
2943 err_rx_desc:
2944         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2945                 em_dma_free(adapter, &rxr->rxdma);
2946 err_tx_desc:
2947         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2948                 em_dma_free(adapter, &txr->txdma);
2949         free(adapter->rx_rings, M_DEVBUF);
2950 rx_fail:
2951         buf_ring_free(txr->br, M_DEVBUF);
2952         free(adapter->tx_rings, M_DEVBUF);
2953 fail:
2954         return (error);
2955 }
2956
2957
2958 /*********************************************************************
2959  *
2960  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2961  *  the information needed to transmit a packet on the wire. This is
2962  *  called only once at attach, setup is done every reset.
2963  *
2964  **********************************************************************/
2965 static int
2966 em_allocate_transmit_buffers(struct tx_ring *txr)
2967 {
2968         struct adapter *adapter = txr->adapter;
2969         device_t dev = adapter->dev;
2970         struct em_buffer *txbuf;
2971         int error, i;
2972
2973         /*
2974          * Setup DMA descriptor areas.
2975          */
2976         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
2977                                1, 0,                    /* alignment, bounds */
2978                                BUS_SPACE_MAXADDR,       /* lowaddr */
2979                                BUS_SPACE_MAXADDR,       /* highaddr */
2980                                NULL, NULL,              /* filter, filterarg */
2981                                EM_TSO_SIZE,             /* maxsize */
2982                                EM_MAX_SCATTER,          /* nsegments */
2983                                PAGE_SIZE,               /* maxsegsize */
2984                                0,                       /* flags */
2985                                NULL,                    /* lockfunc */
2986                                NULL,                    /* lockfuncarg */
2987                                &txr->txtag))) {
2988                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2989                 goto fail;
2990         }
2991
2992         if (!(txr->tx_buffers =
2993             (struct em_buffer *) malloc(sizeof(struct em_buffer) *
2994             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2995                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2996                 error = ENOMEM;
2997                 goto fail;
2998         }
2999
3000         /* Create the descriptor buffer dma maps */
3001         txbuf = txr->tx_buffers;
3002         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3003                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3004                 if (error != 0) {
3005                         device_printf(dev, "Unable to create TX DMA map\n");
3006                         goto fail;
3007                 }
3008         }
3009
3010         return 0;
3011 fail:
3012         /* We free all, it handles case where we are in the middle */
3013         em_free_transmit_structures(adapter);
3014         return (error);
3015 }
3016
3017 /*********************************************************************
3018  *
3019  *  Initialize a transmit ring.
3020  *
3021  **********************************************************************/
3022 static void
3023 em_setup_transmit_ring(struct tx_ring *txr)
3024 {
3025         struct adapter *adapter = txr->adapter;
3026         struct em_buffer *txbuf;
3027         int i;
3028
3029         /* Clear the old descriptor contents */
3030         EM_TX_LOCK(txr);
3031         bzero((void *)txr->tx_base,
3032               (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3033         /* Reset indices */
3034         txr->next_avail_desc = 0;
3035         txr->next_to_clean = 0;
3036
3037         /* Free any existing tx buffers. */
3038         txbuf = txr->tx_buffers;
3039         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3040                 if (txbuf->m_head != NULL) {
3041                         bus_dmamap_sync(txr->txtag, txbuf->map,
3042                             BUS_DMASYNC_POSTWRITE);
3043                         bus_dmamap_unload(txr->txtag, txbuf->map);
3044                         m_freem(txbuf->m_head);
3045                         txbuf->m_head = NULL;
3046                 }
3047                 /* clear the watch index */
3048                 txbuf->next_eop = -1;
3049         }
3050
3051         /* Set number of descriptors available */
3052         txr->tx_avail = adapter->num_tx_desc;
3053
3054         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3055             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3056         EM_TX_UNLOCK(txr);
3057 }
3058
3059 /*********************************************************************
3060  *
3061  *  Initialize all transmit rings.
3062  *
3063  **********************************************************************/
3064 static void
3065 em_setup_transmit_structures(struct adapter *adapter)
3066 {
3067         struct tx_ring *txr = adapter->tx_rings;
3068
3069         for (int i = 0; i < adapter->num_queues; i++, txr++)
3070                 em_setup_transmit_ring(txr);
3071
3072         return;
3073 }
3074
3075 /*********************************************************************
3076  *
3077  *  Enable transmit unit.
3078  *
3079  **********************************************************************/
3080 static void
3081 em_initialize_transmit_unit(struct adapter *adapter)
3082 {
3083         struct tx_ring  *txr = adapter->tx_rings;
3084         struct e1000_hw *hw = &adapter->hw;
3085         u32     tctl, tarc, tipg = 0;
3086
3087          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3088
3089         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3090                 u64 bus_addr = txr->txdma.dma_paddr;
3091                 /* Base and Len of TX Ring */
3092                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3093                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3094                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3095                     (u32)(bus_addr >> 32));
3096                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3097                     (u32)bus_addr);
3098                 /* Init the HEAD/TAIL indices */
3099                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3100                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3101
3102                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3103                     E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
3104                     E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
3105
3106                 txr->watchdog_check = FALSE;
3107         }
3108
3109         /* Set the default values for the Tx Inter Packet Gap timer */
3110         switch (adapter->hw.mac.type) {
3111         case e1000_82542:
3112                 tipg = DEFAULT_82542_TIPG_IPGT;
3113                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3114                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3115                 break;
3116         case e1000_80003es2lan:
3117                 tipg = DEFAULT_82543_TIPG_IPGR1;
3118                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3119                     E1000_TIPG_IPGR2_SHIFT;
3120                 break;
3121         default:
3122                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3123                     (adapter->hw.phy.media_type ==
3124                     e1000_media_type_internal_serdes))
3125                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3126                 else
3127                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3128                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3129                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3130         }
3131
3132         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3133         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3134
3135         if(adapter->hw.mac.type >= e1000_82540)
3136                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3137                     adapter->tx_abs_int_delay.value);
3138
3139         if ((adapter->hw.mac.type == e1000_82571) ||
3140             (adapter->hw.mac.type == e1000_82572)) {
3141                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3142                 tarc |= SPEED_MODE_BIT;
3143                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3144         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3145                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3146                 tarc |= 1;
3147                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3148                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3149                 tarc |= 1;
3150                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3151         }
3152
3153         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3154         if (adapter->tx_int_delay.value > 0)
3155                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3156
3157         /* Program the Transmit Control Register */
3158         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3159         tctl &= ~E1000_TCTL_CT;
3160         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3161                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3162
3163         if (adapter->hw.mac.type >= e1000_82571)
3164                 tctl |= E1000_TCTL_MULR;
3165
3166         /* This write will effectively turn on the transmit unit. */
3167         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3168
3169 }
3170
3171
3172 /*********************************************************************
3173  *
3174  *  Free all transmit rings.
3175  *
3176  **********************************************************************/
3177 static void
3178 em_free_transmit_structures(struct adapter *adapter)
3179 {
3180         struct tx_ring *txr = adapter->tx_rings;
3181
3182         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3183                 EM_TX_LOCK(txr);
3184                 em_free_transmit_buffers(txr);
3185                 em_dma_free(adapter, &txr->txdma);
3186                 EM_TX_UNLOCK(txr);
3187                 EM_TX_LOCK_DESTROY(txr);
3188         }
3189
3190         free(adapter->tx_rings, M_DEVBUF);
3191 }
3192
3193 /*********************************************************************
3194  *
3195  *  Free transmit ring related data structures.
3196  *
3197  **********************************************************************/
3198 static void
3199 em_free_transmit_buffers(struct tx_ring *txr)
3200 {
3201         struct adapter          *adapter = txr->adapter;
3202         struct em_buffer        *txbuf;
3203
3204         INIT_DEBUGOUT("free_transmit_ring: begin");
3205
3206         if (txr->tx_buffers == NULL)
3207                 return;
3208
3209         for (int i = 0; i < adapter->num_tx_desc; i++) {
3210                 txbuf = &txr->tx_buffers[i];
3211                 if (txbuf->m_head != NULL) {
3212                         bus_dmamap_sync(txr->txtag, txbuf->map,
3213                             BUS_DMASYNC_POSTWRITE);
3214                         bus_dmamap_unload(txr->txtag,
3215                             txbuf->map);
3216                         m_freem(txbuf->m_head);
3217                         txbuf->m_head = NULL;
3218                         if (txbuf->map != NULL) {
3219                                 bus_dmamap_destroy(txr->txtag,
3220                                     txbuf->map);
3221                                 txbuf->map = NULL;
3222                         }
3223                 } else if (txbuf->map != NULL) {
3224                         bus_dmamap_unload(txr->txtag,
3225                             txbuf->map);
3226                         bus_dmamap_destroy(txr->txtag,
3227                             txbuf->map);
3228                         txbuf->map = NULL;
3229                 }
3230         }
3231 #if __FreeBSD_version >= 800000
3232         if (txr->br != NULL)
3233                 buf_ring_free(txr->br, M_DEVBUF);
3234 #endif
3235         if (txr->tx_buffers != NULL) {
3236                 free(txr->tx_buffers, M_DEVBUF);
3237                 txr->tx_buffers = NULL;
3238         }
3239         if (txr->txtag != NULL) {
3240                 bus_dma_tag_destroy(txr->txtag);
3241                 txr->txtag = NULL;
3242         }
3243         return;
3244 }
3245
3246
3247 /*********************************************************************
3248  *
3249  *  The offload context needs to be set when we transfer the first
3250  *  packet of a particular protocol (TCP/UDP). This routine has been
3251  *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3252  *
3253  *  Added back the old method of keeping the current context type
3254  *  and not setting if unnecessary, as this is reported to be a
3255  *  big performance win.  -jfv
3256  **********************************************************************/
3257 static void
3258 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp,
3259     u32 *txd_upper, u32 *txd_lower)
3260 {
3261         struct adapter                  *adapter = txr->adapter;
3262         struct e1000_context_desc       *TXD = NULL;
3263         struct em_buffer *tx_buffer;
3264         struct ether_vlan_header *eh;
3265         struct ip *ip = NULL;
3266         struct ip6_hdr *ip6;
3267         int cur, ehdrlen;
3268         u32 cmd, hdr_len, ip_hlen;
3269         u16 etype;
3270         u8 ipproto;
3271
3272
3273         cmd = hdr_len = ipproto = 0;
3274         cur = txr->next_avail_desc;
3275
3276         /*
3277          * Determine where frame payload starts.
3278          * Jump over vlan headers if already present,
3279          * helpful for QinQ too.
3280          */
3281         eh = mtod(mp, struct ether_vlan_header *);
3282         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3283                 etype = ntohs(eh->evl_proto);
3284                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3285         } else {
3286                 etype = ntohs(eh->evl_encap_proto);
3287                 ehdrlen = ETHER_HDR_LEN;
3288         }
3289
3290         /*
3291          * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3292          * TODO: Support SCTP too when it hits the tree.
3293          */
3294         switch (etype) {
3295         case ETHERTYPE_IP:
3296                 ip = (struct ip *)(mp->m_data + ehdrlen);
3297                 ip_hlen = ip->ip_hl << 2;
3298
3299                 /* Setup of IP header checksum. */
3300                 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3301                         /*
3302                          * Start offset for header checksum calculation.
3303                          * End offset for header checksum calculation.
3304                          * Offset of place to put the checksum.
3305                          */
3306                         TXD = (struct e1000_context_desc *)
3307                             &txr->tx_base[cur];
3308                         TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3309                         TXD->lower_setup.ip_fields.ipcse =
3310                             htole16(ehdrlen + ip_hlen);
3311                         TXD->lower_setup.ip_fields.ipcso =
3312                             ehdrlen + offsetof(struct ip, ip_sum);
3313                         cmd |= E1000_TXD_CMD_IP;
3314                         *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3315                 }
3316
3317                 if (mp->m_len < ehdrlen + ip_hlen)
3318                         return; /* failure */
3319
3320                 hdr_len = ehdrlen + ip_hlen;
3321                 ipproto = ip->ip_p;
3322
3323                 break;
3324         case ETHERTYPE_IPV6:
3325                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3326                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3327
3328                 if (mp->m_len < ehdrlen + ip_hlen)
3329                         return; /* failure */
3330
3331                 /* IPv6 doesn't have a header checksum. */
3332
3333                 hdr_len = ehdrlen + ip_hlen;
3334                 ipproto = ip6->ip6_nxt;
3335
3336                 break;
3337         default:
3338                 *txd_upper = 0;
3339                 *txd_lower = 0;
3340                 return;
3341         }
3342
3343         switch (ipproto) {
3344         case IPPROTO_TCP:
3345                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3346                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3347                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3348                         /* no need for context if already set */
3349                         if (txr->last_hw_offload == CSUM_TCP)
3350                                 return;
3351                         txr->last_hw_offload = CSUM_TCP;
3352                         /*
3353                          * Start offset for payload checksum calculation.
3354                          * End offset for payload checksum calculation.
3355                          * Offset of place to put the checksum.
3356                          */
3357                         TXD = (struct e1000_context_desc *)
3358                             &txr->tx_base[cur];
3359                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3360                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3361                         TXD->upper_setup.tcp_fields.tucso =
3362                             hdr_len + offsetof(struct tcphdr, th_sum);
3363                         cmd |= E1000_TXD_CMD_TCP;
3364                 }
3365                 break;
3366         case IPPROTO_UDP:
3367         {
3368                 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3369                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3370                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3371                         /* no need for context if already set */
3372                         if (txr->last_hw_offload == CSUM_UDP)
3373                                 return;
3374                         txr->last_hw_offload = CSUM_UDP;
3375                         /*
3376                          * Start offset for header checksum calculation.
3377                          * End offset for header checksum calculation.
3378                          * Offset of place to put the checksum.
3379                          */
3380                         TXD = (struct e1000_context_desc *)
3381                             &txr->tx_base[cur];
3382                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3383                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3384                         TXD->upper_setup.tcp_fields.tucso =
3385                             hdr_len + offsetof(struct udphdr, uh_sum);
3386                 }
3387                 /* Fall Thru */
3388         }
3389         default:
3390                 break;
3391         }
3392
3393         TXD->tcp_seg_setup.data = htole32(0);
3394         TXD->cmd_and_length =
3395             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3396         tx_buffer = &txr->tx_buffers[cur];
3397         tx_buffer->m_head = NULL;
3398         tx_buffer->next_eop = -1;
3399
3400         if (++cur == adapter->num_tx_desc)
3401                 cur = 0;
3402
3403         txr->tx_avail--;
3404         txr->next_avail_desc = cur;
3405 }
3406
3407
3408 /**********************************************************************
3409  *
3410  *  Setup work for hardware segmentation offload (TSO)
3411  *
3412  **********************************************************************/
3413 static bool
3414 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *txd_upper,
3415    u32 *txd_lower)
3416 {
3417         struct adapter                  *adapter = txr->adapter;
3418         struct e1000_context_desc       *TXD;
3419         struct em_buffer                *tx_buffer;
3420         struct ether_vlan_header        *eh;
3421         struct ip                       *ip;
3422         struct ip6_hdr                  *ip6;
3423         struct tcphdr                   *th;
3424         int cur, ehdrlen, hdr_len, ip_hlen, isip6;
3425         u16 etype;
3426
3427         /*
3428          * This function could/should be extended to support IP/IPv6
3429          * fragmentation as well.  But as they say, one step at a time.
3430          */
3431
3432         /*
3433          * Determine where frame payload starts.
3434          * Jump over vlan headers if already present,
3435          * helpful for QinQ too.
3436          */
3437         eh = mtod(mp, struct ether_vlan_header *);
3438         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3439                 etype = ntohs(eh->evl_proto);
3440                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3441         } else {
3442                 etype = ntohs(eh->evl_encap_proto);
3443                 ehdrlen = ETHER_HDR_LEN;
3444         }
3445
3446         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3447         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3448                 return FALSE;   /* -1 */
3449
3450         /*
3451          * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3452          * TODO: Support SCTP too when it hits the tree.
3453          */
3454         switch (etype) {
3455         case ETHERTYPE_IP:
3456                 isip6 = 0;
3457                 ip = (struct ip *)(mp->m_data + ehdrlen);
3458                 if (ip->ip_p != IPPROTO_TCP)
3459                         return FALSE;   /* 0 */
3460                 ip->ip_len = 0;
3461                 ip->ip_sum = 0;
3462                 ip_hlen = ip->ip_hl << 2;
3463                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3464                         return FALSE;   /* -1 */
3465                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3466 #if 1
3467                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3468                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3469 #else
3470                 th->th_sum = mp->m_pkthdr.csum_data;
3471 #endif
3472                 break;
3473         case ETHERTYPE_IPV6:
3474                 isip6 = 1;
3475                 return FALSE;                   /* Not supported yet. */
3476                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3477                 if (ip6->ip6_nxt != IPPROTO_TCP)
3478                         return FALSE;   /* 0 */
3479                 ip6->ip6_plen = 0;
3480                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3481                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3482                         return FALSE;   /* -1 */
3483                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3484 #if 0
3485                 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3486                     htons(IPPROTO_TCP));        /* XXX: function notyet. */
3487 #else
3488                 th->th_sum = mp->m_pkthdr.csum_data;
3489 #endif
3490                 break;
3491         default:
3492                 return FALSE;
3493         }
3494         hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3495
3496         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3497                       E1000_TXD_DTYP_D |        /* Data descr type */
3498                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3499
3500         /* IP and/or TCP header checksum calculation and insertion. */
3501         *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3502                       E1000_TXD_POPTS_TXSM) << 8;
3503
3504         cur = txr->next_avail_desc;
3505         tx_buffer = &txr->tx_buffers[cur];
3506         TXD = (struct e1000_context_desc *) &txr->tx_base[cur];
3507
3508         /* IPv6 doesn't have a header checksum. */
3509         if (!isip6) {
3510                 /*
3511                  * Start offset for header checksum calculation.
3512                  * End offset for header checksum calculation.
3513                  * Offset of place put the checksum.
3514                  */
3515                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3516                 TXD->lower_setup.ip_fields.ipcse =
3517                     htole16(ehdrlen + ip_hlen - 1);
3518                 TXD->lower_setup.ip_fields.ipcso =
3519                     ehdrlen + offsetof(struct ip, ip_sum);
3520         }
3521         /*
3522          * Start offset for payload checksum calculation.
3523          * End offset for payload checksum calculation.
3524          * Offset of place to put the checksum.
3525          */
3526         TXD->upper_setup.tcp_fields.tucss =
3527             ehdrlen + ip_hlen;
3528         TXD->upper_setup.tcp_fields.tucse = 0;
3529         TXD->upper_setup.tcp_fields.tucso =
3530             ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3531         /*
3532          * Payload size per packet w/o any headers.
3533          * Length of all headers up to payload.
3534          */
3535         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3536         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3537
3538         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3539                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3540                                 E1000_TXD_CMD_TSE |     /* TSE context */
3541                                 (isip6 ? 0 : E1000_TXD_CMD_IP) |
3542                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3543                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3544
3545         tx_buffer->m_head = NULL;
3546         tx_buffer->next_eop = -1;
3547
3548         if (++cur == adapter->num_tx_desc)
3549                 cur = 0;
3550
3551         txr->tx_avail--;
3552         txr->next_avail_desc = cur;
3553         txr->tx_tso = TRUE;
3554
3555         return TRUE;
3556 }
3557
3558
3559 /**********************************************************************
3560  *
3561  *  Examine each tx_buffer in the used queue. If the hardware is done
3562  *  processing the packet then free associated resources. The
3563  *  tx_buffer is put back on the free queue.
3564  *
3565  **********************************************************************/
3566 static bool
3567 em_txeof(struct tx_ring *txr)
3568 {
3569         struct adapter  *adapter = txr->adapter;
3570         int first, last, done, num_avail;
3571         struct em_buffer *tx_buffer;
3572         struct e1000_tx_desc   *tx_desc, *eop_desc;
3573         struct ifnet   *ifp = adapter->ifp;
3574
3575         EM_TX_LOCK_ASSERT(txr);
3576
3577         if (txr->tx_avail == adapter->num_tx_desc)
3578                 return (FALSE);
3579
3580         num_avail = txr->tx_avail;
3581         first = txr->next_to_clean;
3582         tx_desc = &txr->tx_base[first];
3583         tx_buffer = &txr->tx_buffers[first];
3584         last = tx_buffer->next_eop;
3585         eop_desc = &txr->tx_base[last];
3586
3587         /*
3588          * What this does is get the index of the
3589          * first descriptor AFTER the EOP of the
3590          * first packet, that way we can do the
3591          * simple comparison on the inner while loop.
3592          */
3593         if (++last == adapter->num_tx_desc)
3594                 last = 0;
3595         done = last;
3596
3597         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3598             BUS_DMASYNC_POSTREAD);
3599
3600         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3601                 /* We clean the range of the packet */
3602                 while (first != done) {
3603                         tx_desc->upper.data = 0;
3604                         tx_desc->lower.data = 0;
3605                         tx_desc->buffer_addr = 0;
3606                         ++num_avail;
3607
3608                         if (tx_buffer->m_head) {
3609                                 ifp->if_opackets++;
3610                                 bus_dmamap_sync(txr->txtag,
3611                                     tx_buffer->map,
3612                                     BUS_DMASYNC_POSTWRITE);
3613                                 bus_dmamap_unload(txr->txtag,
3614                                     tx_buffer->map);
3615
3616                                 m_freem(tx_buffer->m_head);
3617                                 tx_buffer->m_head = NULL;
3618                         }
3619                         tx_buffer->next_eop = -1;
3620                         txr->watchdog_time = ticks;
3621
3622                         if (++first == adapter->num_tx_desc)
3623                                 first = 0;
3624
3625                         tx_buffer = &txr->tx_buffers[first];
3626                         tx_desc = &txr->tx_base[first];
3627                 }
3628                 /* See if we can continue to the next packet */
3629                 last = tx_buffer->next_eop;
3630                 if (last != -1) {
3631                         eop_desc = &txr->tx_base[last];
3632                         /* Get new done point */
3633                         if (++last == adapter->num_tx_desc) last = 0;
3634                         done = last;
3635                 } else
3636                         break;
3637         }
3638         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3639             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3640
3641         txr->next_to_clean = first;
3642
3643         /*
3644          * If we have enough room, clear IFF_DRV_OACTIVE to
3645          * tell the stack that it is OK to send packets.
3646          * If there are no pending descriptors, clear the watchdog.
3647          */
3648         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
3649                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3650                 if (num_avail == adapter->num_tx_desc) {
3651                         txr->watchdog_check = FALSE;
3652                         txr->tx_avail = num_avail;
3653                         return (FALSE);
3654                 }
3655         }
3656
3657         txr->tx_avail = num_avail;
3658         return (TRUE);
3659 }
3660
3661
3662 /*********************************************************************
3663  *
3664  *  Refresh RX descriptor mbufs from system mbuf buffer pool.
3665  *
3666  **********************************************************************/
3667 static void
3668 em_refresh_mbufs(struct rx_ring *rxr, int limit)
3669 {
3670         struct adapter          *adapter = rxr->adapter;
3671         struct mbuf             *m;
3672         bus_dma_segment_t       segs[1];
3673         bus_dmamap_t            map;
3674         struct em_buffer        *rxbuf;
3675         int                     i, error, nsegs, cleaned;
3676
3677         i = rxr->next_to_refresh;
3678         cleaned = -1;
3679         while (i != limit) {
3680                 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
3681                 if (m == NULL)
3682                         goto update;
3683                 m->m_len = m->m_pkthdr.len = MCLBYTES;
3684
3685                 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3686                         m_adj(m, ETHER_ALIGN);
3687
3688                 /*
3689                  * Using memory from the mbuf cluster pool, invoke the
3690                  * bus_dma machinery to arrange the memory mapping.
3691                  */
3692                 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxr->rx_sparemap,
3693                     m, segs, &nsegs, BUS_DMA_NOWAIT);
3694                 if (error != 0) {
3695                         m_free(m);
3696                         goto update;
3697                 }
3698
3699                 /* If nsegs is wrong then the stack is corrupt. */
3700                 KASSERT(nsegs == 1, ("Too many segments returned!"));
3701
3702                 rxbuf = &rxr->rx_buffers[i];
3703                 if (rxbuf->m_head != NULL)
3704                         bus_dmamap_unload(rxr->rxtag, rxbuf->map);
3705
3706                 map = rxbuf->map;
3707                 rxbuf->map = rxr->rx_sparemap;
3708                 rxr->rx_sparemap = map;
3709                 bus_dmamap_sync(rxr->rxtag,
3710                     rxbuf->map, BUS_DMASYNC_PREREAD);
3711                 rxbuf->m_head = m;
3712                 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr);
3713
3714                 cleaned = i;
3715                 /* Calculate next index */
3716                 if (++i == adapter->num_rx_desc)
3717                         i = 0;
3718                 /* This is the work marker for refresh */
3719                 rxr->next_to_refresh = i;
3720         }
3721 update:
3722         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3723             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3724         if (cleaned != -1) /* Update tail index */
3725                 E1000_WRITE_REG(&adapter->hw,
3726                     E1000_RDT(rxr->me), cleaned);
3727
3728         return;
3729 }
3730
3731
3732 /*********************************************************************
3733  *
3734  *  Allocate memory for rx_buffer structures. Since we use one
3735  *  rx_buffer per received packet, the maximum number of rx_buffer's
3736  *  that we'll need is equal to the number of receive descriptors
3737  *  that we've allocated.
3738  *
3739  **********************************************************************/
3740 static int
3741 em_allocate_receive_buffers(struct rx_ring *rxr)
3742 {
3743         struct adapter          *adapter = rxr->adapter;
3744         device_t                dev = adapter->dev;
3745         struct em_buffer        *rxbuf;
3746         int                     error;
3747
3748         rxr->rx_buffers = malloc(sizeof(struct em_buffer) *
3749             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3750         if (rxr->rx_buffers == NULL) {
3751                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3752                 return (ENOMEM);
3753         }
3754
3755         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3756                                 1, 0,                   /* alignment, bounds */
3757                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3758                                 BUS_SPACE_MAXADDR,      /* highaddr */
3759                                 NULL, NULL,             /* filter, filterarg */
3760                                 MCLBYTES,               /* maxsize */
3761                                 1,                      /* nsegments */
3762                                 MCLBYTES,               /* maxsegsize */
3763                                 0,                      /* flags */
3764                                 NULL,                   /* lockfunc */
3765                                 NULL,                   /* lockarg */
3766                                 &rxr->rxtag);
3767         if (error) {
3768                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
3769                     __func__, error);
3770                 goto fail;
3771         }
3772
3773         /* Create the spare map (used by getbuf) */
3774         error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
3775              &rxr->rx_sparemap);
3776         if (error) {
3777                 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3778                     __func__, error);
3779                 goto fail;
3780         }
3781
3782         rxbuf = rxr->rx_buffers;
3783         for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3784                 rxbuf = &rxr->rx_buffers[i];
3785                 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
3786                     &rxbuf->map);
3787                 if (error) {
3788                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3789                             __func__, error);
3790                         goto fail;
3791                 }
3792         }
3793
3794         return (0);
3795
3796 fail:
3797         em_free_receive_structures(adapter);
3798         return (error);
3799 }
3800
3801
3802 /*********************************************************************
3803  *
3804  *  Initialize a receive ring and its buffers.
3805  *
3806  **********************************************************************/
3807 static int
3808 em_setup_receive_ring(struct rx_ring *rxr)
3809 {
3810         struct  adapter         *adapter = rxr->adapter;
3811         struct em_buffer        *rxbuf;
3812         bus_dma_segment_t       seg[1];
3813         int                     rsize, nsegs, error;
3814
3815
3816         /* Clear the ring contents */
3817         EM_RX_LOCK(rxr);
3818         rsize = roundup2(adapter->num_rx_desc *
3819             sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
3820         bzero((void *)rxr->rx_base, rsize);
3821
3822         /*
3823         ** Free current RX buffer structs and their mbufs
3824         */
3825         for (int i = 0; i < adapter->num_rx_desc; i++) {
3826                 rxbuf = &rxr->rx_buffers[i];
3827                 if (rxbuf->m_head != NULL) {
3828                         bus_dmamap_sync(rxr->rxtag, rxbuf->map,
3829                             BUS_DMASYNC_POSTREAD);
3830                         bus_dmamap_unload(rxr->rxtag, rxbuf->map);
3831                         m_freem(rxbuf->m_head);
3832                 }
3833         }
3834
3835         /* Now replenish the mbufs */
3836         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3837
3838                 rxbuf = &rxr->rx_buffers[j];
3839                 rxbuf->m_head = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
3840                 if (rxbuf->m_head == NULL)
3841                         panic("RX ring hdr initialization failed!\n");
3842                 rxbuf->m_head->m_len = MCLBYTES;
3843                 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */
3844                 rxbuf->m_head->m_pkthdr.len = MCLBYTES;
3845
3846                 /* Get the memory mapping */
3847                 error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
3848                     rxbuf->map, rxbuf->m_head, seg,
3849                     &nsegs, BUS_DMA_NOWAIT);
3850                 if (error != 0)
3851                         panic("RX ring dma initialization failed!\n");
3852                 bus_dmamap_sync(rxr->rxtag,
3853                     rxbuf->map, BUS_DMASYNC_PREREAD);
3854
3855                 /* Update descriptor */
3856                 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr);
3857         }
3858
3859
3860         /* Setup our descriptor indices */
3861         rxr->next_to_check = 0;
3862         rxr->next_to_refresh = 0;
3863
3864         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3865             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3866
3867         EM_RX_UNLOCK(rxr);
3868         return (0);
3869 }
3870
3871 /*********************************************************************
3872  *
3873  *  Initialize all receive rings.
3874  *
3875  **********************************************************************/
3876 static int
3877 em_setup_receive_structures(struct adapter *adapter)
3878 {
3879         struct rx_ring *rxr = adapter->rx_rings;
3880         int j;
3881
3882         for (j = 0; j < adapter->num_queues; j++, rxr++)
3883                 if (em_setup_receive_ring(rxr))
3884                         goto fail;
3885
3886         return (0);
3887 fail:
3888         /*
3889          * Free RX buffers allocated so far, we will only handle
3890          * the rings that completed, the failing case will have
3891          * cleaned up for itself. 'j' failed, so its the terminus.
3892          */
3893         for (int i = 0; i < j; ++i) {
3894                 rxr = &adapter->rx_rings[i];
3895                 for (int n = 0; n < adapter->num_rx_desc; n++) {
3896                         struct em_buffer *rxbuf;
3897                         rxbuf = &rxr->rx_buffers[n];
3898                         if (rxbuf->m_head != NULL) {
3899                                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
3900                                   BUS_DMASYNC_POSTREAD);
3901                                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
3902                                 m_freem(rxbuf->m_head);
3903                                 rxbuf->m_head = NULL;
3904                         }
3905                 }
3906         }
3907
3908         return (ENOBUFS);
3909 }
3910
3911 /*********************************************************************
3912  *
3913  *  Free all receive rings.
3914  *
3915  **********************************************************************/
3916 static void
3917 em_free_receive_structures(struct adapter *adapter)
3918 {
3919         struct rx_ring *rxr = adapter->rx_rings;
3920
3921         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3922                 em_free_receive_buffers(rxr);
3923                 /* Free the ring memory as well */
3924                 em_dma_free(adapter, &rxr->rxdma);
3925                 EM_RX_LOCK_DESTROY(rxr);
3926         }
3927
3928         free(adapter->rx_rings, M_DEVBUF);
3929 }
3930
3931
3932 /*********************************************************************
3933  *
3934  *  Free receive ring data structures
3935  *
3936  **********************************************************************/
3937 static void
3938 em_free_receive_buffers(struct rx_ring *rxr)
3939 {
3940         struct adapter          *adapter = rxr->adapter;
3941         struct em_buffer        *rxbuf = NULL;
3942
3943         INIT_DEBUGOUT("free_receive_buffers: begin");
3944
3945         if (rxr->rx_sparemap) {
3946                 bus_dmamap_destroy(rxr->rxtag, rxr->rx_sparemap);
3947                 rxr->rx_sparemap = NULL;
3948         }
3949
3950         if (rxr->rx_buffers != NULL) {
3951                 for (int i = 0; i < adapter->num_rx_desc; i++) {
3952                         rxbuf = &rxr->rx_buffers[i];
3953                         if (rxbuf->map != NULL) {
3954                                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
3955                                     BUS_DMASYNC_POSTREAD);
3956                                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
3957                                 bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
3958                         }
3959                         if (rxbuf->m_head != NULL) {
3960                                 m_freem(rxbuf->m_head);
3961                                 rxbuf->m_head = NULL;
3962                         }
3963                 }
3964                 free(rxr->rx_buffers, M_DEVBUF);
3965                 rxr->rx_buffers = NULL;
3966         }
3967
3968         if (rxr->rxtag != NULL) {
3969                 bus_dma_tag_destroy(rxr->rxtag);
3970                 rxr->rxtag = NULL;
3971         }
3972
3973         return;
3974 }
3975
3976
3977 /*********************************************************************
3978  *
3979  *  Enable receive unit.
3980  *
3981  **********************************************************************/
3982 #define MAX_INTS_PER_SEC        8000
3983 #define DEFAULT_ITR          1000000000/(MAX_INTS_PER_SEC * 256)
3984
3985 static void
3986 em_initialize_receive_unit(struct adapter *adapter)
3987 {
3988         struct rx_ring  *rxr = adapter->rx_rings;
3989         struct ifnet    *ifp = adapter->ifp;
3990         struct e1000_hw *hw = &adapter->hw;
3991         u64     bus_addr;
3992         u32     rctl, rxcsum;
3993
3994         INIT_DEBUGOUT("em_initialize_receive_units: begin");
3995
3996         /*
3997          * Make sure receives are disabled while setting
3998          * up the descriptor ring
3999          */
4000         rctl = E1000_READ_REG(hw, E1000_RCTL);
4001         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4002
4003         E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4004             adapter->rx_abs_int_delay.value);
4005         /*
4006          * Set the interrupt throttling rate. Value is calculated
4007          * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4008          */
4009         E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
4010
4011         /*
4012         ** When using MSIX interrupts we need to throttle
4013         ** using the EITR register (82574 only)
4014         */
4015         if (hw->mac.type == e1000_82574)
4016                 for (int i = 0; i < 4; i++)
4017                         E1000_WRITE_REG(hw, E1000_EITR_82574(i),
4018                             DEFAULT_ITR);
4019
4020         /* Disable accelerated ackknowledge */
4021         if (adapter->hw.mac.type == e1000_82574)
4022                 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4023
4024         if (ifp->if_capenable & IFCAP_RXCSUM) {
4025                 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4026                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4027                 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4028         }
4029
4030         /*
4031         ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4032         ** long latencies are observed, like Lenovo X60. This
4033         ** change eliminates the problem, but since having positive
4034         ** values in RDTR is a known source of problems on other
4035         ** platforms another solution is being sought.
4036         */
4037         if (hw->mac.type == e1000_82573)
4038                 E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
4039
4040         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4041                 /* Setup the Base and Length of the Rx Descriptor Ring */
4042                 bus_addr = rxr->rxdma.dma_paddr;
4043                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4044                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4045                 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
4046                 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
4047                 /* Setup the Head and Tail Descriptor Pointers */
4048                 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4049                 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1);
4050         }
4051
4052         /* Setup the Receive Control Register */
4053         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4054         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
4055             E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
4056             (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4057
4058         /* Strip the CRC */
4059         rctl |= E1000_RCTL_SECRC;
4060
4061         /* Make sure VLAN Filters are off */
4062         rctl &= ~E1000_RCTL_VFE;
4063         rctl &= ~E1000_RCTL_SBP;
4064         rctl |= E1000_RCTL_SZ_2048;
4065         if (ifp->if_mtu > ETHERMTU)
4066                 rctl |= E1000_RCTL_LPE;
4067         else
4068                 rctl &= ~E1000_RCTL_LPE;
4069
4070         /* Write out the settings */
4071         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4072
4073         return;
4074 }
4075
4076
4077 /*********************************************************************
4078  *
4079  *  This routine executes in interrupt context. It replenishes
4080  *  the mbufs in the descriptor and sends data which has been
4081  *  dma'ed into host memory to upper layer.
4082  *
4083  *  We loop at most count times if count is > 0, or until done if
4084  *  count < 0.
4085  *
4086  *  For polling we also now return the number of cleaned packets
4087  *********************************************************************/
4088 static int
4089 em_rxeof(struct rx_ring *rxr, int count)
4090 {
4091         struct adapter          *adapter = rxr->adapter;
4092         struct ifnet            *ifp = adapter->ifp;
4093         struct mbuf             *mp, *sendmp;
4094         u8                      status = 0;
4095         u16                     len;
4096         int                     i, processed, rxdone = 0;
4097         bool                    eop;
4098         struct e1000_rx_desc    *cur;
4099
4100         EM_RX_LOCK(rxr);
4101
4102         for (i = rxr->next_to_check, processed = 0; count != 0;) {
4103
4104                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4105                         break;
4106
4107                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4108                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4109
4110                 cur = &rxr->rx_base[i];
4111                 status = cur->status;
4112                 mp = sendmp = NULL;
4113
4114                 if ((status & E1000_RXD_STAT_DD) == 0)
4115                         break;
4116
4117                 len = le16toh(cur->length);
4118                 eop = (status & E1000_RXD_STAT_EOP) != 0;
4119                 count--;
4120
4121                 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) == 0) {
4122
4123                         /* Assign correct length to the current fragment */
4124                         mp = rxr->rx_buffers[i].m_head;
4125                         mp->m_len = len;
4126
4127                         if (rxr->fmp == NULL) {
4128                                 mp->m_pkthdr.len = len;
4129                                 rxr->fmp = mp; /* Store the first mbuf */
4130                                 rxr->lmp = mp;
4131                         } else {
4132                                 /* Chain mbuf's together */
4133                                 mp->m_flags &= ~M_PKTHDR;
4134                                 rxr->lmp->m_next = mp;
4135                                 rxr->lmp = rxr->lmp->m_next;
4136                                 rxr->fmp->m_pkthdr.len += len;
4137                         }
4138
4139                         if (eop) {
4140                                 rxr->fmp->m_pkthdr.rcvif = ifp;
4141                                 ifp->if_ipackets++;
4142                                 em_receive_checksum(cur, rxr->fmp);
4143 #ifndef __NO_STRICT_ALIGNMENT
4144                                 if (adapter->max_frame_size >
4145                                     (MCLBYTES - ETHER_ALIGN) &&
4146                                     em_fixup_rx(rxr) != 0)
4147                                         goto skip;
4148 #endif
4149                                 if (status & E1000_RXD_STAT_VP) {
4150                                         rxr->fmp->m_pkthdr.ether_vtag =
4151                                             (le16toh(cur->special) &
4152                                             E1000_RXD_SPC_VLAN_MASK);
4153                                         rxr->fmp->m_flags |= M_VLANTAG;
4154                                 }
4155 #ifdef EM_MULTIQUEUE
4156                                 rxr->fmp->m_pkthdr.flowid = curcpu;
4157                                 rxr->fmp->m_flags |= M_FLOWID;
4158 #endif
4159 #ifndef __NO_STRICT_ALIGNMENT
4160 skip:
4161 #endif
4162                                 sendmp = rxr->fmp;
4163                                 rxr->fmp = NULL;
4164                                 rxr->lmp = NULL;
4165                         }
4166                 } else {
4167                         ifp->if_ierrors++;
4168                         /* Reuse loaded DMA map and just update mbuf chain */
4169                         mp = rxr->rx_buffers[i].m_head;
4170                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
4171                         mp->m_data = mp->m_ext.ext_buf;
4172                         mp->m_next = NULL;
4173                         if (adapter->max_frame_size <=
4174                             (MCLBYTES - ETHER_ALIGN))
4175                                 m_adj(mp, ETHER_ALIGN);
4176                         if (rxr->fmp != NULL) {
4177                                 m_freem(rxr->fmp);
4178                                 rxr->fmp = NULL;
4179                                 rxr->lmp = NULL;
4180                         }
4181                         sendmp = NULL;
4182                 }
4183
4184                 /* Zero out the receive descriptors status. */
4185                 cur->status = 0;
4186                 ++rxdone;       /* cumulative for POLL */
4187                 ++processed;
4188
4189                 /* Advance our pointers to the next descriptor. */
4190                 if (++i == adapter->num_rx_desc)
4191                         i = 0;
4192
4193                 /* Send to the stack */
4194                 if (sendmp != NULL) {
4195                         rxr->next_to_check = i;
4196                         EM_RX_UNLOCK(rxr);
4197                         (*ifp->if_input)(ifp, sendmp);
4198                         EM_RX_LOCK(rxr);
4199                         i = rxr->next_to_check;
4200                 }
4201
4202                 /* Only refresh mbufs every 8 descriptors */
4203                 if (processed == 8) {
4204                         em_refresh_mbufs(rxr, i);
4205                         processed = 0;
4206                 }
4207         }
4208
4209         /* Catch any remaining refresh work */
4210         if (processed != 0) {
4211                 em_refresh_mbufs(rxr, i);
4212                 processed = 0;
4213         }
4214
4215         rxr->next_to_check = i;
4216         EM_RX_UNLOCK(rxr);
4217
4218 #ifdef DEVICE_POLLING
4219         return (rxdone);
4220 #else
4221         return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4222 #endif
4223 }
4224
4225 #ifndef __NO_STRICT_ALIGNMENT
4226 /*
4227  * When jumbo frames are enabled we should realign entire payload on
4228  * architecures with strict alignment. This is serious design mistake of 8254x
4229  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4230  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4231  * payload. On architecures without strict alignment restrictions 8254x still
4232  * performs unaligned memory access which would reduce the performance too.
4233  * To avoid copying over an entire frame to align, we allocate a new mbuf and
4234  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4235  * existing mbuf chain.
4236  *
4237  * Be aware, best performance of the 8254x is achived only when jumbo frame is
4238  * not used at all on architectures with strict alignment.
4239  */
4240 static int
4241 em_fixup_rx(struct rx_ring *rxr)
4242 {
4243         struct adapter *adapter = rxr->adapter;
4244         struct mbuf *m, *n;
4245         int error;
4246
4247         error = 0;
4248         m = rxr->fmp;
4249         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4250                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4251                 m->m_data += ETHER_HDR_LEN;
4252         } else {
4253                 MGETHDR(n, M_DONTWAIT, MT_DATA);
4254                 if (n != NULL) {
4255                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4256                         m->m_data += ETHER_HDR_LEN;
4257                         m->m_len -= ETHER_HDR_LEN;
4258                         n->m_len = ETHER_HDR_LEN;
4259                         M_MOVE_PKTHDR(n, m);
4260                         n->m_next = m;
4261                         rxr->fmp = n;
4262                 } else {
4263                         adapter->dropped_pkts++;
4264                         m_freem(rxr->fmp);
4265                         rxr->fmp = NULL;
4266                         error = ENOMEM;
4267                 }
4268         }
4269
4270         return (error);
4271 }
4272 #endif
4273
4274 /*********************************************************************
4275  *
4276  *  Verify that the hardware indicated that the checksum is valid.
4277  *  Inform the stack about the status of checksum so that stack
4278  *  doesn't spend time verifying the checksum.
4279  *
4280  *********************************************************************/
4281 static void
4282 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4283 {
4284         /* Ignore Checksum bit is set */
4285         if (rx_desc->status & E1000_RXD_STAT_IXSM) {
4286                 mp->m_pkthdr.csum_flags = 0;
4287                 return;
4288         }
4289
4290         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4291                 /* Did it pass? */
4292                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4293                         /* IP Checksum Good */
4294                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4295                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4296
4297                 } else {
4298                         mp->m_pkthdr.csum_flags = 0;
4299                 }
4300         }
4301
4302         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4303                 /* Did it pass? */
4304                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4305                         mp->m_pkthdr.csum_flags |=
4306                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4307                         mp->m_pkthdr.csum_data = htons(0xffff);
4308                 }
4309         }
4310 }
4311
4312 /*
4313  * This routine is run via an vlan
4314  * config EVENT
4315  */
4316 static void
4317 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4318 {
4319         struct adapter  *adapter = ifp->if_softc;
4320         u32             index, bit;
4321
4322         if (ifp->if_softc !=  arg)   /* Not our event */
4323                 return;
4324
4325         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
4326                 return;
4327
4328         index = (vtag >> 5) & 0x7F;
4329         bit = vtag & 0x1F;
4330         em_shadow_vfta[index] |= (1 << bit);
4331         ++adapter->num_vlans;
4332         /* Re-init to load the changes */
4333         em_init(adapter);
4334 }
4335
4336 /*
4337  * This routine is run via an vlan
4338  * unconfig EVENT
4339  */
4340 static void
4341 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4342 {
4343         struct adapter  *adapter = ifp->if_softc;
4344         u32             index, bit;
4345
4346         if (ifp->if_softc !=  arg)
4347                 return;
4348
4349         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4350                 return;
4351
4352         index = (vtag >> 5) & 0x7F;
4353         bit = vtag & 0x1F;
4354         em_shadow_vfta[index] &= ~(1 << bit);
4355         --adapter->num_vlans;
4356         /* Re-init to load the changes */
4357         em_init(adapter);
4358 }
4359
4360 static void
4361 em_setup_vlan_hw_support(struct adapter *adapter)
4362 {
4363         struct e1000_hw *hw = &adapter->hw;
4364         u32             reg;
4365
4366         /*
4367         ** We get here thru init_locked, meaning
4368         ** a soft reset, this has already cleared
4369         ** the VFTA and other state, so if there
4370         ** have been no vlan's registered do nothing.
4371         */
4372         if (adapter->num_vlans == 0)
4373                 return;
4374
4375         /*
4376         ** A soft reset zero's out the VFTA, so
4377         ** we need to repopulate it now.
4378         */
4379         for (int i = 0; i < EM_VFTA_SIZE; i++)
4380                 if (em_shadow_vfta[i] != 0)
4381                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4382                             i, em_shadow_vfta[i]);
4383
4384         reg = E1000_READ_REG(hw, E1000_CTRL);
4385         reg |= E1000_CTRL_VME;
4386         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4387
4388         /* Enable the Filter Table */
4389         reg = E1000_READ_REG(hw, E1000_RCTL);
4390         reg &= ~E1000_RCTL_CFIEN;
4391         reg |= E1000_RCTL_VFE;
4392         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4393
4394         /* Update the frame size */
4395         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4396             adapter->max_frame_size + VLAN_TAG_SIZE);
4397 }
4398
4399 static void
4400 em_enable_intr(struct adapter *adapter)
4401 {
4402         struct e1000_hw *hw = &adapter->hw;
4403         u32 ims_mask = IMS_ENABLE_MASK;
4404
4405         if (hw->mac.type == e1000_82574) {
4406                 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4407                 ims_mask |= EM_MSIX_MASK;
4408         }
4409         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4410 }
4411
4412 static void
4413 em_disable_intr(struct adapter *adapter)
4414 {
4415         struct e1000_hw *hw = &adapter->hw;
4416
4417         if (hw->mac.type == e1000_82574)
4418                 E1000_WRITE_REG(hw, EM_EIAC, 0);
4419         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4420 }
4421
4422 /*
4423  * Bit of a misnomer, what this really means is
4424  * to enable OS management of the system... aka
4425  * to disable special hardware management features
4426  */
4427 static void
4428 em_init_manageability(struct adapter *adapter)
4429 {
4430         /* A shared code workaround */
4431 #define E1000_82542_MANC2H E1000_MANC2H
4432         if (adapter->has_manage) {
4433                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4434                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4435
4436                 /* disable hardware interception of ARP */
4437                 manc &= ~(E1000_MANC_ARP_EN);
4438
4439                 /* enable receiving management packets to the host */
4440                 manc |= E1000_MANC_EN_MNG2HOST;
4441 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4442 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4443                 manc2h |= E1000_MNG2HOST_PORT_623;
4444                 manc2h |= E1000_MNG2HOST_PORT_664;
4445                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4446                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4447         }
4448 }
4449
4450 /*
4451  * Give control back to hardware management
4452  * controller if there is one.
4453  */
4454 static void
4455 em_release_manageability(struct adapter *adapter)
4456 {
4457         if (adapter->has_manage) {
4458                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4459
4460                 /* re-enable hardware interception of ARP */
4461                 manc |= E1000_MANC_ARP_EN;
4462                 manc &= ~E1000_MANC_EN_MNG2HOST;
4463
4464                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4465         }
4466 }
4467
4468 /*
4469  * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4470  * For ASF and Pass Through versions of f/w this means
4471  * that the driver is loaded. For AMT version type f/w
4472  * this means that the network i/f is open.
4473  */
4474 static void
4475 em_get_hw_control(struct adapter *adapter)
4476 {
4477         u32 ctrl_ext, swsm;
4478
4479         if (adapter->hw.mac.type == e1000_82573) {
4480                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4481                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4482                     swsm | E1000_SWSM_DRV_LOAD);
4483                 return;
4484         }
4485         /* else */
4486         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4487         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4488             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4489         return;
4490 }
4491
4492 /*
4493  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4494  * For ASF and Pass Through versions of f/w this means that
4495  * the driver is no longer loaded. For AMT versions of the
4496  * f/w this means that the network i/f is closed.
4497  */
4498 static void
4499 em_release_hw_control(struct adapter *adapter)
4500 {
4501         u32 ctrl_ext, swsm;
4502
4503         if (!adapter->has_manage)
4504                 return;
4505
4506         if (adapter->hw.mac.type == e1000_82573) {
4507                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4508                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4509                     swsm & ~E1000_SWSM_DRV_LOAD);
4510                 return;
4511         }
4512         /* else */
4513         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4514         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4515             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4516         return;
4517 }
4518
4519 static int
4520 em_is_valid_ether_addr(u8 *addr)
4521 {
4522         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4523
4524         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4525                 return (FALSE);
4526         }
4527
4528         return (TRUE);
4529 }
4530
4531 /*
4532 ** Parse the interface capabilities with regard
4533 ** to both system management and wake-on-lan for
4534 ** later use.
4535 */
4536 static void
4537 em_get_wakeup(device_t dev)
4538 {
4539         struct adapter  *adapter = device_get_softc(dev);
4540         u16             eeprom_data = 0, device_id, apme_mask;
4541
4542         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4543         apme_mask = EM_EEPROM_APME;
4544
4545         switch (adapter->hw.mac.type) {
4546         case e1000_82573:
4547         case e1000_82583:
4548                 adapter->has_amt = TRUE;
4549                 /* Falls thru */
4550         case e1000_82571:
4551         case e1000_82572:
4552         case e1000_80003es2lan:
4553                 if (adapter->hw.bus.func == 1) {
4554                         e1000_read_nvm(&adapter->hw,
4555                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4556                         break;
4557                 } else
4558                         e1000_read_nvm(&adapter->hw,
4559                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4560                 break;
4561         case e1000_ich8lan:
4562         case e1000_ich9lan:
4563         case e1000_ich10lan:
4564         case e1000_pchlan:
4565                 apme_mask = E1000_WUC_APME;
4566                 adapter->has_amt = TRUE;
4567                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
4568                 break;
4569         default:
4570                 e1000_read_nvm(&adapter->hw,
4571                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4572                 break;
4573         }
4574         if (eeprom_data & apme_mask)
4575                 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4576         /*
4577          * We have the eeprom settings, now apply the special cases
4578          * where the eeprom may be wrong or the board won't support
4579          * wake on lan on a particular port
4580          */
4581         device_id = pci_get_device(dev);
4582         switch (device_id) {
4583         case E1000_DEV_ID_82571EB_FIBER:
4584                 /* Wake events only supported on port A for dual fiber
4585                  * regardless of eeprom setting */
4586                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4587                     E1000_STATUS_FUNC_1)
4588                         adapter->wol = 0;
4589                 break;
4590         case E1000_DEV_ID_82571EB_QUAD_COPPER:
4591         case E1000_DEV_ID_82571EB_QUAD_FIBER:
4592         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
4593                 /* if quad port adapter, disable WoL on all but port A */
4594                 if (global_quad_port_a != 0)
4595                         adapter->wol = 0;
4596                 /* Reset for multiple quad port adapters */
4597                 if (++global_quad_port_a == 4)
4598                         global_quad_port_a = 0;
4599                 break;
4600         }
4601         return;
4602 }
4603
4604
4605 /*
4606  * Enable PCI Wake On Lan capability
4607  */
4608 static void
4609 em_enable_wakeup(device_t dev)
4610 {
4611         struct adapter  *adapter = device_get_softc(dev);
4612         struct ifnet    *ifp = adapter->ifp;
4613         u32             pmc, ctrl, ctrl_ext, rctl;
4614         u16             status;
4615
4616         if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0))
4617                 return;
4618
4619         /* Advertise the wakeup capability */
4620         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4621         ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4622         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4623         E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4624
4625         if ((adapter->hw.mac.type == e1000_ich8lan) ||
4626             (adapter->hw.mac.type == e1000_pchlan) ||
4627             (adapter->hw.mac.type == e1000_ich9lan) ||
4628             (adapter->hw.mac.type == e1000_ich10lan)) {
4629                 e1000_disable_gig_wol_ich8lan(&adapter->hw);
4630                 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw);
4631         }
4632
4633         /* Keep the laser running on Fiber adapters */
4634         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4635             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4636                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4637                 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4638                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4639         }
4640
4641         /*
4642         ** Determine type of Wakeup: note that wol
4643         ** is set with all bits on by default.
4644         */
4645         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0)
4646                 adapter->wol &= ~E1000_WUFC_MAG;
4647
4648         if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0)
4649                 adapter->wol &= ~E1000_WUFC_MC;
4650         else {
4651                 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4652                 rctl |= E1000_RCTL_MPE;
4653                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4654         }
4655
4656         if (adapter->hw.mac.type == e1000_pchlan) {
4657                 if (em_enable_phy_wakeup(adapter))
4658                         return;
4659         } else {
4660                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4661                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4662         }
4663
4664         if (adapter->hw.phy.type == e1000_phy_igp_3)
4665                 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
4666
4667         /* Request PME */
4668         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4669         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4670         if (ifp->if_capenable & IFCAP_WOL)
4671                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4672         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
4673
4674         return;
4675 }
4676
4677 /*
4678 ** WOL in the newer chipset interfaces (pchlan)
4679 ** require thing to be copied into the phy
4680 */
4681 static int
4682 em_enable_phy_wakeup(struct adapter *adapter)
4683 {
4684         struct e1000_hw *hw = &adapter->hw;
4685         u32 mreg, ret = 0;
4686         u16 preg;
4687
4688         /* copy MAC RARs to PHY RARs */
4689         for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) {
4690                 mreg = E1000_READ_REG(hw, E1000_RAL(i));
4691                 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF));
4692                 e1000_write_phy_reg(hw, BM_RAR_M(i),
4693                     (u16)((mreg >> 16) & 0xFFFF));
4694                 mreg = E1000_READ_REG(hw, E1000_RAH(i));
4695                 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF));
4696                 e1000_write_phy_reg(hw, BM_RAR_CTRL(i),
4697                     (u16)((mreg >> 16) & 0xFFFF));
4698         }
4699
4700         /* copy MAC MTA to PHY MTA */
4701         for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
4702                 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
4703                 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
4704                 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
4705                     (u16)((mreg >> 16) & 0xFFFF));
4706         }
4707
4708         /* configure PHY Rx Control register */
4709         e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
4710         mreg = E1000_READ_REG(hw, E1000_RCTL);
4711         if (mreg & E1000_RCTL_UPE)
4712                 preg |= BM_RCTL_UPE;
4713         if (mreg & E1000_RCTL_MPE)
4714                 preg |= BM_RCTL_MPE;
4715         preg &= ~(BM_RCTL_MO_MASK);
4716         if (mreg & E1000_RCTL_MO_3)
4717                 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
4718                                 << BM_RCTL_MO_SHIFT);
4719         if (mreg & E1000_RCTL_BAM)
4720                 preg |= BM_RCTL_BAM;
4721         if (mreg & E1000_RCTL_PMCF)
4722                 preg |= BM_RCTL_PMCF;
4723         mreg = E1000_READ_REG(hw, E1000_CTRL);
4724         if (mreg & E1000_CTRL_RFCE)
4725                 preg |= BM_RCTL_RFCE;
4726         e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
4727
4728         /* enable PHY wakeup in MAC register */
4729         E1000_WRITE_REG(hw, E1000_WUC,
4730             E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
4731         E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
4732
4733         /* configure and enable PHY wakeup in PHY registers */
4734         e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
4735         e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
4736
4737         /* activate PHY wakeup */
4738         ret = hw->phy.ops.acquire(hw);
4739         if (ret) {
4740                 printf("Could not acquire PHY\n");
4741                 return ret;
4742         }
4743         e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4744                                  (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
4745         ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
4746         if (ret) {
4747                 printf("Could not read PHY page 769\n");
4748                 goto out;
4749         }
4750         preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
4751         ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
4752         if (ret)
4753                 printf("Could not set PHY Host Wakeup bit\n");
4754 out:
4755         hw->phy.ops.release(hw);
4756
4757         return ret;
4758 }
4759
4760 static void
4761 em_led_func(void *arg, int onoff)
4762 {
4763         struct adapter  *adapter = arg;
4764
4765         EM_CORE_LOCK(adapter);
4766         if (onoff) {
4767                 e1000_setup_led(&adapter->hw);
4768                 e1000_led_on(&adapter->hw);
4769         } else {
4770                 e1000_led_off(&adapter->hw);
4771                 e1000_cleanup_led(&adapter->hw);
4772         }
4773         EM_CORE_UNLOCK(adapter);
4774 }
4775
4776 /**********************************************************************
4777  *
4778  *  Update the board statistics counters.
4779  *
4780  **********************************************************************/
4781 static void
4782 em_update_stats_counters(struct adapter *adapter)
4783 {
4784         struct ifnet   *ifp;
4785
4786         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
4787            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4788                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4789                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
4790         }
4791         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4792         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4793         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4794         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4795
4796         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4797         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4798         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4799         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4800         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4801         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4802         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4803         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4804         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4805         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4806         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4807         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4808         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4809         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4810         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4811         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4812         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4813         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4814         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4815         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4816
4817         /* For the 64-bit byte counters the low dword must be read first. */
4818         /* Both registers clear on the read of the high dword */
4819
4820         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
4821         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
4822
4823         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4824         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4825         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4826         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4827         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4828
4829         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4830         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4831
4832         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
4833         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
4834         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
4835         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
4836         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
4837         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
4838         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
4839         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
4840         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
4841         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
4842
4843         if (adapter->hw.mac.type >= e1000_82543) {
4844                 adapter->stats.algnerrc +=
4845                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
4846                 adapter->stats.rxerrc +=
4847                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
4848                 adapter->stats.tncrs +=
4849                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
4850                 adapter->stats.cexterr +=
4851                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
4852                 adapter->stats.tsctc +=
4853                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
4854                 adapter->stats.tsctfc +=
4855                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
4856         }
4857         ifp = adapter->ifp;
4858
4859         ifp->if_collisions = adapter->stats.colc;
4860
4861         /* Rx Errors */
4862         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
4863             adapter->stats.crcerrs + adapter->stats.algnerrc +
4864             adapter->stats.ruc + adapter->stats.roc +
4865             adapter->stats.mpc + adapter->stats.cexterr;
4866
4867         /* Tx Errors */
4868         ifp->if_oerrors = adapter->stats.ecol +
4869             adapter->stats.latecol + adapter->watchdog_events;
4870 }
4871
4872
4873 /**********************************************************************
4874  *
4875  *  This routine is called only when em_display_debug_stats is enabled.
4876  *  This routine provides a way to take a look at important statistics
4877  *  maintained by the driver and hardware.
4878  *
4879  **********************************************************************/
4880 static void
4881 em_print_debug_info(struct adapter *adapter)
4882 {
4883         device_t dev = adapter->dev;
4884         u8 *hw_addr = adapter->hw.hw_addr;
4885         struct rx_ring *rxr = adapter->rx_rings;
4886         struct tx_ring *txr = adapter->tx_rings;
4887
4888         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
4889         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
4890             E1000_READ_REG(&adapter->hw, E1000_CTRL),
4891             E1000_READ_REG(&adapter->hw, E1000_RCTL));
4892         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
4893             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
4894             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
4895         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
4896             adapter->hw.fc.high_water,
4897             adapter->hw.fc.low_water);
4898         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
4899             E1000_READ_REG(&adapter->hw, E1000_TIDV),
4900             E1000_READ_REG(&adapter->hw, E1000_TADV));
4901         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
4902             E1000_READ_REG(&adapter->hw, E1000_RDTR),
4903             E1000_READ_REG(&adapter->hw, E1000_RADV));
4904
4905         for (int i = 0; i < adapter->num_queues; i++, txr++) {
4906                 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i,
4907                     E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
4908                     E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
4909                 device_printf(dev, "TX(%d) no descriptors avail event = %ld\n",
4910                     txr->me, txr->no_desc_avail);
4911                 device_printf(dev, "TX(%d) MSIX IRQ Handled = %ld\n",
4912                     txr->me, txr->tx_irq);
4913                 device_printf(dev, "Num Tx descriptors avail = %d\n",
4914                     txr->tx_avail);
4915                 device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
4916                     txr->no_desc_avail);
4917         }
4918         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4919                 device_printf(dev, "RX(%d) MSIX IRQ Handled = %ld\n",
4920                     rxr->me, rxr->rx_irq);
4921                 device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
4922                     E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
4923                     E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
4924         }
4925         device_printf(dev, "Std mbuf failed = %ld\n",
4926             adapter->mbuf_alloc_failed);
4927         device_printf(dev, "Std mbuf cluster failed = %ld\n",
4928             adapter->mbuf_cluster_failed);
4929         device_printf(dev, "Driver dropped packets = %ld\n",
4930             adapter->dropped_pkts);
4931 }
4932
4933 static void
4934 em_print_hw_stats(struct adapter *adapter)
4935 {
4936         device_t dev = adapter->dev;
4937
4938         device_printf(dev, "Excessive collisions = %lld\n",
4939             (long long)adapter->stats.ecol);
4940 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
4941         device_printf(dev, "Symbol errors = %lld\n",
4942             (long long)adapter->stats.symerrs);
4943 #endif
4944         device_printf(dev, "Sequence errors = %lld\n",
4945             (long long)adapter->stats.sec);
4946         device_printf(dev, "Defer count = %lld\n",
4947             (long long)adapter->stats.dc);
4948         device_printf(dev, "Missed Packets = %lld\n",
4949             (long long)adapter->stats.mpc);
4950         device_printf(dev, "Receive No Buffers = %lld\n",
4951             (long long)adapter->stats.rnbc);
4952         /* RLEC is inaccurate on some hardware, calculate our own. */
4953         device_printf(dev, "Receive Length Errors = %lld\n",
4954             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
4955         device_printf(dev, "Receive errors = %lld\n",
4956             (long long)adapter->stats.rxerrc);
4957         device_printf(dev, "Crc errors = %lld\n",
4958             (long long)adapter->stats.crcerrs);
4959         device_printf(dev, "Alignment errors = %lld\n",
4960             (long long)adapter->stats.algnerrc);
4961         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
4962             (long long)adapter->stats.cexterr);
4963         device_printf(dev, "watchdog timeouts = %ld\n",
4964             adapter->watchdog_events);
4965         device_printf(dev, "XON Rcvd = %lld\n",
4966             (long long)adapter->stats.xonrxc);
4967         device_printf(dev, "XON Xmtd = %lld\n",
4968             (long long)adapter->stats.xontxc);
4969         device_printf(dev, "XOFF Rcvd = %lld\n",
4970             (long long)adapter->stats.xoffrxc);
4971         device_printf(dev, "XOFF Xmtd = %lld\n",
4972             (long long)adapter->stats.xofftxc);
4973         device_printf(dev, "Good Packets Rcvd = %lld\n",
4974             (long long)adapter->stats.gprc);
4975         device_printf(dev, "Good Packets Xmtd = %lld\n",
4976             (long long)adapter->stats.gptc);
4977         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
4978             (long long)adapter->stats.tsctc);
4979         device_printf(dev, "TSO Contexts Failed = %lld\n",
4980             (long long)adapter->stats.tsctfc);
4981 }
4982
4983 /**********************************************************************
4984  *
4985  *  This routine provides a way to dump out the adapter eeprom,
4986  *  often a useful debug/service tool. This only dumps the first
4987  *  32 words, stuff that matters is in that extent.
4988  *
4989  **********************************************************************/
4990 static void
4991 em_print_nvm_info(struct adapter *adapter)
4992 {
4993         u16     eeprom_data;
4994         int     i, j, row = 0;
4995
4996         /* Its a bit crude, but it gets the job done */
4997         printf("\nInterface EEPROM Dump:\n");
4998         printf("Offset\n0x0000  ");
4999         for (i = 0, j = 0; i < 32; i++, j++) {
5000                 if (j == 8) { /* Make the offset block */
5001                         j = 0; ++row;
5002                         printf("\n0x00%x0  ",row);
5003                 }
5004                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5005                 printf("%04x ", eeprom_data);
5006         }
5007         printf("\n");
5008 }
5009
5010 static int
5011 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5012 {
5013         struct adapter *adapter;
5014         int error;
5015         int result;
5016
5017         result = -1;
5018         error = sysctl_handle_int(oidp, &result, 0, req);
5019
5020         if (error || !req->newptr)
5021                 return (error);
5022
5023         if (result == 1) {
5024                 adapter = (struct adapter *)arg1;
5025                 em_print_debug_info(adapter);
5026         }
5027         /*
5028          * This value will cause a hex dump of the
5029          * first 32 16-bit words of the EEPROM to
5030          * the screen.
5031          */
5032         if (result == 2) {
5033                 adapter = (struct adapter *)arg1;
5034                 em_print_nvm_info(adapter);
5035         }
5036
5037         return (error);
5038 }
5039
5040
5041 static int
5042 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
5043 {
5044         struct adapter *adapter;
5045         int error;
5046         int result;
5047
5048         result = -1;
5049         error = sysctl_handle_int(oidp, &result, 0, req);
5050
5051         if (error || !req->newptr)
5052                 return (error);
5053
5054         if (result == 1) {
5055                 adapter = (struct adapter *)arg1;
5056                 em_print_hw_stats(adapter);
5057         }
5058
5059         return (error);
5060 }
5061
5062 static int
5063 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5064 {
5065         struct em_int_delay_info *info;
5066         struct adapter *adapter;
5067         u32 regval;
5068         int error, usecs, ticks;
5069
5070         info = (struct em_int_delay_info *)arg1;
5071         usecs = info->value;
5072         error = sysctl_handle_int(oidp, &usecs, 0, req);
5073         if (error != 0 || req->newptr == NULL)
5074                 return (error);
5075         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5076                 return (EINVAL);
5077         info->value = usecs;
5078         ticks = EM_USECS_TO_TICKS(usecs);
5079
5080         adapter = info->adapter;
5081
5082         EM_CORE_LOCK(adapter);
5083         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5084         regval = (regval & ~0xffff) | (ticks & 0xffff);
5085         /* Handle a few special cases. */
5086         switch (info->offset) {
5087         case E1000_RDTR:
5088                 break;
5089         case E1000_TIDV:
5090                 if (ticks == 0) {
5091                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5092                         /* Don't write 0 into the TIDV register. */
5093                         regval++;
5094                 } else
5095                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5096                 break;
5097         }
5098         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5099         EM_CORE_UNLOCK(adapter);
5100         return (0);
5101 }
5102
5103 static void
5104 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5105         const char *description, struct em_int_delay_info *info,
5106         int offset, int value)
5107 {
5108         info->adapter = adapter;
5109         info->offset = offset;
5110         info->value = value;
5111         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5112             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5113             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5114             info, 0, em_sysctl_int_delay, "I", description);
5115 }
5116
5117 static void
5118 em_add_rx_process_limit(struct adapter *adapter, const char *name,
5119         const char *description, int *limit, int value)
5120 {
5121         *limit = value;
5122         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5123             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5124             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5125 }
5126
5127