sys/dev/e1000/if_em.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2011, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #endif
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #if __FreeBSD_version >= 800000
  44 #include <sys/buf_ring.h>
  45 #endif
  46 #include <sys/bus.h>
  47 #include <sys/endian.h>
  48 #include <sys/kernel.h>
  49 #include <sys/kthread.h>
  50 #include <sys/malloc.h>
  51 #include <sys/mbuf.h>
  52 #include <sys/module.h>
  53 #include <sys/rman.h>
  54 #include <sys/socket.h>
  55 #include <sys/sockio.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/taskqueue.h>
  58 #include <sys/eventhandler.h>
  59 #include <machine/bus.h>
  60 #include <machine/resource.h>
  61
  62 #include <net/bpf.h>
  63 #include <net/ethernet.h>
  64 #include <net/if.h>
  65 #include <net/if_arp.h>
  66 #include <net/if_dl.h>
  67 #include <net/if_media.h>
  68
  69 #include <net/if_types.h>
  70 #include <net/if_vlan_var.h>
  71
  72 #include <netinet/in_systm.h>
  73 #include <netinet/in.h>
  74 #include <netinet/if_ether.h>
  75 #include <netinet/ip.h>
  76 #include <netinet/ip6.h>
  77 #include <netinet/tcp.h>
  78 #include <netinet/udp.h>
  79
  80 #include <machine/in_cksum.h>
  81 #include <dev/led/led.h>
  82 #include <dev/pci/pcivar.h>
  83 #include <dev/pci/pcireg.h>
  84
  85 #include "e1000_api.h"
  86 #include "e1000_82571.h"
  87 #include "if_em.h"
  88
  89 /*********************************************************************
  90  *  Set this to one to display debug statistics
  91  *********************************************************************/
  92 int     em_display_debug_stats = 0;
  93
  94 /*********************************************************************
  95  *  Driver version:
  96  *********************************************************************/
  97 char em_driver_version[] = "7.3.2";
  98
  99 /*********************************************************************
 100  *  PCI Device ID Table
 101  *
 102  *  Used by probe to select devices to load on
 103  *  Last field stores an index into e1000_strings
 104  *  Last entry must be all 0s
 105  *
 106  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 107  *********************************************************************/
 108
 109 static em_vendor_info_t em_vendor_info_array[] =
 110 {
 111         /* Intel(R) PRO/1000 Network Connection */
 112         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 113         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 114         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
 116                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
 118                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 119         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 120                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
 122                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
 124                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
 126                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 127         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 128         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 129         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 131
 132         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
 136         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 137                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 138         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 139                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 141                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 142         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 143                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 144         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_ICH8_82567V_3,   PCI_ANY_ID, PCI_ANY_ID, 0},
 152         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 153         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 154         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 155         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 157         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 158         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 159         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 160         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 162         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
 163         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 164         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 165         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 166         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 167         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 168         { 0x8086, E1000_DEV_ID_ICH10_D_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 169         { 0x8086, E1000_DEV_ID_PCH_M_HV_LM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 170         { 0x8086, E1000_DEV_ID_PCH_M_HV_LC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 171         { 0x8086, E1000_DEV_ID_PCH_D_HV_DM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 172         { 0x8086, E1000_DEV_ID_PCH_D_HV_DC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 173         { 0x8086, E1000_DEV_ID_PCH2_LV_LM,      PCI_ANY_ID, PCI_ANY_ID, 0},
 174         { 0x8086, E1000_DEV_ID_PCH2_LV_V,       PCI_ANY_ID, PCI_ANY_ID, 0},
 175         /* required last entry */
 176         { 0, 0, 0, 0, 0}
 177 };
 178
 179 /*********************************************************************
 180  *  Table of branding strings for all supported NICs.
 181  *********************************************************************/
 182
 183 static char *em_strings[] = {
 184         "Intel(R) PRO/1000 Network Connection"
 185 };
 186
 187 /*********************************************************************
 188  *  Function prototypes
 189  *********************************************************************/
 190 static int      em_probe(device_t);
 191 static int      em_attach(device_t);
 192 static int      em_detach(device_t);
 193 static int      em_shutdown(device_t);
 194 static int      em_suspend(device_t);
 195 static int      em_resume(device_t);
 196 static void     em_start(struct ifnet *);
 197 static void     em_start_locked(struct ifnet *, struct tx_ring *);
 198 #ifdef EM_MULTIQUEUE
 199 static int      em_mq_start(struct ifnet *, struct mbuf *);
 200 static int      em_mq_start_locked(struct ifnet *,
 201                     struct tx_ring *, struct mbuf *);
 202 static void     em_qflush(struct ifnet *);
 203 #endif
 204 static int      em_ioctl(struct ifnet *, u_long, caddr_t);
 205 static void     em_init(void *);
 206 static void     em_init_locked(struct adapter *);
 207 static void     em_stop(void *);
 208 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 209 static int      em_media_change(struct ifnet *);
 210 static void     em_identify_hardware(struct adapter *);
 211 static int      em_allocate_pci_resources(struct adapter *);
 212 static int      em_allocate_legacy(struct adapter *);
 213 static int      em_allocate_msix(struct adapter *);
 214 static int      em_allocate_queues(struct adapter *);
 215 static int      em_setup_msix(struct adapter *);
 216 static void     em_free_pci_resources(struct adapter *);
 217 static void     em_local_timer(void *);
 218 static void     em_reset(struct adapter *);
 219 static int      em_setup_interface(device_t, struct adapter *);
 220
 221 static void     em_setup_transmit_structures(struct adapter *);
 222 static void     em_initialize_transmit_unit(struct adapter *);
 223 static int      em_allocate_transmit_buffers(struct tx_ring *);
 224 static void     em_free_transmit_structures(struct adapter *);
 225 static void     em_free_transmit_buffers(struct tx_ring *);
 226
 227 static int      em_setup_receive_structures(struct adapter *);
 228 static int      em_allocate_receive_buffers(struct rx_ring *);
 229 static void     em_initialize_receive_unit(struct adapter *);
 230 static void     em_free_receive_structures(struct adapter *);
 231 static void     em_free_receive_buffers(struct rx_ring *);
 232
 233 static void     em_enable_intr(struct adapter *);
 234 static void     em_disable_intr(struct adapter *);
 235 static void     em_update_stats_counters(struct adapter *);
 236 static void     em_add_hw_stats(struct adapter *adapter);
 237 static bool     em_txeof(struct tx_ring *);
 238 static bool     em_rxeof(struct rx_ring *, int, int *);
 239 #ifndef __NO_STRICT_ALIGNMENT
 240 static int      em_fixup_rx(struct rx_ring *);
 241 #endif
 242 static void     em_receive_checksum(struct e1000_rx_desc *, struct mbuf *);
 243 static void     em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int,
 244                     struct ip *, u32 *, u32 *);
 245 static void     em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *,
 246                     struct tcphdr *, u32 *, u32 *);
 247 static void     em_set_promisc(struct adapter *);
 248 static void     em_disable_promisc(struct adapter *);
 249 static void     em_set_multi(struct adapter *);
 250 static void     em_update_link_status(struct adapter *);
 251 static void     em_refresh_mbufs(struct rx_ring *, int);
 252 static void     em_register_vlan(void *, struct ifnet *, u16);
 253 static void     em_unregister_vlan(void *, struct ifnet *, u16);
 254 static void     em_setup_vlan_hw_support(struct adapter *);
 255 static int      em_xmit(struct tx_ring *, struct mbuf **);
 256 static int      em_dma_malloc(struct adapter *, bus_size_t,
 257                     struct em_dma_alloc *, int);
 258 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 259 static int      em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
 260 static void     em_print_nvm_info(struct adapter *);
 261 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 262 static void     em_print_debug_info(struct adapter *);
 263 static int      em_is_valid_ether_addr(u8 *);
 264 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 265 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 266                     const char *, struct em_int_delay_info *, int, int);
 267 /* Management and WOL Support */
 268 static void     em_init_manageability(struct adapter *);
 269 static void     em_release_manageability(struct adapter *);
 270 static void     em_get_hw_control(struct adapter *);
 271 static void     em_release_hw_control(struct adapter *);
 272 static void     em_get_wakeup(device_t);
 273 static void     em_enable_wakeup(device_t);
 274 static int      em_enable_phy_wakeup(struct adapter *);
 275 static void     em_led_func(void *, int);
 276 static void     em_disable_aspm(struct adapter *);
 277
 278 static int      em_irq_fast(void *);
 279
 280 /* MSIX handlers */
 281 static void     em_msix_tx(void *);
 282 static void     em_msix_rx(void *);
 283 static void     em_msix_link(void *);
 284 static void     em_handle_tx(void *context, int pending);
 285 static void     em_handle_rx(void *context, int pending);
 286 static void     em_handle_link(void *context, int pending);
 287
 288 static void     em_set_sysctl_value(struct adapter *, const char *,
 289                     const char *, int *, int);
 290 static int      em_set_flowcntl(SYSCTL_HANDLER_ARGS);
 291
 292 static __inline void em_rx_discard(struct rx_ring *, int);
 293
 294 #ifdef DEVICE_POLLING
 295 static poll_handler_t em_poll;
 296 #endif /* POLLING */
 297
 298 /*********************************************************************
 299  *  FreeBSD Device Interface Entry Points
 300  *********************************************************************/
 301
 302 static device_method_t em_methods[] = {
 303         /* Device interface */
 304         DEVMETHOD(device_probe, em_probe),
 305         DEVMETHOD(device_attach, em_attach),
 306         DEVMETHOD(device_detach, em_detach),
 307         DEVMETHOD(device_shutdown, em_shutdown),
 308         DEVMETHOD(device_suspend, em_suspend),
 309         DEVMETHOD(device_resume, em_resume),
 310         {0, 0}
 311 };
 312
 313 static driver_t em_driver = {
 314         "em", em_methods, sizeof(struct adapter),
 315 };
 316
 317 devclass_t em_devclass;
 318 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 319 MODULE_DEPEND(em, pci, 1, 1, 1);
 320 MODULE_DEPEND(em, ether, 1, 1, 1);
 321
 322 /*********************************************************************
 323  *  Tunable default values.
 324  *********************************************************************/
 325
 326 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 327 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 328 #define M_TSO_LEN                       66
 329
 330 /* Allow common code without TSO */
 331 #ifndef CSUM_TSO
 332 #define CSUM_TSO        0
 333 #endif
 334
 335 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
 336
 337 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 338 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 339 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 340 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 341 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &em_tx_int_delay_dflt,
 342     0, "Default transmit interrupt delay in usecs");
 343 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt,
 344     0, "Default receive interrupt delay in usecs");
 345
 346 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 347 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 348 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 349 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 350 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN,
 351     &em_tx_abs_int_delay_dflt, 0,
 352     "Default transmit interrupt delay limit in usecs");
 353 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN,
 354     &em_rx_abs_int_delay_dflt, 0,
 355     "Default receive interrupt delay limit in usecs");
 356
 357 static int em_rxd = EM_DEFAULT_RXD;
 358 static int em_txd = EM_DEFAULT_TXD;
 359 TUNABLE_INT("hw.em.rxd", &em_rxd);
 360 TUNABLE_INT("hw.em.txd", &em_txd);
 361 SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0,
 362     "Number of receive descriptors per queue");
 363 SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0,
 364     "Number of transmit descriptors per queue");
 365
 366 static int em_smart_pwr_down = FALSE;
 367 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 368 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down,
 369     0, "Set to true to leave smart power down enabled on newer adapters");
 370
 371 /* Controls whether promiscuous also shows bad packets */
 372 static int em_debug_sbp = FALSE;
 373 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 374 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0,
 375     "Show bad packets in promiscuous mode");
 376
 377 static int em_enable_msix = TRUE;
 378 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix);
 379 SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0,
 380     "Enable MSI-X interrupts");
 381
 382 /* How many packets rxeof tries to clean at a time */
 383 static int em_rx_process_limit = 100;
 384 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
 385 SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
 386     &em_rx_process_limit, 0,
 387     "Maximum number of received packets to process "
 388     "at a time, -1 means unlimited");
 389
 390 /* Energy efficient ethernet - default to OFF */
 391 static int eee_setting = 0;
 392 TUNABLE_INT("hw.em.eee_setting", &eee_setting);
 393 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0,
 394     "Enable Energy Efficient Ethernet");
 395
 396 /* Global used in WOL setup with multiport cards */
 397 static int global_quad_port_a = 0;
 398
 399 /*********************************************************************
 400  *  Device identification routine
 401  *
 402  *  em_probe determines if the driver should be loaded on
 403  *  adapter based on PCI vendor/device id of the adapter.
 404  *
 405  *  return BUS_PROBE_DEFAULT on success, positive on failure
 406  *********************************************************************/
 407
 408 static int
 409 em_probe(device_t dev)
 410 {
 411         char            adapter_name[60];
 412         u16             pci_vendor_id = 0;
 413         u16             pci_device_id = 0;
 414         u16             pci_subvendor_id = 0;
 415         u16             pci_subdevice_id = 0;
 416         em_vendor_info_t *ent;
 417
 418         INIT_DEBUGOUT("em_probe: begin");
 419
 420         pci_vendor_id = pci_get_vendor(dev);
 421         if (pci_vendor_id != EM_VENDOR_ID)
 422                 return (ENXIO);
 423
 424         pci_device_id = pci_get_device(dev);
 425         pci_subvendor_id = pci_get_subvendor(dev);
 426         pci_subdevice_id = pci_get_subdevice(dev);
 427
 428         ent = em_vendor_info_array;
 429         while (ent->vendor_id != 0) {
 430                 if ((pci_vendor_id == ent->vendor_id) &&
 431                     (pci_device_id == ent->device_id) &&
 432
 433                     ((pci_subvendor_id == ent->subvendor_id) ||
 434                     (ent->subvendor_id == PCI_ANY_ID)) &&
 435
 436                     ((pci_subdevice_id == ent->subdevice_id) ||
 437                     (ent->subdevice_id == PCI_ANY_ID))) {
 438                         sprintf(adapter_name, "%s %s",
 439                                 em_strings[ent->index],
 440                                 em_driver_version);
 441                         device_set_desc_copy(dev, adapter_name);
 442                         return (BUS_PROBE_DEFAULT);
 443                 }
 444                 ent++;
 445         }
 446
 447         return (ENXIO);
 448 }
 449
 450 /*********************************************************************
 451  *  Device initialization routine
 452  *
 453  *  The attach entry point is called when the driver is being loaded.
 454  *  This routine identifies the type of hardware, allocates all resources
 455  *  and initializes the hardware.
 456  *
 457  *  return 0 on success, positive on failure
 458  *********************************************************************/
 459
 460 static int
 461 em_attach(device_t dev)
 462 {
 463         struct adapter  *adapter;
 464         struct e1000_hw *hw;
 465         int             error = 0;
 466
 467         INIT_DEBUGOUT("em_attach: begin");
 468
 469         if (resource_disabled("em", device_get_unit(dev))) {
 470                 device_printf(dev, "Disabled by device hint\n");
 471                 return (ENXIO);
 472         }
 473
 474         adapter = device_get_softc(dev);
 475         adapter->dev = adapter->osdep.dev = dev;
 476         hw = &adapter->hw;
 477         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 478
 479         /* SYSCTL stuff */
 480         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 481             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 482             OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 483             em_sysctl_nvm_info, "I", "NVM Information");
 484
 485         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 486             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 487             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 488             em_sysctl_debug_info, "I", "Debug Information");
 489
 490         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 491             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 492             OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 493             em_set_flowcntl, "I", "Flow Control");
 494
 495         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 496
 497         /* Determine hardware and mac info */
 498         em_identify_hardware(adapter);
 499
 500         /* Setup PCI resources */
 501         if (em_allocate_pci_resources(adapter)) {
 502                 device_printf(dev, "Allocation of PCI resources failed\n");
 503                 error = ENXIO;
 504                 goto err_pci;
 505         }
 506
 507         /*
 508         ** For ICH8 and family we need to
 509         ** map the flash memory, and this
 510         ** must happen after the MAC is
 511         ** identified
 512         */
 513         if ((hw->mac.type == e1000_ich8lan) ||
 514             (hw->mac.type == e1000_ich9lan) ||
 515             (hw->mac.type == e1000_ich10lan) ||
 516             (hw->mac.type == e1000_pchlan) ||
 517             (hw->mac.type == e1000_pch2lan)) {
 518                 int rid = EM_BAR_TYPE_FLASH;
 519                 adapter->flash = bus_alloc_resource_any(dev,
 520                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 521                 if (adapter->flash == NULL) {
 522                         device_printf(dev, "Mapping of Flash failed\n");
 523                         error = ENXIO;
 524                         goto err_pci;
 525                 }
 526                 /* This is used in the shared code */
 527                 hw->flash_address = (u8 *)adapter->flash;
 528                 adapter->osdep.flash_bus_space_tag =
 529                     rman_get_bustag(adapter->flash);
 530                 adapter->osdep.flash_bus_space_handle =
 531                     rman_get_bushandle(adapter->flash);
 532         }
 533
 534         /* Do Shared Code initialization */
 535         if (e1000_setup_init_funcs(hw, TRUE)) {
 536                 device_printf(dev, "Setup of Shared code failed\n");
 537                 error = ENXIO;
 538                 goto err_pci;
 539         }
 540
 541         e1000_get_bus_info(hw);
 542
 543         /* Set up some sysctls for the tunable interrupt delays */
 544         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 545             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 546             E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt);
 547         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 548             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 549             E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt);
 550         em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 551             "receive interrupt delay limit in usecs",
 552             &adapter->rx_abs_int_delay,
 553             E1000_REGISTER(hw, E1000_RADV),
 554             em_rx_abs_int_delay_dflt);
 555         em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 556             "transmit interrupt delay limit in usecs",
 557             &adapter->tx_abs_int_delay,
 558             E1000_REGISTER(hw, E1000_TADV),
 559             em_tx_abs_int_delay_dflt);
 560
 561         /* Sysctl for limiting the amount of work done in the taskqueue */
 562         em_set_sysctl_value(adapter, "rx_processing_limit",
 563             "max number of rx packets to process", &adapter->rx_process_limit,
 564             em_rx_process_limit);
 565
 566         /*
 567          * Validate number of transmit and receive descriptors. It
 568          * must not exceed hardware maximum, and must be multiple
 569          * of E1000_DBA_ALIGN.
 570          */
 571         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 572             (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) {
 573                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 574                     EM_DEFAULT_TXD, em_txd);
 575                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 576         } else
 577                 adapter->num_tx_desc = em_txd;
 578
 579         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 580             (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) {
 581                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 582                     EM_DEFAULT_RXD, em_rxd);
 583                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 584         } else
 585                 adapter->num_rx_desc = em_rxd;
 586
 587         hw->mac.autoneg = DO_AUTO_NEG;
 588         hw->phy.autoneg_wait_to_complete = FALSE;
 589         hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 590
 591         /* Copper options */
 592         if (hw->phy.media_type == e1000_media_type_copper) {
 593                 hw->phy.mdix = AUTO_ALL_MODES;
 594                 hw->phy.disable_polarity_correction = FALSE;
 595                 hw->phy.ms_type = EM_MASTER_SLAVE;
 596         }
 597
 598         /*
 599          * Set the frame limits assuming
 600          * standard ethernet sized frames.
 601          */
 602         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 603         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 604
 605         /*
 606          * This controls when hardware reports transmit completion
 607          * status.
 608          */
 609         hw->mac.report_tx_early = 1;
 610
 611         /*
 612         ** Get queue/ring memory
 613         */
 614         if (em_allocate_queues(adapter)) {
 615                 error = ENOMEM;
 616                 goto err_pci;
 617         }
 618
 619         /* Allocate multicast array memory. */
 620         adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
 621             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 622         if (adapter->mta == NULL) {
 623                 device_printf(dev, "Can not allocate multicast setup array\n");
 624                 error = ENOMEM;
 625                 goto err_late;
 626         }
 627
 628         /* Check SOL/IDER usage */
 629         if (e1000_check_reset_block(hw))
 630                 device_printf(dev, "PHY reset is blocked"
 631                     " due to SOL/IDER session.\n");
 632
 633         /* Sysctl for setting Energy Efficient Ethernet */
 634         em_set_sysctl_value(adapter, "eee_control",
 635             "enable Energy Efficient Ethernet",
 636             &hw->dev_spec.ich8lan.eee_disable, eee_setting);
 637
 638         /*
 639         ** Start from a known state, this is
 640         ** important in reading the nvm and
 641         ** mac from that.
 642         */
 643         e1000_reset_hw(hw);
 644
 645
 646         /* Make sure we have a good EEPROM before we read from it */
 647         if (e1000_validate_nvm_checksum(hw) < 0) {
 648                 /*
 649                 ** Some PCI-E parts fail the first check due to
 650                 ** the link being in sleep state, call it again,
 651                 ** if it fails a second time its a real issue.
 652                 */
 653                 if (e1000_validate_nvm_checksum(hw) < 0) {
 654                         device_printf(dev,
 655                             "The EEPROM Checksum Is Not Valid\n");
 656                         error = EIO;
 657                         goto err_late;
 658                 }
 659         }
 660
 661         /* Copy the permanent MAC address out of the EEPROM */
 662         if (e1000_read_mac_addr(hw) < 0) {
 663                 device_printf(dev, "EEPROM read error while reading MAC"
 664                     " address\n");
 665                 error = EIO;
 666                 goto err_late;
 667         }
 668
 669         if (!em_is_valid_ether_addr(hw->mac.addr)) {
 670                 device_printf(dev, "Invalid MAC address\n");
 671                 error = EIO;
 672                 goto err_late;
 673         }
 674
 675         /*
 676         **  Do interrupt configuration
 677         */
 678         if (adapter->msix > 1) /* Do MSIX */
 679                 error = em_allocate_msix(adapter);
 680         else  /* MSI or Legacy */
 681                 error = em_allocate_legacy(adapter);
 682         if (error)
 683                 goto err_late;
 684
 685         /*
 686          * Get Wake-on-Lan and Management info for later use
 687          */
 688         em_get_wakeup(dev);
 689
 690         /* Setup OS specific network interface */
 691         if (em_setup_interface(dev, adapter) != 0)
 692                 goto err_late;
 693
 694         em_reset(adapter);
 695
 696         /* Initialize statistics */
 697         em_update_stats_counters(adapter);
 698
 699         hw->mac.get_link_status = 1;
 700         em_update_link_status(adapter);
 701
 702         /* Register for VLAN events */
 703         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 704             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 705         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 706             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 707
 708         em_add_hw_stats(adapter);
 709
 710         /* Non-AMT based hardware can now take control from firmware */
 711         if (adapter->has_manage && !adapter->has_amt)
 712                 em_get_hw_control(adapter);
 713
 714         /* Tell the stack that the interface is not active */
 715         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 716         adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 717
 718         adapter->led_dev = led_create(em_led_func, adapter,
 719             device_get_nameunit(dev));
 720
 721         INIT_DEBUGOUT("em_attach: end");
 722
 723         return (0);
 724
 725 err_late:
 726         em_free_transmit_structures(adapter);
 727         em_free_receive_structures(adapter);
 728         em_release_hw_control(adapter);
 729         if (adapter->ifp != NULL)
 730                 if_free(adapter->ifp);
 731 err_pci:
 732         em_free_pci_resources(adapter);
 733         free(adapter->mta, M_DEVBUF);
 734         EM_CORE_LOCK_DESTROY(adapter);
 735
 736         return (error);
 737 }
 738
 739 /*********************************************************************
 740  *  Device removal routine
 741  *
 742  *  The detach entry point is called when the driver is being removed.
 743  *  This routine stops the adapter and deallocates all the resources
 744  *  that were allocated for driver operation.
 745  *
 746  *  return 0 on success, positive on failure
 747  *********************************************************************/
 748
 749 static int
 750 em_detach(device_t dev)
 751 {
 752         struct adapter  *adapter = device_get_softc(dev);
 753         struct ifnet    *ifp = adapter->ifp;
 754
 755         INIT_DEBUGOUT("em_detach: begin");
 756
 757         /* Make sure VLANS are not using driver */
 758         if (adapter->ifp->if_vlantrunk != NULL) {
 759                 device_printf(dev,"Vlan in use, detach first\n");
 760                 return (EBUSY);
 761         }
 762
 763 #ifdef DEVICE_POLLING
 764         if (ifp->if_capenable & IFCAP_POLLING)
 765                 ether_poll_deregister(ifp);
 766 #endif
 767
 768         if (adapter->led_dev != NULL)
 769                 led_destroy(adapter->led_dev);
 770
 771         EM_CORE_LOCK(adapter);
 772         adapter->in_detach = 1;
 773         em_stop(adapter);
 774         EM_CORE_UNLOCK(adapter);
 775         EM_CORE_LOCK_DESTROY(adapter);
 776
 777         e1000_phy_hw_reset(&adapter->hw);
 778
 779         em_release_manageability(adapter);
 780         em_release_hw_control(adapter);
 781
 782         /* Unregister VLAN events */
 783         if (adapter->vlan_attach != NULL)
 784                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 785         if (adapter->vlan_detach != NULL)
 786                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 787
 788         ether_ifdetach(adapter->ifp);
 789         callout_drain(&adapter->timer);
 790
 791         em_free_pci_resources(adapter);
 792         bus_generic_detach(dev);
 793         if_free(ifp);
 794
 795         em_free_transmit_structures(adapter);
 796         em_free_receive_structures(adapter);
 797
 798         em_release_hw_control(adapter);
 799         free(adapter->mta, M_DEVBUF);
 800
 801         return (0);
 802 }
 803
 804 /*********************************************************************
 805  *
 806  *  Shutdown entry point
 807  *
 808  **********************************************************************/
 809
 810 static int
 811 em_shutdown(device_t dev)
 812 {
 813         return em_suspend(dev);
 814 }
 815
 816 /*
 817  * Suspend/resume device methods.
 818  */
 819 static int
 820 em_suspend(device_t dev)
 821 {
 822         struct adapter *adapter = device_get_softc(dev);
 823
 824         EM_CORE_LOCK(adapter);
 825
 826         em_release_manageability(adapter);
 827         em_release_hw_control(adapter);
 828         em_enable_wakeup(dev);
 829
 830         EM_CORE_UNLOCK(adapter);
 831
 832         return bus_generic_suspend(dev);
 833 }
 834
 835 static int
 836 em_resume(device_t dev)
 837 {
 838         struct adapter *adapter = device_get_softc(dev);
 839         struct ifnet *ifp = adapter->ifp;
 840
 841         EM_CORE_LOCK(adapter);
 842         if (adapter->hw.mac.type == e1000_pch2lan)
 843                 e1000_resume_workarounds_pchlan(&adapter->hw);
 844         em_init_locked(adapter);
 845         em_init_manageability(adapter);
 846         EM_CORE_UNLOCK(adapter);
 847         em_start(ifp);
 848
 849         return bus_generic_resume(dev);
 850 }
 851
 852
 853 #ifdef EM_MULTIQUEUE
 854 /*********************************************************************
 855  *  Multiqueue Transmit routines
 856  *
 857  *  em_mq_start is called by the stack to initiate a transmit.
 858  *  however, if busy the driver can queue the request rather
 859  *  than do an immediate send. It is this that is an advantage
 860  *  in this driver, rather than also having multiple tx queues.
 861  **********************************************************************/
 862 static int
 863 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 864 {
 865         struct adapter  *adapter = txr->adapter;
 866         struct mbuf     *next;
 867         int             err = 0, enq = 0;
 868
 869         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 870             IFF_DRV_RUNNING || adapter->link_active == 0) {
 871                 if (m != NULL)
 872                         err = drbr_enqueue(ifp, txr->br, m);
 873                 return (err);
 874         }
 875
 876         enq = 0;
 877         if (m == NULL) {
 878                 next = drbr_dequeue(ifp, txr->br);
 879         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 880                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 881                         return (err);
 882                 next = drbr_dequeue(ifp, txr->br);
 883         } else
 884                 next = m;
 885
 886         /* Process the queue */
 887         while (next != NULL) {
 888                 if ((err = em_xmit(txr, &next)) != 0) {
 889                         if (next != NULL)
 890                                 err = drbr_enqueue(ifp, txr->br, next);
 891                         break;
 892                 }
 893                 enq++;
 894                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 895                 ETHER_BPF_MTAP(ifp, next);
 896                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 897                         break;
 898                 next = drbr_dequeue(ifp, txr->br);
 899         }
 900
 901         if (enq > 0) {
 902                 /* Set the watchdog */
 903                 txr->queue_status = EM_QUEUE_WORKING;
 904                 txr->watchdog_time = ticks;
 905         }
 906
 907         if (txr->tx_avail < EM_MAX_SCATTER)
 908                 em_txeof(txr);
 909         if (txr->tx_avail < EM_MAX_SCATTER)
 910                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 911         return (err);
 912 }
 913
 914 /*
 915 ** Multiqueue capable stack interface
 916 */
 917 static int
 918 em_mq_start(struct ifnet *ifp, struct mbuf *m)
 919 {
 920         struct adapter  *adapter = ifp->if_softc;
 921         struct tx_ring  *txr = adapter->tx_rings;
 922         int             error;
 923
 924         if (EM_TX_TRYLOCK(txr)) {
 925                 error = em_mq_start_locked(ifp, txr, m);
 926                 EM_TX_UNLOCK(txr);
 927         } else
 928                 error = drbr_enqueue(ifp, txr->br, m);
 929
 930         return (error);
 931 }
 932
 933 /*
 934 ** Flush all ring buffers
 935 */
 936 static void
 937 em_qflush(struct ifnet *ifp)
 938 {
 939         struct adapter  *adapter = ifp->if_softc;
 940         struct tx_ring  *txr = adapter->tx_rings;
 941         struct mbuf     *m;
 942
 943         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 944                 EM_TX_LOCK(txr);
 945                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 946                         m_freem(m);
 947                 EM_TX_UNLOCK(txr);
 948         }
 949         if_qflush(ifp);
 950 }
 951 #endif /* EM_MULTIQUEUE */
 952
 953 static void
 954 em_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 955 {
 956         struct adapter  *adapter = ifp->if_softc;
 957         struct mbuf     *m_head;
 958
 959         EM_TX_LOCK_ASSERT(txr);
 960
 961         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 962             IFF_DRV_RUNNING)
 963                 return;
 964
 965         if (!adapter->link_active)
 966                 return;
 967
 968         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 969                 /* Call cleanup if number of TX descriptors low */
 970                 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
 971                         em_txeof(txr);
 972                 if (txr->tx_avail < EM_MAX_SCATTER) {
 973                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 974                         break;
 975                 }
 976                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 977                 if (m_head == NULL)
 978                         break;
 979                 /*
 980                  *  Encapsulation can modify our pointer, and or make it
 981                  *  NULL on failure.  In that event, we can't requeue.
 982                  */
 983                 if (em_xmit(txr, &m_head)) {
 984                         if (m_head == NULL)
 985                                 break;
 986                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 987                         break;
 988                 }
 989
 990                 /* Send a copy of the frame to the BPF listener */
 991                 ETHER_BPF_MTAP(ifp, m_head);
 992
 993                 /* Set timeout in case hardware has problems transmitting. */
 994                 txr->watchdog_time = ticks;
 995                 txr->queue_status = EM_QUEUE_WORKING;
 996         }
 997
 998         return;
 999 }
1000
1001 static void
1002 em_start(struct ifnet *ifp)
1003 {
1004         struct adapter  *adapter = ifp->if_softc;
1005         struct tx_ring  *txr = adapter->tx_rings;
1006
1007         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1008                 EM_TX_LOCK(txr);
1009                 em_start_locked(ifp, txr);
1010                 EM_TX_UNLOCK(txr);
1011         }
1012         /*
1013         ** If we went inactive schedule
1014         ** a task to clean up.
1015         */
1016         if (ifp->if_drv_flags & IFF_DRV_OACTIVE)
1017                 taskqueue_enqueue(txr->tq, &txr->tx_task);
1018         return;
1019 }
1020
1021 /*********************************************************************
1022  *  Ioctl entry point
1023  *
1024  *  em_ioctl is called when the user wants to configure the
1025  *  interface.
1026  *
1027  *  return 0 on success, positive on failure
1028  **********************************************************************/
1029
1030 static int
1031 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1032 {
1033         struct adapter  *adapter = ifp->if_softc;
1034         struct ifreq    *ifr = (struct ifreq *)data;
1035 #if defined(INET) || defined(INET6)
1036         struct ifaddr   *ifa = (struct ifaddr *)data;
1037 #endif
1038         bool            avoid_reset = FALSE;
1039         int             error = 0;
1040
1041         if (adapter->in_detach)
1042                 return (error);
1043
1044         switch (command) {
1045         case SIOCSIFADDR:
1046 #ifdef INET
1047                 if (ifa->ifa_addr->sa_family == AF_INET)
1048                         avoid_reset = TRUE;
1049 #endif
1050 #ifdef INET6
1051                 if (ifa->ifa_addr->sa_family == AF_INET6)
1052                         avoid_reset = TRUE;
1053 #endif
1054                 /*
1055                 ** Calling init results in link renegotiation,
1056                 ** so we avoid doing it when possible.
1057                 */
1058                 if (avoid_reset) {
1059                         ifp->if_flags |= IFF_UP;
1060                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1061                                 em_init(adapter);
1062 #ifdef INET
1063                         if (!(ifp->if_flags & IFF_NOARP))
1064                                 arp_ifinit(ifp, ifa);
1065 #endif
1066                 } else
1067                         error = ether_ioctl(ifp, command, data);
1068                 break;
1069         case SIOCSIFMTU:
1070             {
1071                 int max_frame_size;
1072
1073                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1074
1075                 EM_CORE_LOCK(adapter);
1076                 switch (adapter->hw.mac.type) {
1077                 case e1000_82571:
1078                 case e1000_82572:
1079                 case e1000_ich9lan:
1080                 case e1000_ich10lan:
1081                 case e1000_pch2lan:
1082                 case e1000_82574:
1083                 case e1000_82583:
1084                 case e1000_80003es2lan: /* 9K Jumbo Frame size */
1085                         max_frame_size = 9234;
1086                         break;
1087                 case e1000_pchlan:
1088                         max_frame_size = 4096;
1089                         break;
1090                         /* Adapters that do not support jumbo frames */
1091                 case e1000_ich8lan:
1092                         max_frame_size = ETHER_MAX_LEN;
1093                         break;
1094                 default:
1095                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1096                 }
1097                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1098                     ETHER_CRC_LEN) {
1099                         EM_CORE_UNLOCK(adapter);
1100                         error = EINVAL;
1101                         break;
1102                 }
1103
1104                 ifp->if_mtu = ifr->ifr_mtu;
1105                 adapter->max_frame_size =
1106                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1107                 em_init_locked(adapter);
1108                 EM_CORE_UNLOCK(adapter);
1109                 break;
1110             }
1111         case SIOCSIFFLAGS:
1112                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1113                     SIOCSIFFLAGS (Set Interface Flags)");
1114                 EM_CORE_LOCK(adapter);
1115                 if (ifp->if_flags & IFF_UP) {
1116                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1117                                 if ((ifp->if_flags ^ adapter->if_flags) &
1118                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1119                                         em_disable_promisc(adapter);
1120                                         em_set_promisc(adapter);
1121                                 }
1122                         } else
1123                                 em_init_locked(adapter);
1124                 } else
1125                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1126                                 em_stop(adapter);
1127                 adapter->if_flags = ifp->if_flags;
1128                 EM_CORE_UNLOCK(adapter);
1129                 break;
1130         case SIOCADDMULTI:
1131         case SIOCDELMULTI:
1132                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1133                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1134                         EM_CORE_LOCK(adapter);
1135                         em_disable_intr(adapter);
1136                         em_set_multi(adapter);
1137 #ifdef DEVICE_POLLING
1138                         if (!(ifp->if_capenable & IFCAP_POLLING))
1139 #endif
1140                                 em_enable_intr(adapter);
1141                         EM_CORE_UNLOCK(adapter);
1142                 }
1143                 break;
1144         case SIOCSIFMEDIA:
1145                 /* Check SOL/IDER usage */
1146                 EM_CORE_LOCK(adapter);
1147                 if (e1000_check_reset_block(&adapter->hw)) {
1148                         EM_CORE_UNLOCK(adapter);
1149                         device_printf(adapter->dev, "Media change is"
1150                             " blocked due to SOL/IDER session.\n");
1151                         break;
1152                 }
1153                 EM_CORE_UNLOCK(adapter);
1154                 /* falls thru */
1155         case SIOCGIFMEDIA:
1156                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1157                     SIOCxIFMEDIA (Get/Set Interface Media)");
1158                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1159                 break;
1160         case SIOCSIFCAP:
1161             {
1162                 int mask, reinit;
1163
1164                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1165                 reinit = 0;
1166                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1167 #ifdef DEVICE_POLLING
1168                 if (mask & IFCAP_POLLING) {
1169                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1170                                 error = ether_poll_register(em_poll, ifp);
1171                                 if (error)
1172                                         return (error);
1173                                 EM_CORE_LOCK(adapter);
1174                                 em_disable_intr(adapter);
1175                                 ifp->if_capenable |= IFCAP_POLLING;
1176                                 EM_CORE_UNLOCK(adapter);
1177                         } else {
1178                                 error = ether_poll_deregister(ifp);
1179                                 /* Enable interrupt even in error case */
1180                                 EM_CORE_LOCK(adapter);
1181                                 em_enable_intr(adapter);
1182                                 ifp->if_capenable &= ~IFCAP_POLLING;
1183                                 EM_CORE_UNLOCK(adapter);
1184                         }
1185                 }
1186 #endif
1187                 if (mask & IFCAP_HWCSUM) {
1188                         ifp->if_capenable ^= IFCAP_HWCSUM;
1189                         reinit = 1;
1190                 }
1191                 if (mask & IFCAP_TSO4) {
1192                         ifp->if_capenable ^= IFCAP_TSO4;
1193                         reinit = 1;
1194                 }
1195                 if (mask & IFCAP_VLAN_HWTAGGING) {
1196                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1197                         reinit = 1;
1198                 }
1199                 if (mask & IFCAP_VLAN_HWFILTER) {
1200                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1201                         reinit = 1;
1202                 }
1203                 if (mask & IFCAP_VLAN_HWTSO) {
1204                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1205                         reinit = 1;
1206                 }
1207                 if ((mask & IFCAP_WOL) &&
1208                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
1209                         if (mask & IFCAP_WOL_MCAST)
1210                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1211                         if (mask & IFCAP_WOL_MAGIC)
1212                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1213                 }
1214                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1215                         em_init(adapter);
1216                 VLAN_CAPABILITIES(ifp);
1217                 break;
1218             }
1219
1220         default:
1221                 error = ether_ioctl(ifp, command, data);
1222                 break;
1223         }
1224
1225         return (error);
1226 }
1227
1228
1229 /*********************************************************************
1230  *  Init entry point
1231  *
1232  *  This routine is used in two ways. It is used by the stack as
1233  *  init entry point in network interface structure. It is also used
1234  *  by the driver as a hw/sw initialization routine to get to a
1235  *  consistent state.
1236  *
1237  *  return 0 on success, positive on failure
1238  **********************************************************************/
1239
1240 static void
1241 em_init_locked(struct adapter *adapter)
1242 {
1243         struct ifnet    *ifp = adapter->ifp;
1244         device_t        dev = adapter->dev;
1245
1246         INIT_DEBUGOUT("em_init: begin");
1247
1248         EM_CORE_LOCK_ASSERT(adapter);
1249
1250         em_disable_intr(adapter);
1251         callout_stop(&adapter->timer);
1252
1253         /* Get the latest mac address, User can use a LAA */
1254         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1255               ETHER_ADDR_LEN);
1256
1257         /* Put the address into the Receive Address Array */
1258         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1259
1260         /*
1261          * With the 82571 adapter, RAR[0] may be overwritten
1262          * when the other port is reset, we make a duplicate
1263          * in RAR[14] for that eventuality, this assures
1264          * the interface continues to function.
1265          */
1266         if (adapter->hw.mac.type == e1000_82571) {
1267                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1268                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1269                     E1000_RAR_ENTRIES - 1);
1270         }
1271
1272         /* Initialize the hardware */
1273         em_reset(adapter);
1274         em_update_link_status(adapter);
1275
1276         /* Setup VLAN support, basic and offload if available */
1277         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1278
1279         /* Set hardware offload abilities */
1280         ifp->if_hwassist = 0;
1281         if (ifp->if_capenable & IFCAP_TXCSUM)
1282                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1283         if (ifp->if_capenable & IFCAP_TSO4)
1284                 ifp->if_hwassist |= CSUM_TSO;
1285
1286         /* Configure for OS presence */
1287         em_init_manageability(adapter);
1288
1289         /* Prepare transmit descriptors and buffers */
1290         em_setup_transmit_structures(adapter);
1291         em_initialize_transmit_unit(adapter);
1292
1293         /* Setup Multicast table */
1294         em_set_multi(adapter);
1295
1296         /*
1297         ** Figure out the desired mbuf
1298         ** pool for doing jumbos
1299         */
1300         if (adapter->max_frame_size <= 2048)
1301                 adapter->rx_mbuf_sz = MCLBYTES;
1302         else if (adapter->max_frame_size <= 4096)
1303                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1304         else
1305                 adapter->rx_mbuf_sz = MJUM9BYTES;
1306
1307         /* Prepare receive descriptors and buffers */
1308         if (em_setup_receive_structures(adapter)) {
1309                 device_printf(dev, "Could not setup receive structures\n");
1310                 em_stop(adapter);
1311                 return;
1312         }
1313         em_initialize_receive_unit(adapter);
1314
1315         /* Use real VLAN Filter support? */
1316         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1317                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1318                         /* Use real VLAN Filter support */
1319                         em_setup_vlan_hw_support(adapter);
1320                 else {
1321                         u32 ctrl;
1322                         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1323                         ctrl |= E1000_CTRL_VME;
1324                         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1325                 }
1326         }
1327
1328         /* Don't lose promiscuous settings */
1329         em_set_promisc(adapter);
1330
1331         /* Set the interface as ACTIVE */
1332         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1333         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1334
1335         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1336         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1337
1338         /* MSI/X configuration for 82574 */
1339         if (adapter->hw.mac.type == e1000_82574) {
1340                 int tmp;
1341                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1342                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1343                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1344                 /* Set the IVAR - interrupt vector routing. */
1345                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
1346         }
1347
1348 #ifdef DEVICE_POLLING
1349         /*
1350          * Only enable interrupts if we are not polling, make sure
1351          * they are off otherwise.
1352          */
1353         if (ifp->if_capenable & IFCAP_POLLING)
1354                 em_disable_intr(adapter);
1355         else
1356 #endif /* DEVICE_POLLING */
1357                 em_enable_intr(adapter);
1358
1359         /* AMT based hardware can now take control from firmware */
1360         if (adapter->has_manage && adapter->has_amt)
1361                 em_get_hw_control(adapter);
1362 }
1363
1364 static void
1365 em_init(void *arg)
1366 {
1367         struct adapter *adapter = arg;
1368
1369         EM_CORE_LOCK(adapter);
1370         em_init_locked(adapter);
1371         EM_CORE_UNLOCK(adapter);
1372 }
1373
1374
1375 #ifdef DEVICE_POLLING
1376 /*********************************************************************
1377  *
1378  *  Legacy polling routine: note this only works with single queue
1379  *
1380  *********************************************************************/
1381 static int
1382 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1383 {
1384         struct adapter *adapter = ifp->if_softc;
1385         struct tx_ring  *txr = adapter->tx_rings;
1386         struct rx_ring  *rxr = adapter->rx_rings;
1387         u32             reg_icr;
1388         int             rx_done;
1389
1390         EM_CORE_LOCK(adapter);
1391         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1392                 EM_CORE_UNLOCK(adapter);
1393                 return (0);
1394         }
1395
1396         if (cmd == POLL_AND_CHECK_STATUS) {
1397                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1398                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1399                         callout_stop(&adapter->timer);
1400                         adapter->hw.mac.get_link_status = 1;
1401                         em_update_link_status(adapter);
1402                         callout_reset(&adapter->timer, hz,
1403                             em_local_timer, adapter);
1404                 }
1405         }
1406         EM_CORE_UNLOCK(adapter);
1407
1408         em_rxeof(rxr, count, &rx_done);
1409
1410         EM_TX_LOCK(txr);
1411         em_txeof(txr);
1412 #ifdef EM_MULTIQUEUE
1413         if (!drbr_empty(ifp, txr->br))
1414                 em_mq_start_locked(ifp, txr, NULL);
1415 #else
1416         em_start_locked(ifp, txr);
1417 #endif
1418         EM_TX_UNLOCK(txr);
1419
1420         return (rx_done);
1421 }
1422 #endif /* DEVICE_POLLING */
1423
1424
1425 /*********************************************************************
1426  *
1427  *  Fast Legacy/MSI Combined Interrupt Service routine
1428  *
1429  *********************************************************************/
1430 static int
1431 em_irq_fast(void *arg)
1432 {
1433         struct adapter  *adapter = arg;
1434         struct ifnet    *ifp;
1435         u32             reg_icr;
1436
1437         ifp = adapter->ifp;
1438
1439         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1440
1441         /* Hot eject?  */
1442         if (reg_icr == 0xffffffff)
1443                 return FILTER_STRAY;
1444
1445         /* Definitely not our interrupt.  */
1446         if (reg_icr == 0x0)
1447                 return FILTER_STRAY;
1448
1449         /*
1450          * Starting with the 82571 chip, bit 31 should be used to
1451          * determine whether the interrupt belongs to us.
1452          */
1453         if (adapter->hw.mac.type >= e1000_82571 &&
1454             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1455                 return FILTER_STRAY;
1456
1457         em_disable_intr(adapter);
1458         taskqueue_enqueue(adapter->tq, &adapter->que_task);
1459
1460         /* Link status change */
1461         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1462                 adapter->hw.mac.get_link_status = 1;
1463                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1464         }
1465
1466         if (reg_icr & E1000_ICR_RXO)
1467                 adapter->rx_overruns++;
1468         return FILTER_HANDLED;
1469 }
1470
1471 /* Combined RX/TX handler, used by Legacy and MSI */
1472 static void
1473 em_handle_que(void *context, int pending)
1474 {
1475         struct adapter  *adapter = context;
1476         struct ifnet    *ifp = adapter->ifp;
1477         struct tx_ring  *txr = adapter->tx_rings;
1478         struct rx_ring  *rxr = adapter->rx_rings;
1479
1480
1481         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1482                 bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1483                 EM_TX_LOCK(txr);
1484                 em_txeof(txr);
1485 #ifdef EM_MULTIQUEUE
1486                 if (!drbr_empty(ifp, txr->br))
1487                         em_mq_start_locked(ifp, txr, NULL);
1488 #else
1489                 em_start_locked(ifp, txr);
1490 #endif
1491                 EM_TX_UNLOCK(txr);
1492                 if (more || (ifp->if_drv_flags & IFF_DRV_OACTIVE)) {
1493                         taskqueue_enqueue(adapter->tq, &adapter->que_task);
1494                         return;
1495                 }
1496         }
1497
1498         em_enable_intr(adapter);
1499         return;
1500 }
1501
1502
1503 /*********************************************************************
1504  *
1505  *  MSIX Interrupt Service Routines
1506  *
1507  **********************************************************************/
1508 static void
1509 em_msix_tx(void *arg)
1510 {
1511         struct tx_ring *txr = arg;
1512         struct adapter *adapter = txr->adapter;
1513         bool            more;
1514
1515         ++txr->tx_irq;
1516         EM_TX_LOCK(txr);
1517         more = em_txeof(txr);
1518         EM_TX_UNLOCK(txr);
1519         if (more)
1520                 taskqueue_enqueue(txr->tq, &txr->tx_task);
1521         else
1522                 /* Reenable this interrupt */
1523                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1524         return;
1525 }
1526
1527 /*********************************************************************
1528  *
1529  *  MSIX RX Interrupt Service routine
1530  *
1531  **********************************************************************/
1532
1533 static void
1534 em_msix_rx(void *arg)
1535 {
1536         struct rx_ring  *rxr = arg;
1537         struct adapter  *adapter = rxr->adapter;
1538         bool            more;
1539
1540         ++rxr->rx_irq;
1541         more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1542         if (more)
1543                 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1544         else
1545                 /* Reenable this interrupt */
1546                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1547         return;
1548 }
1549
1550 /*********************************************************************
1551  *
1552  *  MSIX Link Fast Interrupt Service routine
1553  *
1554  **********************************************************************/
1555 static void
1556 em_msix_link(void *arg)
1557 {
1558         struct adapter  *adapter = arg;
1559         u32             reg_icr;
1560
1561         ++adapter->link_irq;
1562         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1563
1564         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1565                 adapter->hw.mac.get_link_status = 1;
1566                 em_handle_link(adapter, 0);
1567         } else
1568                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1569                     EM_MSIX_LINK | E1000_IMS_LSC);
1570         return;
1571 }
1572
1573 static void
1574 em_handle_rx(void *context, int pending)
1575 {
1576         struct rx_ring  *rxr = context;
1577         struct adapter  *adapter = rxr->adapter;
1578         bool            more;
1579
1580         more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1581         if (more)
1582                 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1583         else
1584                 /* Reenable this interrupt */
1585                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1586 }
1587
1588 static void
1589 em_handle_tx(void *context, int pending)
1590 {
1591         struct tx_ring  *txr = context;
1592         struct adapter  *adapter = txr->adapter;
1593         struct ifnet    *ifp = adapter->ifp;
1594
1595         EM_TX_LOCK(txr);
1596         em_txeof(txr);
1597 #ifdef EM_MULTIQUEUE
1598         if (!drbr_empty(ifp, txr->br))
1599                 em_mq_start_locked(ifp, txr, NULL);
1600 #else
1601         em_start_locked(ifp, txr);
1602 #endif
1603         E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1604         EM_TX_UNLOCK(txr);
1605 }
1606
1607 static void
1608 em_handle_link(void *context, int pending)
1609 {
1610         struct adapter  *adapter = context;
1611         struct ifnet *ifp = adapter->ifp;
1612
1613         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1614                 return;
1615
1616         EM_CORE_LOCK(adapter);
1617         callout_stop(&adapter->timer);
1618         em_update_link_status(adapter);
1619         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1620         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1621             EM_MSIX_LINK | E1000_IMS_LSC);
1622         EM_CORE_UNLOCK(adapter);
1623 }
1624
1625
1626 /*********************************************************************
1627  *
1628  *  Media Ioctl callback
1629  *
1630  *  This routine is called whenever the user queries the status of
1631  *  the interface using ifconfig.
1632  *
1633  **********************************************************************/
1634 static void
1635 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1636 {
1637         struct adapter *adapter = ifp->if_softc;
1638         u_char fiber_type = IFM_1000_SX;
1639
1640         INIT_DEBUGOUT("em_media_status: begin");
1641
1642         EM_CORE_LOCK(adapter);
1643         em_update_link_status(adapter);
1644
1645         ifmr->ifm_status = IFM_AVALID;
1646         ifmr->ifm_active = IFM_ETHER;
1647
1648         if (!adapter->link_active) {
1649                 EM_CORE_UNLOCK(adapter);
1650                 return;
1651         }
1652
1653         ifmr->ifm_status |= IFM_ACTIVE;
1654
1655         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1656             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1657                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1658         } else {
1659                 switch (adapter->link_speed) {
1660                 case 10:
1661                         ifmr->ifm_active |= IFM_10_T;
1662                         break;
1663                 case 100:
1664                         ifmr->ifm_active |= IFM_100_TX;
1665                         break;
1666                 case 1000:
1667                         ifmr->ifm_active |= IFM_1000_T;
1668                         break;
1669                 }
1670                 if (adapter->link_duplex == FULL_DUPLEX)
1671                         ifmr->ifm_active |= IFM_FDX;
1672                 else
1673                         ifmr->ifm_active |= IFM_HDX;
1674         }
1675         EM_CORE_UNLOCK(adapter);
1676 }
1677
1678 /*********************************************************************
1679  *
1680  *  Media Ioctl callback
1681  *
1682  *  This routine is called when the user changes speed/duplex using
1683  *  media/mediopt option with ifconfig.
1684  *
1685  **********************************************************************/
1686 static int
1687 em_media_change(struct ifnet *ifp)
1688 {
1689         struct adapter *adapter = ifp->if_softc;
1690         struct ifmedia  *ifm = &adapter->media;
1691
1692         INIT_DEBUGOUT("em_media_change: begin");
1693
1694         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1695                 return (EINVAL);
1696
1697         EM_CORE_LOCK(adapter);
1698         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1699         case IFM_AUTO:
1700                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1701                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1702                 break;
1703         case IFM_1000_LX:
1704         case IFM_1000_SX:
1705         case IFM_1000_T:
1706                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1707                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1708                 break;
1709         case IFM_100_TX:
1710                 adapter->hw.mac.autoneg = FALSE;
1711                 adapter->hw.phy.autoneg_advertised = 0;
1712                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1713                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1714                 else
1715                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1716                 break;
1717         case IFM_10_T:
1718                 adapter->hw.mac.autoneg = FALSE;
1719                 adapter->hw.phy.autoneg_advertised = 0;
1720                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1721                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1722                 else
1723                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1724                 break;
1725         default:
1726                 device_printf(adapter->dev, "Unsupported media type\n");
1727         }
1728
1729         em_init_locked(adapter);
1730         EM_CORE_UNLOCK(adapter);
1731
1732         return (0);
1733 }
1734
1735 /*********************************************************************
1736  *
1737  *  This routine maps the mbufs to tx descriptors.
1738  *
1739  *  return 0 on success, positive on failure
1740  **********************************************************************/
1741
1742 static int
1743 em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1744 {
1745         struct adapter          *adapter = txr->adapter;
1746         bus_dma_segment_t       segs[EM_MAX_SCATTER];
1747         bus_dmamap_t            map;
1748         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
1749         struct e1000_tx_desc    *ctxd = NULL;
1750         struct mbuf             *m_head;
1751         struct ether_header     *eh;
1752         struct ip               *ip = NULL;
1753         struct tcphdr           *tp = NULL;
1754         u32                     txd_upper, txd_lower, txd_used, txd_saved;
1755         int                     ip_off, poff;
1756         int                     nsegs, i, j, first, last = 0;
1757         int                     error, do_tso, tso_desc = 0, remap = 1;
1758
1759 retry:
1760         m_head = *m_headp;
1761         txd_upper = txd_lower = txd_used = txd_saved = 0;
1762         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1763         ip_off = poff = 0;
1764
1765         /*
1766          * Intel recommends entire IP/TCP header length reside in a single
1767          * buffer. If multiple descriptors are used to describe the IP and
1768          * TCP header, each descriptor should describe one or more
1769          * complete headers; descriptors referencing only parts of headers
1770          * are not supported. If all layer headers are not coalesced into
1771          * a single buffer, each buffer should not cross a 4KB boundary,
1772          * or be larger than the maximum read request size.
1773          * Controller also requires modifing IP/TCP header to make TSO work
1774          * so we firstly get a writable mbuf chain then coalesce ethernet/
1775          * IP/TCP header into a single buffer to meet the requirement of
1776          * controller. This also simplifies IP/TCP/UDP checksum offloading
1777          * which also has similiar restrictions.
1778          */
1779         if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
1780                 if (do_tso || (m_head->m_next != NULL &&
1781                     m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) {
1782                         if (M_WRITABLE(*m_headp) == 0) {
1783                                 m_head = m_dup(*m_headp, M_DONTWAIT);
1784                                 m_freem(*m_headp);
1785                                 if (m_head == NULL) {
1786                                         *m_headp = NULL;
1787                                         return (ENOBUFS);
1788                                 }
1789                                 *m_headp = m_head;
1790                         }
1791                 }
1792                 /*
1793                  * XXX
1794                  * Assume IPv4, we don't have TSO/checksum offload support
1795                  * for IPv6 yet.
1796                  */
1797                 ip_off = sizeof(struct ether_header);
1798                 m_head = m_pullup(m_head, ip_off);
1799                 if (m_head == NULL) {
1800                         *m_headp = NULL;
1801                         return (ENOBUFS);
1802                 }
1803                 eh = mtod(m_head, struct ether_header *);
1804                 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1805                         ip_off = sizeof(struct ether_vlan_header);
1806                         m_head = m_pullup(m_head, ip_off);
1807                         if (m_head == NULL) {
1808                                 *m_headp = NULL;
1809                                 return (ENOBUFS);
1810                         }
1811                 }
1812                 m_head = m_pullup(m_head, ip_off + sizeof(struct ip));
1813                 if (m_head == NULL) {
1814                         *m_headp = NULL;
1815                         return (ENOBUFS);
1816                 }
1817                 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1818                 poff = ip_off + (ip->ip_hl << 2);
1819                 if (do_tso) {
1820                         m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1821                         if (m_head == NULL) {
1822                                 *m_headp = NULL;
1823                                 return (ENOBUFS);
1824                         }
1825                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1826                         /*
1827                          * TSO workaround:
1828                          *   pull 4 more bytes of data into it.
1829                          */
1830                         m_head = m_pullup(m_head, poff + (tp->th_off << 2) + 4);
1831                         if (m_head == NULL) {
1832                                 *m_headp = NULL;
1833                                 return (ENOBUFS);
1834                         }
1835                         ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1836                         ip->ip_len = 0;
1837                         ip->ip_sum = 0;
1838                         /*
1839                          * The pseudo TCP checksum does not include TCP payload
1840                          * length so driver should recompute the checksum here
1841                          * what hardware expect to see. This is adherence of
1842                          * Microsoft's Large Send specification.
1843                          */
1844                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1845                         tp->th_sum = in_pseudo(ip->ip_src.s_addr,
1846                             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
1847                 } else if (m_head->m_pkthdr.csum_flags & CSUM_TCP) {
1848                         m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1849                         if (m_head == NULL) {
1850                                 *m_headp = NULL;
1851                                 return (ENOBUFS);
1852                         }
1853                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1854                         m_head = m_pullup(m_head, poff + (tp->th_off << 2));
1855                         if (m_head == NULL) {
1856                                 *m_headp = NULL;
1857                                 return (ENOBUFS);
1858                         }
1859                         ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1860                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1861                 } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) {
1862                         m_head = m_pullup(m_head, poff + sizeof(struct udphdr));
1863                         if (m_head == NULL) {
1864                                 *m_headp = NULL;
1865                                 return (ENOBUFS);
1866                         }
1867                         ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1868                 }
1869                 *m_headp = m_head;
1870         }
1871
1872         /*
1873          * Map the packet for DMA
1874          *
1875          * Capture the first descriptor index,
1876          * this descriptor will have the index
1877          * of the EOP which is the only one that
1878          * now gets a DONE bit writeback.
1879          */
1880         first = txr->next_avail_desc;
1881         tx_buffer = &txr->tx_buffers[first];
1882         tx_buffer_mapped = tx_buffer;
1883         map = tx_buffer->map;
1884
1885         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1886             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1887
1888         /*
1889          * There are two types of errors we can (try) to handle:
1890          * - EFBIG means the mbuf chain was too long and bus_dma ran
1891          *   out of segments.  Defragment the mbuf chain and try again.
1892          * - ENOMEM means bus_dma could not obtain enough bounce buffers
1893          *   at this point in time.  Defer sending and try again later.
1894          * All other errors, in particular EINVAL, are fatal and prevent the
1895          * mbuf chain from ever going through.  Drop it and report error.
1896          */
1897         if (error == EFBIG && remap) {
1898                 struct mbuf *m;
1899
1900                 m = m_defrag(*m_headp, M_DONTWAIT);
1901                 if (m == NULL) {
1902                         adapter->mbuf_alloc_failed++;
1903                         m_freem(*m_headp);
1904                         *m_headp = NULL;
1905                         return (ENOBUFS);
1906                 }
1907                 *m_headp = m;
1908
1909                 /* Try it again, but only once */
1910                 remap = 0;
1911                 goto retry;
1912         } else if (error == ENOMEM) {
1913                 adapter->no_tx_dma_setup++;
1914                 return (error);
1915         } else if (error != 0) {
1916                 adapter->no_tx_dma_setup++;
1917                 m_freem(*m_headp);
1918                 *m_headp = NULL;
1919                 return (error);
1920         }
1921
1922         /*
1923          * TSO Hardware workaround, if this packet is not
1924          * TSO, and is only a single descriptor long, and
1925          * it follows a TSO burst, then we need to add a
1926          * sentinel descriptor to prevent premature writeback.
1927          */
1928         if ((do_tso == 0) && (txr->tx_tso == TRUE)) {
1929                 if (nsegs == 1)
1930                         tso_desc = TRUE;
1931                 txr->tx_tso = FALSE;
1932         }
1933
1934         if (nsegs > (txr->tx_avail - 2)) {
1935                 txr->no_desc_avail++;
1936                 bus_dmamap_unload(txr->txtag, map);
1937                 return (ENOBUFS);
1938         }
1939         m_head = *m_headp;
1940
1941         /* Do hardware assists */
1942         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1943                 em_tso_setup(txr, m_head, ip_off, ip, tp,
1944                     &txd_upper, &txd_lower);
1945                 /* we need to make a final sentinel transmit desc */
1946                 tso_desc = TRUE;
1947         } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
1948                 em_transmit_checksum_setup(txr, m_head,
1949                     ip_off, ip, &txd_upper, &txd_lower);
1950
1951         if (m_head->m_flags & M_VLANTAG) {
1952                 /* Set the vlan id. */
1953                 txd_upper |=
1954                     (htole16(m_head->m_pkthdr.ether_vtag) << 16);
1955                 /* Tell hardware to add tag */
1956                 txd_lower |= htole32(E1000_TXD_CMD_VLE);
1957         }
1958
1959         i = txr->next_avail_desc;
1960
1961         /* Set up our transmit descriptors */
1962         for (j = 0; j < nsegs; j++) {
1963                 bus_size_t seg_len;
1964                 bus_addr_t seg_addr;
1965
1966                 tx_buffer = &txr->tx_buffers[i];
1967                 ctxd = &txr->tx_base[i];
1968                 seg_addr = segs[j].ds_addr;
1969                 seg_len  = segs[j].ds_len;
1970                 /*
1971                 ** TSO Workaround:
1972                 ** If this is the last descriptor, we want to
1973                 ** split it so we have a small final sentinel
1974                 */
1975                 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
1976                         seg_len -= 4;
1977                         ctxd->buffer_addr = htole64(seg_addr);
1978                         ctxd->lower.data = htole32(
1979                         adapter->txd_cmd | txd_lower | seg_len);
1980                         ctxd->upper.data =
1981                             htole32(txd_upper);
1982                         if (++i == adapter->num_tx_desc)
1983                                 i = 0;
1984                         /* Now make the sentinel */
1985                         ++txd_used; /* using an extra txd */
1986                         ctxd = &txr->tx_base[i];
1987                         tx_buffer = &txr->tx_buffers[i];
1988                         ctxd->buffer_addr =
1989                             htole64(seg_addr + seg_len);
1990                         ctxd->lower.data = htole32(
1991                         adapter->txd_cmd | txd_lower | 4);
1992                         ctxd->upper.data =
1993                             htole32(txd_upper);
1994                         last = i;
1995                         if (++i == adapter->num_tx_desc)
1996                                 i = 0;
1997                 } else {
1998                         ctxd->buffer_addr = htole64(seg_addr);
1999                         ctxd->lower.data = htole32(
2000                         adapter->txd_cmd | txd_lower | seg_len);
2001                         ctxd->upper.data =
2002                             htole32(txd_upper);
2003                         last = i;
2004                         if (++i == adapter->num_tx_desc)
2005                                 i = 0;
2006                 }
2007                 tx_buffer->m_head = NULL;
2008                 tx_buffer->next_eop = -1;
2009         }
2010
2011         txr->next_avail_desc = i;
2012         txr->tx_avail -= nsegs;
2013         if (tso_desc) /* TSO used an extra for sentinel */
2014                 txr->tx_avail -= txd_used;
2015
2016         tx_buffer->m_head = m_head;
2017         /*
2018         ** Here we swap the map so the last descriptor,
2019         ** which gets the completion interrupt has the
2020         ** real map, and the first descriptor gets the
2021         ** unused map from this descriptor.
2022         */
2023         tx_buffer_mapped->map = tx_buffer->map;
2024         tx_buffer->map = map;
2025         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
2026
2027         /*
2028          * Last Descriptor of Packet
2029          * needs End Of Packet (EOP)
2030          * and Report Status (RS)
2031          */
2032         ctxd->lower.data |=
2033             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2034         /*
2035          * Keep track in the first buffer which
2036          * descriptor will be written back
2037          */
2038         tx_buffer = &txr->tx_buffers[first];
2039         tx_buffer->next_eop = last;
2040         /* Update the watchdog time early and often */
2041         txr->watchdog_time = ticks;
2042
2043         /*
2044          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2045          * that this frame is available to transmit.
2046          */
2047         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2048             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2049         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
2050
2051         return (0);
2052 }
2053
2054 static void
2055 em_set_promisc(struct adapter *adapter)
2056 {
2057         struct ifnet    *ifp = adapter->ifp;
2058         u32             reg_rctl;
2059
2060         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2061
2062         if (ifp->if_flags & IFF_PROMISC) {
2063                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2064                 /* Turn this on if you want to see bad packets */
2065                 if (em_debug_sbp)
2066                         reg_rctl |= E1000_RCTL_SBP;
2067                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2068         } else if (ifp->if_flags & IFF_ALLMULTI) {
2069                 reg_rctl |= E1000_RCTL_MPE;
2070                 reg_rctl &= ~E1000_RCTL_UPE;
2071                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2072         }
2073 }
2074
2075 static void
2076 em_disable_promisc(struct adapter *adapter)
2077 {
2078         u32     reg_rctl;
2079
2080         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2081
2082         reg_rctl &=  (~E1000_RCTL_UPE);
2083         reg_rctl &=  (~E1000_RCTL_MPE);
2084         reg_rctl &=  (~E1000_RCTL_SBP);
2085         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2086 }
2087
2088
2089 /*********************************************************************
2090  *  Multicast Update
2091  *
2092  *  This routine is called whenever multicast address list is updated.
2093  *
2094  **********************************************************************/
2095
2096 static void
2097 em_set_multi(struct adapter *adapter)
2098 {
2099         struct ifnet    *ifp = adapter->ifp;
2100         struct ifmultiaddr *ifma;
2101         u32 reg_rctl = 0;
2102         u8  *mta; /* Multicast array memory */
2103         int mcnt = 0;
2104
2105         IOCTL_DEBUGOUT("em_set_multi: begin");
2106
2107         mta = adapter->mta;
2108         bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
2109
2110         if (adapter->hw.mac.type == e1000_82542 &&
2111             adapter->hw.revision_id == E1000_REVISION_2) {
2112                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2113                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2114                         e1000_pci_clear_mwi(&adapter->hw);
2115                 reg_rctl |= E1000_RCTL_RST;
2116                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2117                 msec_delay(5);
2118         }
2119
2120 #if __FreeBSD_version < 800000
2121         IF_ADDR_LOCK(ifp);
2122 #else
2123         if_maddr_rlock(ifp);
2124 #endif
2125         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2126                 if (ifma->ifma_addr->sa_family != AF_LINK)
2127                         continue;
2128
2129                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2130                         break;
2131
2132                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2133                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2134                 mcnt++;
2135         }
2136 #if __FreeBSD_version < 800000
2137         IF_ADDR_UNLOCK(ifp);
2138 #else
2139         if_maddr_runlock(ifp);
2140 #endif
2141         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2142                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2143                 reg_rctl |= E1000_RCTL_MPE;
2144                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2145         } else
2146                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2147
2148         if (adapter->hw.mac.type == e1000_82542 &&
2149             adapter->hw.revision_id == E1000_REVISION_2) {
2150                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2151                 reg_rctl &= ~E1000_RCTL_RST;
2152                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2153                 msec_delay(5);
2154                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2155                         e1000_pci_set_mwi(&adapter->hw);
2156         }
2157 }
2158
2159
2160 /*********************************************************************
2161  *  Timer routine
2162  *
2163  *  This routine checks for link status and updates statistics.
2164  *
2165  **********************************************************************/
2166
2167 static void
2168 em_local_timer(void *arg)
2169 {
2170         struct adapter  *adapter = arg;
2171         struct ifnet    *ifp = adapter->ifp;
2172         struct tx_ring  *txr = adapter->tx_rings;
2173         struct rx_ring  *rxr = adapter->rx_rings;
2174         u32             trigger;
2175
2176         EM_CORE_LOCK_ASSERT(adapter);
2177
2178         em_update_link_status(adapter);
2179         em_update_stats_counters(adapter);
2180
2181         /* Reset LAA into RAR[0] on 82571 */
2182         if ((adapter->hw.mac.type == e1000_82571) &&
2183             e1000_get_laa_state_82571(&adapter->hw))
2184                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2185
2186         /* Mask to use in the irq trigger */
2187         if (adapter->msix_mem)
2188                 trigger = rxr->ims; /* RX for 82574 */
2189         else
2190                 trigger = E1000_ICS_RXDMT0;
2191
2192         /*
2193         ** Check on the state of the TX queue(s), this
2194         ** can be done without the lock because its RO
2195         ** and the HUNG state will be static if set.
2196         */
2197         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2198                 if ((txr->queue_status == EM_QUEUE_HUNG) &&
2199                     (adapter->pause_frames == 0))
2200                         goto hung;
2201                 /* Schedule a TX tasklet if needed */
2202                 if (txr->tx_avail <= EM_MAX_SCATTER)
2203                         taskqueue_enqueue(txr->tq, &txr->tx_task);
2204         }
2205
2206         adapter->pause_frames = 0;
2207         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2208 #ifndef DEVICE_POLLING
2209         /* Trigger an RX interrupt to guarantee mbuf refresh */
2210         E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger);
2211 #endif
2212         return;
2213 hung:
2214         /* Looks like we're hung */
2215         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2216         device_printf(adapter->dev,
2217             "Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2218             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2219             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2220         device_printf(adapter->dev,"TX(%d) desc avail = %d,"
2221             "Next TX to Clean = %d\n",
2222             txr->me, txr->tx_avail, txr->next_to_clean);
2223         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2224         adapter->watchdog_events++;
2225         adapter->pause_frames = 0;
2226         em_init_locked(adapter);
2227 }
2228
2229
2230 static void
2231 em_update_link_status(struct adapter *adapter)
2232 {
2233         struct e1000_hw *hw = &adapter->hw;
2234         struct ifnet *ifp = adapter->ifp;
2235         device_t dev = adapter->dev;
2236         struct tx_ring *txr = adapter->tx_rings;
2237         u32 link_check = 0;
2238
2239         /* Get the cached link value or read phy for real */
2240         switch (hw->phy.media_type) {
2241         case e1000_media_type_copper:
2242                 if (hw->mac.get_link_status) {
2243                         /* Do the work to read phy */
2244                         e1000_check_for_link(hw);
2245                         link_check = !hw->mac.get_link_status;
2246                         if (link_check) /* ESB2 fix */
2247                                 e1000_cfg_on_link_up(hw);
2248                 } else
2249                         link_check = TRUE;
2250                 break;
2251         case e1000_media_type_fiber:
2252                 e1000_check_for_link(hw);
2253                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2254                                  E1000_STATUS_LU);
2255                 break;
2256         case e1000_media_type_internal_serdes:
2257                 e1000_check_for_link(hw);
2258                 link_check = adapter->hw.mac.serdes_has_link;
2259                 break;
2260         default:
2261         case e1000_media_type_unknown:
2262                 break;
2263         }
2264
2265         /* Now check for a transition */
2266         if (link_check && (adapter->link_active == 0)) {
2267                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2268                     &adapter->link_duplex);
2269                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2270                 if ((adapter->link_speed != SPEED_1000) &&
2271                     ((hw->mac.type == e1000_82571) ||
2272                     (hw->mac.type == e1000_82572))) {
2273                         int tarc0;
2274                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2275                         tarc0 &= ~SPEED_MODE_BIT;
2276                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2277                 }
2278                 if (bootverbose)
2279                         device_printf(dev, "Link is up %d Mbps %s\n",
2280                             adapter->link_speed,
2281                             ((adapter->link_duplex == FULL_DUPLEX) ?
2282                             "Full Duplex" : "Half Duplex"));
2283                 adapter->link_active = 1;
2284                 adapter->smartspeed = 0;
2285                 ifp->if_baudrate = adapter->link_speed * 1000000;
2286                 if_link_state_change(ifp, LINK_STATE_UP);
2287         } else if (!link_check && (adapter->link_active == 1)) {
2288                 ifp->if_baudrate = adapter->link_speed = 0;
2289                 adapter->link_duplex = 0;
2290                 if (bootverbose)
2291                         device_printf(dev, "Link is Down\n");
2292                 adapter->link_active = 0;
2293                 /* Link down, disable watchdog */
2294                 for (int i = 0; i < adapter->num_queues; i++, txr++)
2295                         txr->queue_status = EM_QUEUE_IDLE;
2296                 if_link_state_change(ifp, LINK_STATE_DOWN);
2297         }
2298 }
2299
2300 /*********************************************************************
2301  *
2302  *  This routine disables all traffic on the adapter by issuing a
2303  *  global reset on the MAC and deallocates TX/RX buffers.
2304  *
2305  *  This routine should always be called with BOTH the CORE
2306  *  and TX locks.
2307  **********************************************************************/
2308
2309 static void
2310 em_stop(void *arg)
2311 {
2312         struct adapter  *adapter = arg;
2313         struct ifnet    *ifp = adapter->ifp;
2314         struct tx_ring  *txr = adapter->tx_rings;
2315
2316         EM_CORE_LOCK_ASSERT(adapter);
2317
2318         INIT_DEBUGOUT("em_stop: begin");
2319
2320         em_disable_intr(adapter);
2321         callout_stop(&adapter->timer);
2322
2323         /* Tell the stack that the interface is no longer active */
2324         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2325         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2326
2327         /* Unarm watchdog timer. */
2328         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2329                 EM_TX_LOCK(txr);
2330                 txr->queue_status = EM_QUEUE_IDLE;
2331                 EM_TX_UNLOCK(txr);
2332         }
2333
2334         e1000_reset_hw(&adapter->hw);
2335         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2336
2337         e1000_led_off(&adapter->hw);
2338         e1000_cleanup_led(&adapter->hw);
2339 }
2340
2341
2342 /*********************************************************************
2343  *
2344  *  Determine hardware revision.
2345  *
2346  **********************************************************************/
2347 static void
2348 em_identify_hardware(struct adapter *adapter)
2349 {
2350         device_t dev = adapter->dev;
2351
2352         /* Make sure our PCI config space has the necessary stuff set */
2353         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2354         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2355             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2356                 device_printf(dev, "Memory Access and/or Bus Master bits "
2357                     "were not set!\n");
2358                 adapter->hw.bus.pci_cmd_word |=
2359                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2360                 pci_write_config(dev, PCIR_COMMAND,
2361                     adapter->hw.bus.pci_cmd_word, 2);
2362         }
2363
2364         /* Save off the information about this board */
2365         adapter->hw.vendor_id = pci_get_vendor(dev);
2366         adapter->hw.device_id = pci_get_device(dev);
2367         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2368         adapter->hw.subsystem_vendor_id =
2369             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2370         adapter->hw.subsystem_device_id =
2371             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2372
2373         /* Do Shared Code Init and Setup */
2374         if (e1000_set_mac_type(&adapter->hw)) {
2375                 device_printf(dev, "Setup init failure\n");
2376                 return;
2377         }
2378 }
2379
2380 static int
2381 em_allocate_pci_resources(struct adapter *adapter)
2382 {
2383         device_t        dev = adapter->dev;
2384         int             rid;
2385
2386         rid = PCIR_BAR(0);
2387         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2388             &rid, RF_ACTIVE);
2389         if (adapter->memory == NULL) {
2390                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2391                 return (ENXIO);
2392         }
2393         adapter->osdep.mem_bus_space_tag =
2394             rman_get_bustag(adapter->memory);
2395         adapter->osdep.mem_bus_space_handle =
2396             rman_get_bushandle(adapter->memory);
2397         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2398
2399         /* Default to a single queue */
2400         adapter->num_queues = 1;
2401
2402         /*
2403          * Setup MSI/X or MSI if PCI Express
2404          */
2405         adapter->msix = em_setup_msix(adapter);
2406
2407         adapter->hw.back = &adapter->osdep;
2408
2409         return (0);
2410 }
2411
2412 /*********************************************************************
2413  *
2414  *  Setup the Legacy or MSI Interrupt handler
2415  *
2416  **********************************************************************/
2417 int
2418 em_allocate_legacy(struct adapter *adapter)
2419 {
2420         device_t dev = adapter->dev;
2421         struct tx_ring  *txr = adapter->tx_rings;
2422         int error, rid = 0;
2423
2424         /* Manually turn off all interrupts */
2425         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2426
2427         if (adapter->msix == 1) /* using MSI */
2428                 rid = 1;
2429         /* We allocate a single interrupt resource */
2430         adapter->res = bus_alloc_resource_any(dev,
2431             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2432         if (adapter->res == NULL) {
2433                 device_printf(dev, "Unable to allocate bus resource: "
2434                     "interrupt\n");
2435                 return (ENXIO);
2436         }
2437
2438         /*
2439          * Allocate a fast interrupt and the associated
2440          * deferred processing contexts.
2441          */
2442         TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter);
2443         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2444             taskqueue_thread_enqueue, &adapter->tq);
2445         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que",
2446             device_get_nameunit(adapter->dev));
2447         /* Use a TX only tasklet for local timer */
2448         TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2449         txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2450             taskqueue_thread_enqueue, &txr->tq);
2451         taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2452             device_get_nameunit(adapter->dev));
2453         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2454         if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET,
2455             em_irq_fast, NULL, adapter, &adapter->tag)) != 0) {
2456                 device_printf(dev, "Failed to register fast interrupt "
2457                             "handler: %d\n", error);
2458                 taskqueue_free(adapter->tq);
2459                 adapter->tq = NULL;
2460                 return (error);
2461         }
2462
2463         return (0);
2464 }
2465
2466 /*********************************************************************
2467  *
2468  *  Setup the MSIX Interrupt handlers
2469  *   This is not really Multiqueue, rather
2470  *   its just seperate interrupt vectors
2471  *   for TX, RX, and Link.
2472  *
2473  **********************************************************************/
2474 int
2475 em_allocate_msix(struct adapter *adapter)
2476 {
2477         device_t        dev = adapter->dev;
2478         struct          tx_ring *txr = adapter->tx_rings;
2479         struct          rx_ring *rxr = adapter->rx_rings;
2480         int             error, rid, vector = 0;
2481
2482
2483         /* Make sure all interrupts are disabled */
2484         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2485
2486         /* First set up ring resources */
2487         for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
2488
2489                 /* RX ring */
2490                 rid = vector + 1;
2491
2492                 rxr->res = bus_alloc_resource_any(dev,
2493                     SYS_RES_IRQ, &rid, RF_ACTIVE);
2494                 if (rxr->res == NULL) {
2495                         device_printf(dev,
2496                             "Unable to allocate bus resource: "
2497                             "RX MSIX Interrupt %d\n", i);
2498                         return (ENXIO);
2499                 }
2500                 if ((error = bus_setup_intr(dev, rxr->res,
2501                     INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx,
2502                     rxr, &rxr->tag)) != 0) {
2503                         device_printf(dev, "Failed to register RX handler");
2504                         return (error);
2505                 }
2506 #if __FreeBSD_version >= 800504
2507                 bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i);
2508 #endif
2509                 rxr->msix = vector++; /* NOTE increment vector for TX */
2510                 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr);
2511                 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT,
2512                     taskqueue_thread_enqueue, &rxr->tq);
2513                 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq",
2514                     device_get_nameunit(adapter->dev));
2515                 /*
2516                 ** Set the bit to enable interrupt
2517                 ** in E1000_IMS -- bits 20 and 21
2518                 ** are for RX0 and RX1, note this has
2519                 ** NOTHING to do with the MSIX vector
2520                 */
2521                 rxr->ims = 1 << (20 + i);
2522                 adapter->ivars |= (8 | rxr->msix) << (i * 4);
2523
2524                 /* TX ring */
2525                 rid = vector + 1;
2526                 txr->res = bus_alloc_resource_any(dev,
2527                     SYS_RES_IRQ, &rid, RF_ACTIVE);
2528                 if (txr->res == NULL) {
2529                         device_printf(dev,
2530                             "Unable to allocate bus resource: "
2531                             "TX MSIX Interrupt %d\n", i);
2532                         return (ENXIO);
2533                 }
2534                 if ((error = bus_setup_intr(dev, txr->res,
2535                     INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx,
2536                     txr, &txr->tag)) != 0) {
2537                         device_printf(dev, "Failed to register TX handler");
2538                         return (error);
2539                 }
2540 #if __FreeBSD_version >= 800504
2541                 bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i);
2542 #endif
2543                 txr->msix = vector++; /* Increment vector for next pass */
2544                 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2545                 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2546                     taskqueue_thread_enqueue, &txr->tq);
2547                 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2548                     device_get_nameunit(adapter->dev));
2549                 /*
2550                 ** Set the bit to enable interrupt
2551                 ** in E1000_IMS -- bits 22 and 23
2552                 ** are for TX0 and TX1, note this has
2553                 ** NOTHING to do with the MSIX vector
2554                 */
2555                 txr->ims = 1 << (22 + i);
2556                 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4));
2557         }
2558
2559         /* Link interrupt */
2560         ++rid;
2561         adapter->res = bus_alloc_resource_any(dev,
2562             SYS_RES_IRQ, &rid, RF_ACTIVE);
2563         if (!adapter->res) {
2564                 device_printf(dev,"Unable to allocate "
2565                     "bus resource: Link interrupt [%d]\n", rid);
2566                 return (ENXIO);
2567         }
2568         /* Set the link handler function */
2569         error = bus_setup_intr(dev, adapter->res,
2570             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2571             em_msix_link, adapter, &adapter->tag);
2572         if (error) {
2573                 adapter->res = NULL;
2574                 device_printf(dev, "Failed to register LINK handler");
2575                 return (error);
2576         }
2577 #if __FreeBSD_version >= 800504
2578                 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2579 #endif
2580         adapter->linkvec = vector;
2581         adapter->ivars |=  (8 | vector) << 16;
2582         adapter->ivars |= 0x80000000;
2583
2584         return (0);
2585 }
2586
2587
2588 static void
2589 em_free_pci_resources(struct adapter *adapter)
2590 {
2591         device_t        dev = adapter->dev;
2592         struct tx_ring  *txr;
2593         struct rx_ring  *rxr;
2594         int             rid;
2595
2596
2597         /*
2598         ** Release all the queue interrupt resources:
2599         */
2600         for (int i = 0; i < adapter->num_queues; i++) {
2601                 txr = &adapter->tx_rings[i];
2602                 rxr = &adapter->rx_rings[i];
2603                 /* an early abort? */
2604                 if ((txr == NULL) || (rxr == NULL))
2605                         break;
2606                 rid = txr->msix +1;
2607                 if (txr->tag != NULL) {
2608                         bus_teardown_intr(dev, txr->res, txr->tag);
2609                         txr->tag = NULL;
2610                 }
2611                 if (txr->res != NULL)
2612                         bus_release_resource(dev, SYS_RES_IRQ,
2613                             rid, txr->res);
2614                 rid = rxr->msix +1;
2615                 if (rxr->tag != NULL) {
2616                         bus_teardown_intr(dev, rxr->res, rxr->tag);
2617                         rxr->tag = NULL;
2618                 }
2619                 if (rxr->res != NULL)
2620                         bus_release_resource(dev, SYS_RES_IRQ,
2621                             rid, rxr->res);
2622         }
2623
2624         if (adapter->linkvec) /* we are doing MSIX */
2625                 rid = adapter->linkvec + 1;
2626         else
2627                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2628
2629         if (adapter->tag != NULL) {
2630                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2631                 adapter->tag = NULL;
2632         }
2633
2634         if (adapter->res != NULL)
2635                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2636
2637
2638         if (adapter->msix)
2639                 pci_release_msi(dev);
2640
2641         if (adapter->msix_mem != NULL)
2642                 bus_release_resource(dev, SYS_RES_MEMORY,
2643                     PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2644
2645         if (adapter->memory != NULL)
2646                 bus_release_resource(dev, SYS_RES_MEMORY,
2647                     PCIR_BAR(0), adapter->memory);
2648
2649         if (adapter->flash != NULL)
2650                 bus_release_resource(dev, SYS_RES_MEMORY,
2651                     EM_FLASH, adapter->flash);
2652 }
2653
2654 /*
2655  * Setup MSI or MSI/X
2656  */
2657 static int
2658 em_setup_msix(struct adapter *adapter)
2659 {
2660         device_t dev = adapter->dev;
2661         int val = 0;
2662
2663         /*
2664         ** Setup MSI/X for Hartwell: tests have shown
2665         ** use of two queues to be unstable, and to
2666         ** provide no great gain anyway, so we simply
2667         ** seperate the interrupts and use a single queue.
2668         */
2669         if ((adapter->hw.mac.type == e1000_82574) &&
2670             (em_enable_msix == TRUE)) {
2671                 /* Map the MSIX BAR */
2672                 int rid = PCIR_BAR(EM_MSIX_BAR);
2673                 adapter->msix_mem = bus_alloc_resource_any(dev,
2674                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2675                 if (!adapter->msix_mem) {
2676                         /* May not be enabled */
2677                         device_printf(adapter->dev,
2678                             "Unable to map MSIX table \n");
2679                         goto msi;
2680                 }
2681                 val = pci_msix_count(dev);
2682                 /* We only need 3 vectors */
2683                 if (val > 3)
2684                         val = 3;
2685                 if ((val != 3) && (val != 5)) {
2686                         bus_release_resource(dev, SYS_RES_MEMORY,
2687                             PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2688                         adapter->msix_mem = NULL;
2689                         device_printf(adapter->dev,
2690                             "MSIX: incorrect vectors, using MSI\n");
2691                         goto msi;
2692                 }
2693
2694                 if (pci_alloc_msix(dev, &val) == 0) {
2695                         device_printf(adapter->dev,
2696                             "Using MSIX interrupts "
2697                             "with %d vectors\n", val);
2698                 }
2699
2700                 return (val);
2701         }
2702 msi:
2703         val = pci_msi_count(dev);
2704         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2705                 adapter->msix = 1;
2706                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2707                 return (val);
2708         }
2709         /* Should only happen due to manual configuration */
2710         device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n");
2711         return (0);
2712 }
2713
2714
2715 /*********************************************************************
2716  *
2717  *  Initialize the hardware to a configuration
2718  *  as specified by the adapter structure.
2719  *
2720  **********************************************************************/
2721 static void
2722 em_reset(struct adapter *adapter)
2723 {
2724         device_t        dev = adapter->dev;
2725         struct ifnet    *ifp = adapter->ifp;
2726         struct e1000_hw *hw = &adapter->hw;
2727         u16             rx_buffer_size;
2728         u32             pba;
2729
2730         INIT_DEBUGOUT("em_reset: begin");
2731
2732         /* Set up smart power down as default off on newer adapters. */
2733         if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
2734             hw->mac.type == e1000_82572)) {
2735                 u16 phy_tmp = 0;
2736
2737                 /* Speed up time to link by disabling smart power down. */
2738                 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2739                 phy_tmp &= ~IGP02E1000_PM_SPD;
2740                 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2741         }
2742
2743         /*
2744          * Packet Buffer Allocation (PBA)
2745          * Writing PBA sets the receive portion of the buffer
2746          * the remainder is used for the transmit buffer.
2747          */
2748         switch (hw->mac.type) {
2749         /* Total Packet Buffer on these is 48K */
2750         case e1000_82571:
2751         case e1000_82572:
2752         case e1000_80003es2lan:
2753                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
2754                 break;
2755         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
2756                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
2757                 break;
2758         case e1000_82574:
2759         case e1000_82583:
2760                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
2761                 break;
2762         case e1000_ich8lan:
2763                 pba = E1000_PBA_8K;
2764                 break;
2765         case e1000_ich9lan:
2766         case e1000_ich10lan:
2767                 /* Boost Receive side for jumbo frames */
2768                 if (adapter->max_frame_size > 4096)
2769                         pba = E1000_PBA_14K;
2770                 else
2771                         pba = E1000_PBA_10K;
2772                 break;
2773         case e1000_pchlan:
2774         case e1000_pch2lan:
2775                 pba = E1000_PBA_26K;
2776                 break;
2777         default:
2778                 if (adapter->max_frame_size > 8192)
2779                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
2780                 else
2781                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
2782         }
2783         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
2784
2785         /*
2786          * These parameters control the automatic generation (Tx) and
2787          * response (Rx) to Ethernet PAUSE frames.
2788          * - High water mark should allow for at least two frames to be
2789          *   received after sending an XOFF.
2790          * - Low water mark works best when it is very near the high water mark.
2791          *   This allows the receiver to restart by sending XON when it has
2792          *   drained a bit. Here we use an arbitary value of 1500 which will
2793          *   restart after one full frame is pulled from the buffer. There
2794          *   could be several smaller frames in the buffer and if so they will
2795          *   not trigger the XON until their total number reduces the buffer
2796          *   by 1500.
2797          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2798          */
2799         rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 );
2800         hw->fc.high_water = rx_buffer_size -
2801             roundup2(adapter->max_frame_size, 1024);
2802         hw->fc.low_water = hw->fc.high_water - 1500;
2803
2804         if (adapter->fc) /* locally set flow control value? */
2805                 hw->fc.requested_mode = adapter->fc;
2806         else
2807                 hw->fc.requested_mode = e1000_fc_full;
2808
2809         if (hw->mac.type == e1000_80003es2lan)
2810                 hw->fc.pause_time = 0xFFFF;
2811         else
2812                 hw->fc.pause_time = EM_FC_PAUSE_TIME;
2813
2814         hw->fc.send_xon = TRUE;
2815
2816         /* Device specific overrides/settings */
2817         switch (hw->mac.type) {
2818         case e1000_pchlan:
2819                 /* Workaround: no TX flow ctrl for PCH */
2820                 hw->fc.requested_mode = e1000_fc_rx_pause;
2821                 hw->fc.pause_time = 0xFFFF; /* override */
2822                 if (ifp->if_mtu > ETHERMTU) {
2823                         hw->fc.high_water = 0x3500;
2824                         hw->fc.low_water = 0x1500;
2825                 } else {
2826                         hw->fc.high_water = 0x5000;
2827                         hw->fc.low_water = 0x3000;
2828                 }
2829                 hw->fc.refresh_time = 0x1000;
2830                 break;
2831         case e1000_pch2lan:
2832                 hw->fc.high_water = 0x5C20;
2833                 hw->fc.low_water = 0x5048;
2834                 hw->fc.pause_time = 0x0650;
2835                 hw->fc.refresh_time = 0x0400;
2836                 /* Jumbos need adjusted PBA */
2837                 if (ifp->if_mtu > ETHERMTU)
2838                         E1000_WRITE_REG(hw, E1000_PBA, 12);
2839                 else
2840                         E1000_WRITE_REG(hw, E1000_PBA, 26);
2841                 break;
2842         case e1000_ich9lan:
2843         case e1000_ich10lan:
2844                 if (ifp->if_mtu > ETHERMTU) {
2845                         hw->fc.high_water = 0x2800;
2846                         hw->fc.low_water = hw->fc.high_water - 8;
2847                         break;
2848                 }
2849                 /* else fall thru */
2850         default:
2851                 if (hw->mac.type == e1000_80003es2lan)
2852                         hw->fc.pause_time = 0xFFFF;
2853                 break;
2854         }
2855
2856         /* Issue a global reset */
2857         e1000_reset_hw(hw);
2858         E1000_WRITE_REG(hw, E1000_WUC, 0);
2859         em_disable_aspm(adapter);
2860         /* and a re-init */
2861         if (e1000_init_hw(hw) < 0) {
2862                 device_printf(dev, "Hardware Initialization Failed\n");
2863                 return;
2864         }
2865
2866         E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
2867         e1000_get_phy_info(hw);
2868         e1000_check_for_link(hw);
2869         return;
2870 }
2871
2872 /*********************************************************************
2873  *
2874  *  Setup networking device structure and register an interface.
2875  *
2876  **********************************************************************/
2877 static int
2878 em_setup_interface(device_t dev, struct adapter *adapter)
2879 {
2880         struct ifnet   *ifp;
2881
2882         INIT_DEBUGOUT("em_setup_interface: begin");
2883
2884         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2885         if (ifp == NULL) {
2886                 device_printf(dev, "can not allocate ifnet structure\n");
2887                 return (-1);
2888         }
2889         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2890         ifp->if_init =  em_init;
2891         ifp->if_softc = adapter;
2892         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2893         ifp->if_ioctl = em_ioctl;
2894         ifp->if_start = em_start;
2895         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2896         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2897         IFQ_SET_READY(&ifp->if_snd);
2898
2899         ether_ifattach(ifp, adapter->hw.mac.addr);
2900
2901         ifp->if_capabilities = ifp->if_capenable = 0;
2902
2903 #ifdef EM_MULTIQUEUE
2904         /* Multiqueue stack interface */
2905         ifp->if_transmit = em_mq_start;
2906         ifp->if_qflush = em_qflush;
2907 #endif
2908
2909         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2910         ifp->if_capabilities |= IFCAP_TSO4;
2911         /*
2912          * Tell the upper layer(s) we
2913          * support full VLAN capability
2914          */
2915         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2916         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2917                              |  IFCAP_VLAN_HWTSO
2918                              |  IFCAP_VLAN_MTU;
2919         ifp->if_capenable = ifp->if_capabilities;
2920
2921         /*
2922         ** Don't turn this on by default, if vlans are
2923         ** created on another pseudo device (eg. lagg)
2924         ** then vlan events are not passed thru, breaking
2925         ** operation, but with HW FILTER off it works. If
2926         ** using vlans directly on the em driver you can
2927         ** enable this and get full hardware tag filtering.
2928         */
2929         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2930
2931 #ifdef DEVICE_POLLING
2932         ifp->if_capabilities |= IFCAP_POLLING;
2933 #endif
2934
2935         /* Enable only WOL MAGIC by default */
2936         if (adapter->wol) {
2937                 ifp->if_capabilities |= IFCAP_WOL;
2938                 ifp->if_capenable |= IFCAP_WOL_MAGIC;
2939         }
2940
2941         /*
2942          * Specify the media types supported by this adapter and register
2943          * callbacks to update media and link information
2944          */
2945         ifmedia_init(&adapter->media, IFM_IMASK,
2946             em_media_change, em_media_status);
2947         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2948             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2949                 u_char fiber_type = IFM_1000_SX;        /* default type */
2950
2951                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2952                             0, NULL);
2953                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2954         } else {
2955                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2956                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2957                             0, NULL);
2958                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2959                             0, NULL);
2960                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2961                             0, NULL);
2962                 if (adapter->hw.phy.type != e1000_phy_ife) {
2963                         ifmedia_add(&adapter->media,
2964                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2965                         ifmedia_add(&adapter->media,
2966                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2967                 }
2968         }
2969         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2970         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2971         return (0);
2972 }
2973
2974
2975 /*
2976  * Manage DMA'able memory.
2977  */
2978 static void
2979 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2980 {
2981         if (error)
2982                 return;
2983         *(bus_addr_t *) arg = segs[0].ds_addr;
2984 }
2985
2986 static int
2987 em_dma_malloc(struct adapter *adapter, bus_size_t size,
2988         struct em_dma_alloc *dma, int mapflags)
2989 {
2990         int error;
2991
2992         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2993                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
2994                                 BUS_SPACE_MAXADDR,      /* lowaddr */
2995                                 BUS_SPACE_MAXADDR,      /* highaddr */
2996                                 NULL, NULL,             /* filter, filterarg */
2997                                 size,                   /* maxsize */
2998                                 1,                      /* nsegments */
2999                                 size,                   /* maxsegsize */
3000                                 0,                      /* flags */
3001                                 NULL,                   /* lockfunc */
3002                                 NULL,                   /* lockarg */
3003                                 &dma->dma_tag);
3004         if (error) {
3005                 device_printf(adapter->dev,
3006                     "%s: bus_dma_tag_create failed: %d\n",
3007                     __func__, error);
3008                 goto fail_0;
3009         }
3010
3011         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3012             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3013         if (error) {
3014                 device_printf(adapter->dev,
3015                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3016                     __func__, (uintmax_t)size, error);
3017                 goto fail_2;
3018         }
3019
3020         dma->dma_paddr = 0;
3021         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3022             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3023         if (error || dma->dma_paddr == 0) {
3024                 device_printf(adapter->dev,
3025                     "%s: bus_dmamap_load failed: %d\n",
3026                     __func__, error);
3027                 goto fail_3;
3028         }
3029
3030         return (0);
3031
3032 fail_3:
3033         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3034 fail_2:
3035         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3036         bus_dma_tag_destroy(dma->dma_tag);
3037 fail_0:
3038         dma->dma_map = NULL;
3039         dma->dma_tag = NULL;
3040
3041         return (error);
3042 }
3043
3044 static void
3045 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3046 {
3047         if (dma->dma_tag == NULL)
3048                 return;
3049         if (dma->dma_map != NULL) {
3050                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3051                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3052                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3053                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3054                 dma->dma_map = NULL;
3055         }
3056         bus_dma_tag_destroy(dma->dma_tag);
3057         dma->dma_tag = NULL;
3058 }
3059
3060
3061 /*********************************************************************
3062  *
3063  *  Allocate memory for the transmit and receive rings, and then
3064  *  the descriptors associated with each, called only once at attach.
3065  *
3066  **********************************************************************/
3067 static int
3068 em_allocate_queues(struct adapter *adapter)
3069 {
3070         device_t                dev = adapter->dev;
3071         struct tx_ring          *txr = NULL;
3072         struct rx_ring          *rxr = NULL;
3073         int rsize, tsize, error = E1000_SUCCESS;
3074         int txconf = 0, rxconf = 0;
3075
3076
3077         /* Allocate the TX ring struct memory */
3078         if (!(adapter->tx_rings =
3079             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3080             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3081                 device_printf(dev, "Unable to allocate TX ring memory\n");
3082                 error = ENOMEM;
3083                 goto fail;
3084         }
3085
3086         /* Now allocate the RX */
3087         if (!(adapter->rx_rings =
3088             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3089             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3090                 device_printf(dev, "Unable to allocate RX ring memory\n");
3091                 error = ENOMEM;
3092                 goto rx_fail;
3093         }
3094
3095         tsize = roundup2(adapter->num_tx_desc *
3096             sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
3097         /*
3098          * Now set up the TX queues, txconf is needed to handle the
3099          * possibility that things fail midcourse and we need to
3100          * undo memory gracefully
3101          */
3102         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3103                 /* Set up some basics */
3104                 txr = &adapter->tx_rings[i];
3105                 txr->adapter = adapter;
3106                 txr->me = i;
3107
3108                 /* Initialize the TX lock */
3109                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3110                     device_get_nameunit(dev), txr->me);
3111                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3112
3113                 if (em_dma_malloc(adapter, tsize,
3114                         &txr->txdma, BUS_DMA_NOWAIT)) {
3115                         device_printf(dev,
3116                             "Unable to allocate TX Descriptor memory\n");
3117                         error = ENOMEM;
3118                         goto err_tx_desc;
3119                 }
3120                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
3121                 bzero((void *)txr->tx_base, tsize);
3122
3123                 if (em_allocate_transmit_buffers(txr)) {
3124                         device_printf(dev,
3125                             "Critical Failure setting up transmit buffers\n");
3126                         error = ENOMEM;
3127                         goto err_tx_desc;
3128                 }
3129 #if __FreeBSD_version >= 800000
3130                 /* Allocate a buf ring */
3131                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3132                     M_WAITOK, &txr->tx_mtx);
3133 #endif
3134         }
3135
3136         /*
3137          * Next the RX queues...
3138          */
3139         rsize = roundup2(adapter->num_rx_desc *
3140             sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
3141         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3142                 rxr = &adapter->rx_rings[i];
3143                 rxr->adapter = adapter;
3144                 rxr->me = i;
3145
3146                 /* Initialize the RX lock */
3147                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3148                     device_get_nameunit(dev), txr->me);
3149                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3150
3151                 if (em_dma_malloc(adapter, rsize,
3152                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
3153                         device_printf(dev,
3154                             "Unable to allocate RxDescriptor memory\n");
3155                         error = ENOMEM;
3156                         goto err_rx_desc;
3157                 }
3158                 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr;
3159                 bzero((void *)rxr->rx_base, rsize);
3160
3161                 /* Allocate receive buffers for the ring*/
3162                 if (em_allocate_receive_buffers(rxr)) {
3163                         device_printf(dev,
3164                             "Critical Failure setting up receive buffers\n");
3165                         error = ENOMEM;
3166                         goto err_rx_desc;
3167                 }
3168         }
3169
3170         return (0);
3171
3172 err_rx_desc:
3173         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3174                 em_dma_free(adapter, &rxr->rxdma);
3175 err_tx_desc:
3176         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3177                 em_dma_free(adapter, &txr->txdma);
3178         free(adapter->rx_rings, M_DEVBUF);
3179 rx_fail:
3180 #if __FreeBSD_version >= 800000
3181         buf_ring_free(txr->br, M_DEVBUF);
3182 #endif
3183         free(adapter->tx_rings, M_DEVBUF);
3184 fail:
3185         return (error);
3186 }
3187
3188
3189 /*********************************************************************
3190  *
3191  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3192  *  the information needed to transmit a packet on the wire. This is
3193  *  called only once at attach, setup is done every reset.
3194  *
3195  **********************************************************************/
3196 static int
3197 em_allocate_transmit_buffers(struct tx_ring *txr)
3198 {
3199         struct adapter *adapter = txr->adapter;
3200         device_t dev = adapter->dev;
3201         struct em_buffer *txbuf;
3202         int error, i;
3203
3204         /*
3205          * Setup DMA descriptor areas.
3206          */
3207         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3208                                1, 0,                    /* alignment, bounds */
3209                                BUS_SPACE_MAXADDR,       /* lowaddr */
3210                                BUS_SPACE_MAXADDR,       /* highaddr */
3211                                NULL, NULL,              /* filter, filterarg */
3212                                EM_TSO_SIZE,             /* maxsize */
3213                                EM_MAX_SCATTER,          /* nsegments */
3214                                PAGE_SIZE,               /* maxsegsize */
3215                                0,                       /* flags */
3216                                NULL,                    /* lockfunc */
3217                                NULL,                    /* lockfuncarg */
3218                                &txr->txtag))) {
3219                 device_printf(dev,"Unable to allocate TX DMA tag\n");
3220                 goto fail;
3221         }
3222
3223         if (!(txr->tx_buffers =
3224             (struct em_buffer *) malloc(sizeof(struct em_buffer) *
3225             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3226                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3227                 error = ENOMEM;
3228                 goto fail;
3229         }
3230
3231         /* Create the descriptor buffer dma maps */
3232         txbuf = txr->tx_buffers;
3233         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3234                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3235                 if (error != 0) {
3236                         device_printf(dev, "Unable to create TX DMA map\n");
3237                         goto fail;
3238                 }
3239         }
3240
3241         return 0;
3242 fail:
3243         /* We free all, it handles case where we are in the middle */
3244         em_free_transmit_structures(adapter);
3245         return (error);
3246 }
3247
3248 /*********************************************************************
3249  *
3250  *  Initialize a transmit ring.
3251  *
3252  **********************************************************************/
3253 static void
3254 em_setup_transmit_ring(struct tx_ring *txr)
3255 {
3256         struct adapter *adapter = txr->adapter;
3257         struct em_buffer *txbuf;
3258         int i;
3259
3260         /* Clear the old descriptor contents */
3261         EM_TX_LOCK(txr);
3262         bzero((void *)txr->tx_base,
3263               (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3264         /* Reset indices */
3265         txr->next_avail_desc = 0;
3266         txr->next_to_clean = 0;
3267
3268         /* Free any existing tx buffers. */
3269         txbuf = txr->tx_buffers;
3270         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3271                 if (txbuf->m_head != NULL) {
3272                         bus_dmamap_sync(txr->txtag, txbuf->map,
3273                             BUS_DMASYNC_POSTWRITE);
3274                         bus_dmamap_unload(txr->txtag, txbuf->map);
3275                         m_freem(txbuf->m_head);
3276                         txbuf->m_head = NULL;
3277                 }
3278                 /* clear the watch index */
3279                 txbuf->next_eop = -1;
3280         }
3281
3282         /* Set number of descriptors available */
3283         txr->tx_avail = adapter->num_tx_desc;
3284         txr->queue_status = EM_QUEUE_IDLE;
3285
3286         /* Clear checksum offload context. */
3287         txr->last_hw_offload = 0;
3288         txr->last_hw_ipcss = 0;
3289         txr->last_hw_ipcso = 0;
3290         txr->last_hw_tucss = 0;
3291         txr->last_hw_tucso = 0;
3292
3293         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3294             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3295         EM_TX_UNLOCK(txr);
3296 }
3297
3298 /*********************************************************************
3299  *
3300  *  Initialize all transmit rings.
3301  *
3302  **********************************************************************/
3303 static void
3304 em_setup_transmit_structures(struct adapter *adapter)
3305 {
3306         struct tx_ring *txr = adapter->tx_rings;
3307
3308         for (int i = 0; i < adapter->num_queues; i++, txr++)
3309                 em_setup_transmit_ring(txr);
3310
3311         return;
3312 }
3313
3314 /*********************************************************************
3315  *
3316  *  Enable transmit unit.
3317  *
3318  **********************************************************************/
3319 static void
3320 em_initialize_transmit_unit(struct adapter *adapter)
3321 {
3322         struct tx_ring  *txr = adapter->tx_rings;
3323         struct e1000_hw *hw = &adapter->hw;
3324         u32     tctl, tarc, tipg = 0;
3325
3326          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3327
3328         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3329                 u64 bus_addr = txr->txdma.dma_paddr;
3330                 /* Base and Len of TX Ring */
3331                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3332                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3333                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3334                     (u32)(bus_addr >> 32));
3335                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3336                     (u32)bus_addr);
3337                 /* Init the HEAD/TAIL indices */
3338                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3339                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3340
3341                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3342                     E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
3343                     E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
3344
3345                 txr->queue_status = EM_QUEUE_IDLE;
3346         }
3347
3348         /* Set the default values for the Tx Inter Packet Gap timer */
3349         switch (adapter->hw.mac.type) {
3350         case e1000_80003es2lan:
3351                 tipg = DEFAULT_82543_TIPG_IPGR1;
3352                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3353                     E1000_TIPG_IPGR2_SHIFT;
3354                 break;
3355         default:
3356                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3357                     (adapter->hw.phy.media_type ==
3358                     e1000_media_type_internal_serdes))
3359                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3360                 else
3361                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3362                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3363                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3364         }
3365
3366         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3367         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3368
3369         if(adapter->hw.mac.type >= e1000_82540)
3370                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3371                     adapter->tx_abs_int_delay.value);
3372
3373         if ((adapter->hw.mac.type == e1000_82571) ||
3374             (adapter->hw.mac.type == e1000_82572)) {
3375                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3376                 tarc |= SPEED_MODE_BIT;
3377                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3378         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3379                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3380                 tarc |= 1;
3381                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3382                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3383                 tarc |= 1;
3384                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3385         }
3386
3387         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3388         if (adapter->tx_int_delay.value > 0)
3389                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3390
3391         /* Program the Transmit Control Register */
3392         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3393         tctl &= ~E1000_TCTL_CT;
3394         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3395                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3396
3397         if (adapter->hw.mac.type >= e1000_82571)
3398                 tctl |= E1000_TCTL_MULR;
3399
3400         /* This write will effectively turn on the transmit unit. */
3401         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3402
3403 }
3404
3405
3406 /*********************************************************************
3407  *
3408  *  Free all transmit rings.
3409  *
3410  **********************************************************************/
3411 static void
3412 em_free_transmit_structures(struct adapter *adapter)
3413 {
3414         struct tx_ring *txr = adapter->tx_rings;
3415
3416         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3417                 EM_TX_LOCK(txr);
3418                 em_free_transmit_buffers(txr);
3419                 em_dma_free(adapter, &txr->txdma);
3420                 EM_TX_UNLOCK(txr);
3421                 EM_TX_LOCK_DESTROY(txr);
3422         }
3423
3424         free(adapter->tx_rings, M_DEVBUF);
3425 }
3426
3427 /*********************************************************************
3428  *
3429  *  Free transmit ring related data structures.
3430  *
3431  **********************************************************************/
3432 static void
3433 em_free_transmit_buffers(struct tx_ring *txr)
3434 {
3435         struct adapter          *adapter = txr->adapter;
3436         struct em_buffer        *txbuf;
3437
3438         INIT_DEBUGOUT("free_transmit_ring: begin");
3439
3440         if (txr->tx_buffers == NULL)
3441                 return;
3442
3443         for (int i = 0; i < adapter->num_tx_desc; i++) {
3444                 txbuf = &txr->tx_buffers[i];
3445                 if (txbuf->m_head != NULL) {
3446                         bus_dmamap_sync(txr->txtag, txbuf->map,
3447                             BUS_DMASYNC_POSTWRITE);
3448                         bus_dmamap_unload(txr->txtag,
3449                             txbuf->map);
3450                         m_freem(txbuf->m_head);
3451                         txbuf->m_head = NULL;
3452                         if (txbuf->map != NULL) {
3453                                 bus_dmamap_destroy(txr->txtag,
3454                                     txbuf->map);
3455                                 txbuf->map = NULL;
3456                         }
3457                 } else if (txbuf->map != NULL) {
3458                         bus_dmamap_unload(txr->txtag,
3459                             txbuf->map);
3460                         bus_dmamap_destroy(txr->txtag,
3461                             txbuf->map);
3462                         txbuf->map = NULL;
3463                 }
3464         }
3465 #if __FreeBSD_version >= 800000
3466         if (txr->br != NULL)
3467                 buf_ring_free(txr->br, M_DEVBUF);
3468 #endif
3469         if (txr->tx_buffers != NULL) {
3470                 free(txr->tx_buffers, M_DEVBUF);
3471                 txr->tx_buffers = NULL;
3472         }
3473         if (txr->txtag != NULL) {
3474                 bus_dma_tag_destroy(txr->txtag);
3475                 txr->txtag = NULL;
3476         }
3477         return;
3478 }
3479
3480
3481 /*********************************************************************
3482  *  The offload context is protocol specific (TCP/UDP) and thus
3483  *  only needs to be set when the protocol changes. The occasion
3484  *  of a context change can be a performance detriment, and
3485  *  might be better just disabled. The reason arises in the way
3486  *  in which the controller supports pipelined requests from the
3487  *  Tx data DMA. Up to four requests can be pipelined, and they may
3488  *  belong to the same packet or to multiple packets. However all
3489  *  requests for one packet are issued before a request is issued
3490  *  for a subsequent packet and if a request for the next packet
3491  *  requires a context change, that request will be stalled
3492  *  until the previous request completes. This means setting up
3493  *  a new context effectively disables pipelined Tx data DMA which
3494  *  in turn greatly slow down performance to send small sized
3495  *  frames.
3496  **********************************************************************/
3497 static void
3498 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3499     struct ip *ip, u32 *txd_upper, u32 *txd_lower)
3500 {
3501         struct adapter                  *adapter = txr->adapter;
3502         struct e1000_context_desc       *TXD = NULL;
3503         struct em_buffer                *tx_buffer;
3504         int                             cur, hdr_len;
3505         u32                             cmd = 0;
3506         u16                             offload = 0;
3507         u8                              ipcso, ipcss, tucso, tucss;
3508
3509         ipcss = ipcso = tucss = tucso = 0;
3510         hdr_len = ip_off + (ip->ip_hl << 2);
3511         cur = txr->next_avail_desc;
3512
3513         /* Setup of IP header checksum. */
3514         if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3515                 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3516                 offload |= CSUM_IP;
3517                 ipcss = ip_off;
3518                 ipcso = ip_off + offsetof(struct ip, ip_sum);
3519                 /*
3520                  * Start offset for header checksum calculation.
3521                  * End offset for header checksum calculation.
3522                  * Offset of place to put the checksum.
3523                  */
3524                 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3525                 TXD->lower_setup.ip_fields.ipcss = ipcss;
3526                 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
3527                 TXD->lower_setup.ip_fields.ipcso = ipcso;
3528                 cmd |= E1000_TXD_CMD_IP;
3529         }
3530
3531         if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3532                 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3533                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3534                 offload |= CSUM_TCP;
3535                 tucss = hdr_len;
3536                 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
3537                 /*
3538                  * Setting up new checksum offload context for every frames
3539                  * takes a lot of processing time for hardware. This also
3540                  * reduces performance a lot for small sized frames so avoid
3541                  * it if driver can use previously configured checksum
3542                  * offload context.
3543                  */
3544                 if (txr->last_hw_offload == offload) {
3545                         if (offload & CSUM_IP) {
3546                                 if (txr->last_hw_ipcss == ipcss &&
3547                                     txr->last_hw_ipcso == ipcso &&
3548                                     txr->last_hw_tucss == tucss &&
3549                                     txr->last_hw_tucso == tucso)
3550                                         return;
3551                         } else {
3552                                 if (txr->last_hw_tucss == tucss &&
3553                                     txr->last_hw_tucso == tucso)
3554                                         return;
3555                         }
3556                 }
3557                 txr->last_hw_offload = offload;
3558                 txr->last_hw_tucss = tucss;
3559                 txr->last_hw_tucso = tucso;
3560                 /*
3561                  * Start offset for payload checksum calculation.
3562                  * End offset for payload checksum calculation.
3563                  * Offset of place to put the checksum.
3564                  */
3565                 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3566                 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3567                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3568                 TXD->upper_setup.tcp_fields.tucso = tucso;
3569                 cmd |= E1000_TXD_CMD_TCP;
3570         } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3571                 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3572                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3573                 tucss = hdr_len;
3574                 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
3575                 /*
3576                  * Setting up new checksum offload context for every frames
3577                  * takes a lot of processing time for hardware. This also
3578                  * reduces performance a lot for small sized frames so avoid
3579                  * it if driver can use previously configured checksum
3580                  * offload context.
3581                  */
3582                 if (txr->last_hw_offload == offload) {
3583                         if (offload & CSUM_IP) {
3584                                 if (txr->last_hw_ipcss == ipcss &&
3585                                     txr->last_hw_ipcso == ipcso &&
3586                                     txr->last_hw_tucss == tucss &&
3587                                     txr->last_hw_tucso == tucso)
3588                                         return;
3589                         } else {
3590                                 if (txr->last_hw_tucss == tucss &&
3591                                     txr->last_hw_tucso == tucso)
3592                                         return;
3593                         }
3594                 }
3595                 txr->last_hw_offload = offload;
3596                 txr->last_hw_tucss = tucss;
3597                 txr->last_hw_tucso = tucso;
3598                 /*
3599                  * Start offset for header checksum calculation.
3600                  * End offset for header checksum calculation.
3601                  * Offset of place to put the checksum.
3602                  */
3603                 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3604                 TXD->upper_setup.tcp_fields.tucss = tucss;
3605                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3606                 TXD->upper_setup.tcp_fields.tucso = tucso;
3607         }
3608
3609         if (offload & CSUM_IP) {
3610                 txr->last_hw_ipcss = ipcss;
3611                 txr->last_hw_ipcso = ipcso;
3612         }
3613
3614         TXD->tcp_seg_setup.data = htole32(0);
3615         TXD->cmd_and_length =
3616             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3617         tx_buffer = &txr->tx_buffers[cur];
3618         tx_buffer->m_head = NULL;
3619         tx_buffer->next_eop = -1;
3620
3621         if (++cur == adapter->num_tx_desc)
3622                 cur = 0;
3623
3624         txr->tx_avail--;
3625         txr->next_avail_desc = cur;
3626 }
3627
3628
3629 /**********************************************************************
3630  *
3631  *  Setup work for hardware segmentation offload (TSO)
3632  *
3633  **********************************************************************/
3634 static void
3635 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3636     struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower)
3637 {
3638         struct adapter                  *adapter = txr->adapter;
3639         struct e1000_context_desc       *TXD;
3640         struct em_buffer                *tx_buffer;
3641         int cur, hdr_len;
3642
3643         /*
3644          * In theory we can use the same TSO context if and only if
3645          * frame is the same type(IP/TCP) and the same MSS. However
3646          * checking whether a frame has the same IP/TCP structure is
3647          * hard thing so just ignore that and always restablish a
3648          * new TSO context.
3649          */
3650         hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2);
3651         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3652                       E1000_TXD_DTYP_D |        /* Data descr type */
3653                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3654
3655         /* IP and/or TCP header checksum calculation and insertion. */
3656         *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
3657
3658         cur = txr->next_avail_desc;
3659         tx_buffer = &txr->tx_buffers[cur];
3660         TXD = (struct e1000_context_desc *) &txr->tx_base[cur];
3661
3662         /*
3663          * Start offset for header checksum calculation.
3664          * End offset for header checksum calculation.
3665          * Offset of place put the checksum.
3666          */
3667         TXD->lower_setup.ip_fields.ipcss = ip_off;
3668         TXD->lower_setup.ip_fields.ipcse =
3669             htole16(ip_off + (ip->ip_hl << 2) - 1);
3670         TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum);
3671         /*
3672          * Start offset for payload checksum calculation.
3673          * End offset for payload checksum calculation.
3674          * Offset of place to put the checksum.
3675          */
3676         TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2);
3677         TXD->upper_setup.tcp_fields.tucse = 0;
3678         TXD->upper_setup.tcp_fields.tucso =
3679             ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum);
3680         /*
3681          * Payload size per packet w/o any headers.
3682          * Length of all headers up to payload.
3683          */
3684         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3685         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3686
3687         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3688                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3689                                 E1000_TXD_CMD_TSE |     /* TSE context */
3690                                 E1000_TXD_CMD_IP |      /* Do IP csum */
3691                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3692                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3693
3694         tx_buffer->m_head = NULL;
3695         tx_buffer->next_eop = -1;
3696
3697         if (++cur == adapter->num_tx_desc)
3698                 cur = 0;
3699
3700         txr->tx_avail--;
3701         txr->next_avail_desc = cur;
3702         txr->tx_tso = TRUE;
3703 }
3704
3705
3706 /**********************************************************************
3707  *
3708  *  Examine each tx_buffer in the used queue. If the hardware is done
3709  *  processing the packet then free associated resources. The
3710  *  tx_buffer is put back on the free queue.
3711  *
3712  **********************************************************************/
3713 static bool
3714 em_txeof(struct tx_ring *txr)
3715 {
3716         struct adapter  *adapter = txr->adapter;
3717         int first, last, done, processed;
3718         struct em_buffer *tx_buffer;
3719         struct e1000_tx_desc   *tx_desc, *eop_desc;
3720         struct ifnet   *ifp = adapter->ifp;
3721
3722         EM_TX_LOCK_ASSERT(txr);
3723
3724         /* No work, make sure watchdog is off */
3725         if (txr->tx_avail == adapter->num_tx_desc) {
3726                 txr->queue_status = EM_QUEUE_IDLE;
3727                 return (FALSE);
3728         }
3729
3730         processed = 0;
3731         first = txr->next_to_clean;
3732         tx_desc = &txr->tx_base[first];
3733         tx_buffer = &txr->tx_buffers[first];
3734         last = tx_buffer->next_eop;
3735         eop_desc = &txr->tx_base[last];
3736
3737         /*
3738          * What this does is get the index of the
3739          * first descriptor AFTER the EOP of the
3740          * first packet, that way we can do the
3741          * simple comparison on the inner while loop.
3742          */
3743         if (++last == adapter->num_tx_desc)
3744                 last = 0;
3745         done = last;
3746
3747         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3748             BUS_DMASYNC_POSTREAD);
3749
3750         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3751                 /* We clean the range of the packet */
3752                 while (first != done) {
3753                         tx_desc->upper.data = 0;
3754                         tx_desc->lower.data = 0;
3755                         tx_desc->buffer_addr = 0;
3756                         ++txr->tx_avail;
3757                         ++processed;
3758
3759                         if (tx_buffer->m_head) {
3760                                 bus_dmamap_sync(txr->txtag,
3761                                     tx_buffer->map,
3762                                     BUS_DMASYNC_POSTWRITE);
3763                                 bus_dmamap_unload(txr->txtag,
3764                                     tx_buffer->map);
3765                                 m_freem(tx_buffer->m_head);
3766                                 tx_buffer->m_head = NULL;
3767                         }
3768                         tx_buffer->next_eop = -1;
3769                         txr->watchdog_time = ticks;
3770
3771                         if (++first == adapter->num_tx_desc)
3772                                 first = 0;
3773
3774                         tx_buffer = &txr->tx_buffers[first];
3775                         tx_desc = &txr->tx_base[first];
3776                 }
3777                 ++ifp->if_opackets;
3778                 /* See if we can continue to the next packet */
3779                 last = tx_buffer->next_eop;
3780                 if (last != -1) {
3781                         eop_desc = &txr->tx_base[last];
3782                         /* Get new done point */
3783                         if (++last == adapter->num_tx_desc) last = 0;
3784                         done = last;
3785                 } else
3786                         break;
3787         }
3788         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3789             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3790
3791         txr->next_to_clean = first;
3792
3793         /*
3794         ** Watchdog calculation, we know there's
3795         ** work outstanding or the first return
3796         ** would have been taken, so none processed
3797         ** for too long indicates a hang. local timer
3798         ** will examine this and do a reset if needed.
3799         */
3800         if ((!processed) && ((ticks - txr->watchdog_time) > EM_WATCHDOG))
3801                 txr->queue_status = EM_QUEUE_HUNG;
3802
3803         /*
3804          * If we have a minimum free, clear IFF_DRV_OACTIVE
3805          * to tell the stack that it is OK to send packets.
3806          * Notice that all writes of OACTIVE happen under the
3807          * TX lock which, with a single queue, guarantees
3808          * sanity.
3809          */
3810         if (txr->tx_avail >= EM_MAX_SCATTER)
3811                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3812
3813         /* Disable watchdog if all clean */
3814         if (txr->tx_avail == adapter->num_tx_desc) {
3815                 txr->queue_status = EM_QUEUE_IDLE;
3816                 return (FALSE);
3817         }
3818
3819         return (TRUE);
3820 }
3821
3822
3823 /*********************************************************************
3824  *
3825  *  Refresh RX descriptor mbufs from system mbuf buffer pool.
3826  *
3827  **********************************************************************/
3828 static void
3829 em_refresh_mbufs(struct rx_ring *rxr, int limit)
3830 {
3831         struct adapter          *adapter = rxr->adapter;
3832         struct mbuf             *m;
3833         bus_dma_segment_t       segs[1];
3834         struct em_buffer        *rxbuf;
3835         int                     i, j, error, nsegs;
3836         bool                    cleaned = FALSE;
3837
3838         i = j = rxr->next_to_refresh;
3839         /*
3840         ** Get one descriptor beyond
3841         ** our work mark to control
3842         ** the loop.
3843         */
3844         if (++j == adapter->num_rx_desc)
3845                 j = 0;
3846
3847         while (j != limit) {
3848                 rxbuf = &rxr->rx_buffers[i];
3849                 if (rxbuf->m_head == NULL) {
3850                         m = m_getjcl(M_DONTWAIT, MT_DATA,
3851                             M_PKTHDR, adapter->rx_mbuf_sz);
3852                         /*
3853                         ** If we have a temporary resource shortage
3854                         ** that causes a failure, just abort refresh
3855                         ** for now, we will return to this point when
3856                         ** reinvoked from em_rxeof.
3857                         */
3858                         if (m == NULL)
3859                                 goto update;
3860                 } else
3861                         m = rxbuf->m_head;
3862
3863                 m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz;
3864                 m->m_flags |= M_PKTHDR;
3865                 m->m_data = m->m_ext.ext_buf;
3866
3867                 /* Use bus_dma machinery to setup the memory mapping  */
3868                 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map,
3869                     m, segs, &nsegs, BUS_DMA_NOWAIT);
3870                 if (error != 0) {
3871                         printf("Refresh mbufs: hdr dmamap load"
3872                             " failure - %d\n", error);
3873                         m_free(m);
3874                         rxbuf->m_head = NULL;
3875                         goto update;
3876                 }
3877                 rxbuf->m_head = m;
3878                 bus_dmamap_sync(rxr->rxtag,
3879                     rxbuf->map, BUS_DMASYNC_PREREAD);
3880                 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr);
3881                 cleaned = TRUE;
3882
3883                 i = j; /* Next is precalulated for us */
3884                 rxr->next_to_refresh = i;
3885                 /* Calculate next controlling index */
3886                 if (++j == adapter->num_rx_desc)
3887                         j = 0;
3888         }
3889 update:
3890         /*
3891         ** Update the tail pointer only if,
3892         ** and as far as we have refreshed.
3893         */
3894         if (cleaned)
3895                 E1000_WRITE_REG(&adapter->hw,
3896                     E1000_RDT(rxr->me), rxr->next_to_refresh);
3897
3898         return;
3899 }
3900
3901
3902 /*********************************************************************
3903  *
3904  *  Allocate memory for rx_buffer structures. Since we use one
3905  *  rx_buffer per received packet, the maximum number of rx_buffer's
3906  *  that we'll need is equal to the number of receive descriptors
3907  *  that we've allocated.
3908  *
3909  **********************************************************************/
3910 static int
3911 em_allocate_receive_buffers(struct rx_ring *rxr)
3912 {
3913         struct adapter          *adapter = rxr->adapter;
3914         device_t                dev = adapter->dev;
3915         struct em_buffer        *rxbuf;
3916         int                     error;
3917
3918         rxr->rx_buffers = malloc(sizeof(struct em_buffer) *
3919             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3920         if (rxr->rx_buffers == NULL) {
3921                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3922                 return (ENOMEM);
3923         }
3924
3925         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3926                                 1, 0,                   /* alignment, bounds */
3927                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3928                                 BUS_SPACE_MAXADDR,      /* highaddr */
3929                                 NULL, NULL,             /* filter, filterarg */
3930                                 MJUM9BYTES,             /* maxsize */
3931                                 1,                      /* nsegments */
3932                                 MJUM9BYTES,             /* maxsegsize */
3933                                 0,                      /* flags */
3934                                 NULL,                   /* lockfunc */
3935                                 NULL,                   /* lockarg */
3936                                 &rxr->rxtag);
3937         if (error) {
3938                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
3939                     __func__, error);
3940                 goto fail;
3941         }
3942
3943         rxbuf = rxr->rx_buffers;
3944         for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3945                 rxbuf = &rxr->rx_buffers[i];
3946                 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
3947                     &rxbuf->map);
3948                 if (error) {
3949                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3950                             __func__, error);
3951                         goto fail;
3952                 }
3953         }
3954
3955         return (0);
3956
3957 fail:
3958         em_free_receive_structures(adapter);
3959         return (error);
3960 }
3961
3962
3963 /*********************************************************************
3964  *
3965  *  Initialize a receive ring and its buffers.
3966  *
3967  **********************************************************************/
3968 static int
3969 em_setup_receive_ring(struct rx_ring *rxr)
3970 {
3971         struct  adapter         *adapter = rxr->adapter;
3972         struct em_buffer        *rxbuf;
3973         bus_dma_segment_t       seg[1];
3974         int                     rsize, nsegs, error;
3975
3976
3977         /* Clear the ring contents */
3978         EM_RX_LOCK(rxr);
3979         rsize = roundup2(adapter->num_rx_desc *
3980             sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
3981         bzero((void *)rxr->rx_base, rsize);
3982
3983         /*
3984         ** Free current RX buffer structs and their mbufs
3985         */
3986         for (int i = 0; i < adapter->num_rx_desc; i++) {
3987                 rxbuf = &rxr->rx_buffers[i];
3988                 if (rxbuf->m_head != NULL) {
3989                         bus_dmamap_sync(rxr->rxtag, rxbuf->map,
3990                             BUS_DMASYNC_POSTREAD);
3991                         bus_dmamap_unload(rxr->rxtag, rxbuf->map);
3992                         m_freem(rxbuf->m_head);
3993                         rxbuf->m_head = NULL; /* mark as freed */
3994                 }
3995         }
3996
3997         /* Now replenish the mbufs */
3998         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3999                 rxbuf = &rxr->rx_buffers[j];
4000                 rxbuf->m_head = m_getjcl(M_DONTWAIT, MT_DATA,
4001                     M_PKTHDR, adapter->rx_mbuf_sz);
4002                 if (rxbuf->m_head == NULL) {
4003                         error = ENOBUFS;
4004                         goto fail;
4005                 }
4006                 rxbuf->m_head->m_len = adapter->rx_mbuf_sz;
4007                 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */
4008                 rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz;
4009
4010                 /* Get the memory mapping */
4011                 error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
4012                     rxbuf->map, rxbuf->m_head, seg,
4013                     &nsegs, BUS_DMA_NOWAIT);
4014                 if (error != 0) {
4015                         m_freem(rxbuf->m_head);
4016                         rxbuf->m_head = NULL;
4017                         goto fail;
4018                 }
4019                 bus_dmamap_sync(rxr->rxtag,
4020                     rxbuf->map, BUS_DMASYNC_PREREAD);
4021
4022                 /* Update descriptor */
4023                 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr);
4024         }
4025         rxr->next_to_check = 0;
4026         rxr->next_to_refresh = 0;
4027         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4028             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4029
4030 fail:
4031         EM_RX_UNLOCK(rxr);
4032         return (error);
4033 }
4034
4035 /*********************************************************************
4036  *
4037  *  Initialize all receive rings.
4038  *
4039  **********************************************************************/
4040 static int
4041 em_setup_receive_structures(struct adapter *adapter)
4042 {
4043         struct rx_ring *rxr = adapter->rx_rings;
4044         int q;
4045
4046         for (q = 0; q < adapter->num_queues; q++, rxr++)
4047                 if (em_setup_receive_ring(rxr))
4048                         goto fail;
4049
4050         return (0);
4051 fail:
4052         /*
4053          * Free RX buffers allocated so far, we will only handle
4054          * the rings that completed, the failing case will have
4055          * cleaned up for itself. 'q' failed, so its the terminus.
4056          */
4057         for (int i = 0; i < q; ++i) {
4058                 rxr = &adapter->rx_rings[i];
4059                 for (int n = 0; n < adapter->num_rx_desc; n++) {
4060                         struct em_buffer *rxbuf;
4061                         rxbuf = &rxr->rx_buffers[n];
4062                         if (rxbuf->m_head != NULL) {
4063                                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4064                                   BUS_DMASYNC_POSTREAD);
4065                                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4066                                 m_freem(rxbuf->m_head);
4067                                 rxbuf->m_head = NULL;
4068                         }
4069                 }
4070                 rxr->next_to_check = 0;
4071                 rxr->next_to_refresh = 0;
4072         }
4073
4074         return (ENOBUFS);
4075 }
4076
4077 /*********************************************************************
4078  *
4079  *  Free all receive rings.
4080  *
4081  **********************************************************************/
4082 static void
4083 em_free_receive_structures(struct adapter *adapter)
4084 {
4085         struct rx_ring *rxr = adapter->rx_rings;
4086
4087         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4088                 em_free_receive_buffers(rxr);
4089                 /* Free the ring memory as well */
4090                 em_dma_free(adapter, &rxr->rxdma);
4091                 EM_RX_LOCK_DESTROY(rxr);
4092         }
4093
4094         free(adapter->rx_rings, M_DEVBUF);
4095 }
4096
4097
4098 /*********************************************************************
4099  *
4100  *  Free receive ring data structures
4101  *
4102  **********************************************************************/
4103 static void
4104 em_free_receive_buffers(struct rx_ring *rxr)
4105 {
4106         struct adapter          *adapter = rxr->adapter;
4107         struct em_buffer        *rxbuf = NULL;
4108
4109         INIT_DEBUGOUT("free_receive_buffers: begin");
4110
4111         if (rxr->rx_buffers != NULL) {
4112                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4113                         rxbuf = &rxr->rx_buffers[i];
4114                         if (rxbuf->map != NULL) {
4115                                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4116                                     BUS_DMASYNC_POSTREAD);
4117                                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4118                                 bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
4119                         }
4120                         if (rxbuf->m_head != NULL) {
4121                                 m_freem(rxbuf->m_head);
4122                                 rxbuf->m_head = NULL;
4123                         }
4124                 }
4125                 free(rxr->rx_buffers, M_DEVBUF);
4126                 rxr->rx_buffers = NULL;
4127                 rxr->next_to_check = 0;
4128                 rxr->next_to_refresh = 0;
4129         }
4130
4131         if (rxr->rxtag != NULL) {
4132                 bus_dma_tag_destroy(rxr->rxtag);
4133                 rxr->rxtag = NULL;
4134         }
4135
4136         return;
4137 }
4138
4139
4140 /*********************************************************************
4141  *
4142  *  Enable receive unit.
4143  *
4144  **********************************************************************/
4145 #define MAX_INTS_PER_SEC        8000
4146 #define DEFAULT_ITR          1000000000/(MAX_INTS_PER_SEC * 256)
4147
4148 static void
4149 em_initialize_receive_unit(struct adapter *adapter)
4150 {
4151         struct rx_ring  *rxr = adapter->rx_rings;
4152         struct ifnet    *ifp = adapter->ifp;
4153         struct e1000_hw *hw = &adapter->hw;
4154         u64     bus_addr;
4155         u32     rctl, rxcsum;
4156
4157         INIT_DEBUGOUT("em_initialize_receive_units: begin");
4158
4159         /*
4160          * Make sure receives are disabled while setting
4161          * up the descriptor ring
4162          */
4163         rctl = E1000_READ_REG(hw, E1000_RCTL);
4164         /* Do not disable if ever enabled on this hardware */
4165         if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583))
4166                 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4167
4168         E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4169             adapter->rx_abs_int_delay.value);
4170         /*
4171          * Set the interrupt throttling rate. Value is calculated
4172          * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4173          */
4174         E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
4175
4176         /*
4177         ** When using MSIX interrupts we need to throttle
4178         ** using the EITR register (82574 only)
4179         */
4180         if (hw->mac.type == e1000_82574) {
4181                 for (int i = 0; i < 4; i++)
4182                         E1000_WRITE_REG(hw, E1000_EITR_82574(i),
4183                             DEFAULT_ITR);
4184                 /* Disable accelerated acknowledge */
4185                 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4186         }
4187
4188         if (ifp->if_capenable & IFCAP_RXCSUM) {
4189                 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4190                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4191                 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4192         }
4193
4194         /*
4195         ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4196         ** long latencies are observed, like Lenovo X60. This
4197         ** change eliminates the problem, but since having positive
4198         ** values in RDTR is a known source of problems on other
4199         ** platforms another solution is being sought.
4200         */
4201         if (hw->mac.type == e1000_82573)
4202                 E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
4203
4204         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4205                 /* Setup the Base and Length of the Rx Descriptor Ring */
4206                 bus_addr = rxr->rxdma.dma_paddr;
4207                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4208                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4209                 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
4210                 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
4211                 /* Setup the Head and Tail Descriptor Pointers */
4212                 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4213                 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1);
4214         }
4215
4216         /* Set PTHRESH for improved jumbo performance */
4217         if (((adapter->hw.mac.type == e1000_ich9lan) ||
4218             (adapter->hw.mac.type == e1000_pch2lan) ||
4219             (adapter->hw.mac.type == e1000_ich10lan)) &&
4220             (ifp->if_mtu > ETHERMTU)) {
4221                 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
4222                 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3);
4223         }
4224
4225         if (adapter->hw.mac.type == e1000_pch2lan) {
4226                 if (ifp->if_mtu > ETHERMTU)
4227                         e1000_lv_jumbo_workaround_ich8lan(hw, TRUE);
4228                 else
4229                         e1000_lv_jumbo_workaround_ich8lan(hw, FALSE);
4230         }
4231
4232         /* Setup the Receive Control Register */
4233         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4234         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
4235             E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
4236             (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4237
4238         /* Strip the CRC */
4239         rctl |= E1000_RCTL_SECRC;
4240
4241         /* Make sure VLAN Filters are off */
4242         rctl &= ~E1000_RCTL_VFE;
4243         rctl &= ~E1000_RCTL_SBP;
4244
4245         if (adapter->rx_mbuf_sz == MCLBYTES)
4246                 rctl |= E1000_RCTL_SZ_2048;
4247         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
4248                 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4249         else if (adapter->rx_mbuf_sz > MJUMPAGESIZE)
4250                 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4251
4252         if (ifp->if_mtu > ETHERMTU)
4253                 rctl |= E1000_RCTL_LPE;
4254         else
4255                 rctl &= ~E1000_RCTL_LPE;
4256
4257         /* Write out the settings */
4258         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4259
4260         return;
4261 }
4262
4263
4264 /*********************************************************************
4265  *
4266  *  This routine executes in interrupt context. It replenishes
4267  *  the mbufs in the descriptor and sends data which has been
4268  *  dma'ed into host memory to upper layer.
4269  *
4270  *  We loop at most count times if count is > 0, or until done if
4271  *  count < 0.
4272  *
4273  *  For polling we also now return the number of cleaned packets
4274  *********************************************************************/
4275 static bool
4276 em_rxeof(struct rx_ring *rxr, int count, int *done)
4277 {
4278         struct adapter          *adapter = rxr->adapter;
4279         struct ifnet            *ifp = adapter->ifp;
4280         struct mbuf             *mp, *sendmp;
4281         u8                      status = 0;
4282         u16                     len;
4283         int                     i, processed, rxdone = 0;
4284         bool                    eop;
4285         struct e1000_rx_desc    *cur;
4286
4287         EM_RX_LOCK(rxr);
4288
4289         for (i = rxr->next_to_check, processed = 0; count != 0;) {
4290
4291                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4292                         break;
4293
4294                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4295                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4296
4297                 cur = &rxr->rx_base[i];
4298                 status = cur->status;
4299                 mp = sendmp = NULL;
4300
4301                 if ((status & E1000_RXD_STAT_DD) == 0)
4302                         break;
4303
4304                 len = le16toh(cur->length);
4305                 eop = (status & E1000_RXD_STAT_EOP) != 0;
4306
4307                 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) ||
4308                     (rxr->discard == TRUE)) {
4309                         ifp->if_ierrors++;
4310                         ++rxr->rx_discarded;
4311                         if (!eop) /* Catch subsequent segs */
4312                                 rxr->discard = TRUE;
4313                         else
4314                                 rxr->discard = FALSE;
4315                         em_rx_discard(rxr, i);
4316                         goto next_desc;
4317                 }
4318
4319                 /* Assign correct length to the current fragment */
4320                 mp = rxr->rx_buffers[i].m_head;
4321                 mp->m_len = len;
4322
4323                 /* Trigger for refresh */
4324                 rxr->rx_buffers[i].m_head = NULL;
4325
4326                 /* First segment? */
4327                 if (rxr->fmp == NULL) {
4328                         mp->m_pkthdr.len = len;
4329                         rxr->fmp = rxr->lmp = mp;
4330                 } else {
4331                         /* Chain mbuf's together */
4332                         mp->m_flags &= ~M_PKTHDR;
4333                         rxr->lmp->m_next = mp;
4334                         rxr->lmp = mp;
4335                         rxr->fmp->m_pkthdr.len += len;
4336                 }
4337
4338                 if (eop) {
4339                         --count;
4340                         sendmp = rxr->fmp;
4341                         sendmp->m_pkthdr.rcvif = ifp;
4342                         ifp->if_ipackets++;
4343                         em_receive_checksum(cur, sendmp);
4344 #ifndef __NO_STRICT_ALIGNMENT
4345                         if (adapter->max_frame_size >
4346                             (MCLBYTES - ETHER_ALIGN) &&
4347                             em_fixup_rx(rxr) != 0)
4348                                 goto skip;
4349 #endif
4350                         if (status & E1000_RXD_STAT_VP) {
4351                                 sendmp->m_pkthdr.ether_vtag =
4352                                     le16toh(cur->special);
4353                                 sendmp->m_flags |= M_VLANTAG;
4354                         }
4355 #ifndef __NO_STRICT_ALIGNMENT
4356 skip:
4357 #endif
4358                         rxr->fmp = rxr->lmp = NULL;
4359                 }
4360 next_desc:
4361                 /* Zero out the receive descriptors status. */
4362                 cur->status = 0;
4363                 ++rxdone;       /* cumulative for POLL */
4364                 ++processed;
4365
4366                 /* Advance our pointers to the next descriptor. */
4367                 if (++i == adapter->num_rx_desc)
4368                         i = 0;
4369
4370                 /* Send to the stack */
4371                 if (sendmp != NULL) {
4372                         rxr->next_to_check = i;
4373                         EM_RX_UNLOCK(rxr);
4374                         (*ifp->if_input)(ifp, sendmp);
4375                         EM_RX_LOCK(rxr);
4376                         i = rxr->next_to_check;
4377                 }
4378
4379                 /* Only refresh mbufs every 8 descriptors */
4380                 if (processed == 8) {
4381                         em_refresh_mbufs(rxr, i);
4382                         processed = 0;
4383                 }
4384         }
4385
4386         /* Catch any remaining refresh work */
4387         if (e1000_rx_unrefreshed(rxr))
4388                 em_refresh_mbufs(rxr, i);
4389
4390         rxr->next_to_check = i;
4391         if (done != NULL)
4392                 *done = rxdone;
4393         EM_RX_UNLOCK(rxr);
4394
4395         return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4396 }
4397
4398 static __inline void
4399 em_rx_discard(struct rx_ring *rxr, int i)
4400 {
4401         struct em_buffer        *rbuf;
4402
4403         rbuf = &rxr->rx_buffers[i];
4404         /* Free any previous pieces */
4405         if (rxr->fmp != NULL) {
4406                 rxr->fmp->m_flags |= M_PKTHDR;
4407                 m_freem(rxr->fmp);
4408                 rxr->fmp = NULL;
4409                 rxr->lmp = NULL;
4410         }
4411         /*
4412         ** Free buffer and allow em_refresh_mbufs()
4413         ** to clean up and recharge buffer.
4414         */
4415         if (rbuf->m_head) {
4416                 m_free(rbuf->m_head);
4417                 rbuf->m_head = NULL;
4418         }
4419         return;
4420 }
4421
4422 #ifndef __NO_STRICT_ALIGNMENT
4423 /*
4424  * When jumbo frames are enabled we should realign entire payload on
4425  * architecures with strict alignment. This is serious design mistake of 8254x
4426  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4427  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4428  * payload. On architecures without strict alignment restrictions 8254x still
4429  * performs unaligned memory access which would reduce the performance too.
4430  * To avoid copying over an entire frame to align, we allocate a new mbuf and
4431  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4432  * existing mbuf chain.
4433  *
4434  * Be aware, best performance of the 8254x is achived only when jumbo frame is
4435  * not used at all on architectures with strict alignment.
4436  */
4437 static int
4438 em_fixup_rx(struct rx_ring *rxr)
4439 {
4440         struct adapter *adapter = rxr->adapter;
4441         struct mbuf *m, *n;
4442         int error;
4443
4444         error = 0;
4445         m = rxr->fmp;
4446         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4447                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4448                 m->m_data += ETHER_HDR_LEN;
4449         } else {
4450                 MGETHDR(n, M_DONTWAIT, MT_DATA);
4451                 if (n != NULL) {
4452                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4453                         m->m_data += ETHER_HDR_LEN;
4454                         m->m_len -= ETHER_HDR_LEN;
4455                         n->m_len = ETHER_HDR_LEN;
4456                         M_MOVE_PKTHDR(n, m);
4457                         n->m_next = m;
4458                         rxr->fmp = n;
4459                 } else {
4460                         adapter->dropped_pkts++;
4461                         m_freem(rxr->fmp);
4462                         rxr->fmp = NULL;
4463                         error = ENOMEM;
4464                 }
4465         }
4466
4467         return (error);
4468 }
4469 #endif
4470
4471 /*********************************************************************
4472  *
4473  *  Verify that the hardware indicated that the checksum is valid.
4474  *  Inform the stack about the status of checksum so that stack
4475  *  doesn't spend time verifying the checksum.
4476  *
4477  *********************************************************************/
4478 static void
4479 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4480 {
4481         /* Ignore Checksum bit is set */
4482         if (rx_desc->status & E1000_RXD_STAT_IXSM) {
4483                 mp->m_pkthdr.csum_flags = 0;
4484                 return;
4485         }
4486
4487         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4488                 /* Did it pass? */
4489                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4490                         /* IP Checksum Good */
4491                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4492                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4493
4494                 } else {
4495                         mp->m_pkthdr.csum_flags = 0;
4496                 }
4497         }
4498
4499         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4500                 /* Did it pass? */
4501                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4502                         mp->m_pkthdr.csum_flags |=
4503                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4504                         mp->m_pkthdr.csum_data = htons(0xffff);
4505                 }
4506         }
4507 }
4508
4509 /*
4510  * This routine is run via an vlan
4511  * config EVENT
4512  */
4513 static void
4514 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4515 {
4516         struct adapter  *adapter = ifp->if_softc;
4517         u32             index, bit;
4518
4519         if (ifp->if_softc !=  arg)   /* Not our event */
4520                 return;
4521
4522         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
4523                 return;
4524
4525         EM_CORE_LOCK(adapter);
4526         index = (vtag >> 5) & 0x7F;
4527         bit = vtag & 0x1F;
4528         adapter->shadow_vfta[index] |= (1 << bit);
4529         ++adapter->num_vlans;
4530         /* Re-init to load the changes */
4531         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4532                 em_init_locked(adapter);
4533         EM_CORE_UNLOCK(adapter);
4534 }
4535
4536 /*
4537  * This routine is run via an vlan
4538  * unconfig EVENT
4539  */
4540 static void
4541 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4542 {
4543         struct adapter  *adapter = ifp->if_softc;
4544         u32             index, bit;
4545
4546         if (ifp->if_softc !=  arg)
4547                 return;
4548
4549         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4550                 return;
4551
4552         EM_CORE_LOCK(adapter);
4553         index = (vtag >> 5) & 0x7F;
4554         bit = vtag & 0x1F;
4555         adapter->shadow_vfta[index] &= ~(1 << bit);
4556         --adapter->num_vlans;
4557         /* Re-init to load the changes */
4558         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4559                 em_init_locked(adapter);
4560         EM_CORE_UNLOCK(adapter);
4561 }
4562
4563 static void
4564 em_setup_vlan_hw_support(struct adapter *adapter)
4565 {
4566         struct e1000_hw *hw = &adapter->hw;
4567         u32             reg;
4568
4569         /*
4570         ** We get here thru init_locked, meaning
4571         ** a soft reset, this has already cleared
4572         ** the VFTA and other state, so if there
4573         ** have been no vlan's registered do nothing.
4574         */
4575         if (adapter->num_vlans == 0)
4576                 return;
4577
4578         /*
4579         ** A soft reset zero's out the VFTA, so
4580         ** we need to repopulate it now.
4581         */
4582         for (int i = 0; i < EM_VFTA_SIZE; i++)
4583                 if (adapter->shadow_vfta[i] != 0)
4584                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4585                             i, adapter->shadow_vfta[i]);
4586
4587         reg = E1000_READ_REG(hw, E1000_CTRL);
4588         reg |= E1000_CTRL_VME;
4589         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4590
4591         /* Enable the Filter Table */
4592         reg = E1000_READ_REG(hw, E1000_RCTL);
4593         reg &= ~E1000_RCTL_CFIEN;
4594         reg |= E1000_RCTL_VFE;
4595         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4596 }
4597
4598 static void
4599 em_enable_intr(struct adapter *adapter)
4600 {
4601         struct e1000_hw *hw = &adapter->hw;
4602         u32 ims_mask = IMS_ENABLE_MASK;
4603
4604         if (hw->mac.type == e1000_82574) {
4605                 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4606                 ims_mask |= EM_MSIX_MASK;
4607         }
4608         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4609 }
4610
4611 static void
4612 em_disable_intr(struct adapter *adapter)
4613 {
4614         struct e1000_hw *hw = &adapter->hw;
4615
4616         if (hw->mac.type == e1000_82574)
4617                 E1000_WRITE_REG(hw, EM_EIAC, 0);
4618         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4619 }
4620
4621 /*
4622  * Bit of a misnomer, what this really means is
4623  * to enable OS management of the system... aka
4624  * to disable special hardware management features
4625  */
4626 static void
4627 em_init_manageability(struct adapter *adapter)
4628 {
4629         /* A shared code workaround */
4630 #define E1000_82542_MANC2H E1000_MANC2H
4631         if (adapter->has_manage) {
4632                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4633                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4634
4635                 /* disable hardware interception of ARP */
4636                 manc &= ~(E1000_MANC_ARP_EN);
4637
4638                 /* enable receiving management packets to the host */
4639                 manc |= E1000_MANC_EN_MNG2HOST;
4640 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4641 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4642                 manc2h |= E1000_MNG2HOST_PORT_623;
4643                 manc2h |= E1000_MNG2HOST_PORT_664;
4644                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4645                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4646         }
4647 }
4648
4649 /*
4650  * Give control back to hardware management
4651  * controller if there is one.
4652  */
4653 static void
4654 em_release_manageability(struct adapter *adapter)
4655 {
4656         if (adapter->has_manage) {
4657                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4658
4659                 /* re-enable hardware interception of ARP */
4660                 manc |= E1000_MANC_ARP_EN;
4661                 manc &= ~E1000_MANC_EN_MNG2HOST;
4662
4663                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4664         }
4665 }
4666
4667 /*
4668  * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4669  * For ASF and Pass Through versions of f/w this means
4670  * that the driver is loaded. For AMT version type f/w
4671  * this means that the network i/f is open.
4672  */
4673 static void
4674 em_get_hw_control(struct adapter *adapter)
4675 {
4676         u32 ctrl_ext, swsm;
4677
4678         if (adapter->hw.mac.type == e1000_82573) {
4679                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4680                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4681                     swsm | E1000_SWSM_DRV_LOAD);
4682                 return;
4683         }
4684         /* else */
4685         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4686         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4687             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4688         return;
4689 }
4690
4691 /*
4692  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4693  * For ASF and Pass Through versions of f/w this means that
4694  * the driver is no longer loaded. For AMT versions of the
4695  * f/w this means that the network i/f is closed.
4696  */
4697 static void
4698 em_release_hw_control(struct adapter *adapter)
4699 {
4700         u32 ctrl_ext, swsm;
4701
4702         if (!adapter->has_manage)
4703                 return;
4704
4705         if (adapter->hw.mac.type == e1000_82573) {
4706                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4707                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4708                     swsm & ~E1000_SWSM_DRV_LOAD);
4709                 return;
4710         }
4711         /* else */
4712         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4713         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4714             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4715         return;
4716 }
4717
4718 static int
4719 em_is_valid_ether_addr(u8 *addr)
4720 {
4721         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4722
4723         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4724                 return (FALSE);
4725         }
4726
4727         return (TRUE);
4728 }
4729
4730 /*
4731 ** Parse the interface capabilities with regard
4732 ** to both system management and wake-on-lan for
4733 ** later use.
4734 */
4735 static void
4736 em_get_wakeup(device_t dev)
4737 {
4738         struct adapter  *adapter = device_get_softc(dev);
4739         u16             eeprom_data = 0, device_id, apme_mask;
4740
4741         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4742         apme_mask = EM_EEPROM_APME;
4743
4744         switch (adapter->hw.mac.type) {
4745         case e1000_82573:
4746         case e1000_82583:
4747                 adapter->has_amt = TRUE;
4748                 /* Falls thru */
4749         case e1000_82571:
4750         case e1000_82572:
4751         case e1000_80003es2lan:
4752                 if (adapter->hw.bus.func == 1) {
4753                         e1000_read_nvm(&adapter->hw,
4754                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4755                         break;
4756                 } else
4757                         e1000_read_nvm(&adapter->hw,
4758                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4759                 break;
4760         case e1000_ich8lan:
4761         case e1000_ich9lan:
4762         case e1000_ich10lan:
4763         case e1000_pchlan:
4764         case e1000_pch2lan:
4765                 apme_mask = E1000_WUC_APME;
4766                 adapter->has_amt = TRUE;
4767                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
4768                 break;
4769         default:
4770                 e1000_read_nvm(&adapter->hw,
4771                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4772                 break;
4773         }
4774         if (eeprom_data & apme_mask)
4775                 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4776         /*
4777          * We have the eeprom settings, now apply the special cases
4778          * where the eeprom may be wrong or the board won't support
4779          * wake on lan on a particular port
4780          */
4781         device_id = pci_get_device(dev);
4782         switch (device_id) {
4783         case E1000_DEV_ID_82571EB_FIBER:
4784                 /* Wake events only supported on port A for dual fiber
4785                  * regardless of eeprom setting */
4786                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4787                     E1000_STATUS_FUNC_1)
4788                         adapter->wol = 0;
4789                 break;
4790         case E1000_DEV_ID_82571EB_QUAD_COPPER:
4791         case E1000_DEV_ID_82571EB_QUAD_FIBER:
4792         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
4793                 /* if quad port adapter, disable WoL on all but port A */
4794                 if (global_quad_port_a != 0)
4795                         adapter->wol = 0;
4796                 /* Reset for multiple quad port adapters */
4797                 if (++global_quad_port_a == 4)
4798                         global_quad_port_a = 0;
4799                 break;
4800         }
4801         return;
4802 }
4803
4804
4805 /*
4806  * Enable PCI Wake On Lan capability
4807  */
4808 static void
4809 em_enable_wakeup(device_t dev)
4810 {
4811         struct adapter  *adapter = device_get_softc(dev);
4812         struct ifnet    *ifp = adapter->ifp;
4813         u32             pmc, ctrl, ctrl_ext, rctl;
4814         u16             status;
4815
4816         if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
4817                 return;
4818
4819         /* Advertise the wakeup capability */
4820         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4821         ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4822         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4823         E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4824
4825         if ((adapter->hw.mac.type == e1000_ich8lan) ||
4826             (adapter->hw.mac.type == e1000_pchlan) ||
4827             (adapter->hw.mac.type == e1000_ich9lan) ||
4828             (adapter->hw.mac.type == e1000_ich10lan))
4829                 e1000_suspend_workarounds_ich8lan(&adapter->hw);
4830
4831         /* Keep the laser running on Fiber adapters */
4832         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4833             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4834                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4835                 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4836                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4837         }
4838
4839         /*
4840         ** Determine type of Wakeup: note that wol
4841         ** is set with all bits on by default.
4842         */
4843         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0)
4844                 adapter->wol &= ~E1000_WUFC_MAG;
4845
4846         if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0)
4847                 adapter->wol &= ~E1000_WUFC_MC;
4848         else {
4849                 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4850                 rctl |= E1000_RCTL_MPE;
4851                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4852         }
4853
4854         if ((adapter->hw.mac.type == e1000_pchlan) ||
4855             (adapter->hw.mac.type == e1000_pch2lan)) {
4856                 if (em_enable_phy_wakeup(adapter))
4857                         return;
4858         } else {
4859                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4860                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4861         }
4862
4863         if (adapter->hw.phy.type == e1000_phy_igp_3)
4864                 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
4865
4866         /* Request PME */
4867         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4868         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4869         if (ifp->if_capenable & IFCAP_WOL)
4870                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4871         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
4872
4873         return;
4874 }
4875
4876 /*
4877 ** WOL in the newer chipset interfaces (pchlan)
4878 ** require thing to be copied into the phy
4879 */
4880 static int
4881 em_enable_phy_wakeup(struct adapter *adapter)
4882 {
4883         struct e1000_hw *hw = &adapter->hw;
4884         u32 mreg, ret = 0;
4885         u16 preg;
4886
4887         /* copy MAC RARs to PHY RARs */
4888         e1000_copy_rx_addrs_to_phy_ich8lan(hw);
4889
4890         /* copy MAC MTA to PHY MTA */
4891         for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
4892                 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
4893                 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
4894                 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
4895                     (u16)((mreg >> 16) & 0xFFFF));
4896         }
4897
4898         /* configure PHY Rx Control register */
4899         e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
4900         mreg = E1000_READ_REG(hw, E1000_RCTL);
4901         if (mreg & E1000_RCTL_UPE)
4902                 preg |= BM_RCTL_UPE;
4903         if (mreg & E1000_RCTL_MPE)
4904                 preg |= BM_RCTL_MPE;
4905         preg &= ~(BM_RCTL_MO_MASK);
4906         if (mreg & E1000_RCTL_MO_3)
4907                 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
4908                                 << BM_RCTL_MO_SHIFT);
4909         if (mreg & E1000_RCTL_BAM)
4910                 preg |= BM_RCTL_BAM;
4911         if (mreg & E1000_RCTL_PMCF)
4912                 preg |= BM_RCTL_PMCF;
4913         mreg = E1000_READ_REG(hw, E1000_CTRL);
4914         if (mreg & E1000_CTRL_RFCE)
4915                 preg |= BM_RCTL_RFCE;
4916         e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
4917
4918         /* enable PHY wakeup in MAC register */
4919         E1000_WRITE_REG(hw, E1000_WUC,
4920             E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
4921         E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
4922
4923         /* configure and enable PHY wakeup in PHY registers */
4924         e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
4925         e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
4926
4927         /* activate PHY wakeup */
4928         ret = hw->phy.ops.acquire(hw);
4929         if (ret) {
4930                 printf("Could not acquire PHY\n");
4931                 return ret;
4932         }
4933         e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4934                                  (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
4935         ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
4936         if (ret) {
4937                 printf("Could not read PHY page 769\n");
4938                 goto out;
4939         }
4940         preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
4941         ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
4942         if (ret)
4943                 printf("Could not set PHY Host Wakeup bit\n");
4944 out:
4945         hw->phy.ops.release(hw);
4946
4947         return ret;
4948 }
4949
4950 static void
4951 em_led_func(void *arg, int onoff)
4952 {
4953         struct adapter  *adapter = arg;
4954
4955         EM_CORE_LOCK(adapter);
4956         if (onoff) {
4957                 e1000_setup_led(&adapter->hw);
4958                 e1000_led_on(&adapter->hw);
4959         } else {
4960                 e1000_led_off(&adapter->hw);
4961                 e1000_cleanup_led(&adapter->hw);
4962         }
4963         EM_CORE_UNLOCK(adapter);
4964 }
4965
4966 /*
4967 ** Disable the L0S and L1 LINK states
4968 */
4969 static void
4970 em_disable_aspm(struct adapter *adapter)
4971 {
4972         int             base, reg;
4973         u16             link_cap,link_ctrl;
4974         device_t        dev = adapter->dev;
4975
4976         switch (adapter->hw.mac.type) {
4977                 case e1000_82573:
4978                 case e1000_82574:
4979                 case e1000_82583:
4980                         break;
4981                 default:
4982                         return;
4983         }
4984         if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0)
4985                 return;
4986         reg = base + PCIR_EXPRESS_LINK_CAP;
4987         link_cap = pci_read_config(dev, reg, 2);
4988         if ((link_cap & PCIM_LINK_CAP_ASPM) == 0)
4989                 return;
4990         reg = base + PCIR_EXPRESS_LINK_CTL;
4991         link_ctrl = pci_read_config(dev, reg, 2);
4992         link_ctrl &= 0xFFFC; /* turn off bit 1 and 2 */
4993         pci_write_config(dev, reg, link_ctrl, 2);
4994         return;
4995 }
4996
4997 /**********************************************************************
4998  *
4999  *  Update the board statistics counters.
5000  *
5001  **********************************************************************/
5002 static void
5003 em_update_stats_counters(struct adapter *adapter)
5004 {
5005         struct ifnet   *ifp;
5006
5007         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5008            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5009                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5010                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5011         }
5012         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5013         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5014         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5015         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5016
5017         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5018         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5019         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5020         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5021         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5022         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5023         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5024         /*
5025         ** For watchdog management we need to know if we have been
5026         ** paused during the last interval, so capture that here.
5027         */
5028         adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5029         adapter->stats.xoffrxc += adapter->pause_frames;
5030         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5031         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5032         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5033         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5034         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5035         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5036         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5037         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5038         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5039         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5040         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5041         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5042
5043         /* For the 64-bit byte counters the low dword must be read first. */
5044         /* Both registers clear on the read of the high dword */
5045
5046         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
5047             ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
5048         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
5049             ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
5050
5051         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5052         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5053         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5054         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5055         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5056
5057         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5058         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5059
5060         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5061         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5062         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5063         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5064         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5065         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5066         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5067         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5068         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5069         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5070
5071         /* Interrupt Counts */
5072
5073         adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC);
5074         adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC);
5075         adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC);
5076         adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC);
5077         adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC);
5078         adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC);
5079         adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC);
5080         adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC);
5081         adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC);
5082
5083         if (adapter->hw.mac.type >= e1000_82543) {
5084                 adapter->stats.algnerrc +=
5085                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5086                 adapter->stats.rxerrc +=
5087                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5088                 adapter->stats.tncrs +=
5089                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5090                 adapter->stats.cexterr +=
5091                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5092                 adapter->stats.tsctc +=
5093                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5094                 adapter->stats.tsctfc +=
5095                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5096         }
5097         ifp = adapter->ifp;
5098
5099         ifp->if_collisions = adapter->stats.colc;
5100
5101         /* Rx Errors */
5102         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5103             adapter->stats.crcerrs + adapter->stats.algnerrc +
5104             adapter->stats.ruc + adapter->stats.roc +
5105             adapter->stats.mpc + adapter->stats.cexterr;
5106
5107         /* Tx Errors */
5108         ifp->if_oerrors = adapter->stats.ecol +
5109             adapter->stats.latecol + adapter->watchdog_events;
5110 }
5111
5112 /* Export a single 32-bit register via a read-only sysctl. */
5113 static int
5114 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5115 {
5116         struct adapter *adapter;
5117         u_int val;
5118
5119         adapter = oidp->oid_arg1;
5120         val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5121         return (sysctl_handle_int(oidp, &val, 0, req));
5122 }
5123
5124 /*
5125  * Add sysctl variables, one per statistic, to the system.
5126  */
5127 static void
5128 em_add_hw_stats(struct adapter *adapter)
5129 {
5130         device_t dev = adapter->dev;
5131
5132         struct tx_ring *txr = adapter->tx_rings;
5133         struct rx_ring *rxr = adapter->rx_rings;
5134
5135         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5136         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5137         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5138         struct e1000_hw_stats *stats = &adapter->stats;
5139
5140         struct sysctl_oid *stat_node, *queue_node, *int_node;
5141         struct sysctl_oid_list *stat_list, *queue_list, *int_list;
5142
5143 #define QUEUE_NAME_LEN 32
5144         char namebuf[QUEUE_NAME_LEN];
5145
5146         /* Driver Statistics */
5147         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5148                         CTLFLAG_RD, &adapter->link_irq,
5149                         "Link MSIX IRQ Handled");
5150         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail",
5151                          CTLFLAG_RD, &adapter->mbuf_alloc_failed,
5152                          "Std mbuf failed");
5153         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail",
5154                          CTLFLAG_RD, &adapter->mbuf_cluster_failed,
5155                          "Std mbuf cluster failed");
5156         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5157                         CTLFLAG_RD, &adapter->dropped_pkts,
5158                         "Driver dropped packets");
5159         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5160                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5161                         "Driver tx dma failure in xmit");
5162         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5163                         CTLFLAG_RD, &adapter->rx_overruns,
5164                         "RX overruns");
5165         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5166                         CTLFLAG_RD, &adapter->watchdog_events,
5167                         "Watchdog timeouts");
5168
5169         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
5170                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
5171                         em_sysctl_reg_handler, "IU",
5172                         "Device Control Register");
5173         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
5174                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
5175                         em_sysctl_reg_handler, "IU",
5176                         "Receiver Control Register");
5177         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5178                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5179                         "Flow Control High Watermark");
5180         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5181                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5182                         "Flow Control Low Watermark");
5183
5184         for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5185                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5186                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5187                                             CTLFLAG_RD, NULL, "Queue Name");
5188                 queue_list = SYSCTL_CHILDREN(queue_node);
5189
5190                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5191                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5192                                 E1000_TDH(txr->me),
5193                                 em_sysctl_reg_handler, "IU",
5194                                 "Transmit Descriptor Head");
5195                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5196                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5197                                 E1000_TDT(txr->me),
5198                                 em_sysctl_reg_handler, "IU",
5199                                 "Transmit Descriptor Tail");
5200                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
5201                                 CTLFLAG_RD, &txr->tx_irq,
5202                                 "Queue MSI-X Transmit Interrupts");
5203                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail",
5204                                 CTLFLAG_RD, &txr->no_desc_avail,
5205                                 "Queue No Descriptor Available");
5206
5207                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5208                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5209                                 E1000_RDH(rxr->me),
5210                                 em_sysctl_reg_handler, "IU",
5211                                 "Receive Descriptor Head");
5212                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5213                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5214                                 E1000_RDT(rxr->me),
5215                                 em_sysctl_reg_handler, "IU",
5216                                 "Receive Descriptor Tail");
5217                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
5218                                 CTLFLAG_RD, &rxr->rx_irq,
5219                                 "Queue MSI-X Receive Interrupts");
5220         }
5221
5222         /* MAC stats get their own sub node */
5223
5224         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5225                                     CTLFLAG_RD, NULL, "Statistics");
5226         stat_list = SYSCTL_CHILDREN(stat_node);
5227
5228         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5229                         CTLFLAG_RD, &stats->ecol,
5230                         "Excessive collisions");
5231         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
5232                         CTLFLAG_RD, &stats->scc,
5233                         "Single collisions");
5234         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5235                         CTLFLAG_RD, &stats->mcc,
5236                         "Multiple collisions");
5237         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
5238                         CTLFLAG_RD, &stats->latecol,
5239                         "Late collisions");
5240         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
5241                         CTLFLAG_RD, &stats->colc,
5242                         "Collision Count");
5243         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5244                         CTLFLAG_RD, &adapter->stats.symerrs,
5245                         "Symbol Errors");
5246         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5247                         CTLFLAG_RD, &adapter->stats.sec,
5248                         "Sequence Errors");
5249         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
5250                         CTLFLAG_RD, &adapter->stats.dc,
5251                         "Defer Count");
5252         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5253                         CTLFLAG_RD, &adapter->stats.mpc,
5254                         "Missed Packets");
5255         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5256                         CTLFLAG_RD, &adapter->stats.rnbc,
5257                         "Receive No Buffers");
5258         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5259                         CTLFLAG_RD, &adapter->stats.ruc,
5260                         "Receive Undersize");
5261         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5262                         CTLFLAG_RD, &adapter->stats.rfc,
5263                         "Fragmented Packets Received ");
5264         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5265                         CTLFLAG_RD, &adapter->stats.roc,
5266                         "Oversized Packets Received");
5267         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5268                         CTLFLAG_RD, &adapter->stats.rjc,
5269                         "Recevied Jabber");
5270         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5271                         CTLFLAG_RD, &adapter->stats.rxerrc,
5272                         "Receive Errors");
5273         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5274                         CTLFLAG_RD, &adapter->stats.crcerrs,
5275                         "CRC errors");
5276         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5277                         CTLFLAG_RD, &adapter->stats.algnerrc,
5278                         "Alignment Errors");
5279         /* On 82575 these are collision counts */
5280         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5281                         CTLFLAG_RD, &adapter->stats.cexterr,
5282                         "Collision/Carrier extension errors");
5283         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5284                         CTLFLAG_RD, &adapter->stats.xonrxc,
5285                         "XON Received");
5286         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5287                         CTLFLAG_RD, &adapter->stats.xontxc,
5288                         "XON Transmitted");
5289         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5290                         CTLFLAG_RD, &adapter->stats.xoffrxc,
5291                         "XOFF Received");
5292         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5293                         CTLFLAG_RD, &adapter->stats.xofftxc,
5294                         "XOFF Transmitted");
5295
5296         /* Packet Reception Stats */
5297         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5298                         CTLFLAG_RD, &adapter->stats.tpr,
5299                         "Total Packets Received ");
5300         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5301                         CTLFLAG_RD, &adapter->stats.gprc,
5302                         "Good Packets Received");
5303         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5304                         CTLFLAG_RD, &adapter->stats.bprc,
5305                         "Broadcast Packets Received");
5306         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5307                         CTLFLAG_RD, &adapter->stats.mprc,
5308                         "Multicast Packets Received");
5309         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5310                         CTLFLAG_RD, &adapter->stats.prc64,
5311                         "64 byte frames received ");
5312         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5313                         CTLFLAG_RD, &adapter->stats.prc127,
5314                         "65-127 byte frames received");
5315         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5316                         CTLFLAG_RD, &adapter->stats.prc255,
5317                         "128-255 byte frames received");
5318         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5319                         CTLFLAG_RD, &adapter->stats.prc511,
5320                         "256-511 byte frames received");
5321         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5322                         CTLFLAG_RD, &adapter->stats.prc1023,
5323                         "512-1023 byte frames received");
5324         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5325                         CTLFLAG_RD, &adapter->stats.prc1522,
5326                         "1023-1522 byte frames received");
5327         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5328                         CTLFLAG_RD, &adapter->stats.gorc,
5329                         "Good Octets Received");
5330
5331         /* Packet Transmission Stats */
5332         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5333                         CTLFLAG_RD, &adapter->stats.gotc,
5334                         "Good Octets Transmitted");
5335         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5336                         CTLFLAG_RD, &adapter->stats.tpt,
5337                         "Total Packets Transmitted");
5338         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5339                         CTLFLAG_RD, &adapter->stats.gptc,
5340                         "Good Packets Transmitted");
5341         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5342                         CTLFLAG_RD, &adapter->stats.bptc,
5343                         "Broadcast Packets Transmitted");
5344         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5345                         CTLFLAG_RD, &adapter->stats.mptc,
5346                         "Multicast Packets Transmitted");
5347         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5348                         CTLFLAG_RD, &adapter->stats.ptc64,
5349                         "64 byte frames transmitted ");
5350         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5351                         CTLFLAG_RD, &adapter->stats.ptc127,
5352                         "65-127 byte frames transmitted");
5353         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5354                         CTLFLAG_RD, &adapter->stats.ptc255,
5355                         "128-255 byte frames transmitted");
5356         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5357                         CTLFLAG_RD, &adapter->stats.ptc511,
5358                         "256-511 byte frames transmitted");
5359         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5360                         CTLFLAG_RD, &adapter->stats.ptc1023,
5361                         "512-1023 byte frames transmitted");
5362         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5363                         CTLFLAG_RD, &adapter->stats.ptc1522,
5364                         "1024-1522 byte frames transmitted");
5365         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5366                         CTLFLAG_RD, &adapter->stats.tsctc,
5367                         "TSO Contexts Transmitted");
5368         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5369                         CTLFLAG_RD, &adapter->stats.tsctfc,
5370                         "TSO Contexts Failed");
5371
5372
5373         /* Interrupt Stats */
5374
5375         int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5376                                     CTLFLAG_RD, NULL, "Interrupt Statistics");
5377         int_list = SYSCTL_CHILDREN(int_node);
5378
5379         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts",
5380                         CTLFLAG_RD, &adapter->stats.iac,
5381                         "Interrupt Assertion Count");
5382
5383         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5384                         CTLFLAG_RD, &adapter->stats.icrxptc,
5385                         "Interrupt Cause Rx Pkt Timer Expire Count");
5386
5387         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5388                         CTLFLAG_RD, &adapter->stats.icrxatc,
5389                         "Interrupt Cause Rx Abs Timer Expire Count");
5390
5391         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5392                         CTLFLAG_RD, &adapter->stats.ictxptc,
5393                         "Interrupt Cause Tx Pkt Timer Expire Count");
5394
5395         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5396                         CTLFLAG_RD, &adapter->stats.ictxatc,
5397                         "Interrupt Cause Tx Abs Timer Expire Count");
5398
5399         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5400                         CTLFLAG_RD, &adapter->stats.ictxqec,
5401                         "Interrupt Cause Tx Queue Empty Count");
5402
5403         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5404                         CTLFLAG_RD, &adapter->stats.ictxqmtc,
5405                         "Interrupt Cause Tx Queue Min Thresh Count");
5406
5407         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5408                         CTLFLAG_RD, &adapter->stats.icrxdmtc,
5409                         "Interrupt Cause Rx Desc Min Thresh Count");
5410
5411         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5412                         CTLFLAG_RD, &adapter->stats.icrxoc,
5413                         "Interrupt Cause Receiver Overrun Count");
5414 }
5415
5416 /**********************************************************************
5417  *
5418  *  This routine provides a way to dump out the adapter eeprom,
5419  *  often a useful debug/service tool. This only dumps the first
5420  *  32 words, stuff that matters is in that extent.
5421  *
5422  **********************************************************************/
5423 static int
5424 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5425 {
5426         struct adapter *adapter = (struct adapter *)arg1;
5427         int error;
5428         int result;
5429
5430         result = -1;
5431         error = sysctl_handle_int(oidp, &result, 0, req);
5432
5433         if (error || !req->newptr)
5434                 return (error);
5435
5436         /*
5437          * This value will cause a hex dump of the
5438          * first 32 16-bit words of the EEPROM to
5439          * the screen.
5440          */
5441         if (result == 1)
5442                 em_print_nvm_info(adapter);
5443
5444         return (error);
5445 }
5446
5447 static void
5448 em_print_nvm_info(struct adapter *adapter)
5449 {
5450         u16     eeprom_data;
5451         int     i, j, row = 0;
5452
5453         /* Its a bit crude, but it gets the job done */
5454         printf("\nInterface EEPROM Dump:\n");
5455         printf("Offset\n0x0000  ");
5456         for (i = 0, j = 0; i < 32; i++, j++) {
5457                 if (j == 8) { /* Make the offset block */
5458                         j = 0; ++row;
5459                         printf("\n0x00%x0  ",row);
5460                 }
5461                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5462                 printf("%04x ", eeprom_data);
5463         }
5464         printf("\n");
5465 }
5466
5467 static int
5468 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5469 {
5470         struct em_int_delay_info *info;
5471         struct adapter *adapter;
5472         u32 regval;
5473         int error, usecs, ticks;
5474
5475         info = (struct em_int_delay_info *)arg1;
5476         usecs = info->value;
5477         error = sysctl_handle_int(oidp, &usecs, 0, req);
5478         if (error != 0 || req->newptr == NULL)
5479                 return (error);
5480         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5481                 return (EINVAL);
5482         info->value = usecs;
5483         ticks = EM_USECS_TO_TICKS(usecs);
5484
5485         adapter = info->adapter;
5486
5487         EM_CORE_LOCK(adapter);
5488         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5489         regval = (regval & ~0xffff) | (ticks & 0xffff);
5490         /* Handle a few special cases. */
5491         switch (info->offset) {
5492         case E1000_RDTR:
5493                 break;
5494         case E1000_TIDV:
5495                 if (ticks == 0) {
5496                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5497                         /* Don't write 0 into the TIDV register. */
5498                         regval++;
5499                 } else
5500                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5501                 break;
5502         }
5503         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5504         EM_CORE_UNLOCK(adapter);
5505         return (0);
5506 }
5507
5508 static void
5509 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5510         const char *description, struct em_int_delay_info *info,
5511         int offset, int value)
5512 {
5513         info->adapter = adapter;
5514         info->offset = offset;
5515         info->value = value;
5516         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5517             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5518             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5519             info, 0, em_sysctl_int_delay, "I", description);
5520 }
5521
5522 static void
5523 em_set_sysctl_value(struct adapter *adapter, const char *name,
5524         const char *description, int *limit, int value)
5525 {
5526         *limit = value;
5527         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5528             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5529             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5530 }
5531
5532
5533 /*
5534 ** Set flow control using sysctl:
5535 ** Flow control values:
5536 **      0 - off
5537 **      1 - rx pause
5538 **      2 - tx pause
5539 **      3 - full
5540 */
5541 static int
5542 em_set_flowcntl(SYSCTL_HANDLER_ARGS)
5543 {
5544         int             error;
5545         static int      input = 3; /* default is full */
5546         struct adapter  *adapter = (struct adapter *) arg1;
5547
5548         error = sysctl_handle_int(oidp, &input, 0, req);
5549
5550         if ((error) || (req->newptr == NULL))
5551                 return (error);
5552
5553         if (input == adapter->fc) /* no change? */
5554                 return (error);
5555
5556         switch (input) {
5557                 case e1000_fc_rx_pause:
5558                 case e1000_fc_tx_pause:
5559                 case e1000_fc_full:
5560                 case e1000_fc_none:
5561                         adapter->hw.fc.requested_mode = input;
5562                         adapter->fc = input;
5563                         break;
5564                 default:
5565                         /* Do nothing */
5566                         return (error);
5567         }
5568
5569         adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
5570         e1000_force_mac_fc(&adapter->hw);
5571         return (error);
5572 }
5573
5574
5575 static int
5576 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5577 {
5578         struct adapter *adapter;
5579         int error;
5580         int result;
5581
5582         result = -1;
5583         error = sysctl_handle_int(oidp, &result, 0, req);
5584
5585         if (error || !req->newptr)
5586                 return (error);
5587
5588         if (result == 1) {
5589                 adapter = (struct adapter *)arg1;
5590                 em_print_debug_info(adapter);
5591         }
5592
5593         return (error);
5594 }
5595
5596 /*
5597 ** This routine is meant to be fluid, add whatever is
5598 ** needed for debugging a problem.  -jfv
5599 */
5600 static void
5601 em_print_debug_info(struct adapter *adapter)
5602 {
5603         device_t dev = adapter->dev;
5604         struct tx_ring *txr = adapter->tx_rings;
5605         struct rx_ring *rxr = adapter->rx_rings;
5606
5607         if (adapter->ifp->if_drv_flags & IFF_DRV_RUNNING)
5608                 printf("Interface is RUNNING ");
5609         else
5610                 printf("Interface is NOT RUNNING\n");
5611
5612         if (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE)
5613                 printf("and INACTIVE\n");
5614         else
5615                 printf("and ACTIVE\n");
5616
5617         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5618             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5619             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5620         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5621             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5622             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5623         device_printf(dev, "Tx Queue Status = %d\n", txr->queue_status);
5624         device_printf(dev, "TX descriptors avail = %d\n",
5625             txr->tx_avail);
5626         device_printf(dev, "Tx Descriptors avail failure = %ld\n",
5627             txr->no_desc_avail);
5628         device_printf(dev, "RX discarded packets = %ld\n",
5629             rxr->rx_discarded);
5630         device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check);
5631         device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh);
5632 }