sys/dev/e1000/if_em.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2011, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #endif
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #if __FreeBSD_version >= 800000
  44 #include <sys/buf_ring.h>
  45 #endif
  46 #include <sys/bus.h>
  47 #include <sys/endian.h>
  48 #include <sys/kernel.h>
  49 #include <sys/kthread.h>
  50 #include <sys/malloc.h>
  51 #include <sys/mbuf.h>
  52 #include <sys/module.h>
  53 #include <sys/rman.h>
  54 #include <sys/socket.h>
  55 #include <sys/sockio.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/taskqueue.h>
  58 #include <sys/eventhandler.h>
  59 #include <machine/bus.h>
  60 #include <machine/resource.h>
  61
  62 #include <net/bpf.h>
  63 #include <net/ethernet.h>
  64 #include <net/if.h>
  65 #include <net/if_arp.h>
  66 #include <net/if_dl.h>
  67 #include <net/if_media.h>
  68
  69 #include <net/if_types.h>
  70 #include <net/if_vlan_var.h>
  71
  72 #include <netinet/in_systm.h>
  73 #include <netinet/in.h>
  74 #include <netinet/if_ether.h>
  75 #include <netinet/ip.h>
  76 #include <netinet/ip6.h>
  77 #include <netinet/tcp.h>
  78 #include <netinet/udp.h>
  79
  80 #include <machine/in_cksum.h>
  81 #include <dev/led/led.h>
  82 #include <dev/pci/pcivar.h>
  83 #include <dev/pci/pcireg.h>
  84
  85 #include "e1000_api.h"
  86 #include "e1000_82571.h"
  87 #include "if_em.h"
  88
  89 /*********************************************************************
  90  *  Set this to one to display debug statistics
  91  *********************************************************************/
  92 int     em_display_debug_stats = 0;
  93
  94 /*********************************************************************
  95  *  Driver version:
  96  *********************************************************************/
  97 char em_driver_version[] = "7.3.2";
  98
  99 /*********************************************************************
 100  *  PCI Device ID Table
 101  *
 102  *  Used by probe to select devices to load on
 103  *  Last field stores an index into e1000_strings
 104  *  Last entry must be all 0s
 105  *
 106  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 107  *********************************************************************/
 108
 109 static em_vendor_info_t em_vendor_info_array[] =
 110 {
 111         /* Intel(R) PRO/1000 Network Connection */
 112         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 113         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 114         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
 116                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
 118                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 119         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 120                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
 122                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
 124                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
 126                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 127         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 128         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 129         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 131
 132         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
 136         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 137                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 138         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 139                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 141                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 142         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 143                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 144         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_ICH8_82567V_3,   PCI_ANY_ID, PCI_ANY_ID, 0},
 152         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 153         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 154         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 155         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 157         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 158         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 159         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 160         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 162         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
 163         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 164         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 165         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 166         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 167         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 168         { 0x8086, E1000_DEV_ID_ICH10_D_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 169         { 0x8086, E1000_DEV_ID_PCH_M_HV_LM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 170         { 0x8086, E1000_DEV_ID_PCH_M_HV_LC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 171         { 0x8086, E1000_DEV_ID_PCH_D_HV_DM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 172         { 0x8086, E1000_DEV_ID_PCH_D_HV_DC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 173         { 0x8086, E1000_DEV_ID_PCH2_LV_LM,      PCI_ANY_ID, PCI_ANY_ID, 0},
 174         { 0x8086, E1000_DEV_ID_PCH2_LV_V,       PCI_ANY_ID, PCI_ANY_ID, 0},
 175         /* required last entry */
 176         { 0, 0, 0, 0, 0}
 177 };
 178
 179 /*********************************************************************
 180  *  Table of branding strings for all supported NICs.
 181  *********************************************************************/
 182
 183 static char *em_strings[] = {
 184         "Intel(R) PRO/1000 Network Connection"
 185 };
 186
 187 /*********************************************************************
 188  *  Function prototypes
 189  *********************************************************************/
 190 static int      em_probe(device_t);
 191 static int      em_attach(device_t);
 192 static int      em_detach(device_t);
 193 static int      em_shutdown(device_t);
 194 static int      em_suspend(device_t);
 195 static int      em_resume(device_t);
 196 #ifdef EM_MULTIQUEUE
 197 static int      em_mq_start(struct ifnet *, struct mbuf *);
 198 static int      em_mq_start_locked(struct ifnet *,
 199                     struct tx_ring *, struct mbuf *);
 200 static void     em_qflush(struct ifnet *);
 201 #else
 202 static void     em_start(struct ifnet *);
 203 static void     em_start_locked(struct ifnet *, struct tx_ring *);
 204 #endif
 205 static int      em_ioctl(struct ifnet *, u_long, caddr_t);
 206 static void     em_init(void *);
 207 static void     em_init_locked(struct adapter *);
 208 static void     em_stop(void *);
 209 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 210 static int      em_media_change(struct ifnet *);
 211 static void     em_identify_hardware(struct adapter *);
 212 static int      em_allocate_pci_resources(struct adapter *);
 213 static int      em_allocate_legacy(struct adapter *);
 214 static int      em_allocate_msix(struct adapter *);
 215 static int      em_allocate_queues(struct adapter *);
 216 static int      em_setup_msix(struct adapter *);
 217 static void     em_free_pci_resources(struct adapter *);
 218 static void     em_local_timer(void *);
 219 static void     em_reset(struct adapter *);
 220 static int      em_setup_interface(device_t, struct adapter *);
 221
 222 static void     em_setup_transmit_structures(struct adapter *);
 223 static void     em_initialize_transmit_unit(struct adapter *);
 224 static int      em_allocate_transmit_buffers(struct tx_ring *);
 225 static void     em_free_transmit_structures(struct adapter *);
 226 static void     em_free_transmit_buffers(struct tx_ring *);
 227
 228 static int      em_setup_receive_structures(struct adapter *);
 229 static int      em_allocate_receive_buffers(struct rx_ring *);
 230 static void     em_initialize_receive_unit(struct adapter *);
 231 static void     em_free_receive_structures(struct adapter *);
 232 static void     em_free_receive_buffers(struct rx_ring *);
 233
 234 static void     em_enable_intr(struct adapter *);
 235 static void     em_disable_intr(struct adapter *);
 236 static void     em_update_stats_counters(struct adapter *);
 237 static void     em_add_hw_stats(struct adapter *adapter);
 238 static void     em_txeof(struct tx_ring *);
 239 static bool     em_rxeof(struct rx_ring *, int, int *);
 240 #ifndef __NO_STRICT_ALIGNMENT
 241 static int      em_fixup_rx(struct rx_ring *);
 242 #endif
 243 static void     em_receive_checksum(struct e1000_rx_desc *, struct mbuf *);
 244 static void     em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int,
 245                     struct ip *, u32 *, u32 *);
 246 static void     em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *,
 247                     struct tcphdr *, u32 *, u32 *);
 248 static void     em_set_promisc(struct adapter *);
 249 static void     em_disable_promisc(struct adapter *);
 250 static void     em_set_multi(struct adapter *);
 251 static void     em_update_link_status(struct adapter *);
 252 static void     em_refresh_mbufs(struct rx_ring *, int);
 253 static void     em_register_vlan(void *, struct ifnet *, u16);
 254 static void     em_unregister_vlan(void *, struct ifnet *, u16);
 255 static void     em_setup_vlan_hw_support(struct adapter *);
 256 static int      em_xmit(struct tx_ring *, struct mbuf **);
 257 static int      em_dma_malloc(struct adapter *, bus_size_t,
 258                     struct em_dma_alloc *, int);
 259 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 260 static int      em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
 261 static void     em_print_nvm_info(struct adapter *);
 262 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 263 static void     em_print_debug_info(struct adapter *);
 264 static int      em_is_valid_ether_addr(u8 *);
 265 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 266 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 267                     const char *, struct em_int_delay_info *, int, int);
 268 /* Management and WOL Support */
 269 static void     em_init_manageability(struct adapter *);
 270 static void     em_release_manageability(struct adapter *);
 271 static void     em_get_hw_control(struct adapter *);
 272 static void     em_release_hw_control(struct adapter *);
 273 static void     em_get_wakeup(device_t);
 274 static void     em_enable_wakeup(device_t);
 275 static int      em_enable_phy_wakeup(struct adapter *);
 276 static void     em_led_func(void *, int);
 277 static void     em_disable_aspm(struct adapter *);
 278
 279 static int      em_irq_fast(void *);
 280
 281 /* MSIX handlers */
 282 static void     em_msix_tx(void *);
 283 static void     em_msix_rx(void *);
 284 static void     em_msix_link(void *);
 285 static void     em_handle_tx(void *context, int pending);
 286 static void     em_handle_rx(void *context, int pending);
 287 static void     em_handle_link(void *context, int pending);
 288
 289 static void     em_set_sysctl_value(struct adapter *, const char *,
 290                     const char *, int *, int);
 291 static int      em_set_flowcntl(SYSCTL_HANDLER_ARGS);
 292
 293 static __inline void em_rx_discard(struct rx_ring *, int);
 294
 295 #ifdef DEVICE_POLLING
 296 static poll_handler_t em_poll;
 297 #endif /* POLLING */
 298
 299 /*********************************************************************
 300  *  FreeBSD Device Interface Entry Points
 301  *********************************************************************/
 302
 303 static device_method_t em_methods[] = {
 304         /* Device interface */
 305         DEVMETHOD(device_probe, em_probe),
 306         DEVMETHOD(device_attach, em_attach),
 307         DEVMETHOD(device_detach, em_detach),
 308         DEVMETHOD(device_shutdown, em_shutdown),
 309         DEVMETHOD(device_suspend, em_suspend),
 310         DEVMETHOD(device_resume, em_resume),
 311         {0, 0}
 312 };
 313
 314 static driver_t em_driver = {
 315         "em", em_methods, sizeof(struct adapter),
 316 };
 317
 318 devclass_t em_devclass;
 319 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 320 MODULE_DEPEND(em, pci, 1, 1, 1);
 321 MODULE_DEPEND(em, ether, 1, 1, 1);
 322
 323 /*********************************************************************
 324  *  Tunable default values.
 325  *********************************************************************/
 326
 327 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 328 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 329 #define M_TSO_LEN                       66
 330
 331 /* Allow common code without TSO */
 332 #ifndef CSUM_TSO
 333 #define CSUM_TSO        0
 334 #endif
 335
 336 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
 337
 338 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 339 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 340 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 341 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 342 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &em_tx_int_delay_dflt,
 343     0, "Default transmit interrupt delay in usecs");
 344 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt,
 345     0, "Default receive interrupt delay in usecs");
 346
 347 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 348 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 349 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 350 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 351 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN,
 352     &em_tx_abs_int_delay_dflt, 0,
 353     "Default transmit interrupt delay limit in usecs");
 354 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN,
 355     &em_rx_abs_int_delay_dflt, 0,
 356     "Default receive interrupt delay limit in usecs");
 357
 358 static int em_rxd = EM_DEFAULT_RXD;
 359 static int em_txd = EM_DEFAULT_TXD;
 360 TUNABLE_INT("hw.em.rxd", &em_rxd);
 361 TUNABLE_INT("hw.em.txd", &em_txd);
 362 SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0,
 363     "Number of receive descriptors per queue");
 364 SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0,
 365     "Number of transmit descriptors per queue");
 366
 367 static int em_smart_pwr_down = FALSE;
 368 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 369 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down,
 370     0, "Set to true to leave smart power down enabled on newer adapters");
 371
 372 /* Controls whether promiscuous also shows bad packets */
 373 static int em_debug_sbp = FALSE;
 374 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 375 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0,
 376     "Show bad packets in promiscuous mode");
 377
 378 static int em_enable_msix = TRUE;
 379 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix);
 380 SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0,
 381     "Enable MSI-X interrupts");
 382
 383 /* How many packets rxeof tries to clean at a time */
 384 static int em_rx_process_limit = 100;
 385 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
 386 SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
 387     &em_rx_process_limit, 0,
 388     "Maximum number of received packets to process "
 389     "at a time, -1 means unlimited");
 390
 391 /* Energy efficient ethernet - default to OFF */
 392 static int eee_setting = 0;
 393 TUNABLE_INT("hw.em.eee_setting", &eee_setting);
 394 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0,
 395     "Enable Energy Efficient Ethernet");
 396
 397 /* Global used in WOL setup with multiport cards */
 398 static int global_quad_port_a = 0;
 399
 400 /*********************************************************************
 401  *  Device identification routine
 402  *
 403  *  em_probe determines if the driver should be loaded on
 404  *  adapter based on PCI vendor/device id of the adapter.
 405  *
 406  *  return BUS_PROBE_DEFAULT on success, positive on failure
 407  *********************************************************************/
 408
 409 static int
 410 em_probe(device_t dev)
 411 {
 412         char            adapter_name[60];
 413         u16             pci_vendor_id = 0;
 414         u16             pci_device_id = 0;
 415         u16             pci_subvendor_id = 0;
 416         u16             pci_subdevice_id = 0;
 417         em_vendor_info_t *ent;
 418
 419         INIT_DEBUGOUT("em_probe: begin");
 420
 421         pci_vendor_id = pci_get_vendor(dev);
 422         if (pci_vendor_id != EM_VENDOR_ID)
 423                 return (ENXIO);
 424
 425         pci_device_id = pci_get_device(dev);
 426         pci_subvendor_id = pci_get_subvendor(dev);
 427         pci_subdevice_id = pci_get_subdevice(dev);
 428
 429         ent = em_vendor_info_array;
 430         while (ent->vendor_id != 0) {
 431                 if ((pci_vendor_id == ent->vendor_id) &&
 432                     (pci_device_id == ent->device_id) &&
 433
 434                     ((pci_subvendor_id == ent->subvendor_id) ||
 435                     (ent->subvendor_id == PCI_ANY_ID)) &&
 436
 437                     ((pci_subdevice_id == ent->subdevice_id) ||
 438                     (ent->subdevice_id == PCI_ANY_ID))) {
 439                         sprintf(adapter_name, "%s %s",
 440                                 em_strings[ent->index],
 441                                 em_driver_version);
 442                         device_set_desc_copy(dev, adapter_name);
 443                         return (BUS_PROBE_DEFAULT);
 444                 }
 445                 ent++;
 446         }
 447
 448         return (ENXIO);
 449 }
 450
 451 /*********************************************************************
 452  *  Device initialization routine
 453  *
 454  *  The attach entry point is called when the driver is being loaded.
 455  *  This routine identifies the type of hardware, allocates all resources
 456  *  and initializes the hardware.
 457  *
 458  *  return 0 on success, positive on failure
 459  *********************************************************************/
 460
 461 static int
 462 em_attach(device_t dev)
 463 {
 464         struct adapter  *adapter;
 465         struct e1000_hw *hw;
 466         int             error = 0;
 467
 468         INIT_DEBUGOUT("em_attach: begin");
 469
 470         if (resource_disabled("em", device_get_unit(dev))) {
 471                 device_printf(dev, "Disabled by device hint\n");
 472                 return (ENXIO);
 473         }
 474
 475         adapter = device_get_softc(dev);
 476         adapter->dev = adapter->osdep.dev = dev;
 477         hw = &adapter->hw;
 478         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 479
 480         /* SYSCTL stuff */
 481         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 482             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 483             OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 484             em_sysctl_nvm_info, "I", "NVM Information");
 485
 486         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 487             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 488             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 489             em_sysctl_debug_info, "I", "Debug Information");
 490
 491         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 492             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 493             OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 494             em_set_flowcntl, "I", "Flow Control");
 495
 496         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 497
 498         /* Determine hardware and mac info */
 499         em_identify_hardware(adapter);
 500
 501         /* Setup PCI resources */
 502         if (em_allocate_pci_resources(adapter)) {
 503                 device_printf(dev, "Allocation of PCI resources failed\n");
 504                 error = ENXIO;
 505                 goto err_pci;
 506         }
 507
 508         /*
 509         ** For ICH8 and family we need to
 510         ** map the flash memory, and this
 511         ** must happen after the MAC is
 512         ** identified
 513         */
 514         if ((hw->mac.type == e1000_ich8lan) ||
 515             (hw->mac.type == e1000_ich9lan) ||
 516             (hw->mac.type == e1000_ich10lan) ||
 517             (hw->mac.type == e1000_pchlan) ||
 518             (hw->mac.type == e1000_pch2lan)) {
 519                 int rid = EM_BAR_TYPE_FLASH;
 520                 adapter->flash = bus_alloc_resource_any(dev,
 521                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 522                 if (adapter->flash == NULL) {
 523                         device_printf(dev, "Mapping of Flash failed\n");
 524                         error = ENXIO;
 525                         goto err_pci;
 526                 }
 527                 /* This is used in the shared code */
 528                 hw->flash_address = (u8 *)adapter->flash;
 529                 adapter->osdep.flash_bus_space_tag =
 530                     rman_get_bustag(adapter->flash);
 531                 adapter->osdep.flash_bus_space_handle =
 532                     rman_get_bushandle(adapter->flash);
 533         }
 534
 535         /* Do Shared Code initialization */
 536         if (e1000_setup_init_funcs(hw, TRUE)) {
 537                 device_printf(dev, "Setup of Shared code failed\n");
 538                 error = ENXIO;
 539                 goto err_pci;
 540         }
 541
 542         e1000_get_bus_info(hw);
 543
 544         /* Set up some sysctls for the tunable interrupt delays */
 545         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 546             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 547             E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt);
 548         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 549             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 550             E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt);
 551         em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 552             "receive interrupt delay limit in usecs",
 553             &adapter->rx_abs_int_delay,
 554             E1000_REGISTER(hw, E1000_RADV),
 555             em_rx_abs_int_delay_dflt);
 556         em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 557             "transmit interrupt delay limit in usecs",
 558             &adapter->tx_abs_int_delay,
 559             E1000_REGISTER(hw, E1000_TADV),
 560             em_tx_abs_int_delay_dflt);
 561
 562         /* Sysctl for limiting the amount of work done in the taskqueue */
 563         em_set_sysctl_value(adapter, "rx_processing_limit",
 564             "max number of rx packets to process", &adapter->rx_process_limit,
 565             em_rx_process_limit);
 566
 567         /*
 568          * Validate number of transmit and receive descriptors. It
 569          * must not exceed hardware maximum, and must be multiple
 570          * of E1000_DBA_ALIGN.
 571          */
 572         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 573             (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) {
 574                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 575                     EM_DEFAULT_TXD, em_txd);
 576                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 577         } else
 578                 adapter->num_tx_desc = em_txd;
 579
 580         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 581             (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) {
 582                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 583                     EM_DEFAULT_RXD, em_rxd);
 584                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 585         } else
 586                 adapter->num_rx_desc = em_rxd;
 587
 588         hw->mac.autoneg = DO_AUTO_NEG;
 589         hw->phy.autoneg_wait_to_complete = FALSE;
 590         hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 591
 592         /* Copper options */
 593         if (hw->phy.media_type == e1000_media_type_copper) {
 594                 hw->phy.mdix = AUTO_ALL_MODES;
 595                 hw->phy.disable_polarity_correction = FALSE;
 596                 hw->phy.ms_type = EM_MASTER_SLAVE;
 597         }
 598
 599         /*
 600          * Set the frame limits assuming
 601          * standard ethernet sized frames.
 602          */
 603         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 604         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 605
 606         /*
 607          * This controls when hardware reports transmit completion
 608          * status.
 609          */
 610         hw->mac.report_tx_early = 1;
 611
 612         /*
 613         ** Get queue/ring memory
 614         */
 615         if (em_allocate_queues(adapter)) {
 616                 error = ENOMEM;
 617                 goto err_pci;
 618         }
 619
 620         /* Allocate multicast array memory. */
 621         adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
 622             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 623         if (adapter->mta == NULL) {
 624                 device_printf(dev, "Can not allocate multicast setup array\n");
 625                 error = ENOMEM;
 626                 goto err_late;
 627         }
 628
 629         /* Check SOL/IDER usage */
 630         if (e1000_check_reset_block(hw))
 631                 device_printf(dev, "PHY reset is blocked"
 632                     " due to SOL/IDER session.\n");
 633
 634         /* Sysctl for setting Energy Efficient Ethernet */
 635         em_set_sysctl_value(adapter, "eee_control",
 636             "enable Energy Efficient Ethernet",
 637             &hw->dev_spec.ich8lan.eee_disable, eee_setting);
 638
 639         /*
 640         ** Start from a known state, this is
 641         ** important in reading the nvm and
 642         ** mac from that.
 643         */
 644         e1000_reset_hw(hw);
 645
 646
 647         /* Make sure we have a good EEPROM before we read from it */
 648         if (e1000_validate_nvm_checksum(hw) < 0) {
 649                 /*
 650                 ** Some PCI-E parts fail the first check due to
 651                 ** the link being in sleep state, call it again,
 652                 ** if it fails a second time its a real issue.
 653                 */
 654                 if (e1000_validate_nvm_checksum(hw) < 0) {
 655                         device_printf(dev,
 656                             "The EEPROM Checksum Is Not Valid\n");
 657                         error = EIO;
 658                         goto err_late;
 659                 }
 660         }
 661
 662         /* Copy the permanent MAC address out of the EEPROM */
 663         if (e1000_read_mac_addr(hw) < 0) {
 664                 device_printf(dev, "EEPROM read error while reading MAC"
 665                     " address\n");
 666                 error = EIO;
 667                 goto err_late;
 668         }
 669
 670         if (!em_is_valid_ether_addr(hw->mac.addr)) {
 671                 device_printf(dev, "Invalid MAC address\n");
 672                 error = EIO;
 673                 goto err_late;
 674         }
 675
 676         /*
 677         **  Do interrupt configuration
 678         */
 679         if (adapter->msix > 1) /* Do MSIX */
 680                 error = em_allocate_msix(adapter);
 681         else  /* MSI or Legacy */
 682                 error = em_allocate_legacy(adapter);
 683         if (error)
 684                 goto err_late;
 685
 686         /*
 687          * Get Wake-on-Lan and Management info for later use
 688          */
 689         em_get_wakeup(dev);
 690
 691         /* Setup OS specific network interface */
 692         if (em_setup_interface(dev, adapter) != 0)
 693                 goto err_late;
 694
 695         em_reset(adapter);
 696
 697         /* Initialize statistics */
 698         em_update_stats_counters(adapter);
 699
 700         hw->mac.get_link_status = 1;
 701         em_update_link_status(adapter);
 702
 703         /* Register for VLAN events */
 704         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 705             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 706         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 707             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 708
 709         em_add_hw_stats(adapter);
 710
 711         /* Non-AMT based hardware can now take control from firmware */
 712         if (adapter->has_manage && !adapter->has_amt)
 713                 em_get_hw_control(adapter);
 714
 715         /* Tell the stack that the interface is not active */
 716         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 717         adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 718
 719         adapter->led_dev = led_create(em_led_func, adapter,
 720             device_get_nameunit(dev));
 721
 722         INIT_DEBUGOUT("em_attach: end");
 723
 724         return (0);
 725
 726 err_late:
 727         em_free_transmit_structures(adapter);
 728         em_free_receive_structures(adapter);
 729         em_release_hw_control(adapter);
 730         if (adapter->ifp != NULL)
 731                 if_free(adapter->ifp);
 732 err_pci:
 733         em_free_pci_resources(adapter);
 734         free(adapter->mta, M_DEVBUF);
 735         EM_CORE_LOCK_DESTROY(adapter);
 736
 737         return (error);
 738 }
 739
 740 /*********************************************************************
 741  *  Device removal routine
 742  *
 743  *  The detach entry point is called when the driver is being removed.
 744  *  This routine stops the adapter and deallocates all the resources
 745  *  that were allocated for driver operation.
 746  *
 747  *  return 0 on success, positive on failure
 748  *********************************************************************/
 749
 750 static int
 751 em_detach(device_t dev)
 752 {
 753         struct adapter  *adapter = device_get_softc(dev);
 754         struct ifnet    *ifp = adapter->ifp;
 755
 756         INIT_DEBUGOUT("em_detach: begin");
 757
 758         /* Make sure VLANS are not using driver */
 759         if (adapter->ifp->if_vlantrunk != NULL) {
 760                 device_printf(dev,"Vlan in use, detach first\n");
 761                 return (EBUSY);
 762         }
 763
 764 #ifdef DEVICE_POLLING
 765         if (ifp->if_capenable & IFCAP_POLLING)
 766                 ether_poll_deregister(ifp);
 767 #endif
 768
 769         if (adapter->led_dev != NULL)
 770                 led_destroy(adapter->led_dev);
 771
 772         EM_CORE_LOCK(adapter);
 773         adapter->in_detach = 1;
 774         em_stop(adapter);
 775         EM_CORE_UNLOCK(adapter);
 776         EM_CORE_LOCK_DESTROY(adapter);
 777
 778         e1000_phy_hw_reset(&adapter->hw);
 779
 780         em_release_manageability(adapter);
 781         em_release_hw_control(adapter);
 782
 783         /* Unregister VLAN events */
 784         if (adapter->vlan_attach != NULL)
 785                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 786         if (adapter->vlan_detach != NULL)
 787                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 788
 789         ether_ifdetach(adapter->ifp);
 790         callout_drain(&adapter->timer);
 791
 792         em_free_pci_resources(adapter);
 793         bus_generic_detach(dev);
 794         if_free(ifp);
 795
 796         em_free_transmit_structures(adapter);
 797         em_free_receive_structures(adapter);
 798
 799         em_release_hw_control(adapter);
 800         free(adapter->mta, M_DEVBUF);
 801
 802         return (0);
 803 }
 804
 805 /*********************************************************************
 806  *
 807  *  Shutdown entry point
 808  *
 809  **********************************************************************/
 810
 811 static int
 812 em_shutdown(device_t dev)
 813 {
 814         return em_suspend(dev);
 815 }
 816
 817 /*
 818  * Suspend/resume device methods.
 819  */
 820 static int
 821 em_suspend(device_t dev)
 822 {
 823         struct adapter *adapter = device_get_softc(dev);
 824
 825         EM_CORE_LOCK(adapter);
 826
 827         em_release_manageability(adapter);
 828         em_release_hw_control(adapter);
 829         em_enable_wakeup(dev);
 830
 831         EM_CORE_UNLOCK(adapter);
 832
 833         return bus_generic_suspend(dev);
 834 }
 835
 836 static int
 837 em_resume(device_t dev)
 838 {
 839         struct adapter *adapter = device_get_softc(dev);
 840         struct tx_ring  *txr = adapter->tx_rings;
 841         struct ifnet *ifp = adapter->ifp;
 842
 843         EM_CORE_LOCK(adapter);
 844         if (adapter->hw.mac.type == e1000_pch2lan)
 845                 e1000_resume_workarounds_pchlan(&adapter->hw);
 846         em_init_locked(adapter);
 847         em_init_manageability(adapter);
 848
 849         if ((ifp->if_flags & IFF_UP) &&
 850             (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
 851                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
 852                         EM_TX_LOCK(txr);
 853 #ifdef EM_MULTIQUEUE
 854                         if (!drbr_empty(ifp, txr->br))
 855                                 em_mq_start_locked(ifp, txr, NULL);
 856 #else
 857                         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 858                                 em_start_locked(ifp, txr);
 859 #endif
 860                         EM_TX_UNLOCK(txr);
 861                 }
 862         }
 863         EM_CORE_UNLOCK(adapter);
 864
 865         return bus_generic_resume(dev);
 866 }
 867
 868
 869 #ifdef EM_MULTIQUEUE
 870 /*********************************************************************
 871  *  Multiqueue Transmit routines
 872  *
 873  *  em_mq_start is called by the stack to initiate a transmit.
 874  *  however, if busy the driver can queue the request rather
 875  *  than do an immediate send. It is this that is an advantage
 876  *  in this driver, rather than also having multiple tx queues.
 877  **********************************************************************/
 878 static int
 879 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 880 {
 881         struct adapter  *adapter = txr->adapter;
 882         struct mbuf     *next;
 883         int             err = 0, enq = 0;
 884
 885         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 886             IFF_DRV_RUNNING || adapter->link_active == 0) {
 887                 if (m != NULL)
 888                         err = drbr_enqueue(ifp, txr->br, m);
 889                 return (err);
 890         }
 891
 892         enq = 0;
 893         if (m == NULL) {
 894                 next = drbr_dequeue(ifp, txr->br);
 895         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 896                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 897                         return (err);
 898                 next = drbr_dequeue(ifp, txr->br);
 899         } else
 900                 next = m;
 901
 902         /* Process the queue */
 903         while (next != NULL) {
 904                 if ((err = em_xmit(txr, &next)) != 0) {
 905                         if (next != NULL)
 906                                 err = drbr_enqueue(ifp, txr->br, next);
 907                         break;
 908                 }
 909                 enq++;
 910                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 911                 ETHER_BPF_MTAP(ifp, next);
 912                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 913                         break;
 914                 next = drbr_dequeue(ifp, txr->br);
 915         }
 916
 917         if (enq > 0) {
 918                 /* Set the watchdog */
 919                 txr->queue_status = EM_QUEUE_WORKING;
 920                 txr->watchdog_time = ticks;
 921         }
 922
 923         if (txr->tx_avail < EM_MAX_SCATTER)
 924                 em_txeof(txr);
 925         if (txr->tx_avail < EM_MAX_SCATTER)
 926                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 927         return (err);
 928 }
 929
 930 /*
 931 ** Multiqueue capable stack interface
 932 */
 933 static int
 934 em_mq_start(struct ifnet *ifp, struct mbuf *m)
 935 {
 936         struct adapter  *adapter = ifp->if_softc;
 937         struct tx_ring  *txr = adapter->tx_rings;
 938         int             error;
 939
 940         if (EM_TX_TRYLOCK(txr)) {
 941                 error = em_mq_start_locked(ifp, txr, m);
 942                 EM_TX_UNLOCK(txr);
 943         } else
 944                 error = drbr_enqueue(ifp, txr->br, m);
 945
 946         return (error);
 947 }
 948
 949 /*
 950 ** Flush all ring buffers
 951 */
 952 static void
 953 em_qflush(struct ifnet *ifp)
 954 {
 955         struct adapter  *adapter = ifp->if_softc;
 956         struct tx_ring  *txr = adapter->tx_rings;
 957         struct mbuf     *m;
 958
 959         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 960                 EM_TX_LOCK(txr);
 961                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 962                         m_freem(m);
 963                 EM_TX_UNLOCK(txr);
 964         }
 965         if_qflush(ifp);
 966 }
 967 #else  /* !EM_MULTIQUEUE */
 968
 969 static void
 970 em_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 971 {
 972         struct adapter  *adapter = ifp->if_softc;
 973         struct mbuf     *m_head;
 974
 975         EM_TX_LOCK_ASSERT(txr);
 976
 977         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 978             IFF_DRV_RUNNING)
 979                 return;
 980
 981         if (!adapter->link_active)
 982                 return;
 983
 984         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 985                 /* Call cleanup if number of TX descriptors low */
 986                 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
 987                         em_txeof(txr);
 988                 if (txr->tx_avail < EM_MAX_SCATTER) {
 989                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 990                         break;
 991                 }
 992                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 993                 if (m_head == NULL)
 994                         break;
 995                 /*
 996                  *  Encapsulation can modify our pointer, and or make it
 997                  *  NULL on failure.  In that event, we can't requeue.
 998                  */
 999                 if (em_xmit(txr, &m_head)) {
1000                         if (m_head == NULL)
1001                                 break;
1002                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1003                         break;
1004                 }
1005
1006                 /* Send a copy of the frame to the BPF listener */
1007                 ETHER_BPF_MTAP(ifp, m_head);
1008
1009                 /* Set timeout in case hardware has problems transmitting. */
1010                 txr->watchdog_time = ticks;
1011                 txr->queue_status = EM_QUEUE_WORKING;
1012         }
1013
1014         return;
1015 }
1016
1017 static void
1018 em_start(struct ifnet *ifp)
1019 {
1020         struct adapter  *adapter = ifp->if_softc;
1021         struct tx_ring  *txr = adapter->tx_rings;
1022
1023         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1024                 EM_TX_LOCK(txr);
1025                 em_start_locked(ifp, txr);
1026                 EM_TX_UNLOCK(txr);
1027         }
1028         return;
1029 }
1030 #endif /* EM_MULTIQUEUE */
1031
1032 /*********************************************************************
1033  *  Ioctl entry point
1034  *
1035  *  em_ioctl is called when the user wants to configure the
1036  *  interface.
1037  *
1038  *  return 0 on success, positive on failure
1039  **********************************************************************/
1040
1041 static int
1042 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1043 {
1044         struct adapter  *adapter = ifp->if_softc;
1045         struct ifreq    *ifr = (struct ifreq *)data;
1046 #if defined(INET) || defined(INET6)
1047         struct ifaddr   *ifa = (struct ifaddr *)data;
1048 #endif
1049         bool            avoid_reset = FALSE;
1050         int             error = 0;
1051
1052         if (adapter->in_detach)
1053                 return (error);
1054
1055         switch (command) {
1056         case SIOCSIFADDR:
1057 #ifdef INET
1058                 if (ifa->ifa_addr->sa_family == AF_INET)
1059                         avoid_reset = TRUE;
1060 #endif
1061 #ifdef INET6
1062                 if (ifa->ifa_addr->sa_family == AF_INET6)
1063                         avoid_reset = TRUE;
1064 #endif
1065                 /*
1066                 ** Calling init results in link renegotiation,
1067                 ** so we avoid doing it when possible.
1068                 */
1069                 if (avoid_reset) {
1070                         ifp->if_flags |= IFF_UP;
1071                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1072                                 em_init(adapter);
1073 #ifdef INET
1074                         if (!(ifp->if_flags & IFF_NOARP))
1075                                 arp_ifinit(ifp, ifa);
1076 #endif
1077                 } else
1078                         error = ether_ioctl(ifp, command, data);
1079                 break;
1080         case SIOCSIFMTU:
1081             {
1082                 int max_frame_size;
1083
1084                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1085
1086                 EM_CORE_LOCK(adapter);
1087                 switch (adapter->hw.mac.type) {
1088                 case e1000_82571:
1089                 case e1000_82572:
1090                 case e1000_ich9lan:
1091                 case e1000_ich10lan:
1092                 case e1000_pch2lan:
1093                 case e1000_82574:
1094                 case e1000_82583:
1095                 case e1000_80003es2lan: /* 9K Jumbo Frame size */
1096                         max_frame_size = 9234;
1097                         break;
1098                 case e1000_pchlan:
1099                         max_frame_size = 4096;
1100                         break;
1101                         /* Adapters that do not support jumbo frames */
1102                 case e1000_ich8lan:
1103                         max_frame_size = ETHER_MAX_LEN;
1104                         break;
1105                 default:
1106                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1107                 }
1108                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1109                     ETHER_CRC_LEN) {
1110                         EM_CORE_UNLOCK(adapter);
1111                         error = EINVAL;
1112                         break;
1113                 }
1114
1115                 ifp->if_mtu = ifr->ifr_mtu;
1116                 adapter->max_frame_size =
1117                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1118                 em_init_locked(adapter);
1119                 EM_CORE_UNLOCK(adapter);
1120                 break;
1121             }
1122         case SIOCSIFFLAGS:
1123                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1124                     SIOCSIFFLAGS (Set Interface Flags)");
1125                 EM_CORE_LOCK(adapter);
1126                 if (ifp->if_flags & IFF_UP) {
1127                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1128                                 if ((ifp->if_flags ^ adapter->if_flags) &
1129                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1130                                         em_disable_promisc(adapter);
1131                                         em_set_promisc(adapter);
1132                                 }
1133                         } else
1134                                 em_init_locked(adapter);
1135                 } else
1136                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1137                                 em_stop(adapter);
1138                 adapter->if_flags = ifp->if_flags;
1139                 EM_CORE_UNLOCK(adapter);
1140                 break;
1141         case SIOCADDMULTI:
1142         case SIOCDELMULTI:
1143                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1144                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1145                         EM_CORE_LOCK(adapter);
1146                         em_disable_intr(adapter);
1147                         em_set_multi(adapter);
1148 #ifdef DEVICE_POLLING
1149                         if (!(ifp->if_capenable & IFCAP_POLLING))
1150 #endif
1151                                 em_enable_intr(adapter);
1152                         EM_CORE_UNLOCK(adapter);
1153                 }
1154                 break;
1155         case SIOCSIFMEDIA:
1156                 /* Check SOL/IDER usage */
1157                 EM_CORE_LOCK(adapter);
1158                 if (e1000_check_reset_block(&adapter->hw)) {
1159                         EM_CORE_UNLOCK(adapter);
1160                         device_printf(adapter->dev, "Media change is"
1161                             " blocked due to SOL/IDER session.\n");
1162                         break;
1163                 }
1164                 EM_CORE_UNLOCK(adapter);
1165                 /* falls thru */
1166         case SIOCGIFMEDIA:
1167                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1168                     SIOCxIFMEDIA (Get/Set Interface Media)");
1169                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1170                 break;
1171         case SIOCSIFCAP:
1172             {
1173                 int mask, reinit;
1174
1175                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1176                 reinit = 0;
1177                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1178 #ifdef DEVICE_POLLING
1179                 if (mask & IFCAP_POLLING) {
1180                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1181                                 error = ether_poll_register(em_poll, ifp);
1182                                 if (error)
1183                                         return (error);
1184                                 EM_CORE_LOCK(adapter);
1185                                 em_disable_intr(adapter);
1186                                 ifp->if_capenable |= IFCAP_POLLING;
1187                                 EM_CORE_UNLOCK(adapter);
1188                         } else {
1189                                 error = ether_poll_deregister(ifp);
1190                                 /* Enable interrupt even in error case */
1191                                 EM_CORE_LOCK(adapter);
1192                                 em_enable_intr(adapter);
1193                                 ifp->if_capenable &= ~IFCAP_POLLING;
1194                                 EM_CORE_UNLOCK(adapter);
1195                         }
1196                 }
1197 #endif
1198                 if (mask & IFCAP_HWCSUM) {
1199                         ifp->if_capenable ^= IFCAP_HWCSUM;
1200                         reinit = 1;
1201                 }
1202                 if (mask & IFCAP_TSO4) {
1203                         ifp->if_capenable ^= IFCAP_TSO4;
1204                         reinit = 1;
1205                 }
1206                 if (mask & IFCAP_VLAN_HWTAGGING) {
1207                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1208                         reinit = 1;
1209                 }
1210                 if (mask & IFCAP_VLAN_HWFILTER) {
1211                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1212                         reinit = 1;
1213                 }
1214                 if (mask & IFCAP_VLAN_HWTSO) {
1215                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1216                         reinit = 1;
1217                 }
1218                 if ((mask & IFCAP_WOL) &&
1219                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
1220                         if (mask & IFCAP_WOL_MCAST)
1221                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1222                         if (mask & IFCAP_WOL_MAGIC)
1223                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1224                 }
1225                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1226                         em_init(adapter);
1227                 VLAN_CAPABILITIES(ifp);
1228                 break;
1229             }
1230
1231         default:
1232                 error = ether_ioctl(ifp, command, data);
1233                 break;
1234         }
1235
1236         return (error);
1237 }
1238
1239
1240 /*********************************************************************
1241  *  Init entry point
1242  *
1243  *  This routine is used in two ways. It is used by the stack as
1244  *  init entry point in network interface structure. It is also used
1245  *  by the driver as a hw/sw initialization routine to get to a
1246  *  consistent state.
1247  *
1248  *  return 0 on success, positive on failure
1249  **********************************************************************/
1250
1251 static void
1252 em_init_locked(struct adapter *adapter)
1253 {
1254         struct ifnet    *ifp = adapter->ifp;
1255         device_t        dev = adapter->dev;
1256
1257         INIT_DEBUGOUT("em_init: begin");
1258
1259         EM_CORE_LOCK_ASSERT(adapter);
1260
1261         em_disable_intr(adapter);
1262         callout_stop(&adapter->timer);
1263
1264         /* Get the latest mac address, User can use a LAA */
1265         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1266               ETHER_ADDR_LEN);
1267
1268         /* Put the address into the Receive Address Array */
1269         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1270
1271         /*
1272          * With the 82571 adapter, RAR[0] may be overwritten
1273          * when the other port is reset, we make a duplicate
1274          * in RAR[14] for that eventuality, this assures
1275          * the interface continues to function.
1276          */
1277         if (adapter->hw.mac.type == e1000_82571) {
1278                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1279                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1280                     E1000_RAR_ENTRIES - 1);
1281         }
1282
1283         /* Initialize the hardware */
1284         em_reset(adapter);
1285         em_update_link_status(adapter);
1286
1287         /* Setup VLAN support, basic and offload if available */
1288         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1289
1290         /* Set hardware offload abilities */
1291         ifp->if_hwassist = 0;
1292         if (ifp->if_capenable & IFCAP_TXCSUM)
1293                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1294         if (ifp->if_capenable & IFCAP_TSO4)
1295                 ifp->if_hwassist |= CSUM_TSO;
1296
1297         /* Configure for OS presence */
1298         em_init_manageability(adapter);
1299
1300         /* Prepare transmit descriptors and buffers */
1301         em_setup_transmit_structures(adapter);
1302         em_initialize_transmit_unit(adapter);
1303
1304         /* Setup Multicast table */
1305         em_set_multi(adapter);
1306
1307         /*
1308         ** Figure out the desired mbuf
1309         ** pool for doing jumbos
1310         */
1311         if (adapter->max_frame_size <= 2048)
1312                 adapter->rx_mbuf_sz = MCLBYTES;
1313         else if (adapter->max_frame_size <= 4096)
1314                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1315         else
1316                 adapter->rx_mbuf_sz = MJUM9BYTES;
1317
1318         /* Prepare receive descriptors and buffers */
1319         if (em_setup_receive_structures(adapter)) {
1320                 device_printf(dev, "Could not setup receive structures\n");
1321                 em_stop(adapter);
1322                 return;
1323         }
1324         em_initialize_receive_unit(adapter);
1325
1326         /* Use real VLAN Filter support? */
1327         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1328                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1329                         /* Use real VLAN Filter support */
1330                         em_setup_vlan_hw_support(adapter);
1331                 else {
1332                         u32 ctrl;
1333                         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1334                         ctrl |= E1000_CTRL_VME;
1335                         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1336                 }
1337         }
1338
1339         /* Don't lose promiscuous settings */
1340         em_set_promisc(adapter);
1341
1342         /* Set the interface as ACTIVE */
1343         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1344         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1345
1346         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1347         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1348
1349         /* MSI/X configuration for 82574 */
1350         if (adapter->hw.mac.type == e1000_82574) {
1351                 int tmp;
1352                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1353                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1354                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1355                 /* Set the IVAR - interrupt vector routing. */
1356                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
1357         }
1358
1359 #ifdef DEVICE_POLLING
1360         /*
1361          * Only enable interrupts if we are not polling, make sure
1362          * they are off otherwise.
1363          */
1364         if (ifp->if_capenable & IFCAP_POLLING)
1365                 em_disable_intr(adapter);
1366         else
1367 #endif /* DEVICE_POLLING */
1368                 em_enable_intr(adapter);
1369
1370         /* AMT based hardware can now take control from firmware */
1371         if (adapter->has_manage && adapter->has_amt)
1372                 em_get_hw_control(adapter);
1373 }
1374
1375 static void
1376 em_init(void *arg)
1377 {
1378         struct adapter *adapter = arg;
1379
1380         EM_CORE_LOCK(adapter);
1381         em_init_locked(adapter);
1382         EM_CORE_UNLOCK(adapter);
1383 }
1384
1385
1386 #ifdef DEVICE_POLLING
1387 /*********************************************************************
1388  *
1389  *  Legacy polling routine: note this only works with single queue
1390  *
1391  *********************************************************************/
1392 static int
1393 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1394 {
1395         struct adapter *adapter = ifp->if_softc;
1396         struct tx_ring  *txr = adapter->tx_rings;
1397         struct rx_ring  *rxr = adapter->rx_rings;
1398         u32             reg_icr;
1399         int             rx_done;
1400
1401         EM_CORE_LOCK(adapter);
1402         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1403                 EM_CORE_UNLOCK(adapter);
1404                 return (0);
1405         }
1406
1407         if (cmd == POLL_AND_CHECK_STATUS) {
1408                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1409                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1410                         callout_stop(&adapter->timer);
1411                         adapter->hw.mac.get_link_status = 1;
1412                         em_update_link_status(adapter);
1413                         callout_reset(&adapter->timer, hz,
1414                             em_local_timer, adapter);
1415                 }
1416         }
1417         EM_CORE_UNLOCK(adapter);
1418
1419         em_rxeof(rxr, count, &rx_done);
1420
1421         EM_TX_LOCK(txr);
1422         em_txeof(txr);
1423 #ifdef EM_MULTIQUEUE
1424         if (!drbr_empty(ifp, txr->br))
1425                 em_mq_start_locked(ifp, txr, NULL);
1426 #else
1427         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1428                 em_start_locked(ifp, txr);
1429 #endif
1430         EM_TX_UNLOCK(txr);
1431
1432         return (rx_done);
1433 }
1434 #endif /* DEVICE_POLLING */
1435
1436
1437 /*********************************************************************
1438  *
1439  *  Fast Legacy/MSI Combined Interrupt Service routine
1440  *
1441  *********************************************************************/
1442 static int
1443 em_irq_fast(void *arg)
1444 {
1445         struct adapter  *adapter = arg;
1446         struct ifnet    *ifp;
1447         u32             reg_icr;
1448
1449         ifp = adapter->ifp;
1450
1451         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1452
1453         /* Hot eject?  */
1454         if (reg_icr == 0xffffffff)
1455                 return FILTER_STRAY;
1456
1457         /* Definitely not our interrupt.  */
1458         if (reg_icr == 0x0)
1459                 return FILTER_STRAY;
1460
1461         /*
1462          * Starting with the 82571 chip, bit 31 should be used to
1463          * determine whether the interrupt belongs to us.
1464          */
1465         if (adapter->hw.mac.type >= e1000_82571 &&
1466             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1467                 return FILTER_STRAY;
1468
1469         em_disable_intr(adapter);
1470         taskqueue_enqueue(adapter->tq, &adapter->que_task);
1471
1472         /* Link status change */
1473         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1474                 adapter->hw.mac.get_link_status = 1;
1475                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1476         }
1477
1478         if (reg_icr & E1000_ICR_RXO)
1479                 adapter->rx_overruns++;
1480         return FILTER_HANDLED;
1481 }
1482
1483 /* Combined RX/TX handler, used by Legacy and MSI */
1484 static void
1485 em_handle_que(void *context, int pending)
1486 {
1487         struct adapter  *adapter = context;
1488         struct ifnet    *ifp = adapter->ifp;
1489         struct tx_ring  *txr = adapter->tx_rings;
1490         struct rx_ring  *rxr = adapter->rx_rings;
1491
1492
1493         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1494                 bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1495                 EM_TX_LOCK(txr);
1496                 em_txeof(txr);
1497 #ifdef EM_MULTIQUEUE
1498                 if (!drbr_empty(ifp, txr->br))
1499                         em_mq_start_locked(ifp, txr, NULL);
1500 #else
1501                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1502                         em_start_locked(ifp, txr);
1503 #endif
1504                 EM_TX_UNLOCK(txr);
1505                 if (more) {
1506                         taskqueue_enqueue(adapter->tq, &adapter->que_task);
1507                         return;
1508                 }
1509         }
1510
1511         em_enable_intr(adapter);
1512         return;
1513 }
1514
1515
1516 /*********************************************************************
1517  *
1518  *  MSIX Interrupt Service Routines
1519  *
1520  **********************************************************************/
1521 static void
1522 em_msix_tx(void *arg)
1523 {
1524         struct tx_ring *txr = arg;
1525         struct adapter *adapter = txr->adapter;
1526         struct ifnet    *ifp = adapter->ifp;
1527
1528         ++txr->tx_irq;
1529         EM_TX_LOCK(txr);
1530         em_txeof(txr);
1531 #ifdef EM_MULTIQUEUE
1532         if (!drbr_empty(ifp, txr->br))
1533                 em_mq_start_locked(ifp, txr, NULL);
1534 #else
1535         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1536                 em_start_locked(ifp, txr);
1537 #endif
1538         /* Reenable this interrupt */
1539         E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1540         EM_TX_UNLOCK(txr);
1541         return;
1542 }
1543
1544 /*********************************************************************
1545  *
1546  *  MSIX RX Interrupt Service routine
1547  *
1548  **********************************************************************/
1549
1550 static void
1551 em_msix_rx(void *arg)
1552 {
1553         struct rx_ring  *rxr = arg;
1554         struct adapter  *adapter = rxr->adapter;
1555         bool            more;
1556
1557         ++rxr->rx_irq;
1558         more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1559         if (more)
1560                 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1561         else
1562                 /* Reenable this interrupt */
1563                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1564         return;
1565 }
1566
1567 /*********************************************************************
1568  *
1569  *  MSIX Link Fast Interrupt Service routine
1570  *
1571  **********************************************************************/
1572 static void
1573 em_msix_link(void *arg)
1574 {
1575         struct adapter  *adapter = arg;
1576         u32             reg_icr;
1577
1578         ++adapter->link_irq;
1579         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1580
1581         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1582                 adapter->hw.mac.get_link_status = 1;
1583                 em_handle_link(adapter, 0);
1584         } else
1585                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1586                     EM_MSIX_LINK | E1000_IMS_LSC);
1587         return;
1588 }
1589
1590 static void
1591 em_handle_rx(void *context, int pending)
1592 {
1593         struct rx_ring  *rxr = context;
1594         struct adapter  *adapter = rxr->adapter;
1595         bool            more;
1596
1597         more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1598         if (more)
1599                 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1600         else
1601                 /* Reenable this interrupt */
1602                 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1603 }
1604
1605 static void
1606 em_handle_tx(void *context, int pending)
1607 {
1608         struct tx_ring  *txr = context;
1609         struct adapter  *adapter = txr->adapter;
1610         struct ifnet    *ifp = adapter->ifp;
1611
1612         EM_TX_LOCK(txr);
1613         em_txeof(txr);
1614 #ifdef EM_MULTIQUEUE
1615         if (!drbr_empty(ifp, txr->br))
1616                 em_mq_start_locked(ifp, txr, NULL);
1617 #else
1618         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1619                 em_start_locked(ifp, txr);
1620 #endif
1621         E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1622         EM_TX_UNLOCK(txr);
1623 }
1624
1625 static void
1626 em_handle_link(void *context, int pending)
1627 {
1628         struct adapter  *adapter = context;
1629         struct tx_ring  *txr = adapter->tx_rings;
1630         struct ifnet *ifp = adapter->ifp;
1631
1632         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1633                 return;
1634
1635         EM_CORE_LOCK(adapter);
1636         callout_stop(&adapter->timer);
1637         em_update_link_status(adapter);
1638         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1639         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1640             EM_MSIX_LINK | E1000_IMS_LSC);
1641         if (adapter->link_active) {
1642                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1643                         EM_TX_LOCK(txr);
1644 #ifdef EM_MULTIQUEUE
1645                         if (!drbr_empty(ifp, txr->br))
1646                                 em_mq_start_locked(ifp, txr, NULL);
1647 #else
1648                         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1649                                 em_start_locked(ifp, txr);
1650 #endif
1651                         EM_TX_UNLOCK(txr);
1652                 }
1653         }
1654         EM_CORE_UNLOCK(adapter);
1655 }
1656
1657
1658 /*********************************************************************
1659  *
1660  *  Media Ioctl callback
1661  *
1662  *  This routine is called whenever the user queries the status of
1663  *  the interface using ifconfig.
1664  *
1665  **********************************************************************/
1666 static void
1667 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1668 {
1669         struct adapter *adapter = ifp->if_softc;
1670         u_char fiber_type = IFM_1000_SX;
1671
1672         INIT_DEBUGOUT("em_media_status: begin");
1673
1674         EM_CORE_LOCK(adapter);
1675         em_update_link_status(adapter);
1676
1677         ifmr->ifm_status = IFM_AVALID;
1678         ifmr->ifm_active = IFM_ETHER;
1679
1680         if (!adapter->link_active) {
1681                 EM_CORE_UNLOCK(adapter);
1682                 return;
1683         }
1684
1685         ifmr->ifm_status |= IFM_ACTIVE;
1686
1687         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1688             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1689                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1690         } else {
1691                 switch (adapter->link_speed) {
1692                 case 10:
1693                         ifmr->ifm_active |= IFM_10_T;
1694                         break;
1695                 case 100:
1696                         ifmr->ifm_active |= IFM_100_TX;
1697                         break;
1698                 case 1000:
1699                         ifmr->ifm_active |= IFM_1000_T;
1700                         break;
1701                 }
1702                 if (adapter->link_duplex == FULL_DUPLEX)
1703                         ifmr->ifm_active |= IFM_FDX;
1704                 else
1705                         ifmr->ifm_active |= IFM_HDX;
1706         }
1707         EM_CORE_UNLOCK(adapter);
1708 }
1709
1710 /*********************************************************************
1711  *
1712  *  Media Ioctl callback
1713  *
1714  *  This routine is called when the user changes speed/duplex using
1715  *  media/mediopt option with ifconfig.
1716  *
1717  **********************************************************************/
1718 static int
1719 em_media_change(struct ifnet *ifp)
1720 {
1721         struct adapter *adapter = ifp->if_softc;
1722         struct ifmedia  *ifm = &adapter->media;
1723
1724         INIT_DEBUGOUT("em_media_change: begin");
1725
1726         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1727                 return (EINVAL);
1728
1729         EM_CORE_LOCK(adapter);
1730         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1731         case IFM_AUTO:
1732                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1733                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1734                 break;
1735         case IFM_1000_LX:
1736         case IFM_1000_SX:
1737         case IFM_1000_T:
1738                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1739                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1740                 break;
1741         case IFM_100_TX:
1742                 adapter->hw.mac.autoneg = FALSE;
1743                 adapter->hw.phy.autoneg_advertised = 0;
1744                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1745                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1746                 else
1747                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1748                 break;
1749         case IFM_10_T:
1750                 adapter->hw.mac.autoneg = FALSE;
1751                 adapter->hw.phy.autoneg_advertised = 0;
1752                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1753                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1754                 else
1755                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1756                 break;
1757         default:
1758                 device_printf(adapter->dev, "Unsupported media type\n");
1759         }
1760
1761         em_init_locked(adapter);
1762         EM_CORE_UNLOCK(adapter);
1763
1764         return (0);
1765 }
1766
1767 /*********************************************************************
1768  *
1769  *  This routine maps the mbufs to tx descriptors.
1770  *
1771  *  return 0 on success, positive on failure
1772  **********************************************************************/
1773
1774 static int
1775 em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1776 {
1777         struct adapter          *adapter = txr->adapter;
1778         bus_dma_segment_t       segs[EM_MAX_SCATTER];
1779         bus_dmamap_t            map;
1780         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
1781         struct e1000_tx_desc    *ctxd = NULL;
1782         struct mbuf             *m_head;
1783         struct ether_header     *eh;
1784         struct ip               *ip = NULL;
1785         struct tcphdr           *tp = NULL;
1786         u32                     txd_upper, txd_lower, txd_used, txd_saved;
1787         int                     ip_off, poff;
1788         int                     nsegs, i, j, first, last = 0;
1789         int                     error, do_tso, tso_desc = 0, remap = 1;
1790
1791 retry:
1792         m_head = *m_headp;
1793         txd_upper = txd_lower = txd_used = txd_saved = 0;
1794         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1795         ip_off = poff = 0;
1796
1797         /*
1798          * Intel recommends entire IP/TCP header length reside in a single
1799          * buffer. If multiple descriptors are used to describe the IP and
1800          * TCP header, each descriptor should describe one or more
1801          * complete headers; descriptors referencing only parts of headers
1802          * are not supported. If all layer headers are not coalesced into
1803          * a single buffer, each buffer should not cross a 4KB boundary,
1804          * or be larger than the maximum read request size.
1805          * Controller also requires modifing IP/TCP header to make TSO work
1806          * so we firstly get a writable mbuf chain then coalesce ethernet/
1807          * IP/TCP header into a single buffer to meet the requirement of
1808          * controller. This also simplifies IP/TCP/UDP checksum offloading
1809          * which also has similiar restrictions.
1810          */
1811         if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
1812                 if (do_tso || (m_head->m_next != NULL &&
1813                     m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) {
1814                         if (M_WRITABLE(*m_headp) == 0) {
1815                                 m_head = m_dup(*m_headp, M_DONTWAIT);
1816                                 m_freem(*m_headp);
1817                                 if (m_head == NULL) {
1818                                         *m_headp = NULL;
1819                                         return (ENOBUFS);
1820                                 }
1821                                 *m_headp = m_head;
1822                         }
1823                 }
1824                 /*
1825                  * XXX
1826                  * Assume IPv4, we don't have TSO/checksum offload support
1827                  * for IPv6 yet.
1828                  */
1829                 ip_off = sizeof(struct ether_header);
1830                 m_head = m_pullup(m_head, ip_off);
1831                 if (m_head == NULL) {
1832                         *m_headp = NULL;
1833                         return (ENOBUFS);
1834                 }
1835                 eh = mtod(m_head, struct ether_header *);
1836                 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1837                         ip_off = sizeof(struct ether_vlan_header);
1838                         m_head = m_pullup(m_head, ip_off);
1839                         if (m_head == NULL) {
1840                                 *m_headp = NULL;
1841                                 return (ENOBUFS);
1842                         }
1843                 }
1844                 m_head = m_pullup(m_head, ip_off + sizeof(struct ip));
1845                 if (m_head == NULL) {
1846                         *m_headp = NULL;
1847                         return (ENOBUFS);
1848                 }
1849                 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1850                 poff = ip_off + (ip->ip_hl << 2);
1851                 if (do_tso) {
1852                         m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1853                         if (m_head == NULL) {
1854                                 *m_headp = NULL;
1855                                 return (ENOBUFS);
1856                         }
1857                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1858                         /*
1859                          * TSO workaround:
1860                          *   pull 4 more bytes of data into it.
1861                          */
1862                         m_head = m_pullup(m_head, poff + (tp->th_off << 2) + 4);
1863                         if (m_head == NULL) {
1864                                 *m_headp = NULL;
1865                                 return (ENOBUFS);
1866                         }
1867                         ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1868                         ip->ip_len = 0;
1869                         ip->ip_sum = 0;
1870                         /*
1871                          * The pseudo TCP checksum does not include TCP payload
1872                          * length so driver should recompute the checksum here
1873                          * what hardware expect to see. This is adherence of
1874                          * Microsoft's Large Send specification.
1875                          */
1876                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1877                         tp->th_sum = in_pseudo(ip->ip_src.s_addr,
1878                             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
1879                 } else if (m_head->m_pkthdr.csum_flags & CSUM_TCP) {
1880                         m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1881                         if (m_head == NULL) {
1882                                 *m_headp = NULL;
1883                                 return (ENOBUFS);
1884                         }
1885                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1886                         m_head = m_pullup(m_head, poff + (tp->th_off << 2));
1887                         if (m_head == NULL) {
1888                                 *m_headp = NULL;
1889                                 return (ENOBUFS);
1890                         }
1891                         ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1892                         tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1893                 } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) {
1894                         m_head = m_pullup(m_head, poff + sizeof(struct udphdr));
1895                         if (m_head == NULL) {
1896                                 *m_headp = NULL;
1897                                 return (ENOBUFS);
1898                         }
1899                         ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1900                 }
1901                 *m_headp = m_head;
1902         }
1903
1904         /*
1905          * Map the packet for DMA
1906          *
1907          * Capture the first descriptor index,
1908          * this descriptor will have the index
1909          * of the EOP which is the only one that
1910          * now gets a DONE bit writeback.
1911          */
1912         first = txr->next_avail_desc;
1913         tx_buffer = &txr->tx_buffers[first];
1914         tx_buffer_mapped = tx_buffer;
1915         map = tx_buffer->map;
1916
1917         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1918             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1919
1920         /*
1921          * There are two types of errors we can (try) to handle:
1922          * - EFBIG means the mbuf chain was too long and bus_dma ran
1923          *   out of segments.  Defragment the mbuf chain and try again.
1924          * - ENOMEM means bus_dma could not obtain enough bounce buffers
1925          *   at this point in time.  Defer sending and try again later.
1926          * All other errors, in particular EINVAL, are fatal and prevent the
1927          * mbuf chain from ever going through.  Drop it and report error.
1928          */
1929         if (error == EFBIG && remap) {
1930                 struct mbuf *m;
1931
1932                 m = m_defrag(*m_headp, M_DONTWAIT);
1933                 if (m == NULL) {
1934                         adapter->mbuf_alloc_failed++;
1935                         m_freem(*m_headp);
1936                         *m_headp = NULL;
1937                         return (ENOBUFS);
1938                 }
1939                 *m_headp = m;
1940
1941                 /* Try it again, but only once */
1942                 remap = 0;
1943                 goto retry;
1944         } else if (error == ENOMEM) {
1945                 adapter->no_tx_dma_setup++;
1946                 return (error);
1947         } else if (error != 0) {
1948                 adapter->no_tx_dma_setup++;
1949                 m_freem(*m_headp);
1950                 *m_headp = NULL;
1951                 return (error);
1952         }
1953
1954         /*
1955          * TSO Hardware workaround, if this packet is not
1956          * TSO, and is only a single descriptor long, and
1957          * it follows a TSO burst, then we need to add a
1958          * sentinel descriptor to prevent premature writeback.
1959          */
1960         if ((do_tso == 0) && (txr->tx_tso == TRUE)) {
1961                 if (nsegs == 1)
1962                         tso_desc = TRUE;
1963                 txr->tx_tso = FALSE;
1964         }
1965
1966         if (nsegs > (txr->tx_avail - 2)) {
1967                 txr->no_desc_avail++;
1968                 bus_dmamap_unload(txr->txtag, map);
1969                 return (ENOBUFS);
1970         }
1971         m_head = *m_headp;
1972
1973         /* Do hardware assists */
1974         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1975                 em_tso_setup(txr, m_head, ip_off, ip, tp,
1976                     &txd_upper, &txd_lower);
1977                 /* we need to make a final sentinel transmit desc */
1978                 tso_desc = TRUE;
1979         } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
1980                 em_transmit_checksum_setup(txr, m_head,
1981                     ip_off, ip, &txd_upper, &txd_lower);
1982
1983         if (m_head->m_flags & M_VLANTAG) {
1984                 /* Set the vlan id. */
1985                 txd_upper |=
1986                     (htole16(m_head->m_pkthdr.ether_vtag) << 16);
1987                 /* Tell hardware to add tag */
1988                 txd_lower |= htole32(E1000_TXD_CMD_VLE);
1989         }
1990
1991         i = txr->next_avail_desc;
1992
1993         /* Set up our transmit descriptors */
1994         for (j = 0; j < nsegs; j++) {
1995                 bus_size_t seg_len;
1996                 bus_addr_t seg_addr;
1997
1998                 tx_buffer = &txr->tx_buffers[i];
1999                 ctxd = &txr->tx_base[i];
2000                 seg_addr = segs[j].ds_addr;
2001                 seg_len  = segs[j].ds_len;
2002                 /*
2003                 ** TSO Workaround:
2004                 ** If this is the last descriptor, we want to
2005                 ** split it so we have a small final sentinel
2006                 */
2007                 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2008                         seg_len -= 4;
2009                         ctxd->buffer_addr = htole64(seg_addr);
2010                         ctxd->lower.data = htole32(
2011                         adapter->txd_cmd | txd_lower | seg_len);
2012                         ctxd->upper.data =
2013                             htole32(txd_upper);
2014                         if (++i == adapter->num_tx_desc)
2015                                 i = 0;
2016                         /* Now make the sentinel */
2017                         ++txd_used; /* using an extra txd */
2018                         ctxd = &txr->tx_base[i];
2019                         tx_buffer = &txr->tx_buffers[i];
2020                         ctxd->buffer_addr =
2021                             htole64(seg_addr + seg_len);
2022                         ctxd->lower.data = htole32(
2023                         adapter->txd_cmd | txd_lower | 4);
2024                         ctxd->upper.data =
2025                             htole32(txd_upper);
2026                         last = i;
2027                         if (++i == adapter->num_tx_desc)
2028                                 i = 0;
2029                 } else {
2030                         ctxd->buffer_addr = htole64(seg_addr);
2031                         ctxd->lower.data = htole32(
2032                         adapter->txd_cmd | txd_lower | seg_len);
2033                         ctxd->upper.data =
2034                             htole32(txd_upper);
2035                         last = i;
2036                         if (++i == adapter->num_tx_desc)
2037                                 i = 0;
2038                 }
2039                 tx_buffer->m_head = NULL;
2040                 tx_buffer->next_eop = -1;
2041         }
2042
2043         txr->next_avail_desc = i;
2044         txr->tx_avail -= nsegs;
2045         if (tso_desc) /* TSO used an extra for sentinel */
2046                 txr->tx_avail -= txd_used;
2047
2048         tx_buffer->m_head = m_head;
2049         /*
2050         ** Here we swap the map so the last descriptor,
2051         ** which gets the completion interrupt has the
2052         ** real map, and the first descriptor gets the
2053         ** unused map from this descriptor.
2054         */
2055         tx_buffer_mapped->map = tx_buffer->map;
2056         tx_buffer->map = map;
2057         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
2058
2059         /*
2060          * Last Descriptor of Packet
2061          * needs End Of Packet (EOP)
2062          * and Report Status (RS)
2063          */
2064         ctxd->lower.data |=
2065             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2066         /*
2067          * Keep track in the first buffer which
2068          * descriptor will be written back
2069          */
2070         tx_buffer = &txr->tx_buffers[first];
2071         tx_buffer->next_eop = last;
2072         /* Update the watchdog time early and often */
2073         txr->watchdog_time = ticks;
2074
2075         /*
2076          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2077          * that this frame is available to transmit.
2078          */
2079         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2080             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2081         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
2082
2083         return (0);
2084 }
2085
2086 static void
2087 em_set_promisc(struct adapter *adapter)
2088 {
2089         struct ifnet    *ifp = adapter->ifp;
2090         u32             reg_rctl;
2091
2092         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2093
2094         if (ifp->if_flags & IFF_PROMISC) {
2095                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2096                 /* Turn this on if you want to see bad packets */
2097                 if (em_debug_sbp)
2098                         reg_rctl |= E1000_RCTL_SBP;
2099                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2100         } else if (ifp->if_flags & IFF_ALLMULTI) {
2101                 reg_rctl |= E1000_RCTL_MPE;
2102                 reg_rctl &= ~E1000_RCTL_UPE;
2103                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2104         }
2105 }
2106
2107 static void
2108 em_disable_promisc(struct adapter *adapter)
2109 {
2110         u32     reg_rctl;
2111
2112         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2113
2114         reg_rctl &=  (~E1000_RCTL_UPE);
2115         reg_rctl &=  (~E1000_RCTL_MPE);
2116         reg_rctl &=  (~E1000_RCTL_SBP);
2117         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2118 }
2119
2120
2121 /*********************************************************************
2122  *  Multicast Update
2123  *
2124  *  This routine is called whenever multicast address list is updated.
2125  *
2126  **********************************************************************/
2127
2128 static void
2129 em_set_multi(struct adapter *adapter)
2130 {
2131         struct ifnet    *ifp = adapter->ifp;
2132         struct ifmultiaddr *ifma;
2133         u32 reg_rctl = 0;
2134         u8  *mta; /* Multicast array memory */
2135         int mcnt = 0;
2136
2137         IOCTL_DEBUGOUT("em_set_multi: begin");
2138
2139         mta = adapter->mta;
2140         bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
2141
2142         if (adapter->hw.mac.type == e1000_82542 &&
2143             adapter->hw.revision_id == E1000_REVISION_2) {
2144                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2145                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2146                         e1000_pci_clear_mwi(&adapter->hw);
2147                 reg_rctl |= E1000_RCTL_RST;
2148                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2149                 msec_delay(5);
2150         }
2151
2152 #if __FreeBSD_version < 800000
2153         IF_ADDR_LOCK(ifp);
2154 #else
2155         if_maddr_rlock(ifp);
2156 #endif
2157         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2158                 if (ifma->ifma_addr->sa_family != AF_LINK)
2159                         continue;
2160
2161                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2162                         break;
2163
2164                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2165                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2166                 mcnt++;
2167         }
2168 #if __FreeBSD_version < 800000
2169         IF_ADDR_UNLOCK(ifp);
2170 #else
2171         if_maddr_runlock(ifp);
2172 #endif
2173         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2174                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2175                 reg_rctl |= E1000_RCTL_MPE;
2176                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2177         } else
2178                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2179
2180         if (adapter->hw.mac.type == e1000_82542 &&
2181             adapter->hw.revision_id == E1000_REVISION_2) {
2182                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2183                 reg_rctl &= ~E1000_RCTL_RST;
2184                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2185                 msec_delay(5);
2186                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2187                         e1000_pci_set_mwi(&adapter->hw);
2188         }
2189 }
2190
2191
2192 /*********************************************************************
2193  *  Timer routine
2194  *
2195  *  This routine checks for link status and updates statistics.
2196  *
2197  **********************************************************************/
2198
2199 static void
2200 em_local_timer(void *arg)
2201 {
2202         struct adapter  *adapter = arg;
2203         struct ifnet    *ifp = adapter->ifp;
2204         struct tx_ring  *txr = adapter->tx_rings;
2205         struct rx_ring  *rxr = adapter->rx_rings;
2206         u32             trigger;
2207
2208         EM_CORE_LOCK_ASSERT(adapter);
2209
2210         em_update_link_status(adapter);
2211         em_update_stats_counters(adapter);
2212
2213         /* Reset LAA into RAR[0] on 82571 */
2214         if ((adapter->hw.mac.type == e1000_82571) &&
2215             e1000_get_laa_state_82571(&adapter->hw))
2216                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2217
2218         /* Mask to use in the irq trigger */
2219         if (adapter->msix_mem)
2220                 trigger = rxr->ims; /* RX for 82574 */
2221         else
2222                 trigger = E1000_ICS_RXDMT0;
2223
2224         /*
2225         ** Check on the state of the TX queue(s), this
2226         ** can be done without the lock because its RO
2227         ** and the HUNG state will be static if set.
2228         */
2229         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2230                 if ((txr->queue_status == EM_QUEUE_HUNG) &&
2231                     (adapter->pause_frames == 0))
2232                         goto hung;
2233                 /* Schedule a TX tasklet if needed */
2234                 if (txr->tx_avail <= EM_MAX_SCATTER)
2235                         taskqueue_enqueue(txr->tq, &txr->tx_task);
2236         }
2237
2238         adapter->pause_frames = 0;
2239         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2240 #ifndef DEVICE_POLLING
2241         /* Trigger an RX interrupt to guarantee mbuf refresh */
2242         E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger);
2243 #endif
2244         return;
2245 hung:
2246         /* Looks like we're hung */
2247         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2248         device_printf(adapter->dev,
2249             "Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2250             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2251             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2252         device_printf(adapter->dev,"TX(%d) desc avail = %d,"
2253             "Next TX to Clean = %d\n",
2254             txr->me, txr->tx_avail, txr->next_to_clean);
2255         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2256         adapter->watchdog_events++;
2257         adapter->pause_frames = 0;
2258         em_init_locked(adapter);
2259 }
2260
2261
2262 static void
2263 em_update_link_status(struct adapter *adapter)
2264 {
2265         struct e1000_hw *hw = &adapter->hw;
2266         struct ifnet *ifp = adapter->ifp;
2267         device_t dev = adapter->dev;
2268         struct tx_ring *txr = adapter->tx_rings;
2269         u32 link_check = 0;
2270
2271         /* Get the cached link value or read phy for real */
2272         switch (hw->phy.media_type) {
2273         case e1000_media_type_copper:
2274                 if (hw->mac.get_link_status) {
2275                         /* Do the work to read phy */
2276                         e1000_check_for_link(hw);
2277                         link_check = !hw->mac.get_link_status;
2278                         if (link_check) /* ESB2 fix */
2279                                 e1000_cfg_on_link_up(hw);
2280                 } else
2281                         link_check = TRUE;
2282                 break;
2283         case e1000_media_type_fiber:
2284                 e1000_check_for_link(hw);
2285                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2286                                  E1000_STATUS_LU);
2287                 break;
2288         case e1000_media_type_internal_serdes:
2289                 e1000_check_for_link(hw);
2290                 link_check = adapter->hw.mac.serdes_has_link;
2291                 break;
2292         default:
2293         case e1000_media_type_unknown:
2294                 break;
2295         }
2296
2297         /* Now check for a transition */
2298         if (link_check && (adapter->link_active == 0)) {
2299                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2300                     &adapter->link_duplex);
2301                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2302                 if ((adapter->link_speed != SPEED_1000) &&
2303                     ((hw->mac.type == e1000_82571) ||
2304                     (hw->mac.type == e1000_82572))) {
2305                         int tarc0;
2306                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2307                         tarc0 &= ~SPEED_MODE_BIT;
2308                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2309                 }
2310                 if (bootverbose)
2311                         device_printf(dev, "Link is up %d Mbps %s\n",
2312                             adapter->link_speed,
2313                             ((adapter->link_duplex == FULL_DUPLEX) ?
2314                             "Full Duplex" : "Half Duplex"));
2315                 adapter->link_active = 1;
2316                 adapter->smartspeed = 0;
2317                 ifp->if_baudrate = adapter->link_speed * 1000000;
2318                 if_link_state_change(ifp, LINK_STATE_UP);
2319         } else if (!link_check && (adapter->link_active == 1)) {
2320                 ifp->if_baudrate = adapter->link_speed = 0;
2321                 adapter->link_duplex = 0;
2322                 if (bootverbose)
2323                         device_printf(dev, "Link is Down\n");
2324                 adapter->link_active = 0;
2325                 /* Link down, disable watchdog */
2326                 for (int i = 0; i < adapter->num_queues; i++, txr++)
2327                         txr->queue_status = EM_QUEUE_IDLE;
2328                 if_link_state_change(ifp, LINK_STATE_DOWN);
2329         }
2330 }
2331
2332 /*********************************************************************
2333  *
2334  *  This routine disables all traffic on the adapter by issuing a
2335  *  global reset on the MAC and deallocates TX/RX buffers.
2336  *
2337  *  This routine should always be called with BOTH the CORE
2338  *  and TX locks.
2339  **********************************************************************/
2340
2341 static void
2342 em_stop(void *arg)
2343 {
2344         struct adapter  *adapter = arg;
2345         struct ifnet    *ifp = adapter->ifp;
2346         struct tx_ring  *txr = adapter->tx_rings;
2347
2348         EM_CORE_LOCK_ASSERT(adapter);
2349
2350         INIT_DEBUGOUT("em_stop: begin");
2351
2352         em_disable_intr(adapter);
2353         callout_stop(&adapter->timer);
2354
2355         /* Tell the stack that the interface is no longer active */
2356         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2357         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2358
2359         /* Unarm watchdog timer. */
2360         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2361                 EM_TX_LOCK(txr);
2362                 txr->queue_status = EM_QUEUE_IDLE;
2363                 EM_TX_UNLOCK(txr);
2364         }
2365
2366         e1000_reset_hw(&adapter->hw);
2367         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2368
2369         e1000_led_off(&adapter->hw);
2370         e1000_cleanup_led(&adapter->hw);
2371 }
2372
2373
2374 /*********************************************************************
2375  *
2376  *  Determine hardware revision.
2377  *
2378  **********************************************************************/
2379 static void
2380 em_identify_hardware(struct adapter *adapter)
2381 {
2382         device_t dev = adapter->dev;
2383
2384         /* Make sure our PCI config space has the necessary stuff set */
2385         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2386         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2387             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2388                 device_printf(dev, "Memory Access and/or Bus Master bits "
2389                     "were not set!\n");
2390                 adapter->hw.bus.pci_cmd_word |=
2391                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2392                 pci_write_config(dev, PCIR_COMMAND,
2393                     adapter->hw.bus.pci_cmd_word, 2);
2394         }
2395
2396         /* Save off the information about this board */
2397         adapter->hw.vendor_id = pci_get_vendor(dev);
2398         adapter->hw.device_id = pci_get_device(dev);
2399         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2400         adapter->hw.subsystem_vendor_id =
2401             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2402         adapter->hw.subsystem_device_id =
2403             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2404
2405         /* Do Shared Code Init and Setup */
2406         if (e1000_set_mac_type(&adapter->hw)) {
2407                 device_printf(dev, "Setup init failure\n");
2408                 return;
2409         }
2410 }
2411
2412 static int
2413 em_allocate_pci_resources(struct adapter *adapter)
2414 {
2415         device_t        dev = adapter->dev;
2416         int             rid;
2417
2418         rid = PCIR_BAR(0);
2419         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2420             &rid, RF_ACTIVE);
2421         if (adapter->memory == NULL) {
2422                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2423                 return (ENXIO);
2424         }
2425         adapter->osdep.mem_bus_space_tag =
2426             rman_get_bustag(adapter->memory);
2427         adapter->osdep.mem_bus_space_handle =
2428             rman_get_bushandle(adapter->memory);
2429         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2430
2431         /* Default to a single queue */
2432         adapter->num_queues = 1;
2433
2434         /*
2435          * Setup MSI/X or MSI if PCI Express
2436          */
2437         adapter->msix = em_setup_msix(adapter);
2438
2439         adapter->hw.back = &adapter->osdep;
2440
2441         return (0);
2442 }
2443
2444 /*********************************************************************
2445  *
2446  *  Setup the Legacy or MSI Interrupt handler
2447  *
2448  **********************************************************************/
2449 int
2450 em_allocate_legacy(struct adapter *adapter)
2451 {
2452         device_t dev = adapter->dev;
2453         struct tx_ring  *txr = adapter->tx_rings;
2454         int error, rid = 0;
2455
2456         /* Manually turn off all interrupts */
2457         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2458
2459         if (adapter->msix == 1) /* using MSI */
2460                 rid = 1;
2461         /* We allocate a single interrupt resource */
2462         adapter->res = bus_alloc_resource_any(dev,
2463             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2464         if (adapter->res == NULL) {
2465                 device_printf(dev, "Unable to allocate bus resource: "
2466                     "interrupt\n");
2467                 return (ENXIO);
2468         }
2469
2470         /*
2471          * Allocate a fast interrupt and the associated
2472          * deferred processing contexts.
2473          */
2474         TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter);
2475         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2476             taskqueue_thread_enqueue, &adapter->tq);
2477         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que",
2478             device_get_nameunit(adapter->dev));
2479         /* Use a TX only tasklet for local timer */
2480         TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2481         txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2482             taskqueue_thread_enqueue, &txr->tq);
2483         taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2484             device_get_nameunit(adapter->dev));
2485         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2486         if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET,
2487             em_irq_fast, NULL, adapter, &adapter->tag)) != 0) {
2488                 device_printf(dev, "Failed to register fast interrupt "
2489                             "handler: %d\n", error);
2490                 taskqueue_free(adapter->tq);
2491                 adapter->tq = NULL;
2492                 return (error);
2493         }
2494
2495         return (0);
2496 }
2497
2498 /*********************************************************************
2499  *
2500  *  Setup the MSIX Interrupt handlers
2501  *   This is not really Multiqueue, rather
2502  *   its just seperate interrupt vectors
2503  *   for TX, RX, and Link.
2504  *
2505  **********************************************************************/
2506 int
2507 em_allocate_msix(struct adapter *adapter)
2508 {
2509         device_t        dev = adapter->dev;
2510         struct          tx_ring *txr = adapter->tx_rings;
2511         struct          rx_ring *rxr = adapter->rx_rings;
2512         int             error, rid, vector = 0;
2513
2514
2515         /* Make sure all interrupts are disabled */
2516         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2517
2518         /* First set up ring resources */
2519         for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
2520
2521                 /* RX ring */
2522                 rid = vector + 1;
2523
2524                 rxr->res = bus_alloc_resource_any(dev,
2525                     SYS_RES_IRQ, &rid, RF_ACTIVE);
2526                 if (rxr->res == NULL) {
2527                         device_printf(dev,
2528                             "Unable to allocate bus resource: "
2529                             "RX MSIX Interrupt %d\n", i);
2530                         return (ENXIO);
2531                 }
2532                 if ((error = bus_setup_intr(dev, rxr->res,
2533                     INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx,
2534                     rxr, &rxr->tag)) != 0) {
2535                         device_printf(dev, "Failed to register RX handler");
2536                         return (error);
2537                 }
2538 #if __FreeBSD_version >= 800504
2539                 bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i);
2540 #endif
2541                 rxr->msix = vector++; /* NOTE increment vector for TX */
2542                 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr);
2543                 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT,
2544                     taskqueue_thread_enqueue, &rxr->tq);
2545                 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq",
2546                     device_get_nameunit(adapter->dev));
2547                 /*
2548                 ** Set the bit to enable interrupt
2549                 ** in E1000_IMS -- bits 20 and 21
2550                 ** are for RX0 and RX1, note this has
2551                 ** NOTHING to do with the MSIX vector
2552                 */
2553                 rxr->ims = 1 << (20 + i);
2554                 adapter->ivars |= (8 | rxr->msix) << (i * 4);
2555
2556                 /* TX ring */
2557                 rid = vector + 1;
2558                 txr->res = bus_alloc_resource_any(dev,
2559                     SYS_RES_IRQ, &rid, RF_ACTIVE);
2560                 if (txr->res == NULL) {
2561                         device_printf(dev,
2562                             "Unable to allocate bus resource: "
2563                             "TX MSIX Interrupt %d\n", i);
2564                         return (ENXIO);
2565                 }
2566                 if ((error = bus_setup_intr(dev, txr->res,
2567                     INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx,
2568                     txr, &txr->tag)) != 0) {
2569                         device_printf(dev, "Failed to register TX handler");
2570                         return (error);
2571                 }
2572 #if __FreeBSD_version >= 800504
2573                 bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i);
2574 #endif
2575                 txr->msix = vector++; /* Increment vector for next pass */
2576                 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2577                 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2578                     taskqueue_thread_enqueue, &txr->tq);
2579                 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2580                     device_get_nameunit(adapter->dev));
2581                 /*
2582                 ** Set the bit to enable interrupt
2583                 ** in E1000_IMS -- bits 22 and 23
2584                 ** are for TX0 and TX1, note this has
2585                 ** NOTHING to do with the MSIX vector
2586                 */
2587                 txr->ims = 1 << (22 + i);
2588                 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4));
2589         }
2590
2591         /* Link interrupt */
2592         ++rid;
2593         adapter->res = bus_alloc_resource_any(dev,
2594             SYS_RES_IRQ, &rid, RF_ACTIVE);
2595         if (!adapter->res) {
2596                 device_printf(dev,"Unable to allocate "
2597                     "bus resource: Link interrupt [%d]\n", rid);
2598                 return (ENXIO);
2599         }
2600         /* Set the link handler function */
2601         error = bus_setup_intr(dev, adapter->res,
2602             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2603             em_msix_link, adapter, &adapter->tag);
2604         if (error) {
2605                 adapter->res = NULL;
2606                 device_printf(dev, "Failed to register LINK handler");
2607                 return (error);
2608         }
2609 #if __FreeBSD_version >= 800504
2610                 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2611 #endif
2612         adapter->linkvec = vector;
2613         adapter->ivars |=  (8 | vector) << 16;
2614         adapter->ivars |= 0x80000000;
2615
2616         return (0);
2617 }
2618
2619
2620 static void
2621 em_free_pci_resources(struct adapter *adapter)
2622 {
2623         device_t        dev = adapter->dev;
2624         struct tx_ring  *txr;
2625         struct rx_ring  *rxr;
2626         int             rid;
2627
2628
2629         /*
2630         ** Release all the queue interrupt resources:
2631         */
2632         for (int i = 0; i < adapter->num_queues; i++) {
2633                 txr = &adapter->tx_rings[i];
2634                 rxr = &adapter->rx_rings[i];
2635                 /* an early abort? */
2636                 if ((txr == NULL) || (rxr == NULL))
2637                         break;
2638                 rid = txr->msix +1;
2639                 if (txr->tag != NULL) {
2640                         bus_teardown_intr(dev, txr->res, txr->tag);
2641                         txr->tag = NULL;
2642                 }
2643                 if (txr->res != NULL)
2644                         bus_release_resource(dev, SYS_RES_IRQ,
2645                             rid, txr->res);
2646                 rid = rxr->msix +1;
2647                 if (rxr->tag != NULL) {
2648                         bus_teardown_intr(dev, rxr->res, rxr->tag);
2649                         rxr->tag = NULL;
2650                 }
2651                 if (rxr->res != NULL)
2652                         bus_release_resource(dev, SYS_RES_IRQ,
2653                             rid, rxr->res);
2654         }
2655
2656         if (adapter->linkvec) /* we are doing MSIX */
2657                 rid = adapter->linkvec + 1;
2658         else
2659                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2660
2661         if (adapter->tag != NULL) {
2662                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2663                 adapter->tag = NULL;
2664         }
2665
2666         if (adapter->res != NULL)
2667                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2668
2669
2670         if (adapter->msix)
2671                 pci_release_msi(dev);
2672
2673         if (adapter->msix_mem != NULL)
2674                 bus_release_resource(dev, SYS_RES_MEMORY,
2675                     PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2676
2677         if (adapter->memory != NULL)
2678                 bus_release_resource(dev, SYS_RES_MEMORY,
2679                     PCIR_BAR(0), adapter->memory);
2680
2681         if (adapter->flash != NULL)
2682                 bus_release_resource(dev, SYS_RES_MEMORY,
2683                     EM_FLASH, adapter->flash);
2684 }
2685
2686 /*
2687  * Setup MSI or MSI/X
2688  */
2689 static int
2690 em_setup_msix(struct adapter *adapter)
2691 {
2692         device_t dev = adapter->dev;
2693         int val = 0;
2694
2695         /*
2696         ** Setup MSI/X for Hartwell: tests have shown
2697         ** use of two queues to be unstable, and to
2698         ** provide no great gain anyway, so we simply
2699         ** seperate the interrupts and use a single queue.
2700         */
2701         if ((adapter->hw.mac.type == e1000_82574) &&
2702             (em_enable_msix == TRUE)) {
2703                 /* Map the MSIX BAR */
2704                 int rid = PCIR_BAR(EM_MSIX_BAR);
2705                 adapter->msix_mem = bus_alloc_resource_any(dev,
2706                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2707                 if (!adapter->msix_mem) {
2708                         /* May not be enabled */
2709                         device_printf(adapter->dev,
2710                             "Unable to map MSIX table \n");
2711                         goto msi;
2712                 }
2713                 val = pci_msix_count(dev);
2714                 /* We only need 3 vectors */
2715                 if (val > 3)
2716                         val = 3;
2717                 if ((val != 3) && (val != 5)) {
2718                         bus_release_resource(dev, SYS_RES_MEMORY,
2719                             PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2720                         adapter->msix_mem = NULL;
2721                         device_printf(adapter->dev,
2722                             "MSIX: incorrect vectors, using MSI\n");
2723                         goto msi;
2724                 }
2725
2726                 if (pci_alloc_msix(dev, &val) == 0) {
2727                         device_printf(adapter->dev,
2728                             "Using MSIX interrupts "
2729                             "with %d vectors\n", val);
2730                 }
2731
2732                 return (val);
2733         }
2734 msi:
2735         val = pci_msi_count(dev);
2736         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2737                 adapter->msix = 1;
2738                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2739                 return (val);
2740         }
2741         /* Should only happen due to manual configuration */
2742         device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n");
2743         return (0);
2744 }
2745
2746
2747 /*********************************************************************
2748  *
2749  *  Initialize the hardware to a configuration
2750  *  as specified by the adapter structure.
2751  *
2752  **********************************************************************/
2753 static void
2754 em_reset(struct adapter *adapter)
2755 {
2756         device_t        dev = adapter->dev;
2757         struct ifnet    *ifp = adapter->ifp;
2758         struct e1000_hw *hw = &adapter->hw;
2759         u16             rx_buffer_size;
2760         u32             pba;
2761
2762         INIT_DEBUGOUT("em_reset: begin");
2763
2764         /* Set up smart power down as default off on newer adapters. */
2765         if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
2766             hw->mac.type == e1000_82572)) {
2767                 u16 phy_tmp = 0;
2768
2769                 /* Speed up time to link by disabling smart power down. */
2770                 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2771                 phy_tmp &= ~IGP02E1000_PM_SPD;
2772                 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2773         }
2774
2775         /*
2776          * Packet Buffer Allocation (PBA)
2777          * Writing PBA sets the receive portion of the buffer
2778          * the remainder is used for the transmit buffer.
2779          */
2780         switch (hw->mac.type) {
2781         /* Total Packet Buffer on these is 48K */
2782         case e1000_82571:
2783         case e1000_82572:
2784         case e1000_80003es2lan:
2785                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
2786                 break;
2787         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
2788                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
2789                 break;
2790         case e1000_82574:
2791         case e1000_82583:
2792                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
2793                 break;
2794         case e1000_ich8lan:
2795                 pba = E1000_PBA_8K;
2796                 break;
2797         case e1000_ich9lan:
2798         case e1000_ich10lan:
2799                 /* Boost Receive side for jumbo frames */
2800                 if (adapter->max_frame_size > 4096)
2801                         pba = E1000_PBA_14K;
2802                 else
2803                         pba = E1000_PBA_10K;
2804                 break;
2805         case e1000_pchlan:
2806         case e1000_pch2lan:
2807                 pba = E1000_PBA_26K;
2808                 break;
2809         default:
2810                 if (adapter->max_frame_size > 8192)
2811                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
2812                 else
2813                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
2814         }
2815         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
2816
2817         /*
2818          * These parameters control the automatic generation (Tx) and
2819          * response (Rx) to Ethernet PAUSE frames.
2820          * - High water mark should allow for at least two frames to be
2821          *   received after sending an XOFF.
2822          * - Low water mark works best when it is very near the high water mark.
2823          *   This allows the receiver to restart by sending XON when it has
2824          *   drained a bit. Here we use an arbitary value of 1500 which will
2825          *   restart after one full frame is pulled from the buffer. There
2826          *   could be several smaller frames in the buffer and if so they will
2827          *   not trigger the XON until their total number reduces the buffer
2828          *   by 1500.
2829          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2830          */
2831         rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 );
2832         hw->fc.high_water = rx_buffer_size -
2833             roundup2(adapter->max_frame_size, 1024);
2834         hw->fc.low_water = hw->fc.high_water - 1500;
2835
2836         if (adapter->fc) /* locally set flow control value? */
2837                 hw->fc.requested_mode = adapter->fc;
2838         else
2839                 hw->fc.requested_mode = e1000_fc_full;
2840
2841         if (hw->mac.type == e1000_80003es2lan)
2842                 hw->fc.pause_time = 0xFFFF;
2843         else
2844                 hw->fc.pause_time = EM_FC_PAUSE_TIME;
2845
2846         hw->fc.send_xon = TRUE;
2847
2848         /* Device specific overrides/settings */
2849         switch (hw->mac.type) {
2850         case e1000_pchlan:
2851                 /* Workaround: no TX flow ctrl for PCH */
2852                 hw->fc.requested_mode = e1000_fc_rx_pause;
2853                 hw->fc.pause_time = 0xFFFF; /* override */
2854                 if (ifp->if_mtu > ETHERMTU) {
2855                         hw->fc.high_water = 0x3500;
2856                         hw->fc.low_water = 0x1500;
2857                 } else {
2858                         hw->fc.high_water = 0x5000;
2859                         hw->fc.low_water = 0x3000;
2860                 }
2861                 hw->fc.refresh_time = 0x1000;
2862                 break;
2863         case e1000_pch2lan:
2864                 hw->fc.high_water = 0x5C20;
2865                 hw->fc.low_water = 0x5048;
2866                 hw->fc.pause_time = 0x0650;
2867                 hw->fc.refresh_time = 0x0400;
2868                 /* Jumbos need adjusted PBA */
2869                 if (ifp->if_mtu > ETHERMTU)
2870                         E1000_WRITE_REG(hw, E1000_PBA, 12);
2871                 else
2872                         E1000_WRITE_REG(hw, E1000_PBA, 26);
2873                 break;
2874         case e1000_ich9lan:
2875         case e1000_ich10lan:
2876                 if (ifp->if_mtu > ETHERMTU) {
2877                         hw->fc.high_water = 0x2800;
2878                         hw->fc.low_water = hw->fc.high_water - 8;
2879                         break;
2880                 }
2881                 /* else fall thru */
2882         default:
2883                 if (hw->mac.type == e1000_80003es2lan)
2884                         hw->fc.pause_time = 0xFFFF;
2885                 break;
2886         }
2887
2888         /* Issue a global reset */
2889         e1000_reset_hw(hw);
2890         E1000_WRITE_REG(hw, E1000_WUC, 0);
2891         em_disable_aspm(adapter);
2892         /* and a re-init */
2893         if (e1000_init_hw(hw) < 0) {
2894                 device_printf(dev, "Hardware Initialization Failed\n");
2895                 return;
2896         }
2897
2898         E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
2899         e1000_get_phy_info(hw);
2900         e1000_check_for_link(hw);
2901         return;
2902 }
2903
2904 /*********************************************************************
2905  *
2906  *  Setup networking device structure and register an interface.
2907  *
2908  **********************************************************************/
2909 static int
2910 em_setup_interface(device_t dev, struct adapter *adapter)
2911 {
2912         struct ifnet   *ifp;
2913
2914         INIT_DEBUGOUT("em_setup_interface: begin");
2915
2916         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2917         if (ifp == NULL) {
2918                 device_printf(dev, "can not allocate ifnet structure\n");
2919                 return (-1);
2920         }
2921         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2922         ifp->if_init =  em_init;
2923         ifp->if_softc = adapter;
2924         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2925         ifp->if_ioctl = em_ioctl;
2926 #ifdef EM_MULTIQUEUE
2927         /* Multiqueue stack interface */
2928         ifp->if_transmit = em_mq_start;
2929         ifp->if_qflush = em_qflush;
2930 #else
2931         ifp->if_start = em_start;
2932         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2933         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2934         IFQ_SET_READY(&ifp->if_snd);
2935 #endif
2936
2937         ether_ifattach(ifp, adapter->hw.mac.addr);
2938
2939         ifp->if_capabilities = ifp->if_capenable = 0;
2940
2941
2942         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2943         ifp->if_capabilities |= IFCAP_TSO4;
2944         /*
2945          * Tell the upper layer(s) we
2946          * support full VLAN capability
2947          */
2948         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2949         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2950                              |  IFCAP_VLAN_HWTSO
2951                              |  IFCAP_VLAN_MTU;
2952         ifp->if_capenable = ifp->if_capabilities;
2953
2954         /*
2955         ** Don't turn this on by default, if vlans are
2956         ** created on another pseudo device (eg. lagg)
2957         ** then vlan events are not passed thru, breaking
2958         ** operation, but with HW FILTER off it works. If
2959         ** using vlans directly on the em driver you can
2960         ** enable this and get full hardware tag filtering.
2961         */
2962         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2963
2964 #ifdef DEVICE_POLLING
2965         ifp->if_capabilities |= IFCAP_POLLING;
2966 #endif
2967
2968         /* Enable only WOL MAGIC by default */
2969         if (adapter->wol) {
2970                 ifp->if_capabilities |= IFCAP_WOL;
2971                 ifp->if_capenable |= IFCAP_WOL_MAGIC;
2972         }
2973
2974         /*
2975          * Specify the media types supported by this adapter and register
2976          * callbacks to update media and link information
2977          */
2978         ifmedia_init(&adapter->media, IFM_IMASK,
2979             em_media_change, em_media_status);
2980         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2981             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2982                 u_char fiber_type = IFM_1000_SX;        /* default type */
2983
2984                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2985                             0, NULL);
2986                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2987         } else {
2988                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2989                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2990                             0, NULL);
2991                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2992                             0, NULL);
2993                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2994                             0, NULL);
2995                 if (adapter->hw.phy.type != e1000_phy_ife) {
2996                         ifmedia_add(&adapter->media,
2997                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2998                         ifmedia_add(&adapter->media,
2999                                 IFM_ETHER | IFM_1000_T, 0, NULL);
3000                 }
3001         }
3002         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3003         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3004         return (0);
3005 }
3006
3007
3008 /*
3009  * Manage DMA'able memory.
3010  */
3011 static void
3012 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3013 {
3014         if (error)
3015                 return;
3016         *(bus_addr_t *) arg = segs[0].ds_addr;
3017 }
3018
3019 static int
3020 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3021         struct em_dma_alloc *dma, int mapflags)
3022 {
3023         int error;
3024
3025         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3026                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
3027                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3028                                 BUS_SPACE_MAXADDR,      /* highaddr */
3029                                 NULL, NULL,             /* filter, filterarg */
3030                                 size,                   /* maxsize */
3031                                 1,                      /* nsegments */
3032                                 size,                   /* maxsegsize */
3033                                 0,                      /* flags */
3034                                 NULL,                   /* lockfunc */
3035                                 NULL,                   /* lockarg */
3036                                 &dma->dma_tag);
3037         if (error) {
3038                 device_printf(adapter->dev,
3039                     "%s: bus_dma_tag_create failed: %d\n",
3040                     __func__, error);
3041                 goto fail_0;
3042         }
3043
3044         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3045             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3046         if (error) {
3047                 device_printf(adapter->dev,
3048                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3049                     __func__, (uintmax_t)size, error);
3050                 goto fail_2;
3051         }
3052
3053         dma->dma_paddr = 0;
3054         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3055             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3056         if (error || dma->dma_paddr == 0) {
3057                 device_printf(adapter->dev,
3058                     "%s: bus_dmamap_load failed: %d\n",
3059                     __func__, error);
3060                 goto fail_3;
3061         }
3062
3063         return (0);
3064
3065 fail_3:
3066         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3067 fail_2:
3068         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3069         bus_dma_tag_destroy(dma->dma_tag);
3070 fail_0:
3071         dma->dma_map = NULL;
3072         dma->dma_tag = NULL;
3073
3074         return (error);
3075 }
3076
3077 static void
3078 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3079 {
3080         if (dma->dma_tag == NULL)
3081                 return;
3082         if (dma->dma_map != NULL) {
3083                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3084                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3085                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3086                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3087                 dma->dma_map = NULL;
3088         }
3089         bus_dma_tag_destroy(dma->dma_tag);
3090         dma->dma_tag = NULL;
3091 }
3092
3093
3094 /*********************************************************************
3095  *
3096  *  Allocate memory for the transmit and receive rings, and then
3097  *  the descriptors associated with each, called only once at attach.
3098  *
3099  **********************************************************************/
3100 static int
3101 em_allocate_queues(struct adapter *adapter)
3102 {
3103         device_t                dev = adapter->dev;
3104         struct tx_ring          *txr = NULL;
3105         struct rx_ring          *rxr = NULL;
3106         int rsize, tsize, error = E1000_SUCCESS;
3107         int txconf = 0, rxconf = 0;
3108
3109
3110         /* Allocate the TX ring struct memory */
3111         if (!(adapter->tx_rings =
3112             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3113             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3114                 device_printf(dev, "Unable to allocate TX ring memory\n");
3115                 error = ENOMEM;
3116                 goto fail;
3117         }
3118
3119         /* Now allocate the RX */
3120         if (!(adapter->rx_rings =
3121             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3122             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3123                 device_printf(dev, "Unable to allocate RX ring memory\n");
3124                 error = ENOMEM;
3125                 goto rx_fail;
3126         }
3127
3128         tsize = roundup2(adapter->num_tx_desc *
3129             sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
3130         /*
3131          * Now set up the TX queues, txconf is needed to handle the
3132          * possibility that things fail midcourse and we need to
3133          * undo memory gracefully
3134          */
3135         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3136                 /* Set up some basics */
3137                 txr = &adapter->tx_rings[i];
3138                 txr->adapter = adapter;
3139                 txr->me = i;
3140
3141                 /* Initialize the TX lock */
3142                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3143                     device_get_nameunit(dev), txr->me);
3144                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3145
3146                 if (em_dma_malloc(adapter, tsize,
3147                         &txr->txdma, BUS_DMA_NOWAIT)) {
3148                         device_printf(dev,
3149                             "Unable to allocate TX Descriptor memory\n");
3150                         error = ENOMEM;
3151                         goto err_tx_desc;
3152                 }
3153                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
3154                 bzero((void *)txr->tx_base, tsize);
3155
3156                 if (em_allocate_transmit_buffers(txr)) {
3157                         device_printf(dev,
3158                             "Critical Failure setting up transmit buffers\n");
3159                         error = ENOMEM;
3160                         goto err_tx_desc;
3161                 }
3162 #if __FreeBSD_version >= 800000
3163                 /* Allocate a buf ring */
3164                 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3165                     M_WAITOK, &txr->tx_mtx);
3166 #endif
3167         }
3168
3169         /*
3170          * Next the RX queues...
3171          */
3172         rsize = roundup2(adapter->num_rx_desc *
3173             sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
3174         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3175                 rxr = &adapter->rx_rings[i];
3176                 rxr->adapter = adapter;
3177                 rxr->me = i;
3178
3179                 /* Initialize the RX lock */
3180                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3181                     device_get_nameunit(dev), txr->me);
3182                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3183
3184                 if (em_dma_malloc(adapter, rsize,
3185                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
3186                         device_printf(dev,
3187                             "Unable to allocate RxDescriptor memory\n");
3188                         error = ENOMEM;
3189                         goto err_rx_desc;
3190                 }
3191                 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr;
3192                 bzero((void *)rxr->rx_base, rsize);
3193
3194                 /* Allocate receive buffers for the ring*/
3195                 if (em_allocate_receive_buffers(rxr)) {
3196                         device_printf(dev,
3197                             "Critical Failure setting up receive buffers\n");
3198                         error = ENOMEM;
3199                         goto err_rx_desc;
3200                 }
3201         }
3202
3203         return (0);
3204
3205 err_rx_desc:
3206         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3207                 em_dma_free(adapter, &rxr->rxdma);
3208 err_tx_desc:
3209         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3210                 em_dma_free(adapter, &txr->txdma);
3211         free(adapter->rx_rings, M_DEVBUF);
3212 rx_fail:
3213 #if __FreeBSD_version >= 800000
3214         buf_ring_free(txr->br, M_DEVBUF);
3215 #endif
3216         free(adapter->tx_rings, M_DEVBUF);
3217 fail:
3218         return (error);
3219 }
3220
3221
3222 /*********************************************************************
3223  *
3224  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3225  *  the information needed to transmit a packet on the wire. This is
3226  *  called only once at attach, setup is done every reset.
3227  *
3228  **********************************************************************/
3229 static int
3230 em_allocate_transmit_buffers(struct tx_ring *txr)
3231 {
3232         struct adapter *adapter = txr->adapter;
3233         device_t dev = adapter->dev;
3234         struct em_buffer *txbuf;
3235         int error, i;
3236
3237         /*
3238          * Setup DMA descriptor areas.
3239          */
3240         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3241                                1, 0,                    /* alignment, bounds */
3242                                BUS_SPACE_MAXADDR,       /* lowaddr */
3243                                BUS_SPACE_MAXADDR,       /* highaddr */
3244                                NULL, NULL,              /* filter, filterarg */
3245                                EM_TSO_SIZE,             /* maxsize */
3246                                EM_MAX_SCATTER,          /* nsegments */
3247                                PAGE_SIZE,               /* maxsegsize */
3248                                0,                       /* flags */
3249                                NULL,                    /* lockfunc */
3250                                NULL,                    /* lockfuncarg */
3251                                &txr->txtag))) {
3252                 device_printf(dev,"Unable to allocate TX DMA tag\n");
3253                 goto fail;
3254         }
3255
3256         if (!(txr->tx_buffers =
3257             (struct em_buffer *) malloc(sizeof(struct em_buffer) *
3258             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3259                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3260                 error = ENOMEM;
3261                 goto fail;
3262         }
3263
3264         /* Create the descriptor buffer dma maps */
3265         txbuf = txr->tx_buffers;
3266         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3267                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3268                 if (error != 0) {
3269                         device_printf(dev, "Unable to create TX DMA map\n");
3270                         goto fail;
3271                 }
3272         }
3273
3274         return 0;
3275 fail:
3276         /* We free all, it handles case where we are in the middle */
3277         em_free_transmit_structures(adapter);
3278         return (error);
3279 }
3280
3281 /*********************************************************************
3282  *
3283  *  Initialize a transmit ring.
3284  *
3285  **********************************************************************/
3286 static void
3287 em_setup_transmit_ring(struct tx_ring *txr)
3288 {
3289         struct adapter *adapter = txr->adapter;
3290         struct em_buffer *txbuf;
3291         int i;
3292
3293         /* Clear the old descriptor contents */
3294         EM_TX_LOCK(txr);
3295         bzero((void *)txr->tx_base,
3296               (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3297         /* Reset indices */
3298         txr->next_avail_desc = 0;
3299         txr->next_to_clean = 0;
3300
3301         /* Free any existing tx buffers. */
3302         txbuf = txr->tx_buffers;
3303         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3304                 if (txbuf->m_head != NULL) {
3305                         bus_dmamap_sync(txr->txtag, txbuf->map,
3306                             BUS_DMASYNC_POSTWRITE);
3307                         bus_dmamap_unload(txr->txtag, txbuf->map);
3308                         m_freem(txbuf->m_head);
3309                         txbuf->m_head = NULL;
3310                 }
3311                 /* clear the watch index */
3312                 txbuf->next_eop = -1;
3313         }
3314
3315         /* Set number of descriptors available */
3316         txr->tx_avail = adapter->num_tx_desc;
3317         txr->queue_status = EM_QUEUE_IDLE;
3318
3319         /* Clear checksum offload context. */
3320         txr->last_hw_offload = 0;
3321         txr->last_hw_ipcss = 0;
3322         txr->last_hw_ipcso = 0;
3323         txr->last_hw_tucss = 0;
3324         txr->last_hw_tucso = 0;
3325
3326         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3327             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3328         EM_TX_UNLOCK(txr);
3329 }
3330
3331 /*********************************************************************
3332  *
3333  *  Initialize all transmit rings.
3334  *
3335  **********************************************************************/
3336 static void
3337 em_setup_transmit_structures(struct adapter *adapter)
3338 {
3339         struct tx_ring *txr = adapter->tx_rings;
3340
3341         for (int i = 0; i < adapter->num_queues; i++, txr++)
3342                 em_setup_transmit_ring(txr);
3343
3344         return;
3345 }
3346
3347 /*********************************************************************
3348  *
3349  *  Enable transmit unit.
3350  *
3351  **********************************************************************/
3352 static void
3353 em_initialize_transmit_unit(struct adapter *adapter)
3354 {
3355         struct tx_ring  *txr = adapter->tx_rings;
3356         struct e1000_hw *hw = &adapter->hw;
3357         u32     tctl, tarc, tipg = 0;
3358
3359          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3360
3361         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3362                 u64 bus_addr = txr->txdma.dma_paddr;
3363                 /* Base and Len of TX Ring */
3364                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3365                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3366                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3367                     (u32)(bus_addr >> 32));
3368                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3369                     (u32)bus_addr);
3370                 /* Init the HEAD/TAIL indices */
3371                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3372                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3373
3374                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3375                     E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
3376                     E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
3377
3378                 txr->queue_status = EM_QUEUE_IDLE;
3379         }
3380
3381         /* Set the default values for the Tx Inter Packet Gap timer */
3382         switch (adapter->hw.mac.type) {
3383         case e1000_80003es2lan:
3384                 tipg = DEFAULT_82543_TIPG_IPGR1;
3385                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3386                     E1000_TIPG_IPGR2_SHIFT;
3387                 break;
3388         default:
3389                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3390                     (adapter->hw.phy.media_type ==
3391                     e1000_media_type_internal_serdes))
3392                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3393                 else
3394                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3395                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3396                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3397         }
3398
3399         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3400         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3401
3402         if(adapter->hw.mac.type >= e1000_82540)
3403                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3404                     adapter->tx_abs_int_delay.value);
3405
3406         if ((adapter->hw.mac.type == e1000_82571) ||
3407             (adapter->hw.mac.type == e1000_82572)) {
3408                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3409                 tarc |= SPEED_MODE_BIT;
3410                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3411         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3412                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3413                 tarc |= 1;
3414                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3415                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3416                 tarc |= 1;
3417                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3418         }
3419
3420         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3421         if (adapter->tx_int_delay.value > 0)
3422                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3423
3424         /* Program the Transmit Control Register */
3425         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3426         tctl &= ~E1000_TCTL_CT;
3427         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3428                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3429
3430         if (adapter->hw.mac.type >= e1000_82571)
3431                 tctl |= E1000_TCTL_MULR;
3432
3433         /* This write will effectively turn on the transmit unit. */
3434         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3435
3436 }
3437
3438
3439 /*********************************************************************
3440  *
3441  *  Free all transmit rings.
3442  *
3443  **********************************************************************/
3444 static void
3445 em_free_transmit_structures(struct adapter *adapter)
3446 {
3447         struct tx_ring *txr = adapter->tx_rings;
3448
3449         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3450                 EM_TX_LOCK(txr);
3451                 em_free_transmit_buffers(txr);
3452                 em_dma_free(adapter, &txr->txdma);
3453                 EM_TX_UNLOCK(txr);
3454                 EM_TX_LOCK_DESTROY(txr);
3455         }
3456
3457         free(adapter->tx_rings, M_DEVBUF);
3458 }
3459
3460 /*********************************************************************
3461  *
3462  *  Free transmit ring related data structures.
3463  *
3464  **********************************************************************/
3465 static void
3466 em_free_transmit_buffers(struct tx_ring *txr)
3467 {
3468         struct adapter          *adapter = txr->adapter;
3469         struct em_buffer        *txbuf;
3470
3471         INIT_DEBUGOUT("free_transmit_ring: begin");
3472
3473         if (txr->tx_buffers == NULL)
3474                 return;
3475
3476         for (int i = 0; i < adapter->num_tx_desc; i++) {
3477                 txbuf = &txr->tx_buffers[i];
3478                 if (txbuf->m_head != NULL) {
3479                         bus_dmamap_sync(txr->txtag, txbuf->map,
3480                             BUS_DMASYNC_POSTWRITE);
3481                         bus_dmamap_unload(txr->txtag,
3482                             txbuf->map);
3483                         m_freem(txbuf->m_head);
3484                         txbuf->m_head = NULL;
3485                         if (txbuf->map != NULL) {
3486                                 bus_dmamap_destroy(txr->txtag,
3487                                     txbuf->map);
3488                                 txbuf->map = NULL;
3489                         }
3490                 } else if (txbuf->map != NULL) {
3491                         bus_dmamap_unload(txr->txtag,
3492                             txbuf->map);
3493                         bus_dmamap_destroy(txr->txtag,
3494                             txbuf->map);
3495                         txbuf->map = NULL;
3496                 }
3497         }
3498 #if __FreeBSD_version >= 800000
3499         if (txr->br != NULL)
3500                 buf_ring_free(txr->br, M_DEVBUF);
3501 #endif
3502         if (txr->tx_buffers != NULL) {
3503                 free(txr->tx_buffers, M_DEVBUF);
3504                 txr->tx_buffers = NULL;
3505         }
3506         if (txr->txtag != NULL) {
3507                 bus_dma_tag_destroy(txr->txtag);
3508                 txr->txtag = NULL;
3509         }
3510         return;
3511 }
3512
3513
3514 /*********************************************************************
3515  *  The offload context is protocol specific (TCP/UDP) and thus
3516  *  only needs to be set when the protocol changes. The occasion
3517  *  of a context change can be a performance detriment, and
3518  *  might be better just disabled. The reason arises in the way
3519  *  in which the controller supports pipelined requests from the
3520  *  Tx data DMA. Up to four requests can be pipelined, and they may
3521  *  belong to the same packet or to multiple packets. However all
3522  *  requests for one packet are issued before a request is issued
3523  *  for a subsequent packet and if a request for the next packet
3524  *  requires a context change, that request will be stalled
3525  *  until the previous request completes. This means setting up
3526  *  a new context effectively disables pipelined Tx data DMA which
3527  *  in turn greatly slow down performance to send small sized
3528  *  frames.
3529  **********************************************************************/
3530 static void
3531 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3532     struct ip *ip, u32 *txd_upper, u32 *txd_lower)
3533 {
3534         struct adapter                  *adapter = txr->adapter;
3535         struct e1000_context_desc       *TXD = NULL;
3536         struct em_buffer                *tx_buffer;
3537         int                             cur, hdr_len;
3538         u32                             cmd = 0;
3539         u16                             offload = 0;
3540         u8                              ipcso, ipcss, tucso, tucss;
3541
3542         ipcss = ipcso = tucss = tucso = 0;
3543         hdr_len = ip_off + (ip->ip_hl << 2);
3544         cur = txr->next_avail_desc;
3545
3546         /* Setup of IP header checksum. */
3547         if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3548                 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3549                 offload |= CSUM_IP;
3550                 ipcss = ip_off;
3551                 ipcso = ip_off + offsetof(struct ip, ip_sum);
3552                 /*
3553                  * Start offset for header checksum calculation.
3554                  * End offset for header checksum calculation.
3555                  * Offset of place to put the checksum.
3556                  */
3557                 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3558                 TXD->lower_setup.ip_fields.ipcss = ipcss;
3559                 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
3560                 TXD->lower_setup.ip_fields.ipcso = ipcso;
3561                 cmd |= E1000_TXD_CMD_IP;
3562         }
3563
3564         if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3565                 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3566                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3567                 offload |= CSUM_TCP;
3568                 tucss = hdr_len;
3569                 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
3570                 /*
3571                  * Setting up new checksum offload context for every frames
3572                  * takes a lot of processing time for hardware. This also
3573                  * reduces performance a lot for small sized frames so avoid
3574                  * it if driver can use previously configured checksum
3575                  * offload context.
3576                  */
3577                 if (txr->last_hw_offload == offload) {
3578                         if (offload & CSUM_IP) {
3579                                 if (txr->last_hw_ipcss == ipcss &&
3580                                     txr->last_hw_ipcso == ipcso &&
3581                                     txr->last_hw_tucss == tucss &&
3582                                     txr->last_hw_tucso == tucso)
3583                                         return;
3584                         } else {
3585                                 if (txr->last_hw_tucss == tucss &&
3586                                     txr->last_hw_tucso == tucso)
3587                                         return;
3588                         }
3589                 }
3590                 txr->last_hw_offload = offload;
3591                 txr->last_hw_tucss = tucss;
3592                 txr->last_hw_tucso = tucso;
3593                 /*
3594                  * Start offset for payload checksum calculation.
3595                  * End offset for payload checksum calculation.
3596                  * Offset of place to put the checksum.
3597                  */
3598                 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3599                 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3600                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3601                 TXD->upper_setup.tcp_fields.tucso = tucso;
3602                 cmd |= E1000_TXD_CMD_TCP;
3603         } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3604                 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3605                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3606                 tucss = hdr_len;
3607                 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
3608                 /*
3609                  * Setting up new checksum offload context for every frames
3610                  * takes a lot of processing time for hardware. This also
3611                  * reduces performance a lot for small sized frames so avoid
3612                  * it if driver can use previously configured checksum
3613                  * offload context.
3614                  */
3615                 if (txr->last_hw_offload == offload) {
3616                         if (offload & CSUM_IP) {
3617                                 if (txr->last_hw_ipcss == ipcss &&
3618                                     txr->last_hw_ipcso == ipcso &&
3619                                     txr->last_hw_tucss == tucss &&
3620                                     txr->last_hw_tucso == tucso)
3621                                         return;
3622                         } else {
3623                                 if (txr->last_hw_tucss == tucss &&
3624                                     txr->last_hw_tucso == tucso)
3625                                         return;
3626                         }
3627                 }
3628                 txr->last_hw_offload = offload;
3629                 txr->last_hw_tucss = tucss;
3630                 txr->last_hw_tucso = tucso;
3631                 /*
3632                  * Start offset for header checksum calculation.
3633                  * End offset for header checksum calculation.
3634                  * Offset of place to put the checksum.
3635                  */
3636                 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3637                 TXD->upper_setup.tcp_fields.tucss = tucss;
3638                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3639                 TXD->upper_setup.tcp_fields.tucso = tucso;
3640         }
3641
3642         if (offload & CSUM_IP) {
3643                 txr->last_hw_ipcss = ipcss;
3644                 txr->last_hw_ipcso = ipcso;
3645         }
3646
3647         TXD->tcp_seg_setup.data = htole32(0);
3648         TXD->cmd_and_length =
3649             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3650         tx_buffer = &txr->tx_buffers[cur];
3651         tx_buffer->m_head = NULL;
3652         tx_buffer->next_eop = -1;
3653
3654         if (++cur == adapter->num_tx_desc)
3655                 cur = 0;
3656
3657         txr->tx_avail--;
3658         txr->next_avail_desc = cur;
3659 }
3660
3661
3662 /**********************************************************************
3663  *
3664  *  Setup work for hardware segmentation offload (TSO)
3665  *
3666  **********************************************************************/
3667 static void
3668 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3669     struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower)
3670 {
3671         struct adapter                  *adapter = txr->adapter;
3672         struct e1000_context_desc       *TXD;
3673         struct em_buffer                *tx_buffer;
3674         int cur, hdr_len;
3675
3676         /*
3677          * In theory we can use the same TSO context if and only if
3678          * frame is the same type(IP/TCP) and the same MSS. However
3679          * checking whether a frame has the same IP/TCP structure is
3680          * hard thing so just ignore that and always restablish a
3681          * new TSO context.
3682          */
3683         hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2);
3684         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3685                       E1000_TXD_DTYP_D |        /* Data descr type */
3686                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3687
3688         /* IP and/or TCP header checksum calculation and insertion. */
3689         *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
3690
3691         cur = txr->next_avail_desc;
3692         tx_buffer = &txr->tx_buffers[cur];
3693         TXD = (struct e1000_context_desc *) &txr->tx_base[cur];
3694
3695         /*
3696          * Start offset for header checksum calculation.
3697          * End offset for header checksum calculation.
3698          * Offset of place put the checksum.
3699          */
3700         TXD->lower_setup.ip_fields.ipcss = ip_off;
3701         TXD->lower_setup.ip_fields.ipcse =
3702             htole16(ip_off + (ip->ip_hl << 2) - 1);
3703         TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum);
3704         /*
3705          * Start offset for payload checksum calculation.
3706          * End offset for payload checksum calculation.
3707          * Offset of place to put the checksum.
3708          */
3709         TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2);
3710         TXD->upper_setup.tcp_fields.tucse = 0;
3711         TXD->upper_setup.tcp_fields.tucso =
3712             ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum);
3713         /*
3714          * Payload size per packet w/o any headers.
3715          * Length of all headers up to payload.
3716          */
3717         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3718         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3719
3720         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3721                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3722                                 E1000_TXD_CMD_TSE |     /* TSE context */
3723                                 E1000_TXD_CMD_IP |      /* Do IP csum */
3724                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3725                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3726
3727         tx_buffer->m_head = NULL;
3728         tx_buffer->next_eop = -1;
3729
3730         if (++cur == adapter->num_tx_desc)
3731                 cur = 0;
3732
3733         txr->tx_avail--;
3734         txr->next_avail_desc = cur;
3735         txr->tx_tso = TRUE;
3736 }
3737
3738
3739 /**********************************************************************
3740  *
3741  *  Examine each tx_buffer in the used queue. If the hardware is done
3742  *  processing the packet then free associated resources. The
3743  *  tx_buffer is put back on the free queue.
3744  *
3745  **********************************************************************/
3746 static void
3747 em_txeof(struct tx_ring *txr)
3748 {
3749         struct adapter  *adapter = txr->adapter;
3750         int first, last, done, processed;
3751         struct em_buffer *tx_buffer;
3752         struct e1000_tx_desc   *tx_desc, *eop_desc;
3753         struct ifnet   *ifp = adapter->ifp;
3754
3755         EM_TX_LOCK_ASSERT(txr);
3756
3757         /* No work, make sure watchdog is off */
3758         if (txr->tx_avail == adapter->num_tx_desc) {
3759                 txr->queue_status = EM_QUEUE_IDLE;
3760                 return;
3761         }
3762
3763         processed = 0;
3764         first = txr->next_to_clean;
3765         tx_desc = &txr->tx_base[first];
3766         tx_buffer = &txr->tx_buffers[first];
3767         last = tx_buffer->next_eop;
3768         eop_desc = &txr->tx_base[last];
3769
3770         /*
3771          * What this does is get the index of the
3772          * first descriptor AFTER the EOP of the
3773          * first packet, that way we can do the
3774          * simple comparison on the inner while loop.
3775          */
3776         if (++last == adapter->num_tx_desc)
3777                 last = 0;
3778         done = last;
3779
3780         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3781             BUS_DMASYNC_POSTREAD);
3782
3783         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3784                 /* We clean the range of the packet */
3785                 while (first != done) {
3786                         tx_desc->upper.data = 0;
3787                         tx_desc->lower.data = 0;
3788                         tx_desc->buffer_addr = 0;
3789                         ++txr->tx_avail;
3790                         ++processed;
3791
3792                         if (tx_buffer->m_head) {
3793                                 bus_dmamap_sync(txr->txtag,
3794                                     tx_buffer->map,
3795                                     BUS_DMASYNC_POSTWRITE);
3796                                 bus_dmamap_unload(txr->txtag,
3797                                     tx_buffer->map);
3798                                 m_freem(tx_buffer->m_head);
3799                                 tx_buffer->m_head = NULL;
3800                         }
3801                         tx_buffer->next_eop = -1;
3802                         txr->watchdog_time = ticks;
3803
3804                         if (++first == adapter->num_tx_desc)
3805                                 first = 0;
3806
3807                         tx_buffer = &txr->tx_buffers[first];
3808                         tx_desc = &txr->tx_base[first];
3809                 }
3810                 ++ifp->if_opackets;
3811                 /* See if we can continue to the next packet */
3812                 last = tx_buffer->next_eop;
3813                 if (last != -1) {
3814                         eop_desc = &txr->tx_base[last];
3815                         /* Get new done point */
3816                         if (++last == adapter->num_tx_desc) last = 0;
3817                         done = last;
3818                 } else
3819                         break;
3820         }
3821         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3822             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3823
3824         txr->next_to_clean = first;
3825
3826         /*
3827         ** Watchdog calculation, we know there's
3828         ** work outstanding or the first return
3829         ** would have been taken, so none processed
3830         ** for too long indicates a hang. local timer
3831         ** will examine this and do a reset if needed.
3832         */
3833         if ((!processed) && ((ticks - txr->watchdog_time) > EM_WATCHDOG))
3834                 txr->queue_status = EM_QUEUE_HUNG;
3835
3836         /*
3837          * If we have a minimum free, clear IFF_DRV_OACTIVE
3838          * to tell the stack that it is OK to send packets.
3839          * Notice that all writes of OACTIVE happen under the
3840          * TX lock which, with a single queue, guarantees
3841          * sanity.
3842          */
3843         if (txr->tx_avail >= EM_MAX_SCATTER)
3844                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3845
3846         /* Disable watchdog if all clean */
3847         if (txr->tx_avail == adapter->num_tx_desc) {
3848                 txr->queue_status = EM_QUEUE_IDLE;
3849         }
3850 }
3851
3852
3853 /*********************************************************************
3854  *
3855  *  Refresh RX descriptor mbufs from system mbuf buffer pool.
3856  *
3857  **********************************************************************/
3858 static void
3859 em_refresh_mbufs(struct rx_ring *rxr, int limit)
3860 {
3861         struct adapter          *adapter = rxr->adapter;
3862         struct mbuf             *m;
3863         bus_dma_segment_t       segs[1];
3864         struct em_buffer        *rxbuf;
3865         int                     i, j, error, nsegs;
3866         bool                    cleaned = FALSE;
3867
3868         i = j = rxr->next_to_refresh;
3869         /*
3870         ** Get one descriptor beyond
3871         ** our work mark to control
3872         ** the loop.
3873         */
3874         if (++j == adapter->num_rx_desc)
3875                 j = 0;
3876
3877         while (j != limit) {
3878                 rxbuf = &rxr->rx_buffers[i];
3879                 if (rxbuf->m_head == NULL) {
3880                         m = m_getjcl(M_DONTWAIT, MT_DATA,
3881                             M_PKTHDR, adapter->rx_mbuf_sz);
3882                         /*
3883                         ** If we have a temporary resource shortage
3884                         ** that causes a failure, just abort refresh
3885                         ** for now, we will return to this point when
3886                         ** reinvoked from em_rxeof.
3887                         */
3888                         if (m == NULL)
3889                                 goto update;
3890                 } else
3891                         m = rxbuf->m_head;
3892
3893                 m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz;
3894                 m->m_flags |= M_PKTHDR;
3895                 m->m_data = m->m_ext.ext_buf;
3896
3897                 /* Use bus_dma machinery to setup the memory mapping  */
3898                 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map,
3899                     m, segs, &nsegs, BUS_DMA_NOWAIT);
3900                 if (error != 0) {
3901                         printf("Refresh mbufs: hdr dmamap load"
3902                             " failure - %d\n", error);
3903                         m_free(m);
3904                         rxbuf->m_head = NULL;
3905                         goto update;
3906                 }
3907                 rxbuf->m_head = m;
3908                 bus_dmamap_sync(rxr->rxtag,
3909                     rxbuf->map, BUS_DMASYNC_PREREAD);
3910                 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr);
3911                 cleaned = TRUE;
3912
3913                 i = j; /* Next is precalulated for us */
3914                 rxr->next_to_refresh = i;
3915                 /* Calculate next controlling index */
3916                 if (++j == adapter->num_rx_desc)
3917                         j = 0;
3918         }
3919 update:
3920         /*
3921         ** Update the tail pointer only if,
3922         ** and as far as we have refreshed.
3923         */
3924         if (cleaned)
3925                 E1000_WRITE_REG(&adapter->hw,
3926                     E1000_RDT(rxr->me), rxr->next_to_refresh);
3927
3928         return;
3929 }
3930
3931
3932 /*********************************************************************
3933  *
3934  *  Allocate memory for rx_buffer structures. Since we use one
3935  *  rx_buffer per received packet, the maximum number of rx_buffer's
3936  *  that we'll need is equal to the number of receive descriptors
3937  *  that we've allocated.
3938  *
3939  **********************************************************************/
3940 static int
3941 em_allocate_receive_buffers(struct rx_ring *rxr)
3942 {
3943         struct adapter          *adapter = rxr->adapter;
3944         device_t                dev = adapter->dev;
3945         struct em_buffer        *rxbuf;
3946         int                     error;
3947
3948         rxr->rx_buffers = malloc(sizeof(struct em_buffer) *
3949             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3950         if (rxr->rx_buffers == NULL) {
3951                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3952                 return (ENOMEM);
3953         }
3954
3955         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3956                                 1, 0,                   /* alignment, bounds */
3957                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3958                                 BUS_SPACE_MAXADDR,      /* highaddr */
3959                                 NULL, NULL,             /* filter, filterarg */
3960                                 MJUM9BYTES,             /* maxsize */
3961                                 1,                      /* nsegments */
3962                                 MJUM9BYTES,             /* maxsegsize */
3963                                 0,                      /* flags */
3964                                 NULL,                   /* lockfunc */
3965                                 NULL,                   /* lockarg */
3966                                 &rxr->rxtag);
3967         if (error) {
3968                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
3969                     __func__, error);
3970                 goto fail;
3971         }
3972
3973         rxbuf = rxr->rx_buffers;
3974         for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3975                 rxbuf = &rxr->rx_buffers[i];
3976                 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
3977                     &rxbuf->map);
3978                 if (error) {
3979                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3980                             __func__, error);
3981                         goto fail;
3982                 }
3983         }
3984
3985         return (0);
3986
3987 fail:
3988         em_free_receive_structures(adapter);
3989         return (error);
3990 }
3991
3992
3993 /*********************************************************************
3994  *
3995  *  Initialize a receive ring and its buffers.
3996  *
3997  **********************************************************************/
3998 static int
3999 em_setup_receive_ring(struct rx_ring *rxr)
4000 {
4001         struct  adapter         *adapter = rxr->adapter;
4002         struct em_buffer        *rxbuf;
4003         bus_dma_segment_t       seg[1];
4004         int                     rsize, nsegs, error;
4005
4006
4007         /* Clear the ring contents */
4008         EM_RX_LOCK(rxr);
4009         rsize = roundup2(adapter->num_rx_desc *
4010             sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
4011         bzero((void *)rxr->rx_base, rsize);
4012
4013         /*
4014         ** Free current RX buffer structs and their mbufs
4015         */
4016         for (int i = 0; i < adapter->num_rx_desc; i++) {
4017                 rxbuf = &rxr->rx_buffers[i];
4018                 if (rxbuf->m_head != NULL) {
4019                         bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4020                             BUS_DMASYNC_POSTREAD);
4021                         bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4022                         m_freem(rxbuf->m_head);
4023                         rxbuf->m_head = NULL; /* mark as freed */
4024                 }
4025         }
4026
4027         /* Now replenish the mbufs */
4028         for (int j = 0; j != adapter->num_rx_desc; ++j) {
4029                 rxbuf = &rxr->rx_buffers[j];
4030                 rxbuf->m_head = m_getjcl(M_DONTWAIT, MT_DATA,
4031                     M_PKTHDR, adapter->rx_mbuf_sz);
4032                 if (rxbuf->m_head == NULL) {
4033                         error = ENOBUFS;
4034                         goto fail;
4035                 }
4036                 rxbuf->m_head->m_len = adapter->rx_mbuf_sz;
4037                 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */
4038                 rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz;
4039
4040                 /* Get the memory mapping */
4041                 error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
4042                     rxbuf->map, rxbuf->m_head, seg,
4043                     &nsegs, BUS_DMA_NOWAIT);
4044                 if (error != 0) {
4045                         m_freem(rxbuf->m_head);
4046                         rxbuf->m_head = NULL;
4047                         goto fail;
4048                 }
4049                 bus_dmamap_sync(rxr->rxtag,
4050                     rxbuf->map, BUS_DMASYNC_PREREAD);
4051
4052                 /* Update descriptor */
4053                 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr);
4054         }
4055         rxr->next_to_check = 0;
4056         rxr->next_to_refresh = 0;
4057         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4058             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4059
4060 fail:
4061         EM_RX_UNLOCK(rxr);
4062         return (error);
4063 }
4064
4065 /*********************************************************************
4066  *
4067  *  Initialize all receive rings.
4068  *
4069  **********************************************************************/
4070 static int
4071 em_setup_receive_structures(struct adapter *adapter)
4072 {
4073         struct rx_ring *rxr = adapter->rx_rings;
4074         int q;
4075
4076         for (q = 0; q < adapter->num_queues; q++, rxr++)
4077                 if (em_setup_receive_ring(rxr))
4078                         goto fail;
4079
4080         return (0);
4081 fail:
4082         /*
4083          * Free RX buffers allocated so far, we will only handle
4084          * the rings that completed, the failing case will have
4085          * cleaned up for itself. 'q' failed, so its the terminus.
4086          */
4087         for (int i = 0; i < q; ++i) {
4088                 rxr = &adapter->rx_rings[i];
4089                 for (int n = 0; n < adapter->num_rx_desc; n++) {
4090                         struct em_buffer *rxbuf;
4091                         rxbuf = &rxr->rx_buffers[n];
4092                         if (rxbuf->m_head != NULL) {
4093                                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4094                                   BUS_DMASYNC_POSTREAD);
4095                                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4096                                 m_freem(rxbuf->m_head);
4097                                 rxbuf->m_head = NULL;
4098                         }
4099                 }
4100                 rxr->next_to_check = 0;
4101                 rxr->next_to_refresh = 0;
4102         }
4103
4104         return (ENOBUFS);
4105 }
4106
4107 /*********************************************************************
4108  *
4109  *  Free all receive rings.
4110  *
4111  **********************************************************************/
4112 static void
4113 em_free_receive_structures(struct adapter *adapter)
4114 {
4115         struct rx_ring *rxr = adapter->rx_rings;
4116
4117         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4118                 em_free_receive_buffers(rxr);
4119                 /* Free the ring memory as well */
4120                 em_dma_free(adapter, &rxr->rxdma);
4121                 EM_RX_LOCK_DESTROY(rxr);
4122         }
4123
4124         free(adapter->rx_rings, M_DEVBUF);
4125 }
4126
4127
4128 /*********************************************************************
4129  *
4130  *  Free receive ring data structures
4131  *
4132  **********************************************************************/
4133 static void
4134 em_free_receive_buffers(struct rx_ring *rxr)
4135 {
4136         struct adapter          *adapter = rxr->adapter;
4137         struct em_buffer        *rxbuf = NULL;
4138
4139         INIT_DEBUGOUT("free_receive_buffers: begin");
4140
4141         if (rxr->rx_buffers != NULL) {
4142                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4143                         rxbuf = &rxr->rx_buffers[i];
4144                         if (rxbuf->map != NULL) {
4145                                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4146                                     BUS_DMASYNC_POSTREAD);
4147                                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4148                                 bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
4149                         }
4150                         if (rxbuf->m_head != NULL) {
4151                                 m_freem(rxbuf->m_head);
4152                                 rxbuf->m_head = NULL;
4153                         }
4154                 }
4155                 free(rxr->rx_buffers, M_DEVBUF);
4156                 rxr->rx_buffers = NULL;
4157                 rxr->next_to_check = 0;
4158                 rxr->next_to_refresh = 0;
4159         }
4160
4161         if (rxr->rxtag != NULL) {
4162                 bus_dma_tag_destroy(rxr->rxtag);
4163                 rxr->rxtag = NULL;
4164         }
4165
4166         return;
4167 }
4168
4169
4170 /*********************************************************************
4171  *
4172  *  Enable receive unit.
4173  *
4174  **********************************************************************/
4175 #define MAX_INTS_PER_SEC        8000
4176 #define DEFAULT_ITR          1000000000/(MAX_INTS_PER_SEC * 256)
4177
4178 static void
4179 em_initialize_receive_unit(struct adapter *adapter)
4180 {
4181         struct rx_ring  *rxr = adapter->rx_rings;
4182         struct ifnet    *ifp = adapter->ifp;
4183         struct e1000_hw *hw = &adapter->hw;
4184         u64     bus_addr;
4185         u32     rctl, rxcsum;
4186
4187         INIT_DEBUGOUT("em_initialize_receive_units: begin");
4188
4189         /*
4190          * Make sure receives are disabled while setting
4191          * up the descriptor ring
4192          */
4193         rctl = E1000_READ_REG(hw, E1000_RCTL);
4194         /* Do not disable if ever enabled on this hardware */
4195         if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583))
4196                 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4197
4198         E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4199             adapter->rx_abs_int_delay.value);
4200         /*
4201          * Set the interrupt throttling rate. Value is calculated
4202          * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4203          */
4204         E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
4205
4206         /*
4207         ** When using MSIX interrupts we need to throttle
4208         ** using the EITR register (82574 only)
4209         */
4210         if (hw->mac.type == e1000_82574) {
4211                 for (int i = 0; i < 4; i++)
4212                         E1000_WRITE_REG(hw, E1000_EITR_82574(i),
4213                             DEFAULT_ITR);
4214                 /* Disable accelerated acknowledge */
4215                 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4216         }
4217
4218         if (ifp->if_capenable & IFCAP_RXCSUM) {
4219                 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4220                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4221                 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4222         }
4223
4224         /*
4225         ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4226         ** long latencies are observed, like Lenovo X60. This
4227         ** change eliminates the problem, but since having positive
4228         ** values in RDTR is a known source of problems on other
4229         ** platforms another solution is being sought.
4230         */
4231         if (hw->mac.type == e1000_82573)
4232                 E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
4233
4234         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4235                 /* Setup the Base and Length of the Rx Descriptor Ring */
4236                 bus_addr = rxr->rxdma.dma_paddr;
4237                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4238                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4239                 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
4240                 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
4241                 /* Setup the Head and Tail Descriptor Pointers */
4242                 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4243                 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1);
4244         }
4245
4246         /* Set PTHRESH for improved jumbo performance */
4247         if (((adapter->hw.mac.type == e1000_ich9lan) ||
4248             (adapter->hw.mac.type == e1000_pch2lan) ||
4249             (adapter->hw.mac.type == e1000_ich10lan)) &&
4250             (ifp->if_mtu > ETHERMTU)) {
4251                 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
4252                 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3);
4253         }
4254
4255         if (adapter->hw.mac.type == e1000_pch2lan) {
4256                 if (ifp->if_mtu > ETHERMTU)
4257                         e1000_lv_jumbo_workaround_ich8lan(hw, TRUE);
4258                 else
4259                         e1000_lv_jumbo_workaround_ich8lan(hw, FALSE);
4260         }
4261
4262         /* Setup the Receive Control Register */
4263         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4264         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
4265             E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
4266             (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4267
4268         /* Strip the CRC */
4269         rctl |= E1000_RCTL_SECRC;
4270
4271         /* Make sure VLAN Filters are off */
4272         rctl &= ~E1000_RCTL_VFE;
4273         rctl &= ~E1000_RCTL_SBP;
4274
4275         if (adapter->rx_mbuf_sz == MCLBYTES)
4276                 rctl |= E1000_RCTL_SZ_2048;
4277         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
4278                 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4279         else if (adapter->rx_mbuf_sz > MJUMPAGESIZE)
4280                 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4281
4282         if (ifp->if_mtu > ETHERMTU)
4283                 rctl |= E1000_RCTL_LPE;
4284         else
4285                 rctl &= ~E1000_RCTL_LPE;
4286
4287         /* Write out the settings */
4288         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4289
4290         return;
4291 }
4292
4293
4294 /*********************************************************************
4295  *
4296  *  This routine executes in interrupt context. It replenishes
4297  *  the mbufs in the descriptor and sends data which has been
4298  *  dma'ed into host memory to upper layer.
4299  *
4300  *  We loop at most count times if count is > 0, or until done if
4301  *  count < 0.
4302  *
4303  *  For polling we also now return the number of cleaned packets
4304  *********************************************************************/
4305 static bool
4306 em_rxeof(struct rx_ring *rxr, int count, int *done)
4307 {
4308         struct adapter          *adapter = rxr->adapter;
4309         struct ifnet            *ifp = adapter->ifp;
4310         struct mbuf             *mp, *sendmp;
4311         u8                      status = 0;
4312         u16                     len;
4313         int                     i, processed, rxdone = 0;
4314         bool                    eop;
4315         struct e1000_rx_desc    *cur;
4316
4317         EM_RX_LOCK(rxr);
4318
4319         for (i = rxr->next_to_check, processed = 0; count != 0;) {
4320
4321                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4322                         break;
4323
4324                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4325                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4326
4327                 cur = &rxr->rx_base[i];
4328                 status = cur->status;
4329                 mp = sendmp = NULL;
4330
4331                 if ((status & E1000_RXD_STAT_DD) == 0)
4332                         break;
4333
4334                 len = le16toh(cur->length);
4335                 eop = (status & E1000_RXD_STAT_EOP) != 0;
4336
4337                 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) ||
4338                     (rxr->discard == TRUE)) {
4339                         ifp->if_ierrors++;
4340                         ++rxr->rx_discarded;
4341                         if (!eop) /* Catch subsequent segs */
4342                                 rxr->discard = TRUE;
4343                         else
4344                                 rxr->discard = FALSE;
4345                         em_rx_discard(rxr, i);
4346                         goto next_desc;
4347                 }
4348
4349                 /* Assign correct length to the current fragment */
4350                 mp = rxr->rx_buffers[i].m_head;
4351                 mp->m_len = len;
4352
4353                 /* Trigger for refresh */
4354                 rxr->rx_buffers[i].m_head = NULL;
4355
4356                 /* First segment? */
4357                 if (rxr->fmp == NULL) {
4358                         mp->m_pkthdr.len = len;
4359                         rxr->fmp = rxr->lmp = mp;
4360                 } else {
4361                         /* Chain mbuf's together */
4362                         mp->m_flags &= ~M_PKTHDR;
4363                         rxr->lmp->m_next = mp;
4364                         rxr->lmp = mp;
4365                         rxr->fmp->m_pkthdr.len += len;
4366                 }
4367
4368                 if (eop) {
4369                         --count;
4370                         sendmp = rxr->fmp;
4371                         sendmp->m_pkthdr.rcvif = ifp;
4372                         ifp->if_ipackets++;
4373                         em_receive_checksum(cur, sendmp);
4374 #ifndef __NO_STRICT_ALIGNMENT
4375                         if (adapter->max_frame_size >
4376                             (MCLBYTES - ETHER_ALIGN) &&
4377                             em_fixup_rx(rxr) != 0)
4378                                 goto skip;
4379 #endif
4380                         if (status & E1000_RXD_STAT_VP) {
4381                                 sendmp->m_pkthdr.ether_vtag =
4382                                     le16toh(cur->special);
4383                                 sendmp->m_flags |= M_VLANTAG;
4384                         }
4385 #ifndef __NO_STRICT_ALIGNMENT
4386 skip:
4387 #endif
4388                         rxr->fmp = rxr->lmp = NULL;
4389                 }
4390 next_desc:
4391                 /* Zero out the receive descriptors status. */
4392                 cur->status = 0;
4393                 ++rxdone;       /* cumulative for POLL */
4394                 ++processed;
4395
4396                 /* Advance our pointers to the next descriptor. */
4397                 if (++i == adapter->num_rx_desc)
4398                         i = 0;
4399
4400                 /* Send to the stack */
4401                 if (sendmp != NULL) {
4402                         rxr->next_to_check = i;
4403                         EM_RX_UNLOCK(rxr);
4404                         (*ifp->if_input)(ifp, sendmp);
4405                         EM_RX_LOCK(rxr);
4406                         i = rxr->next_to_check;
4407                 }
4408
4409                 /* Only refresh mbufs every 8 descriptors */
4410                 if (processed == 8) {
4411                         em_refresh_mbufs(rxr, i);
4412                         processed = 0;
4413                 }
4414         }
4415
4416         /* Catch any remaining refresh work */
4417         if (e1000_rx_unrefreshed(rxr))
4418                 em_refresh_mbufs(rxr, i);
4419
4420         rxr->next_to_check = i;
4421         if (done != NULL)
4422                 *done = rxdone;
4423         EM_RX_UNLOCK(rxr);
4424
4425         return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4426 }
4427
4428 static __inline void
4429 em_rx_discard(struct rx_ring *rxr, int i)
4430 {
4431         struct em_buffer        *rbuf;
4432
4433         rbuf = &rxr->rx_buffers[i];
4434         /* Free any previous pieces */
4435         if (rxr->fmp != NULL) {
4436                 rxr->fmp->m_flags |= M_PKTHDR;
4437                 m_freem(rxr->fmp);
4438                 rxr->fmp = NULL;
4439                 rxr->lmp = NULL;
4440         }
4441         /*
4442         ** Free buffer and allow em_refresh_mbufs()
4443         ** to clean up and recharge buffer.
4444         */
4445         if (rbuf->m_head) {
4446                 m_free(rbuf->m_head);
4447                 rbuf->m_head = NULL;
4448         }
4449         return;
4450 }
4451
4452 #ifndef __NO_STRICT_ALIGNMENT
4453 /*
4454  * When jumbo frames are enabled we should realign entire payload on
4455  * architecures with strict alignment. This is serious design mistake of 8254x
4456  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4457  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4458  * payload. On architecures without strict alignment restrictions 8254x still
4459  * performs unaligned memory access which would reduce the performance too.
4460  * To avoid copying over an entire frame to align, we allocate a new mbuf and
4461  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4462  * existing mbuf chain.
4463  *
4464  * Be aware, best performance of the 8254x is achived only when jumbo frame is
4465  * not used at all on architectures with strict alignment.
4466  */
4467 static int
4468 em_fixup_rx(struct rx_ring *rxr)
4469 {
4470         struct adapter *adapter = rxr->adapter;
4471         struct mbuf *m, *n;
4472         int error;
4473
4474         error = 0;
4475         m = rxr->fmp;
4476         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4477                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4478                 m->m_data += ETHER_HDR_LEN;
4479         } else {
4480                 MGETHDR(n, M_DONTWAIT, MT_DATA);
4481                 if (n != NULL) {
4482                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4483                         m->m_data += ETHER_HDR_LEN;
4484                         m->m_len -= ETHER_HDR_LEN;
4485                         n->m_len = ETHER_HDR_LEN;
4486                         M_MOVE_PKTHDR(n, m);
4487                         n->m_next = m;
4488                         rxr->fmp = n;
4489                 } else {
4490                         adapter->dropped_pkts++;
4491                         m_freem(rxr->fmp);
4492                         rxr->fmp = NULL;
4493                         error = ENOMEM;
4494                 }
4495         }
4496
4497         return (error);
4498 }
4499 #endif
4500
4501 /*********************************************************************
4502  *
4503  *  Verify that the hardware indicated that the checksum is valid.
4504  *  Inform the stack about the status of checksum so that stack
4505  *  doesn't spend time verifying the checksum.
4506  *
4507  *********************************************************************/
4508 static void
4509 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4510 {
4511         /* Ignore Checksum bit is set */
4512         if (rx_desc->status & E1000_RXD_STAT_IXSM) {
4513                 mp->m_pkthdr.csum_flags = 0;
4514                 return;
4515         }
4516
4517         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4518                 /* Did it pass? */
4519                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4520                         /* IP Checksum Good */
4521                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4522                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4523
4524                 } else {
4525                         mp->m_pkthdr.csum_flags = 0;
4526                 }
4527         }
4528
4529         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4530                 /* Did it pass? */
4531                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4532                         mp->m_pkthdr.csum_flags |=
4533                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4534                         mp->m_pkthdr.csum_data = htons(0xffff);
4535                 }
4536         }
4537 }
4538
4539 /*
4540  * This routine is run via an vlan
4541  * config EVENT
4542  */
4543 static void
4544 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4545 {
4546         struct adapter  *adapter = ifp->if_softc;
4547         u32             index, bit;
4548
4549         if (ifp->if_softc !=  arg)   /* Not our event */
4550                 return;
4551
4552         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
4553                 return;
4554
4555         EM_CORE_LOCK(adapter);
4556         index = (vtag >> 5) & 0x7F;
4557         bit = vtag & 0x1F;
4558         adapter->shadow_vfta[index] |= (1 << bit);
4559         ++adapter->num_vlans;
4560         /* Re-init to load the changes */
4561         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4562                 em_init_locked(adapter);
4563         EM_CORE_UNLOCK(adapter);
4564 }
4565
4566 /*
4567  * This routine is run via an vlan
4568  * unconfig EVENT
4569  */
4570 static void
4571 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4572 {
4573         struct adapter  *adapter = ifp->if_softc;
4574         u32             index, bit;
4575
4576         if (ifp->if_softc !=  arg)
4577                 return;
4578
4579         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4580                 return;
4581
4582         EM_CORE_LOCK(adapter);
4583         index = (vtag >> 5) & 0x7F;
4584         bit = vtag & 0x1F;
4585         adapter->shadow_vfta[index] &= ~(1 << bit);
4586         --adapter->num_vlans;
4587         /* Re-init to load the changes */
4588         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4589                 em_init_locked(adapter);
4590         EM_CORE_UNLOCK(adapter);
4591 }
4592
4593 static void
4594 em_setup_vlan_hw_support(struct adapter *adapter)
4595 {
4596         struct e1000_hw *hw = &adapter->hw;
4597         u32             reg;
4598
4599         /*
4600         ** We get here thru init_locked, meaning
4601         ** a soft reset, this has already cleared
4602         ** the VFTA and other state, so if there
4603         ** have been no vlan's registered do nothing.
4604         */
4605         if (adapter->num_vlans == 0)
4606                 return;
4607
4608         /*
4609         ** A soft reset zero's out the VFTA, so
4610         ** we need to repopulate it now.
4611         */
4612         for (int i = 0; i < EM_VFTA_SIZE; i++)
4613                 if (adapter->shadow_vfta[i] != 0)
4614                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4615                             i, adapter->shadow_vfta[i]);
4616
4617         reg = E1000_READ_REG(hw, E1000_CTRL);
4618         reg |= E1000_CTRL_VME;
4619         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4620
4621         /* Enable the Filter Table */
4622         reg = E1000_READ_REG(hw, E1000_RCTL);
4623         reg &= ~E1000_RCTL_CFIEN;
4624         reg |= E1000_RCTL_VFE;
4625         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4626 }
4627
4628 static void
4629 em_enable_intr(struct adapter *adapter)
4630 {
4631         struct e1000_hw *hw = &adapter->hw;
4632         u32 ims_mask = IMS_ENABLE_MASK;
4633
4634         if (hw->mac.type == e1000_82574) {
4635                 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4636                 ims_mask |= EM_MSIX_MASK;
4637         }
4638         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4639 }
4640
4641 static void
4642 em_disable_intr(struct adapter *adapter)
4643 {
4644         struct e1000_hw *hw = &adapter->hw;
4645
4646         if (hw->mac.type == e1000_82574)
4647                 E1000_WRITE_REG(hw, EM_EIAC, 0);
4648         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4649 }
4650
4651 /*
4652  * Bit of a misnomer, what this really means is
4653  * to enable OS management of the system... aka
4654  * to disable special hardware management features
4655  */
4656 static void
4657 em_init_manageability(struct adapter *adapter)
4658 {
4659         /* A shared code workaround */
4660 #define E1000_82542_MANC2H E1000_MANC2H
4661         if (adapter->has_manage) {
4662                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4663                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4664
4665                 /* disable hardware interception of ARP */
4666                 manc &= ~(E1000_MANC_ARP_EN);
4667
4668                 /* enable receiving management packets to the host */
4669                 manc |= E1000_MANC_EN_MNG2HOST;
4670 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4671 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4672                 manc2h |= E1000_MNG2HOST_PORT_623;
4673                 manc2h |= E1000_MNG2HOST_PORT_664;
4674                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4675                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4676         }
4677 }
4678
4679 /*
4680  * Give control back to hardware management
4681  * controller if there is one.
4682  */
4683 static void
4684 em_release_manageability(struct adapter *adapter)
4685 {
4686         if (adapter->has_manage) {
4687                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4688
4689                 /* re-enable hardware interception of ARP */
4690                 manc |= E1000_MANC_ARP_EN;
4691                 manc &= ~E1000_MANC_EN_MNG2HOST;
4692
4693                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4694         }
4695 }
4696
4697 /*
4698  * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4699  * For ASF and Pass Through versions of f/w this means
4700  * that the driver is loaded. For AMT version type f/w
4701  * this means that the network i/f is open.
4702  */
4703 static void
4704 em_get_hw_control(struct adapter *adapter)
4705 {
4706         u32 ctrl_ext, swsm;
4707
4708         if (adapter->hw.mac.type == e1000_82573) {
4709                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4710                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4711                     swsm | E1000_SWSM_DRV_LOAD);
4712                 return;
4713         }
4714         /* else */
4715         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4716         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4717             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4718         return;
4719 }
4720
4721 /*
4722  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4723  * For ASF and Pass Through versions of f/w this means that
4724  * the driver is no longer loaded. For AMT versions of the
4725  * f/w this means that the network i/f is closed.
4726  */
4727 static void
4728 em_release_hw_control(struct adapter *adapter)
4729 {
4730         u32 ctrl_ext, swsm;
4731
4732         if (!adapter->has_manage)
4733                 return;
4734
4735         if (adapter->hw.mac.type == e1000_82573) {
4736                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4737                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4738                     swsm & ~E1000_SWSM_DRV_LOAD);
4739                 return;
4740         }
4741         /* else */
4742         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4743         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4744             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4745         return;
4746 }
4747
4748 static int
4749 em_is_valid_ether_addr(u8 *addr)
4750 {
4751         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4752
4753         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4754                 return (FALSE);
4755         }
4756
4757         return (TRUE);
4758 }
4759
4760 /*
4761 ** Parse the interface capabilities with regard
4762 ** to both system management and wake-on-lan for
4763 ** later use.
4764 */
4765 static void
4766 em_get_wakeup(device_t dev)
4767 {
4768         struct adapter  *adapter = device_get_softc(dev);
4769         u16             eeprom_data = 0, device_id, apme_mask;
4770
4771         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4772         apme_mask = EM_EEPROM_APME;
4773
4774         switch (adapter->hw.mac.type) {
4775         case e1000_82573:
4776         case e1000_82583:
4777                 adapter->has_amt = TRUE;
4778                 /* Falls thru */
4779         case e1000_82571:
4780         case e1000_82572:
4781         case e1000_80003es2lan:
4782                 if (adapter->hw.bus.func == 1) {
4783                         e1000_read_nvm(&adapter->hw,
4784                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4785                         break;
4786                 } else
4787                         e1000_read_nvm(&adapter->hw,
4788                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4789                 break;
4790         case e1000_ich8lan:
4791         case e1000_ich9lan:
4792         case e1000_ich10lan:
4793         case e1000_pchlan:
4794         case e1000_pch2lan:
4795                 apme_mask = E1000_WUC_APME;
4796                 adapter->has_amt = TRUE;
4797                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
4798                 break;
4799         default:
4800                 e1000_read_nvm(&adapter->hw,
4801                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4802                 break;
4803         }
4804         if (eeprom_data & apme_mask)
4805                 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4806         /*
4807          * We have the eeprom settings, now apply the special cases
4808          * where the eeprom may be wrong or the board won't support
4809          * wake on lan on a particular port
4810          */
4811         device_id = pci_get_device(dev);
4812         switch (device_id) {
4813         case E1000_DEV_ID_82571EB_FIBER:
4814                 /* Wake events only supported on port A for dual fiber
4815                  * regardless of eeprom setting */
4816                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4817                     E1000_STATUS_FUNC_1)
4818                         adapter->wol = 0;
4819                 break;
4820         case E1000_DEV_ID_82571EB_QUAD_COPPER:
4821         case E1000_DEV_ID_82571EB_QUAD_FIBER:
4822         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
4823                 /* if quad port adapter, disable WoL on all but port A */
4824                 if (global_quad_port_a != 0)
4825                         adapter->wol = 0;
4826                 /* Reset for multiple quad port adapters */
4827                 if (++global_quad_port_a == 4)
4828                         global_quad_port_a = 0;
4829                 break;
4830         }
4831         return;
4832 }
4833
4834
4835 /*
4836  * Enable PCI Wake On Lan capability
4837  */
4838 static void
4839 em_enable_wakeup(device_t dev)
4840 {
4841         struct adapter  *adapter = device_get_softc(dev);
4842         struct ifnet    *ifp = adapter->ifp;
4843         u32             pmc, ctrl, ctrl_ext, rctl;
4844         u16             status;
4845
4846         if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
4847                 return;
4848
4849         /* Advertise the wakeup capability */
4850         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4851         ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4852         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4853         E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4854
4855         if ((adapter->hw.mac.type == e1000_ich8lan) ||
4856             (adapter->hw.mac.type == e1000_pchlan) ||
4857             (adapter->hw.mac.type == e1000_ich9lan) ||
4858             (adapter->hw.mac.type == e1000_ich10lan))
4859                 e1000_suspend_workarounds_ich8lan(&adapter->hw);
4860
4861         /* Keep the laser running on Fiber adapters */
4862         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4863             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4864                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4865                 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4866                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4867         }
4868
4869         /*
4870         ** Determine type of Wakeup: note that wol
4871         ** is set with all bits on by default.
4872         */
4873         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0)
4874                 adapter->wol &= ~E1000_WUFC_MAG;
4875
4876         if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0)
4877                 adapter->wol &= ~E1000_WUFC_MC;
4878         else {
4879                 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4880                 rctl |= E1000_RCTL_MPE;
4881                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4882         }
4883
4884         if ((adapter->hw.mac.type == e1000_pchlan) ||
4885             (adapter->hw.mac.type == e1000_pch2lan)) {
4886                 if (em_enable_phy_wakeup(adapter))
4887                         return;
4888         } else {
4889                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4890                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4891         }
4892
4893         if (adapter->hw.phy.type == e1000_phy_igp_3)
4894                 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
4895
4896         /* Request PME */
4897         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4898         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4899         if (ifp->if_capenable & IFCAP_WOL)
4900                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4901         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
4902
4903         return;
4904 }
4905
4906 /*
4907 ** WOL in the newer chipset interfaces (pchlan)
4908 ** require thing to be copied into the phy
4909 */
4910 static int
4911 em_enable_phy_wakeup(struct adapter *adapter)
4912 {
4913         struct e1000_hw *hw = &adapter->hw;
4914         u32 mreg, ret = 0;
4915         u16 preg;
4916
4917         /* copy MAC RARs to PHY RARs */
4918         e1000_copy_rx_addrs_to_phy_ich8lan(hw);
4919
4920         /* copy MAC MTA to PHY MTA */
4921         for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
4922                 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
4923                 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
4924                 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
4925                     (u16)((mreg >> 16) & 0xFFFF));
4926         }
4927
4928         /* configure PHY Rx Control register */
4929         e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
4930         mreg = E1000_READ_REG(hw, E1000_RCTL);
4931         if (mreg & E1000_RCTL_UPE)
4932                 preg |= BM_RCTL_UPE;
4933         if (mreg & E1000_RCTL_MPE)
4934                 preg |= BM_RCTL_MPE;
4935         preg &= ~(BM_RCTL_MO_MASK);
4936         if (mreg & E1000_RCTL_MO_3)
4937                 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
4938                                 << BM_RCTL_MO_SHIFT);
4939         if (mreg & E1000_RCTL_BAM)
4940                 preg |= BM_RCTL_BAM;
4941         if (mreg & E1000_RCTL_PMCF)
4942                 preg |= BM_RCTL_PMCF;
4943         mreg = E1000_READ_REG(hw, E1000_CTRL);
4944         if (mreg & E1000_CTRL_RFCE)
4945                 preg |= BM_RCTL_RFCE;
4946         e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
4947
4948         /* enable PHY wakeup in MAC register */
4949         E1000_WRITE_REG(hw, E1000_WUC,
4950             E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
4951         E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
4952
4953         /* configure and enable PHY wakeup in PHY registers */
4954         e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
4955         e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
4956
4957         /* activate PHY wakeup */
4958         ret = hw->phy.ops.acquire(hw);
4959         if (ret) {
4960                 printf("Could not acquire PHY\n");
4961                 return ret;
4962         }
4963         e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4964                                  (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
4965         ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
4966         if (ret) {
4967                 printf("Could not read PHY page 769\n");
4968                 goto out;
4969         }
4970         preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
4971         ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
4972         if (ret)
4973                 printf("Could not set PHY Host Wakeup bit\n");
4974 out:
4975         hw->phy.ops.release(hw);
4976
4977         return ret;
4978 }
4979
4980 static void
4981 em_led_func(void *arg, int onoff)
4982 {
4983         struct adapter  *adapter = arg;
4984
4985         EM_CORE_LOCK(adapter);
4986         if (onoff) {
4987                 e1000_setup_led(&adapter->hw);
4988                 e1000_led_on(&adapter->hw);
4989         } else {
4990                 e1000_led_off(&adapter->hw);
4991                 e1000_cleanup_led(&adapter->hw);
4992         }
4993         EM_CORE_UNLOCK(adapter);
4994 }
4995
4996 /*
4997 ** Disable the L0S and L1 LINK states
4998 */
4999 static void
5000 em_disable_aspm(struct adapter *adapter)
5001 {
5002         int             base, reg;
5003         u16             link_cap,link_ctrl;
5004         device_t        dev = adapter->dev;
5005
5006         switch (adapter->hw.mac.type) {
5007                 case e1000_82573:
5008                 case e1000_82574:
5009                 case e1000_82583:
5010                         break;
5011                 default:
5012                         return;
5013         }
5014         if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0)
5015                 return;
5016         reg = base + PCIR_EXPRESS_LINK_CAP;
5017         link_cap = pci_read_config(dev, reg, 2);
5018         if ((link_cap & PCIM_LINK_CAP_ASPM) == 0)
5019                 return;
5020         reg = base + PCIR_EXPRESS_LINK_CTL;
5021         link_ctrl = pci_read_config(dev, reg, 2);
5022         link_ctrl &= 0xFFFC; /* turn off bit 1 and 2 */
5023         pci_write_config(dev, reg, link_ctrl, 2);
5024         return;
5025 }
5026
5027 /**********************************************************************
5028  *
5029  *  Update the board statistics counters.
5030  *
5031  **********************************************************************/
5032 static void
5033 em_update_stats_counters(struct adapter *adapter)
5034 {
5035         struct ifnet   *ifp;
5036
5037         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5038            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5039                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5040                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5041         }
5042         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5043         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5044         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5045         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5046
5047         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5048         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5049         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5050         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5051         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5052         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5053         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5054         /*
5055         ** For watchdog management we need to know if we have been
5056         ** paused during the last interval, so capture that here.
5057         */
5058         adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5059         adapter->stats.xoffrxc += adapter->pause_frames;
5060         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5061         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5062         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5063         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5064         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5065         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5066         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5067         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5068         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5069         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5070         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5071         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5072
5073         /* For the 64-bit byte counters the low dword must be read first. */
5074         /* Both registers clear on the read of the high dword */
5075
5076         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
5077             ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
5078         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
5079             ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
5080
5081         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5082         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5083         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5084         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5085         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5086
5087         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5088         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5089
5090         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5091         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5092         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5093         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5094         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5095         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5096         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5097         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5098         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5099         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5100
5101         /* Interrupt Counts */
5102
5103         adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC);
5104         adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC);
5105         adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC);
5106         adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC);
5107         adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC);
5108         adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC);
5109         adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC);
5110         adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC);
5111         adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC);
5112
5113         if (adapter->hw.mac.type >= e1000_82543) {
5114                 adapter->stats.algnerrc +=
5115                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5116                 adapter->stats.rxerrc +=
5117                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5118                 adapter->stats.tncrs +=
5119                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5120                 adapter->stats.cexterr +=
5121                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5122                 adapter->stats.tsctc +=
5123                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5124                 adapter->stats.tsctfc +=
5125                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5126         }
5127         ifp = adapter->ifp;
5128
5129         ifp->if_collisions = adapter->stats.colc;
5130
5131         /* Rx Errors */
5132         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5133             adapter->stats.crcerrs + adapter->stats.algnerrc +
5134             adapter->stats.ruc + adapter->stats.roc +
5135             adapter->stats.mpc + adapter->stats.cexterr;
5136
5137         /* Tx Errors */
5138         ifp->if_oerrors = adapter->stats.ecol +
5139             adapter->stats.latecol + adapter->watchdog_events;
5140 }
5141
5142 /* Export a single 32-bit register via a read-only sysctl. */
5143 static int
5144 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5145 {
5146         struct adapter *adapter;
5147         u_int val;
5148
5149         adapter = oidp->oid_arg1;
5150         val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5151         return (sysctl_handle_int(oidp, &val, 0, req));
5152 }
5153
5154 /*
5155  * Add sysctl variables, one per statistic, to the system.
5156  */
5157 static void
5158 em_add_hw_stats(struct adapter *adapter)
5159 {
5160         device_t dev = adapter->dev;
5161
5162         struct tx_ring *txr = adapter->tx_rings;
5163         struct rx_ring *rxr = adapter->rx_rings;
5164
5165         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5166         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5167         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5168         struct e1000_hw_stats *stats = &adapter->stats;
5169
5170         struct sysctl_oid *stat_node, *queue_node, *int_node;
5171         struct sysctl_oid_list *stat_list, *queue_list, *int_list;
5172
5173 #define QUEUE_NAME_LEN 32
5174         char namebuf[QUEUE_NAME_LEN];
5175
5176         /* Driver Statistics */
5177         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5178                         CTLFLAG_RD, &adapter->link_irq,
5179                         "Link MSIX IRQ Handled");
5180         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail",
5181                          CTLFLAG_RD, &adapter->mbuf_alloc_failed,
5182                          "Std mbuf failed");
5183         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail",
5184                          CTLFLAG_RD, &adapter->mbuf_cluster_failed,
5185                          "Std mbuf cluster failed");
5186         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5187                         CTLFLAG_RD, &adapter->dropped_pkts,
5188                         "Driver dropped packets");
5189         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5190                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5191                         "Driver tx dma failure in xmit");
5192         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5193                         CTLFLAG_RD, &adapter->rx_overruns,
5194                         "RX overruns");
5195         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5196                         CTLFLAG_RD, &adapter->watchdog_events,
5197                         "Watchdog timeouts");
5198
5199         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
5200                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
5201                         em_sysctl_reg_handler, "IU",
5202                         "Device Control Register");
5203         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
5204                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
5205                         em_sysctl_reg_handler, "IU",
5206                         "Receiver Control Register");
5207         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5208                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5209                         "Flow Control High Watermark");
5210         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5211                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5212                         "Flow Control Low Watermark");
5213
5214         for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5215                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5216                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5217                                             CTLFLAG_RD, NULL, "Queue Name");
5218                 queue_list = SYSCTL_CHILDREN(queue_node);
5219
5220                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5221                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5222                                 E1000_TDH(txr->me),
5223                                 em_sysctl_reg_handler, "IU",
5224                                 "Transmit Descriptor Head");
5225                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5226                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5227                                 E1000_TDT(txr->me),
5228                                 em_sysctl_reg_handler, "IU",
5229                                 "Transmit Descriptor Tail");
5230                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
5231                                 CTLFLAG_RD, &txr->tx_irq,
5232                                 "Queue MSI-X Transmit Interrupts");
5233                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail",
5234                                 CTLFLAG_RD, &txr->no_desc_avail,
5235                                 "Queue No Descriptor Available");
5236
5237                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5238                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5239                                 E1000_RDH(rxr->me),
5240                                 em_sysctl_reg_handler, "IU",
5241                                 "Receive Descriptor Head");
5242                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5243                                 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5244                                 E1000_RDT(rxr->me),
5245                                 em_sysctl_reg_handler, "IU",
5246                                 "Receive Descriptor Tail");
5247                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
5248                                 CTLFLAG_RD, &rxr->rx_irq,
5249                                 "Queue MSI-X Receive Interrupts");
5250         }
5251
5252         /* MAC stats get their own sub node */
5253
5254         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5255                                     CTLFLAG_RD, NULL, "Statistics");
5256         stat_list = SYSCTL_CHILDREN(stat_node);
5257
5258         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5259                         CTLFLAG_RD, &stats->ecol,
5260                         "Excessive collisions");
5261         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
5262                         CTLFLAG_RD, &stats->scc,
5263                         "Single collisions");
5264         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5265                         CTLFLAG_RD, &stats->mcc,
5266                         "Multiple collisions");
5267         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
5268                         CTLFLAG_RD, &stats->latecol,
5269                         "Late collisions");
5270         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
5271                         CTLFLAG_RD, &stats->colc,
5272                         "Collision Count");
5273         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5274                         CTLFLAG_RD, &adapter->stats.symerrs,
5275                         "Symbol Errors");
5276         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5277                         CTLFLAG_RD, &adapter->stats.sec,
5278                         "Sequence Errors");
5279         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
5280                         CTLFLAG_RD, &adapter->stats.dc,
5281                         "Defer Count");
5282         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5283                         CTLFLAG_RD, &adapter->stats.mpc,
5284                         "Missed Packets");
5285         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5286                         CTLFLAG_RD, &adapter->stats.rnbc,
5287                         "Receive No Buffers");
5288         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5289                         CTLFLAG_RD, &adapter->stats.ruc,
5290                         "Receive Undersize");
5291         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5292                         CTLFLAG_RD, &adapter->stats.rfc,
5293                         "Fragmented Packets Received ");
5294         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5295                         CTLFLAG_RD, &adapter->stats.roc,
5296                         "Oversized Packets Received");
5297         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5298                         CTLFLAG_RD, &adapter->stats.rjc,
5299                         "Recevied Jabber");
5300         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5301                         CTLFLAG_RD, &adapter->stats.rxerrc,
5302                         "Receive Errors");
5303         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5304                         CTLFLAG_RD, &adapter->stats.crcerrs,
5305                         "CRC errors");
5306         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5307                         CTLFLAG_RD, &adapter->stats.algnerrc,
5308                         "Alignment Errors");
5309         /* On 82575 these are collision counts */
5310         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5311                         CTLFLAG_RD, &adapter->stats.cexterr,
5312                         "Collision/Carrier extension errors");
5313         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5314                         CTLFLAG_RD, &adapter->stats.xonrxc,
5315                         "XON Received");
5316         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5317                         CTLFLAG_RD, &adapter->stats.xontxc,
5318                         "XON Transmitted");
5319         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5320                         CTLFLAG_RD, &adapter->stats.xoffrxc,
5321                         "XOFF Received");
5322         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5323                         CTLFLAG_RD, &adapter->stats.xofftxc,
5324                         "XOFF Transmitted");
5325
5326         /* Packet Reception Stats */
5327         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5328                         CTLFLAG_RD, &adapter->stats.tpr,
5329                         "Total Packets Received ");
5330         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5331                         CTLFLAG_RD, &adapter->stats.gprc,
5332                         "Good Packets Received");
5333         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5334                         CTLFLAG_RD, &adapter->stats.bprc,
5335                         "Broadcast Packets Received");
5336         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5337                         CTLFLAG_RD, &adapter->stats.mprc,
5338                         "Multicast Packets Received");
5339         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5340                         CTLFLAG_RD, &adapter->stats.prc64,
5341                         "64 byte frames received ");
5342         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5343                         CTLFLAG_RD, &adapter->stats.prc127,
5344                         "65-127 byte frames received");
5345         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5346                         CTLFLAG_RD, &adapter->stats.prc255,
5347                         "128-255 byte frames received");
5348         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5349                         CTLFLAG_RD, &adapter->stats.prc511,
5350                         "256-511 byte frames received");
5351         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5352                         CTLFLAG_RD, &adapter->stats.prc1023,
5353                         "512-1023 byte frames received");
5354         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5355                         CTLFLAG_RD, &adapter->stats.prc1522,
5356                         "1023-1522 byte frames received");
5357         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5358                         CTLFLAG_RD, &adapter->stats.gorc,
5359                         "Good Octets Received");
5360
5361         /* Packet Transmission Stats */
5362         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5363                         CTLFLAG_RD, &adapter->stats.gotc,
5364                         "Good Octets Transmitted");
5365         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5366                         CTLFLAG_RD, &adapter->stats.tpt,
5367                         "Total Packets Transmitted");
5368         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5369                         CTLFLAG_RD, &adapter->stats.gptc,
5370                         "Good Packets Transmitted");
5371         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5372                         CTLFLAG_RD, &adapter->stats.bptc,
5373                         "Broadcast Packets Transmitted");
5374         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5375                         CTLFLAG_RD, &adapter->stats.mptc,
5376                         "Multicast Packets Transmitted");
5377         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5378                         CTLFLAG_RD, &adapter->stats.ptc64,
5379                         "64 byte frames transmitted ");
5380         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5381                         CTLFLAG_RD, &adapter->stats.ptc127,
5382                         "65-127 byte frames transmitted");
5383         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5384                         CTLFLAG_RD, &adapter->stats.ptc255,
5385                         "128-255 byte frames transmitted");
5386         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5387                         CTLFLAG_RD, &adapter->stats.ptc511,
5388                         "256-511 byte frames transmitted");
5389         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5390                         CTLFLAG_RD, &adapter->stats.ptc1023,
5391                         "512-1023 byte frames transmitted");
5392         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5393                         CTLFLAG_RD, &adapter->stats.ptc1522,
5394                         "1024-1522 byte frames transmitted");
5395         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5396                         CTLFLAG_RD, &adapter->stats.tsctc,
5397                         "TSO Contexts Transmitted");
5398         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5399                         CTLFLAG_RD, &adapter->stats.tsctfc,
5400                         "TSO Contexts Failed");
5401
5402
5403         /* Interrupt Stats */
5404
5405         int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5406                                     CTLFLAG_RD, NULL, "Interrupt Statistics");
5407         int_list = SYSCTL_CHILDREN(int_node);
5408
5409         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts",
5410                         CTLFLAG_RD, &adapter->stats.iac,
5411                         "Interrupt Assertion Count");
5412
5413         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5414                         CTLFLAG_RD, &adapter->stats.icrxptc,
5415                         "Interrupt Cause Rx Pkt Timer Expire Count");
5416
5417         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5418                         CTLFLAG_RD, &adapter->stats.icrxatc,
5419                         "Interrupt Cause Rx Abs Timer Expire Count");
5420
5421         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5422                         CTLFLAG_RD, &adapter->stats.ictxptc,
5423                         "Interrupt Cause Tx Pkt Timer Expire Count");
5424
5425         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5426                         CTLFLAG_RD, &adapter->stats.ictxatc,
5427                         "Interrupt Cause Tx Abs Timer Expire Count");
5428
5429         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5430                         CTLFLAG_RD, &adapter->stats.ictxqec,
5431                         "Interrupt Cause Tx Queue Empty Count");
5432
5433         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5434                         CTLFLAG_RD, &adapter->stats.ictxqmtc,
5435                         "Interrupt Cause Tx Queue Min Thresh Count");
5436
5437         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5438                         CTLFLAG_RD, &adapter->stats.icrxdmtc,
5439                         "Interrupt Cause Rx Desc Min Thresh Count");
5440
5441         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5442                         CTLFLAG_RD, &adapter->stats.icrxoc,
5443                         "Interrupt Cause Receiver Overrun Count");
5444 }
5445
5446 /**********************************************************************
5447  *
5448  *  This routine provides a way to dump out the adapter eeprom,
5449  *  often a useful debug/service tool. This only dumps the first
5450  *  32 words, stuff that matters is in that extent.
5451  *
5452  **********************************************************************/
5453 static int
5454 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5455 {
5456         struct adapter *adapter = (struct adapter *)arg1;
5457         int error;
5458         int result;
5459
5460         result = -1;
5461         error = sysctl_handle_int(oidp, &result, 0, req);
5462
5463         if (error || !req->newptr)
5464                 return (error);
5465
5466         /*
5467          * This value will cause a hex dump of the
5468          * first 32 16-bit words of the EEPROM to
5469          * the screen.
5470          */
5471         if (result == 1)
5472                 em_print_nvm_info(adapter);
5473
5474         return (error);
5475 }
5476
5477 static void
5478 em_print_nvm_info(struct adapter *adapter)
5479 {
5480         u16     eeprom_data;
5481         int     i, j, row = 0;
5482
5483         /* Its a bit crude, but it gets the job done */
5484         printf("\nInterface EEPROM Dump:\n");
5485         printf("Offset\n0x0000  ");
5486         for (i = 0, j = 0; i < 32; i++, j++) {
5487                 if (j == 8) { /* Make the offset block */
5488                         j = 0; ++row;
5489                         printf("\n0x00%x0  ",row);
5490                 }
5491                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5492                 printf("%04x ", eeprom_data);
5493         }
5494         printf("\n");
5495 }
5496
5497 static int
5498 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5499 {
5500         struct em_int_delay_info *info;
5501         struct adapter *adapter;
5502         u32 regval;
5503         int error, usecs, ticks;
5504
5505         info = (struct em_int_delay_info *)arg1;
5506         usecs = info->value;
5507         error = sysctl_handle_int(oidp, &usecs, 0, req);
5508         if (error != 0 || req->newptr == NULL)
5509                 return (error);
5510         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5511                 return (EINVAL);
5512         info->value = usecs;
5513         ticks = EM_USECS_TO_TICKS(usecs);
5514
5515         adapter = info->adapter;
5516
5517         EM_CORE_LOCK(adapter);
5518         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5519         regval = (regval & ~0xffff) | (ticks & 0xffff);
5520         /* Handle a few special cases. */
5521         switch (info->offset) {
5522         case E1000_RDTR:
5523                 break;
5524         case E1000_TIDV:
5525                 if (ticks == 0) {
5526                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5527                         /* Don't write 0 into the TIDV register. */
5528                         regval++;
5529                 } else
5530                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5531                 break;
5532         }
5533         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5534         EM_CORE_UNLOCK(adapter);
5535         return (0);
5536 }
5537
5538 static void
5539 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5540         const char *description, struct em_int_delay_info *info,
5541         int offset, int value)
5542 {
5543         info->adapter = adapter;
5544         info->offset = offset;
5545         info->value = value;
5546         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5547             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5548             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5549             info, 0, em_sysctl_int_delay, "I", description);
5550 }
5551
5552 static void
5553 em_set_sysctl_value(struct adapter *adapter, const char *name,
5554         const char *description, int *limit, int value)
5555 {
5556         *limit = value;
5557         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5558             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5559             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5560 }
5561
5562
5563 /*
5564 ** Set flow control using sysctl:
5565 ** Flow control values:
5566 **      0 - off
5567 **      1 - rx pause
5568 **      2 - tx pause
5569 **      3 - full
5570 */
5571 static int
5572 em_set_flowcntl(SYSCTL_HANDLER_ARGS)
5573 {
5574         int             error;
5575         static int      input = 3; /* default is full */
5576         struct adapter  *adapter = (struct adapter *) arg1;
5577
5578         error = sysctl_handle_int(oidp, &input, 0, req);
5579
5580         if ((error) || (req->newptr == NULL))
5581                 return (error);
5582
5583         if (input == adapter->fc) /* no change? */
5584                 return (error);
5585
5586         switch (input) {
5587                 case e1000_fc_rx_pause:
5588                 case e1000_fc_tx_pause:
5589                 case e1000_fc_full:
5590                 case e1000_fc_none:
5591                         adapter->hw.fc.requested_mode = input;
5592                         adapter->fc = input;
5593                         break;
5594                 default:
5595                         /* Do nothing */
5596                         return (error);
5597         }
5598
5599         adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
5600         e1000_force_mac_fc(&adapter->hw);
5601         return (error);
5602 }
5603
5604
5605 static int
5606 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5607 {
5608         struct adapter *adapter;
5609         int error;
5610         int result;
5611
5612         result = -1;
5613         error = sysctl_handle_int(oidp, &result, 0, req);
5614
5615         if (error || !req->newptr)
5616                 return (error);
5617
5618         if (result == 1) {
5619                 adapter = (struct adapter *)arg1;
5620                 em_print_debug_info(adapter);
5621         }
5622
5623         return (error);
5624 }
5625
5626 /*
5627 ** This routine is meant to be fluid, add whatever is
5628 ** needed for debugging a problem.  -jfv
5629 */
5630 static void
5631 em_print_debug_info(struct adapter *adapter)
5632 {
5633         device_t dev = adapter->dev;
5634         struct tx_ring *txr = adapter->tx_rings;
5635         struct rx_ring *rxr = adapter->rx_rings;
5636
5637         if (adapter->ifp->if_drv_flags & IFF_DRV_RUNNING)
5638                 printf("Interface is RUNNING ");
5639         else
5640                 printf("Interface is NOT RUNNING\n");
5641
5642         if (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE)
5643                 printf("and INACTIVE\n");
5644         else
5645                 printf("and ACTIVE\n");
5646
5647         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5648             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5649             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5650         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5651             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5652             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5653         device_printf(dev, "Tx Queue Status = %d\n", txr->queue_status);
5654         device_printf(dev, "TX descriptors avail = %d\n",
5655             txr->tx_avail);
5656         device_printf(dev, "Tx Descriptors avail failure = %ld\n",
5657             txr->no_desc_avail);
5658         device_printf(dev, "RX discarded packets = %ld\n",
5659             rxr->rx_discarded);
5660         device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check);
5661         device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh);
5662 }