sys/dev/e1000/if_em.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2009, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #endif
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #if __FreeBSD_version >= 800000
  43 #include <sys/buf_ring.h>
  44 #endif
  45 #include <sys/bus.h>
  46 #include <sys/endian.h>
  47 #include <sys/kernel.h>
  48 #include <sys/kthread.h>
  49 #include <sys/malloc.h>
  50 #include <sys/mbuf.h>
  51 #include <sys/module.h>
  52 #include <sys/rman.h>
  53 #include <sys/socket.h>
  54 #include <sys/sockio.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/taskqueue.h>
  57 #if __FreeBSD_version >= 700029
  58 #include <sys/eventhandler.h>
  59 #endif
  60 #include <machine/bus.h>
  61 #include <machine/resource.h>
  62
  63 #include <net/bpf.h>
  64 #include <net/ethernet.h>
  65 #include <net/if.h>
  66 #include <net/if_arp.h>
  67 #include <net/if_dl.h>
  68 #include <net/if_media.h>
  69
  70 #include <net/if_types.h>
  71 #include <net/if_vlan_var.h>
  72
  73 #include <netinet/in_systm.h>
  74 #include <netinet/in.h>
  75 #include <netinet/if_ether.h>
  76 #include <netinet/ip.h>
  77 #include <netinet/ip6.h>
  78 #include <netinet/tcp.h>
  79 #include <netinet/udp.h>
  80
  81 #include <machine/in_cksum.h>
  82 #include <dev/pci/pcivar.h>
  83 #include <dev/pci/pcireg.h>
  84
  85 #include "e1000_api.h"
  86 #include "e1000_82571.h"
  87 #include "if_em.h"
  88
  89 /*********************************************************************
  90  *  Set this to one to display debug statistics
  91  *********************************************************************/
  92 int     em_display_debug_stats = 0;
  93
  94 /*********************************************************************
  95  *  Driver version:
  96  *********************************************************************/
  97 char em_driver_version[] = "6.9.14";
  98
  99
 100 /*********************************************************************
 101  *  PCI Device ID Table
 102  *
 103  *  Used by probe to select devices to load on
 104  *  Last field stores an index into e1000_strings
 105  *  Last entry must be all 0s
 106  *
 107  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 108  *********************************************************************/
 109
 110 static em_vendor_info_t em_vendor_info_array[] =
 111 {
 112         /* Intel(R) PRO/1000 Network Connection */
 113         { 0x8086, E1000_DEV_ID_82540EM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 114         { 0x8086, E1000_DEV_ID_82540EM_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82540EP,         PCI_ANY_ID, PCI_ANY_ID, 0},
 116         { 0x8086, E1000_DEV_ID_82540EP_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82540EP_LP,      PCI_ANY_ID, PCI_ANY_ID, 0},
 118
 119         { 0x8086, E1000_DEV_ID_82541EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 120         { 0x8086, E1000_DEV_ID_82541ER,         PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82541ER_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 122         { 0x8086, E1000_DEV_ID_82541EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82541GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 124         { 0x8086, E1000_DEV_ID_82541GI_LF,      PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82541GI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 126
 127         { 0x8086, E1000_DEV_ID_82542,           PCI_ANY_ID, PCI_ANY_ID, 0},
 128
 129         { 0x8086, E1000_DEV_ID_82543GC_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82543GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 131
 132         { 0x8086, E1000_DEV_ID_82544EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82544EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82544GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82544GC_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 136
 137         { 0x8086, E1000_DEV_ID_82545EM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 138         { 0x8086, E1000_DEV_ID_82545EM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 139         { 0x8086, E1000_DEV_ID_82545GM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_82545GM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 141         { 0x8086, E1000_DEV_ID_82545GM_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 142
 143         { 0x8086, E1000_DEV_ID_82546EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 144         { 0x8086, E1000_DEV_ID_82546EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_82546GB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_82546GB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_82546GB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_82546GB_PCIE,    PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
 152                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 153
 154         { 0x8086, E1000_DEV_ID_82547EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 155         { 0x8086, E1000_DEV_ID_82547EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_82547GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 157
 158         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 159         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 160         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
 162                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 163         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
 164                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 165         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 166                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 167         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
 168                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 169         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
 170                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 171         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
 172                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 173         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 174         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 175         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 176         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 177
 178         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 179         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 180         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 181         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
 182         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 183                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 184         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 185                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 186         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 187                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 188         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 189                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 190         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 191         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 192         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 193         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 194         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 195         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 196         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 197
 198         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 199         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 200         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 201         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 202         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 203         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 204         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 205         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 206         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 207         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 208         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
 209         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 210         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 211         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 212         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 213         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 214         /* required last entry */
 215         { 0, 0, 0, 0, 0}
 216 };
 217
 218 /*********************************************************************
 219  *  Table of branding strings for all supported NICs.
 220  *********************************************************************/
 221
 222 static char *em_strings[] = {
 223         "Intel(R) PRO/1000 Network Connection"
 224 };
 225
 226 /*********************************************************************
 227  *  Function prototypes
 228  *********************************************************************/
 229 static int      em_probe(device_t);
 230 static int      em_attach(device_t);
 231 static int      em_detach(device_t);
 232 static int      em_shutdown(device_t);
 233 static int      em_suspend(device_t);
 234 static int      em_resume(device_t);
 235 static void     em_start(struct ifnet *);
 236 static void     em_start_locked(struct ifnet *ifp);
 237 #if __FreeBSD_version >= 800000
 238 static int      em_mq_start(struct ifnet *, struct mbuf *);
 239 static int      em_mq_start_locked(struct ifnet *, struct mbuf *);
 240 static void     em_qflush(struct ifnet *);
 241 #endif
 242 static int      em_ioctl(struct ifnet *, u_long, caddr_t);
 243 static void     em_watchdog(struct adapter *);
 244 static void     em_init(void *);
 245 static void     em_init_locked(struct adapter *);
 246 static void     em_stop(void *);
 247 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 248 static int      em_media_change(struct ifnet *);
 249 static void     em_identify_hardware(struct adapter *);
 250 static int      em_allocate_pci_resources(struct adapter *);
 251 static int      em_allocate_legacy(struct adapter *adapter);
 252 static int      em_allocate_msix(struct adapter *adapter);
 253 static int      em_setup_msix(struct adapter *);
 254 static void     em_free_pci_resources(struct adapter *);
 255 static void     em_local_timer(void *);
 256 static int      em_hardware_init(struct adapter *);
 257 static void     em_setup_interface(device_t, struct adapter *);
 258 static void     em_setup_transmit_structures(struct adapter *);
 259 static void     em_initialize_transmit_unit(struct adapter *);
 260 static int      em_setup_receive_structures(struct adapter *);
 261 static void     em_initialize_receive_unit(struct adapter *);
 262 static void     em_enable_intr(struct adapter *);
 263 static void     em_disable_intr(struct adapter *);
 264 static void     em_free_transmit_structures(struct adapter *);
 265 static void     em_free_receive_structures(struct adapter *);
 266 static void     em_update_stats_counters(struct adapter *);
 267 static void     em_txeof(struct adapter *);
 268 static void     em_tx_purge(struct adapter *);
 269 static int      em_allocate_receive_structures(struct adapter *);
 270 static int      em_allocate_transmit_structures(struct adapter *);
 271 static int      em_rxeof(struct adapter *, int);
 272 #ifndef __NO_STRICT_ALIGNMENT
 273 static int      em_fixup_rx(struct adapter *);
 274 #endif
 275 static void     em_receive_checksum(struct adapter *, struct e1000_rx_desc *,
 276                     struct mbuf *);
 277 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 278                     u32 *, u32 *);
 279 #if __FreeBSD_version >= 700000
 280 static bool     em_tso_setup(struct adapter *, struct mbuf *,
 281                     u32 *, u32 *);
 282 #endif /* FreeBSD_version >= 700000 */
 283 static void     em_set_promisc(struct adapter *);
 284 static void     em_disable_promisc(struct adapter *);
 285 static void     em_set_multi(struct adapter *);
 286 static void     em_print_hw_stats(struct adapter *);
 287 static void     em_update_link_status(struct adapter *);
 288 static int      em_get_buf(struct adapter *, int);
 289 #if __FreeBSD_version >= 700029
 290 static void     em_register_vlan(void *, struct ifnet *, u16);
 291 static void     em_unregister_vlan(void *, struct ifnet *, u16);
 292 static void     em_setup_vlan_hw_support(struct adapter *);
 293 #endif
 294 static int      em_xmit(struct adapter *, struct mbuf **);
 295 static void     em_smartspeed(struct adapter *);
 296 static int      em_82547_fifo_workaround(struct adapter *, int);
 297 static void     em_82547_update_fifo_head(struct adapter *, int);
 298 static int      em_82547_tx_fifo_reset(struct adapter *);
 299 static void     em_82547_move_tail(void *);
 300 static int      em_dma_malloc(struct adapter *, bus_size_t,
 301                     struct em_dma_alloc *, int);
 302 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 303 static void     em_print_debug_info(struct adapter *);
 304 static void     em_print_nvm_info(struct adapter *);
 305 static int      em_is_valid_ether_addr(u8 *);
 306 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 307 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 308 static u32      em_fill_descriptors (bus_addr_t address, u32 length,
 309                     PDESC_ARRAY desc_array);
 310 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 311 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 312                     const char *, struct em_int_delay_info *, int, int);
 313 /* Management and WOL Support */
 314 static void     em_init_manageability(struct adapter *);
 315 static void     em_release_manageability(struct adapter *);
 316 static void     em_get_hw_control(struct adapter *);
 317 static void     em_release_hw_control(struct adapter *);
 318 static void     em_enable_wakeup(device_t);
 319
 320 #ifdef EM_LEGACY_IRQ
 321 static void     em_intr(void *);
 322 #else /* FAST IRQ */
 323 #if __FreeBSD_version < 700000
 324 static void     em_irq_fast(void *);
 325 #else
 326 static int      em_irq_fast(void *);
 327 #endif
 328
 329 /* MSIX handlers */
 330 static void     em_msix_tx(void *);
 331 static void     em_msix_rx(void *);
 332 static void     em_msix_link(void *);
 333 static void     em_handle_rx(void *context, int pending);
 334 static void     em_handle_tx(void *context, int pending);
 335
 336 static void     em_handle_rxtx(void *context, int pending);
 337 static void     em_handle_link(void *context, int pending);
 338 static void     em_add_rx_process_limit(struct adapter *, const char *,
 339                     const char *, int *, int);
 340 #endif /* ~EM_LEGACY_IRQ */
 341
 342 #ifdef DEVICE_POLLING
 343 static poll_handler_t em_poll;
 344 #endif /* POLLING */
 345
 346 /*********************************************************************
 347  *  FreeBSD Device Interface Entry Points
 348  *********************************************************************/
 349
 350 static device_method_t em_methods[] = {
 351         /* Device interface */
 352         DEVMETHOD(device_probe, em_probe),
 353         DEVMETHOD(device_attach, em_attach),
 354         DEVMETHOD(device_detach, em_detach),
 355         DEVMETHOD(device_shutdown, em_shutdown),
 356         DEVMETHOD(device_suspend, em_suspend),
 357         DEVMETHOD(device_resume, em_resume),
 358         {0, 0}
 359 };
 360
 361 static driver_t em_driver = {
 362         "em", em_methods, sizeof(struct adapter),
 363 };
 364
 365 static devclass_t em_devclass;
 366 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 367 MODULE_DEPEND(em, pci, 1, 1, 1);
 368 MODULE_DEPEND(em, ether, 1, 1, 1);
 369
 370 /*********************************************************************
 371  *  Tunable default values.
 372  *********************************************************************/
 373
 374 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 375 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 376 #define M_TSO_LEN                       66
 377
 378 /* Allow common code without TSO */
 379 #ifndef CSUM_TSO
 380 #define CSUM_TSO        0
 381 #endif
 382
 383 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 384 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 385 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 386 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 387 static int em_rxd = EM_DEFAULT_RXD;
 388 static int em_txd = EM_DEFAULT_TXD;
 389 static int em_smart_pwr_down = FALSE;
 390 /* Controls whether promiscuous also shows bad packets */
 391 static int em_debug_sbp = FALSE;
 392 /* Local switch for MSI/MSIX */
 393 static int em_enable_msi = TRUE;
 394
 395 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 396 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 397 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 398 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 399 TUNABLE_INT("hw.em.rxd", &em_rxd);
 400 TUNABLE_INT("hw.em.txd", &em_txd);
 401 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 402 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 403 TUNABLE_INT("hw.em.enable_msi", &em_enable_msi);
 404
 405 #ifndef EM_LEGACY_IRQ
 406 /* How many packets rxeof tries to clean at a time */
 407 static int em_rx_process_limit = 100;
 408 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
 409 #endif
 410
 411 /* Flow control setting - default to FULL */
 412 static int em_fc_setting = e1000_fc_full;
 413 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting);
 414
 415 /*
 416 ** Shadow VFTA table, this is needed because
 417 ** the real vlan filter table gets cleared during
 418 ** a soft reset and the driver needs to be able
 419 ** to repopulate it.
 420 */
 421 static u32 em_shadow_vfta[EM_VFTA_SIZE];
 422
 423 /* Global used in WOL setup with multiport cards */
 424 static int global_quad_port_a = 0;
 425
 426 /*********************************************************************
 427  *  Device identification routine
 428  *
 429  *  em_probe determines if the driver should be loaded on
 430  *  adapter based on PCI vendor/device id of the adapter.
 431  *
 432  *  return BUS_PROBE_DEFAULT on success, positive on failure
 433  *********************************************************************/
 434
 435 static int
 436 em_probe(device_t dev)
 437 {
 438         char            adapter_name[60];
 439         u16             pci_vendor_id = 0;
 440         u16             pci_device_id = 0;
 441         u16             pci_subvendor_id = 0;
 442         u16             pci_subdevice_id = 0;
 443         em_vendor_info_t *ent;
 444
 445         INIT_DEBUGOUT("em_probe: begin");
 446
 447         pci_vendor_id = pci_get_vendor(dev);
 448         if (pci_vendor_id != EM_VENDOR_ID)
 449                 return (ENXIO);
 450
 451         pci_device_id = pci_get_device(dev);
 452         pci_subvendor_id = pci_get_subvendor(dev);
 453         pci_subdevice_id = pci_get_subdevice(dev);
 454
 455         ent = em_vendor_info_array;
 456         while (ent->vendor_id != 0) {
 457                 if ((pci_vendor_id == ent->vendor_id) &&
 458                     (pci_device_id == ent->device_id) &&
 459
 460                     ((pci_subvendor_id == ent->subvendor_id) ||
 461                     (ent->subvendor_id == PCI_ANY_ID)) &&
 462
 463                     ((pci_subdevice_id == ent->subdevice_id) ||
 464                     (ent->subdevice_id == PCI_ANY_ID))) {
 465                         sprintf(adapter_name, "%s %s",
 466                                 em_strings[ent->index],
 467                                 em_driver_version);
 468                         device_set_desc_copy(dev, adapter_name);
 469                         return (BUS_PROBE_DEFAULT);
 470                 }
 471                 ent++;
 472         }
 473
 474         return (ENXIO);
 475 }
 476
 477 /*********************************************************************
 478  *  Device initialization routine
 479  *
 480  *  The attach entry point is called when the driver is being loaded.
 481  *  This routine identifies the type of hardware, allocates all resources
 482  *  and initializes the hardware.
 483  *
 484  *  return 0 on success, positive on failure
 485  *********************************************************************/
 486
 487 static int
 488 em_attach(device_t dev)
 489 {
 490         struct adapter  *adapter;
 491         int             tsize, rsize;
 492         int             error = 0;
 493         u16             eeprom_data, device_id;
 494
 495         INIT_DEBUGOUT("em_attach: begin");
 496
 497         adapter = device_get_softc(dev);
 498         adapter->dev = adapter->osdep.dev = dev;
 499         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 500         EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
 501         EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
 502
 503         /* SYSCTL stuff */
 504         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 505             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 506             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 507             em_sysctl_debug_info, "I", "Debug Information");
 508
 509         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 510             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 511             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 512             em_sysctl_stats, "I", "Statistics");
 513
 514         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 515         callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
 516
 517         /* Determine hardware and mac info */
 518         em_identify_hardware(adapter);
 519
 520         /* Setup PCI resources */
 521         if (em_allocate_pci_resources(adapter)) {
 522                 device_printf(dev, "Allocation of PCI resources failed\n");
 523                 error = ENXIO;
 524                 goto err_pci;
 525         }
 526
 527         /*
 528         ** For ICH8 and family we need to
 529         ** map the flash memory, and this
 530         ** must happen after the MAC is
 531         ** identified
 532         */
 533         if ((adapter->hw.mac.type == e1000_ich8lan) ||
 534             (adapter->hw.mac.type == e1000_ich9lan) ||
 535             (adapter->hw.mac.type == e1000_ich10lan)) {
 536                 int rid = EM_BAR_TYPE_FLASH;
 537                 adapter->flash = bus_alloc_resource_any(dev,
 538                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 539                 if (adapter->flash == NULL) {
 540                         device_printf(dev, "Mapping of Flash failed\n");
 541                         error = ENXIO;
 542                         goto err_pci;
 543                 }
 544                 /* This is used in the shared code */
 545                 adapter->hw.flash_address = (u8 *)adapter->flash;
 546                 adapter->osdep.flash_bus_space_tag =
 547                     rman_get_bustag(adapter->flash);
 548                 adapter->osdep.flash_bus_space_handle =
 549                     rman_get_bushandle(adapter->flash);
 550         }
 551
 552         /* Do Shared Code initialization */
 553         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 554                 device_printf(dev, "Setup of Shared code failed\n");
 555                 error = ENXIO;
 556                 goto err_pci;
 557         }
 558
 559         e1000_get_bus_info(&adapter->hw);
 560
 561         /* Set up some sysctls for the tunable interrupt delays */
 562         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 563             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 564             E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
 565         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 566             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 567             E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
 568         if (adapter->hw.mac.type >= e1000_82540) {
 569                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 570                     "receive interrupt delay limit in usecs",
 571                     &adapter->rx_abs_int_delay,
 572                     E1000_REGISTER(&adapter->hw, E1000_RADV),
 573                     em_rx_abs_int_delay_dflt);
 574                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 575                     "transmit interrupt delay limit in usecs",
 576                     &adapter->tx_abs_int_delay,
 577                     E1000_REGISTER(&adapter->hw, E1000_TADV),
 578                     em_tx_abs_int_delay_dflt);
 579         }
 580
 581 #ifndef EM_LEGACY_IRQ
 582         /* Sysctls for limiting the amount of work done in the taskqueue */
 583         em_add_rx_process_limit(adapter, "rx_processing_limit",
 584             "max number of rx packets to process", &adapter->rx_process_limit,
 585             em_rx_process_limit);
 586 #endif
 587
 588         /*
 589          * Validate number of transmit and receive descriptors. It
 590          * must not exceed hardware maximum, and must be multiple
 591          * of E1000_DBA_ALIGN.
 592          */
 593         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 594             (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
 595             (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
 596             (em_txd < EM_MIN_TXD)) {
 597                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 598                     EM_DEFAULT_TXD, em_txd);
 599                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 600         } else
 601                 adapter->num_tx_desc = em_txd;
 602         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 603             (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
 604             (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
 605             (em_rxd < EM_MIN_RXD)) {
 606                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 607                     EM_DEFAULT_RXD, em_rxd);
 608                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 609         } else
 610                 adapter->num_rx_desc = em_rxd;
 611
 612         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 613         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 614         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 615         adapter->rx_buffer_len = 2048;
 616
 617         e1000_init_script_state_82541(&adapter->hw, TRUE);
 618         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 619
 620         /* Copper options */
 621         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 622                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 623                 adapter->hw.phy.disable_polarity_correction = FALSE;
 624                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 625         }
 626
 627         /*
 628          * Set the frame limits assuming
 629          * standard ethernet sized frames.
 630          */
 631         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 632         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 633
 634         /*
 635          * This controls when hardware reports transmit completion
 636          * status.
 637          */
 638         adapter->hw.mac.report_tx_early = 1;
 639
 640         tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
 641             EM_DBA_ALIGN);
 642
 643         /* Allocate Transmit Descriptor ring */
 644         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
 645                 device_printf(dev, "Unable to allocate tx_desc memory\n");
 646                 error = ENOMEM;
 647                 goto err_tx_desc;
 648         }
 649         adapter->tx_desc_base =
 650             (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
 651
 652         rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
 653             EM_DBA_ALIGN);
 654
 655         /* Allocate Receive Descriptor ring */
 656         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 657                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 658                 error = ENOMEM;
 659                 goto err_rx_desc;
 660         }
 661         adapter->rx_desc_base =
 662             (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
 663
 664         /*
 665         ** Start from a known state, this is
 666         ** important in reading the nvm and
 667         ** mac from that.
 668         */
 669         e1000_reset_hw(&adapter->hw);
 670
 671         /* Make sure we have a good EEPROM before we read from it */
 672         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 673                 /*
 674                 ** Some PCI-E parts fail the first check due to
 675                 ** the link being in sleep state, call it again,
 676                 ** if it fails a second time its a real issue.
 677                 */
 678                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 679                         device_printf(dev,
 680                             "The EEPROM Checksum Is Not Valid\n");
 681                         error = EIO;
 682                         goto err_hw_init;
 683                 }
 684         }
 685
 686         /* Copy the permanent MAC address out of the EEPROM */
 687         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 688                 device_printf(dev, "EEPROM read error while reading MAC"
 689                     " address\n");
 690                 error = EIO;
 691                 goto err_hw_init;
 692         }
 693
 694         if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
 695                 device_printf(dev, "Invalid MAC address\n");
 696                 error = EIO;
 697                 goto err_hw_init;
 698         }
 699
 700         /* Initialize the hardware */
 701         if (em_hardware_init(adapter)) {
 702                 device_printf(dev, "Unable to initialize the hardware\n");
 703                 error = EIO;
 704                 goto err_hw_init;
 705         }
 706
 707         /* Allocate transmit descriptors and buffers */
 708         if (em_allocate_transmit_structures(adapter)) {
 709                 device_printf(dev, "Could not setup transmit structures\n");
 710                 error = ENOMEM;
 711                 goto err_tx_struct;
 712         }
 713
 714         /* Allocate receive descriptors and buffers */
 715         if (em_allocate_receive_structures(adapter)) {
 716                 device_printf(dev, "Could not setup receive structures\n");
 717                 error = ENOMEM;
 718                 goto err_rx_struct;
 719         }
 720
 721         /*
 722         **  Do interrupt configuration
 723         */
 724         if (adapter->msi > 1) /* Do MSI/X */
 725                 error = em_allocate_msix(adapter);
 726         else  /* MSI or Legacy */
 727                 error = em_allocate_legacy(adapter);
 728         if (error)
 729                 goto err_rx_struct;
 730
 731         /* Setup OS specific network interface */
 732         em_setup_interface(dev, adapter);
 733
 734         /* Initialize statistics */
 735         em_update_stats_counters(adapter);
 736
 737         adapter->hw.mac.get_link_status = 1;
 738         em_update_link_status(adapter);
 739
 740         /* Indicate SOL/IDER usage */
 741         if (e1000_check_reset_block(&adapter->hw))
 742                 device_printf(dev,
 743                     "PHY reset is blocked due to SOL/IDER session.\n");
 744
 745         /* Determine if we have to control management hardware */
 746         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
 747
 748         /*
 749          * Setup Wake-on-Lan
 750          */
 751         switch (adapter->hw.mac.type) {
 752
 753         case e1000_82542:
 754         case e1000_82543:
 755                 break;
 756         case e1000_82546:
 757         case e1000_82546_rev_3:
 758         case e1000_82571:
 759         case e1000_80003es2lan:
 760                 if (adapter->hw.bus.func == 1)
 761                         e1000_read_nvm(&adapter->hw,
 762                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
 763                 else
 764                         e1000_read_nvm(&adapter->hw,
 765                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 766                 eeprom_data &= EM_EEPROM_APME;
 767                 break;
 768         default:
 769                 /* APME bit in EEPROM is mapped to WUC.APME */
 770                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) &
 771                     E1000_WUC_APME;
 772                 break;
 773         }
 774         if (eeprom_data)
 775                 adapter->wol = E1000_WUFC_MAG;
 776         /*
 777          * We have the eeprom settings, now apply the special cases
 778          * where the eeprom may be wrong or the board won't support
 779          * wake on lan on a particular port
 780          */
 781         device_id = pci_get_device(dev);
 782         switch (device_id) {
 783         case E1000_DEV_ID_82546GB_PCIE:
 784                 adapter->wol = 0;
 785                 break;
 786         case E1000_DEV_ID_82546EB_FIBER:
 787         case E1000_DEV_ID_82546GB_FIBER:
 788         case E1000_DEV_ID_82571EB_FIBER:
 789                 /* Wake events only supported on port A for dual fiber
 790                  * regardless of eeprom setting */
 791                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
 792                     E1000_STATUS_FUNC_1)
 793                         adapter->wol = 0;
 794                 break;
 795         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
 796         case E1000_DEV_ID_82571EB_QUAD_COPPER:
 797         case E1000_DEV_ID_82571EB_QUAD_FIBER:
 798         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
 799                 /* if quad port adapter, disable WoL on all but port A */
 800                 if (global_quad_port_a != 0)
 801                         adapter->wol = 0;
 802                 /* Reset for multiple quad port adapters */
 803                 if (++global_quad_port_a == 4)
 804                         global_quad_port_a = 0;
 805                 break;
 806         }
 807
 808         /* Do we need workaround for 82544 PCI-X adapter? */
 809         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
 810             adapter->hw.mac.type == e1000_82544)
 811                 adapter->pcix_82544 = TRUE;
 812         else
 813                 adapter->pcix_82544 = FALSE;
 814
 815 #if __FreeBSD_version >= 700029
 816         /* Register for VLAN events */
 817         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 818             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 819         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 820             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 821 #endif
 822
 823         /* Tell the stack that the interface is not active */
 824         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 825
 826         INIT_DEBUGOUT("em_attach: end");
 827
 828         return (0);
 829
 830 err_rx_struct:
 831         em_free_transmit_structures(adapter);
 832 err_tx_struct:
 833 err_hw_init:
 834         em_release_hw_control(adapter);
 835         em_dma_free(adapter, &adapter->rxdma);
 836 err_rx_desc:
 837         em_dma_free(adapter, &adapter->txdma);
 838 err_tx_desc:
 839 err_pci:
 840         em_free_pci_resources(adapter);
 841         EM_TX_LOCK_DESTROY(adapter);
 842         EM_RX_LOCK_DESTROY(adapter);
 843         EM_CORE_LOCK_DESTROY(adapter);
 844
 845         return (error);
 846 }
 847
 848 /*********************************************************************
 849  *  Device removal routine
 850  *
 851  *  The detach entry point is called when the driver is being removed.
 852  *  This routine stops the adapter and deallocates all the resources
 853  *  that were allocated for driver operation.
 854  *
 855  *  return 0 on success, positive on failure
 856  *********************************************************************/
 857
 858 static int
 859 em_detach(device_t dev)
 860 {
 861         struct adapter  *adapter = device_get_softc(dev);
 862         struct ifnet    *ifp = adapter->ifp;
 863
 864         INIT_DEBUGOUT("em_detach: begin");
 865
 866         /* Make sure VLANS are not using driver */
 867 #if __FreeBSD_version >= 700000
 868         if (adapter->ifp->if_vlantrunk != NULL) {
 869 #else
 870         if (adapter->ifp->if_nvlans != 0) {
 871 #endif
 872                 device_printf(dev,"Vlan in use, detach first\n");
 873                 return (EBUSY);
 874         }
 875
 876 #ifdef DEVICE_POLLING
 877         if (ifp->if_capenable & IFCAP_POLLING)
 878                 ether_poll_deregister(ifp);
 879 #endif
 880
 881         EM_CORE_LOCK(adapter);
 882         EM_TX_LOCK(adapter);
 883         adapter->in_detach = 1;
 884         em_stop(adapter);
 885         e1000_phy_hw_reset(&adapter->hw);
 886
 887         em_release_manageability(adapter);
 888
 889         if (((adapter->hw.mac.type == e1000_82573) ||
 890             (adapter->hw.mac.type == e1000_82583) ||
 891             (adapter->hw.mac.type == e1000_ich8lan) ||
 892             (adapter->hw.mac.type == e1000_ich10lan) ||
 893             (adapter->hw.mac.type == e1000_ich9lan)) &&
 894             e1000_check_mng_mode(&adapter->hw))
 895                 em_release_hw_control(adapter);
 896
 897         if (adapter->wol) {
 898                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 899                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 900                 em_enable_wakeup(dev);
 901         }
 902
 903         EM_TX_UNLOCK(adapter);
 904         EM_CORE_UNLOCK(adapter);
 905
 906 #if __FreeBSD_version >= 700029
 907         /* Unregister VLAN events */
 908         if (adapter->vlan_attach != NULL)
 909                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 910         if (adapter->vlan_detach != NULL)
 911                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 912 #endif
 913
 914         ether_ifdetach(adapter->ifp);
 915         callout_drain(&adapter->timer);
 916         callout_drain(&adapter->tx_fifo_timer);
 917
 918         em_free_pci_resources(adapter);
 919         bus_generic_detach(dev);
 920         if_free(ifp);
 921
 922         em_free_transmit_structures(adapter);
 923         em_free_receive_structures(adapter);
 924
 925         /* Free Transmit Descriptor ring */
 926         if (adapter->tx_desc_base) {
 927                 em_dma_free(adapter, &adapter->txdma);
 928                 adapter->tx_desc_base = NULL;
 929         }
 930
 931         /* Free Receive Descriptor ring */
 932         if (adapter->rx_desc_base) {
 933                 em_dma_free(adapter, &adapter->rxdma);
 934                 adapter->rx_desc_base = NULL;
 935         }
 936
 937         EM_TX_LOCK_DESTROY(adapter);
 938         EM_RX_LOCK_DESTROY(adapter);
 939         EM_CORE_LOCK_DESTROY(adapter);
 940
 941         return (0);
 942 }
 943
 944 /*********************************************************************
 945  *
 946  *  Shutdown entry point
 947  *
 948  **********************************************************************/
 949
 950 static int
 951 em_shutdown(device_t dev)
 952 {
 953         return em_suspend(dev);
 954 }
 955
 956 /*
 957  * Suspend/resume device methods.
 958  */
 959 static int
 960 em_suspend(device_t dev)
 961 {
 962         struct adapter *adapter = device_get_softc(dev);
 963
 964         EM_CORE_LOCK(adapter);
 965
 966         EM_TX_LOCK(adapter);
 967         em_stop(adapter);
 968         EM_TX_UNLOCK(adapter);
 969
 970         em_release_manageability(adapter);
 971
 972         if (((adapter->hw.mac.type == e1000_82573) ||
 973             (adapter->hw.mac.type == e1000_82583) ||
 974             (adapter->hw.mac.type == e1000_ich8lan) ||
 975             (adapter->hw.mac.type == e1000_ich10lan) ||
 976             (adapter->hw.mac.type == e1000_ich9lan)) &&
 977             e1000_check_mng_mode(&adapter->hw))
 978                 em_release_hw_control(adapter);
 979
 980         if (adapter->wol) {
 981                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 982                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 983                 em_enable_wakeup(dev);
 984         }
 985
 986         EM_CORE_UNLOCK(adapter);
 987
 988         return bus_generic_suspend(dev);
 989 }
 990
 991 static int
 992 em_resume(device_t dev)
 993 {
 994         struct adapter *adapter = device_get_softc(dev);
 995         struct ifnet *ifp = adapter->ifp;
 996
 997         EM_CORE_LOCK(adapter);
 998         em_init_locked(adapter);
 999         em_init_manageability(adapter);
1000         EM_CORE_UNLOCK(adapter);
1001         em_start(ifp);
1002
1003         return bus_generic_resume(dev);
1004 }
1005
1006
1007 /*********************************************************************
1008  *  Transmit entry point
1009  *
1010  *  em_start is called by the stack to initiate a transmit.
1011  *  The driver will remain in this routine as long as there are
1012  *  packets to transmit and transmit resources are available.
1013  *  In case resources are not available stack is notified and
1014  *  the packet is requeued.
1015  **********************************************************************/
1016
1017 #if __FreeBSD_version >= 800000
1018 static int
1019 em_mq_start_locked(struct ifnet *ifp, struct mbuf *m)
1020 {
1021         struct adapter  *adapter = ifp->if_softc;
1022         struct mbuf     *next;
1023         int error = E1000_SUCCESS;
1024
1025         EM_TX_LOCK_ASSERT(adapter);
1026         /* To allow being called from a tasklet */
1027         if (m == NULL)
1028                 goto process;
1029
1030         if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
1031             IFF_DRV_RUNNING)
1032             || (!adapter->link_active)) {
1033                 error = drbr_enqueue(ifp, adapter->br, m);
1034                 return (error);
1035         } else if (drbr_empty(ifp, adapter->br) &&
1036             (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) {
1037                 if (em_xmit(adapter, &m)) {
1038                         if (m && (error = drbr_enqueue(ifp, adapter->br, m)) != 0)
1039                                 return (error);
1040                 } else {
1041                         /*
1042                          * We've bypassed the buf ring so we need to update
1043                          * ifp directly
1044                          */
1045                         drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
1046                         /*
1047                         ** Send a copy of the frame to the BPF
1048                         ** listener and set the watchdog on.
1049                         */
1050                         ETHER_BPF_MTAP(ifp, m);
1051                         adapter->watchdog_timer = EM_TX_TIMEOUT;
1052                 }
1053         } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0)
1054                 return (error);
1055
1056 process:
1057         if (drbr_empty(ifp, adapter->br))
1058                 return(error);
1059         /* Process the queue */
1060         while (TRUE) {
1061                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1062                         break;
1063                 next = drbr_dequeue(ifp, adapter->br);
1064                 if (next == NULL)
1065                         break;
1066                 if (em_xmit(adapter, &next))
1067                         break;
1068                 ETHER_BPF_MTAP(ifp, next);
1069                 /* Set the watchdog */
1070                 adapter->watchdog_timer = EM_TX_TIMEOUT;
1071         }
1072
1073         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1074                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1075
1076         return (0);
1077 }
1078
1079 /*
1080 ** Multiqueue capable stack interface, this is not
1081 ** yet truely multiqueue, but that is coming...
1082 */
1083 static int
1084 em_mq_start(struct ifnet *ifp, struct mbuf *m)
1085 {
1086
1087         struct adapter *adapter = ifp->if_softc;
1088         int error = 0;
1089
1090         if (EM_TX_TRYLOCK(adapter)) {
1091                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1092                         error = em_mq_start_locked(ifp, m);
1093                 EM_TX_UNLOCK(adapter);
1094         } else
1095                 error = drbr_enqueue(ifp, adapter->br, m);
1096
1097         return (error);
1098 }
1099
1100 static void
1101 em_qflush(struct ifnet *ifp)
1102 {
1103         struct mbuf *m;
1104         struct adapter *adapter = (struct adapter *)ifp->if_softc;
1105
1106         EM_TX_LOCK(adapter);
1107         while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL)
1108                 m_freem(m);
1109         if_qflush(ifp);
1110         EM_TX_UNLOCK(adapter);
1111 }
1112 #endif /* FreeBSD_version */
1113
1114 static void
1115 em_start_locked(struct ifnet *ifp)
1116 {
1117         struct adapter  *adapter = ifp->if_softc;
1118         struct mbuf     *m_head;
1119
1120         EM_TX_LOCK_ASSERT(adapter);
1121
1122         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
1123             IFF_DRV_RUNNING)
1124                 return;
1125         if (!adapter->link_active)
1126                 return;
1127
1128         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
1129
1130                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
1131                 if (m_head == NULL)
1132                         break;
1133                 /*
1134                  *  Encapsulation can modify our pointer, and or make it
1135                  *  NULL on failure.  In that event, we can't requeue.
1136                  */
1137                 if (em_xmit(adapter, &m_head)) {
1138                         if (m_head == NULL)
1139                                 break;
1140                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1141                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1142                         break;
1143                 }
1144
1145                 /* Send a copy of the frame to the BPF listener */
1146                 ETHER_BPF_MTAP(ifp, m_head);
1147
1148                 /* Set timeout in case hardware has problems transmitting. */
1149                 adapter->watchdog_timer = EM_TX_TIMEOUT;
1150         }
1151         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1152                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1153
1154         return;
1155 }
1156
1157 static void
1158 em_start(struct ifnet *ifp)
1159 {
1160         struct adapter *adapter = ifp->if_softc;
1161
1162         EM_TX_LOCK(adapter);
1163         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1164                 em_start_locked(ifp);
1165         EM_TX_UNLOCK(adapter);
1166 }
1167
1168 /*********************************************************************
1169  *  Ioctl entry point
1170  *
1171  *  em_ioctl is called when the user wants to configure the
1172  *  interface.
1173  *
1174  *  return 0 on success, positive on failure
1175  **********************************************************************/
1176
1177 static int
1178 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1179 {
1180         struct adapter  *adapter = ifp->if_softc;
1181         struct ifreq *ifr = (struct ifreq *)data;
1182 #ifdef INET
1183         struct ifaddr *ifa = (struct ifaddr *)data;
1184 #endif
1185         int error = 0;
1186
1187         if (adapter->in_detach)
1188                 return (error);
1189
1190         switch (command) {
1191         case SIOCSIFADDR:
1192 #ifdef INET
1193                 if (ifa->ifa_addr->sa_family == AF_INET) {
1194                         /*
1195                          * XXX
1196                          * Since resetting hardware takes a very long time
1197                          * and results in link renegotiation we only
1198                          * initialize the hardware only when it is absolutely
1199                          * required.
1200                          */
1201                         ifp->if_flags |= IFF_UP;
1202                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1203                                 EM_CORE_LOCK(adapter);
1204                                 em_init_locked(adapter);
1205                                 EM_CORE_UNLOCK(adapter);
1206                         }
1207                         if (!(ifp->if_flags & IFF_NOARP))
1208                                 arp_ifinit(ifp, ifa);
1209                 } else
1210 #endif
1211                         error = ether_ioctl(ifp, command, data);
1212                 break;
1213         case SIOCSIFMTU:
1214             {
1215                 int max_frame_size;
1216                 u16 eeprom_data = 0;
1217
1218                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1219
1220                 EM_CORE_LOCK(adapter);
1221                 switch (adapter->hw.mac.type) {
1222                 case e1000_82573:
1223                         /*
1224                          * 82573 only supports jumbo frames
1225                          * if ASPM is disabled.
1226                          */
1227                         e1000_read_nvm(&adapter->hw,
1228                             NVM_INIT_3GIO_3, 1, &eeprom_data);
1229                         if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1230                                 max_frame_size = ETHER_MAX_LEN;
1231                                 break;
1232                         }
1233                         /* Allow Jumbo frames - fall thru */
1234                 case e1000_82571:
1235                 case e1000_82572:
1236                 case e1000_ich9lan:
1237                 case e1000_ich10lan:
1238                 case e1000_82574:
1239                 case e1000_80003es2lan: /* Limit Jumbo Frame size */
1240                         max_frame_size = 9234;
1241                         break;
1242                         /* Adapters that do not support jumbo frames */
1243                 case e1000_82542:
1244                 case e1000_82583:
1245                 case e1000_ich8lan:
1246                         max_frame_size = ETHER_MAX_LEN;
1247                         break;
1248                 default:
1249                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1250                 }
1251                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1252                     ETHER_CRC_LEN) {
1253                         EM_CORE_UNLOCK(adapter);
1254                         error = EINVAL;
1255                         break;
1256                 }
1257
1258                 ifp->if_mtu = ifr->ifr_mtu;
1259                 adapter->max_frame_size =
1260                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1261                 em_init_locked(adapter);
1262                 EM_CORE_UNLOCK(adapter);
1263                 break;
1264             }
1265         case SIOCSIFFLAGS:
1266                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1267                     SIOCSIFFLAGS (Set Interface Flags)");
1268                 EM_CORE_LOCK(adapter);
1269                 if (ifp->if_flags & IFF_UP) {
1270                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1271                                 if ((ifp->if_flags ^ adapter->if_flags) &
1272                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1273                                         em_disable_promisc(adapter);
1274                                         em_set_promisc(adapter);
1275                                 }
1276                         } else
1277                                 em_init_locked(adapter);
1278                 } else
1279                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1280                                 EM_TX_LOCK(adapter);
1281                                 em_stop(adapter);
1282                                 EM_TX_UNLOCK(adapter);
1283                         }
1284                 adapter->if_flags = ifp->if_flags;
1285                 EM_CORE_UNLOCK(adapter);
1286                 break;
1287         case SIOCADDMULTI:
1288         case SIOCDELMULTI:
1289                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1290                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1291                         EM_CORE_LOCK(adapter);
1292                         em_disable_intr(adapter);
1293                         em_set_multi(adapter);
1294                         if (adapter->hw.mac.type == e1000_82542 &&
1295                             adapter->hw.revision_id == E1000_REVISION_2) {
1296                                 em_initialize_receive_unit(adapter);
1297                         }
1298 #ifdef DEVICE_POLLING
1299                         if (!(ifp->if_capenable & IFCAP_POLLING))
1300 #endif
1301                                 em_enable_intr(adapter);
1302                         EM_CORE_UNLOCK(adapter);
1303                 }
1304                 break;
1305         case SIOCSIFMEDIA:
1306                 /* Check SOL/IDER usage */
1307                 EM_CORE_LOCK(adapter);
1308                 if (e1000_check_reset_block(&adapter->hw)) {
1309                         EM_CORE_UNLOCK(adapter);
1310                         device_printf(adapter->dev, "Media change is"
1311                             " blocked due to SOL/IDER session.\n");
1312                         break;
1313                 }
1314                 EM_CORE_UNLOCK(adapter);
1315         case SIOCGIFMEDIA:
1316                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1317                     SIOCxIFMEDIA (Get/Set Interface Media)");
1318                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1319                 break;
1320         case SIOCSIFCAP:
1321             {
1322                 int mask, reinit;
1323
1324                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1325                 reinit = 0;
1326                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1327 #ifdef DEVICE_POLLING
1328                 if (mask & IFCAP_POLLING) {
1329                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1330                                 error = ether_poll_register(em_poll, ifp);
1331                                 if (error)
1332                                         return (error);
1333                                 EM_CORE_LOCK(adapter);
1334                                 em_disable_intr(adapter);
1335                                 ifp->if_capenable |= IFCAP_POLLING;
1336                                 EM_CORE_UNLOCK(adapter);
1337                         } else {
1338                                 error = ether_poll_deregister(ifp);
1339                                 /* Enable interrupt even in error case */
1340                                 EM_CORE_LOCK(adapter);
1341                                 em_enable_intr(adapter);
1342                                 ifp->if_capenable &= ~IFCAP_POLLING;
1343                                 EM_CORE_UNLOCK(adapter);
1344                         }
1345                 }
1346 #endif
1347                 if (mask & IFCAP_HWCSUM) {
1348                         ifp->if_capenable ^= IFCAP_HWCSUM;
1349                         reinit = 1;
1350                 }
1351 #if __FreeBSD_version >= 700000
1352                 if (mask & IFCAP_TSO4) {
1353                         ifp->if_capenable ^= IFCAP_TSO4;
1354                         reinit = 1;
1355                 }
1356 #endif
1357
1358                 if (mask & IFCAP_VLAN_HWTAGGING) {
1359                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1360                         reinit = 1;
1361                 }
1362                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1363                         em_init(adapter);
1364 #if __FreeBSD_version >= 700000
1365                 VLAN_CAPABILITIES(ifp);
1366 #endif
1367                 break;
1368             }
1369
1370         default:
1371                 error = ether_ioctl(ifp, command, data);
1372                 break;
1373         }
1374
1375         return (error);
1376 }
1377
1378 /*********************************************************************
1379  *  Watchdog timer:
1380  *
1381  *  This routine is called from the local timer every second.
1382  *  As long as transmit descriptors are being cleaned the value
1383  *  is non-zero and we do nothing. Reaching 0 indicates a tx hang
1384  *  and we then reset the device.
1385  *
1386  **********************************************************************/
1387
1388 static void
1389 em_watchdog(struct adapter *adapter)
1390 {
1391
1392         EM_CORE_LOCK_ASSERT(adapter);
1393
1394         /*
1395         ** The timer is set to 5 every time start queues a packet.
1396         ** Then txeof keeps resetting it as long as it cleans at
1397         ** least one descriptor.
1398         ** Finally, anytime all descriptors are clean the timer is
1399         ** set to 0.
1400         */
1401         EM_TX_LOCK(adapter);
1402         if ((adapter->watchdog_timer == 0) || (--adapter->watchdog_timer)) {
1403                 EM_TX_UNLOCK(adapter);
1404                 return;
1405         }
1406
1407         /* If we are in this routine because of pause frames, then
1408          * don't reset the hardware.
1409          */
1410         if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1411             E1000_STATUS_TXOFF) {
1412                 adapter->watchdog_timer = EM_TX_TIMEOUT;
1413                 EM_TX_UNLOCK(adapter);
1414                 return;
1415         }
1416
1417         if (e1000_check_for_link(&adapter->hw) == 0)
1418                 device_printf(adapter->dev, "watchdog timeout -- resetting\n");
1419         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1420         adapter->watchdog_events++;
1421         EM_TX_UNLOCK(adapter);
1422
1423         em_init_locked(adapter);
1424 }
1425
1426 /*********************************************************************
1427  *  Init entry point
1428  *
1429  *  This routine is used in two ways. It is used by the stack as
1430  *  init entry point in network interface structure. It is also used
1431  *  by the driver as a hw/sw initialization routine to get to a
1432  *  consistent state.
1433  *
1434  *  return 0 on success, positive on failure
1435  **********************************************************************/
1436
1437 static void
1438 em_init_locked(struct adapter *adapter)
1439 {
1440         struct ifnet    *ifp = adapter->ifp;
1441         device_t        dev = adapter->dev;
1442         u32             pba;
1443
1444         INIT_DEBUGOUT("em_init: begin");
1445
1446         EM_CORE_LOCK_ASSERT(adapter);
1447
1448         EM_TX_LOCK(adapter);
1449         em_stop(adapter);
1450         EM_TX_UNLOCK(adapter);
1451
1452         /*
1453          * Packet Buffer Allocation (PBA)
1454          * Writing PBA sets the receive portion of the buffer
1455          * the remainder is used for the transmit buffer.
1456          *
1457          * Devices before the 82547 had a Packet Buffer of 64K.
1458          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1459          * After the 82547 the buffer was reduced to 40K.
1460          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1461          *   Note: default does not leave enough room for Jumbo Frame >10k.
1462          */
1463         switch (adapter->hw.mac.type) {
1464         case e1000_82547:
1465         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1466                 if (adapter->max_frame_size > 8192)
1467                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1468                 else
1469                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1470                 adapter->tx_fifo_head = 0;
1471                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1472                 adapter->tx_fifo_size =
1473                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1474                 break;
1475         /* Total Packet Buffer on these is 48K */
1476         case e1000_82571:
1477         case e1000_82572:
1478         case e1000_80003es2lan:
1479                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1480                 break;
1481         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1482                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1483                 break;
1484         case e1000_82574:
1485         case e1000_82583:
1486                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1487                 break;
1488         case e1000_ich9lan:
1489         case e1000_ich10lan:
1490         case e1000_ich8lan:
1491                 pba = E1000_PBA_8K;
1492                 break;
1493         default:
1494                 /* Devices before 82547 had a Packet Buffer of 64K.   */
1495                 if (adapter->max_frame_size > 8192)
1496                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1497                 else
1498                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1499         }
1500
1501         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1502         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1503
1504         /* Get the latest mac address, User can use a LAA */
1505         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1506               ETHER_ADDR_LEN);
1507
1508         /* Put the address into the Receive Address Array */
1509         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1510
1511         /*
1512          * With the 82571 adapter, RAR[0] may be overwritten
1513          * when the other port is reset, we make a duplicate
1514          * in RAR[14] for that eventuality, this assures
1515          * the interface continues to function.
1516          */
1517         if (adapter->hw.mac.type == e1000_82571) {
1518                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1519                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1520                     E1000_RAR_ENTRIES - 1);
1521         }
1522
1523         /* Initialize the hardware */
1524         if (em_hardware_init(adapter)) {
1525                 device_printf(dev, "Unable to initialize the hardware\n");
1526                 return;
1527         }
1528         em_update_link_status(adapter);
1529
1530         /* Setup VLAN support, basic and offload if available */
1531         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1532
1533 #if __FreeBSD_version < 700029
1534         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1535                 u32 ctrl;
1536                 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1537                 ctrl |= E1000_CTRL_VME;
1538                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1539         }
1540 #else
1541         /* Use real VLAN Filter support */
1542         em_setup_vlan_hw_support(adapter);
1543 #endif
1544
1545         /* Set hardware offload abilities */
1546         ifp->if_hwassist = 0;
1547         if (adapter->hw.mac.type >= e1000_82543) {
1548                 if (ifp->if_capenable & IFCAP_TXCSUM)
1549                         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1550 #if __FreeBSD_version >= 700000
1551                 if (ifp->if_capenable & IFCAP_TSO4)
1552                         ifp->if_hwassist |= CSUM_TSO;
1553 #endif
1554         }
1555
1556         /* Configure for OS presence */
1557         em_init_manageability(adapter);
1558
1559         /* Prepare transmit descriptors and buffers */
1560         em_setup_transmit_structures(adapter);
1561         em_initialize_transmit_unit(adapter);
1562
1563         /* Setup Multicast table */
1564         em_set_multi(adapter);
1565
1566         /* Prepare receive descriptors and buffers */
1567         if (em_setup_receive_structures(adapter)) {
1568                 device_printf(dev, "Could not setup receive structures\n");
1569                 EM_TX_LOCK(adapter);
1570                 em_stop(adapter);
1571                 EM_TX_UNLOCK(adapter);
1572                 return;
1573         }
1574         em_initialize_receive_unit(adapter);
1575
1576         /* Don't lose promiscuous settings */
1577         em_set_promisc(adapter);
1578
1579         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1580         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1581
1582         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1583         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1584
1585         /* MSI/X configuration for 82574 */
1586         if (adapter->hw.mac.type == e1000_82574) {
1587                 int tmp;
1588                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1589                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1590                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1591                 /*
1592                 ** Set the IVAR - interrupt vector routing.
1593                 ** Each nibble represents a vector, high bit
1594                 ** is enable, other 3 bits are the MSIX table
1595                 ** entry, we map RXQ0 to 0, TXQ0 to 1, and
1596                 ** Link (other) to 2, hence the magic number.
1597                 */
1598                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1599         }
1600
1601 #ifdef DEVICE_POLLING
1602         /*
1603          * Only enable interrupts if we are not polling, make sure
1604          * they are off otherwise.
1605          */
1606         if (ifp->if_capenable & IFCAP_POLLING)
1607                 em_disable_intr(adapter);
1608         else
1609 #endif /* DEVICE_POLLING */
1610                 em_enable_intr(adapter);
1611
1612         /* Don't reset the phy next time init gets called */
1613         adapter->hw.phy.reset_disable = TRUE;
1614 }
1615
1616 static void
1617 em_init(void *arg)
1618 {
1619         struct adapter *adapter = arg;
1620
1621         EM_CORE_LOCK(adapter);
1622         em_init_locked(adapter);
1623         EM_CORE_UNLOCK(adapter);
1624 }
1625
1626
1627 #ifdef DEVICE_POLLING
1628 /*********************************************************************
1629  *
1630  *  Legacy polling routine
1631  *
1632  *********************************************************************/
1633 static int
1634 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1635 {
1636         struct adapter *adapter = ifp->if_softc;
1637         u32             reg_icr, rx_done = 0;
1638
1639         EM_CORE_LOCK(adapter);
1640         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1641                 EM_CORE_UNLOCK(adapter);
1642                 return (rx_done);
1643         }
1644
1645         if (cmd == POLL_AND_CHECK_STATUS) {
1646                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1647                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1648                         callout_stop(&adapter->timer);
1649                         adapter->hw.mac.get_link_status = 1;
1650                         em_update_link_status(adapter);
1651                         callout_reset(&adapter->timer, hz,
1652                             em_local_timer, adapter);
1653                 }
1654         }
1655         EM_CORE_UNLOCK(adapter);
1656
1657         rx_done = em_rxeof(adapter, count);
1658
1659         EM_TX_LOCK(adapter);
1660         em_txeof(adapter);
1661 #if __FreeBSD_version >= 800000
1662         if (!drbr_empty(ifp, adapter->br))
1663                 em_mq_start_locked(ifp, NULL);
1664 #else
1665         if (!IFQ_DRV_IS_EMPTY(&ifp->snd))
1666                 em_start_locked(ifp);
1667 #endif
1668         EM_TX_UNLOCK(adapter);
1669         return (rx_done);
1670 }
1671 #endif /* DEVICE_POLLING */
1672
1673 #ifdef EM_LEGACY_IRQ
1674 /*********************************************************************
1675  *
1676  *  Legacy Interrupt Service routine
1677  *
1678  *********************************************************************/
1679
1680 static void
1681 em_intr(void *arg)
1682 {
1683         struct adapter  *adapter = arg;
1684         struct ifnet    *ifp = adapter->ifp;
1685         u32             reg_icr;
1686
1687
1688         if (ifp->if_capenable & IFCAP_POLLING)
1689                 return;
1690
1691         EM_CORE_LOCK(adapter);
1692         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1693         if (reg_icr & E1000_ICR_RXO)
1694                 adapter->rx_overruns++;
1695         if ((reg_icr == 0xffffffff) || (reg_icr == 0)||
1696             (adapter->hw.mac.type >= e1000_82571 &&
1697             (reg_icr & E1000_ICR_INT_ASSERTED) == 0))
1698                         goto out;
1699
1700         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1701                         goto out;
1702
1703         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1704                 callout_stop(&adapter->timer);
1705                 adapter->hw.mac.get_link_status = 1;
1706                 em_update_link_status(adapter);
1707                 /* Deal with TX cruft when link lost */
1708                 em_tx_purge(adapter);
1709                 callout_reset(&adapter->timer, hz,
1710                     em_local_timer, adapter);
1711                 goto out;
1712         }
1713
1714         EM_TX_LOCK(adapter);
1715         em_txeof(adapter);
1716         em_rxeof(adapter, -1);
1717         em_txeof(adapter);
1718         if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
1719             !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1720                 em_start_locked(ifp);
1721         EM_TX_UNLOCK(adapter);
1722
1723 out:
1724         EM_CORE_UNLOCK(adapter);
1725         return;
1726 }
1727
1728 #else /* EM_FAST_IRQ, then fast interrupt routines only */
1729
1730 static void
1731 em_handle_link(void *context, int pending)
1732 {
1733         struct adapter  *adapter = context;
1734         struct ifnet *ifp = adapter->ifp;
1735
1736         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1737                 return;
1738
1739         EM_CORE_LOCK(adapter);
1740         callout_stop(&adapter->timer);
1741         em_update_link_status(adapter);
1742         /* Deal with TX cruft when link lost */
1743         em_tx_purge(adapter);
1744         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1745         EM_CORE_UNLOCK(adapter);
1746 }
1747
1748
1749 /* Combined RX/TX handler, used by Legacy and MSI */
1750 static void
1751 em_handle_rxtx(void *context, int pending)
1752 {
1753         struct adapter  *adapter = context;
1754         struct ifnet    *ifp = adapter->ifp;
1755
1756
1757         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1758                 if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1759                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1760                 EM_TX_LOCK(adapter);
1761                 em_txeof(adapter);
1762
1763 #if __FreeBSD_version >= 800000
1764                 if (!drbr_empty(ifp, adapter->br))
1765                         em_mq_start_locked(ifp, NULL);
1766 #else
1767                 if (!IFQ_DRV_IS_EMPTY(&ifp->snd))
1768                         em_start_locked(ifp);
1769 #endif
1770                 EM_TX_UNLOCK(adapter);
1771         }
1772
1773         em_enable_intr(adapter);
1774 }
1775
1776 /*********************************************************************
1777  *
1778  *  Fast Legacy/MSI Combined Interrupt Service routine
1779  *
1780  *********************************************************************/
1781 #if __FreeBSD_version < 700000
1782 #define FILTER_STRAY
1783 #define FILTER_HANDLED
1784 static void
1785 #else
1786 static int
1787 #endif
1788 em_irq_fast(void *arg)
1789 {
1790         struct adapter  *adapter = arg;
1791         struct ifnet    *ifp;
1792         u32             reg_icr;
1793
1794         ifp = adapter->ifp;
1795
1796         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1797
1798         /* Hot eject?  */
1799         if (reg_icr == 0xffffffff)
1800                 return FILTER_STRAY;
1801
1802         /* Definitely not our interrupt.  */
1803         if (reg_icr == 0x0)
1804                 return FILTER_STRAY;
1805
1806         /*
1807          * Starting with the 82571 chip, bit 31 should be used to
1808          * determine whether the interrupt belongs to us.
1809          */
1810         if (adapter->hw.mac.type >= e1000_82571 &&
1811             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1812                 return FILTER_STRAY;
1813
1814         /*
1815          * Mask interrupts until the taskqueue is finished running.  This is
1816          * cheap, just assume that it is needed.  This also works around the
1817          * MSI message reordering errata on certain systems.
1818          */
1819         em_disable_intr(adapter);
1820         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1821
1822         /* Link status change */
1823         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1824                 adapter->hw.mac.get_link_status = 1;
1825                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1826         }
1827
1828         if (reg_icr & E1000_ICR_RXO)
1829                 adapter->rx_overruns++;
1830         return FILTER_HANDLED;
1831 }
1832
1833 /*********************************************************************
1834  *
1835  *  MSIX Interrupt Service Routines
1836  *
1837  **********************************************************************/
1838 #define EM_MSIX_TX      0x00040000
1839 #define EM_MSIX_RX      0x00010000
1840 #define EM_MSIX_LINK    0x00100000
1841
1842 static void
1843 em_msix_tx(void *arg)
1844 {
1845         struct adapter *adapter = arg;
1846         struct ifnet    *ifp = adapter->ifp;
1847
1848         ++adapter->tx_irq;
1849         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1850                 EM_TX_LOCK(adapter);
1851                 em_txeof(adapter);
1852                 EM_TX_UNLOCK(adapter);
1853                 taskqueue_enqueue(adapter->tq, &adapter->tx_task);
1854         }
1855         /* Reenable this interrupt */
1856         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX);
1857         return;
1858 }
1859
1860 /*********************************************************************
1861  *
1862  *  MSIX RX Interrupt Service routine
1863  *
1864  **********************************************************************/
1865
1866 static void
1867 em_msix_rx(void *arg)
1868 {
1869         struct adapter *adapter = arg;
1870         struct ifnet    *ifp = adapter->ifp;
1871
1872         ++adapter->rx_irq;
1873         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1874             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1875                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1876         /* Reenable this interrupt */
1877         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX);
1878         return;
1879 }
1880
1881 /*********************************************************************
1882  *
1883  *  MSIX Link Fast Interrupt Service routine
1884  *
1885  **********************************************************************/
1886
1887 static void
1888 em_msix_link(void *arg)
1889 {
1890         struct adapter  *adapter = arg;
1891         u32             reg_icr;
1892
1893         ++adapter->link_irq;
1894         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1895
1896         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1897                 adapter->hw.mac.get_link_status = 1;
1898                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1899         }
1900         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1901             EM_MSIX_LINK | E1000_IMS_LSC);
1902         return;
1903 }
1904
1905 static void
1906 em_handle_rx(void *context, int pending)
1907 {
1908         struct adapter  *adapter = context;
1909         struct ifnet    *ifp = adapter->ifp;
1910
1911         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1912             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1913                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1914
1915 }
1916
1917 static void
1918 em_handle_tx(void *context, int pending)
1919 {
1920         struct adapter  *adapter = context;
1921         struct ifnet    *ifp = adapter->ifp;
1922
1923         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1924                 if (!EM_TX_TRYLOCK(adapter))
1925                         return;
1926                 em_txeof(adapter);
1927 #if __FreeBSD_version >= 800000
1928                 if (!drbr_empty(ifp, adapter->br))
1929                         em_mq_start_locked(ifp, NULL);
1930 #else
1931                 if (!IFQ_DRV_IS_EMPTY(&ifp->snd))
1932                         em_start_locked(ifp);
1933 #endif
1934                 EM_TX_UNLOCK(adapter);
1935         }
1936 }
1937 #endif /* EM_FAST_IRQ */
1938
1939 /*********************************************************************
1940  *
1941  *  Media Ioctl callback
1942  *
1943  *  This routine is called whenever the user queries the status of
1944  *  the interface using ifconfig.
1945  *
1946  **********************************************************************/
1947 static void
1948 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1949 {
1950         struct adapter *adapter = ifp->if_softc;
1951         u_char fiber_type = IFM_1000_SX;
1952
1953         INIT_DEBUGOUT("em_media_status: begin");
1954
1955         EM_CORE_LOCK(adapter);
1956         em_update_link_status(adapter);
1957
1958         ifmr->ifm_status = IFM_AVALID;
1959         ifmr->ifm_active = IFM_ETHER;
1960
1961         if (!adapter->link_active) {
1962                 EM_CORE_UNLOCK(adapter);
1963                 return;
1964         }
1965
1966         ifmr->ifm_status |= IFM_ACTIVE;
1967
1968         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1969             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1970                 if (adapter->hw.mac.type == e1000_82545)
1971                         fiber_type = IFM_1000_LX;
1972                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1973         } else {
1974                 switch (adapter->link_speed) {
1975                 case 10:
1976                         ifmr->ifm_active |= IFM_10_T;
1977                         break;
1978                 case 100:
1979                         ifmr->ifm_active |= IFM_100_TX;
1980                         break;
1981                 case 1000:
1982                         ifmr->ifm_active |= IFM_1000_T;
1983                         break;
1984                 }
1985                 if (adapter->link_duplex == FULL_DUPLEX)
1986                         ifmr->ifm_active |= IFM_FDX;
1987                 else
1988                         ifmr->ifm_active |= IFM_HDX;
1989         }
1990         EM_CORE_UNLOCK(adapter);
1991 }
1992
1993 /*********************************************************************
1994  *
1995  *  Media Ioctl callback
1996  *
1997  *  This routine is called when the user changes speed/duplex using
1998  *  media/mediopt option with ifconfig.
1999  *
2000  **********************************************************************/
2001 static int
2002 em_media_change(struct ifnet *ifp)
2003 {
2004         struct adapter *adapter = ifp->if_softc;
2005         struct ifmedia  *ifm = &adapter->media;
2006
2007         INIT_DEBUGOUT("em_media_change: begin");
2008
2009         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2010                 return (EINVAL);
2011
2012         EM_CORE_LOCK(adapter);
2013         switch (IFM_SUBTYPE(ifm->ifm_media)) {
2014         case IFM_AUTO:
2015                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
2016                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
2017                 break;
2018         case IFM_1000_LX:
2019         case IFM_1000_SX:
2020         case IFM_1000_T:
2021                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
2022                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
2023                 break;
2024         case IFM_100_TX:
2025                 adapter->hw.mac.autoneg = FALSE;
2026                 adapter->hw.phy.autoneg_advertised = 0;
2027                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
2028                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
2029                 else
2030                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
2031                 break;
2032         case IFM_10_T:
2033                 adapter->hw.mac.autoneg = FALSE;
2034                 adapter->hw.phy.autoneg_advertised = 0;
2035                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
2036                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
2037                 else
2038                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
2039                 break;
2040         default:
2041                 device_printf(adapter->dev, "Unsupported media type\n");
2042         }
2043
2044         /* As the speed/duplex settings my have changed we need to
2045          * reset the PHY.
2046          */
2047         adapter->hw.phy.reset_disable = FALSE;
2048
2049         em_init_locked(adapter);
2050         EM_CORE_UNLOCK(adapter);
2051
2052         return (0);
2053 }
2054
2055 /*********************************************************************
2056  *
2057  *  This routine maps the mbufs to tx descriptors.
2058  *
2059  *  return 0 on success, positive on failure
2060  **********************************************************************/
2061
2062 static int
2063 em_xmit(struct adapter *adapter, struct mbuf **m_headp)
2064 {
2065         bus_dma_segment_t       segs[EM_MAX_SCATTER];
2066         bus_dmamap_t            map;
2067         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
2068         struct e1000_tx_desc    *ctxd = NULL;
2069         struct mbuf             *m_head;
2070         u32                     txd_upper, txd_lower, txd_used, txd_saved;
2071         int                     nsegs, i, j, first, last = 0;
2072         int                     error, do_tso, tso_desc = 0;
2073 #if __FreeBSD_version < 700000
2074         struct m_tag            *mtag;
2075 #endif
2076         m_head = *m_headp;
2077         txd_upper = txd_lower = txd_used = txd_saved = 0;
2078
2079 #if __FreeBSD_version >= 700000
2080         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
2081 #else
2082         do_tso = 0;
2083 #endif
2084
2085         /*
2086          * Force a cleanup if number of TX descriptors
2087          * available hits the threshold
2088          */
2089         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
2090                 em_txeof(adapter);
2091                 /* Now do we at least have a minimal? */
2092                 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
2093                         adapter->no_tx_desc_avail1++;
2094                         return (ENOBUFS);
2095                 }
2096         }
2097
2098
2099         /*
2100          * TSO workaround:
2101          *  If an mbuf is only header we need
2102          *     to pull 4 bytes of data into it.
2103          */
2104         if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
2105                 m_head = m_pullup(m_head, M_TSO_LEN + 4);
2106                 *m_headp = m_head;
2107                 if (m_head == NULL)
2108                         return (ENOBUFS);
2109         }
2110
2111         /*
2112          * Map the packet for DMA
2113          *
2114          * Capture the first descriptor index,
2115          * this descriptor will have the index
2116          * of the EOP which is the only one that
2117          * now gets a DONE bit writeback.
2118          */
2119         first = adapter->next_avail_tx_desc;
2120         tx_buffer = &adapter->tx_buffer_area[first];
2121         tx_buffer_mapped = tx_buffer;
2122         map = tx_buffer->map;
2123
2124         error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2125             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2126
2127         /*
2128          * There are two types of errors we can (try) to handle:
2129          * - EFBIG means the mbuf chain was too long and bus_dma ran
2130          *   out of segments.  Defragment the mbuf chain and try again.
2131          * - ENOMEM means bus_dma could not obtain enough bounce buffers
2132          *   at this point in time.  Defer sending and try again later.
2133          * All other errors, in particular EINVAL, are fatal and prevent the
2134          * mbuf chain from ever going through.  Drop it and report error.
2135          */
2136         if (error == EFBIG) {
2137                 struct mbuf *m;
2138
2139                 m = m_defrag(*m_headp, M_DONTWAIT);
2140                 if (m == NULL) {
2141                         adapter->mbuf_alloc_failed++;
2142                         m_freem(*m_headp);
2143                         *m_headp = NULL;
2144                         return (ENOBUFS);
2145                 }
2146                 *m_headp = m;
2147
2148                 /* Try it again */
2149                 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2150                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2151
2152                 if (error) {
2153                         adapter->no_tx_dma_setup++;
2154                         m_freem(*m_headp);
2155                         *m_headp = NULL;
2156                         return (error);
2157                 }
2158         } else if (error != 0) {
2159                 adapter->no_tx_dma_setup++;
2160                 return (error);
2161         }
2162
2163         /*
2164          * TSO Hardware workaround, if this packet is not
2165          * TSO, and is only a single descriptor long, and
2166          * it follows a TSO burst, then we need to add a
2167          * sentinel descriptor to prevent premature writeback.
2168          */
2169         if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
2170                 if (nsegs == 1)
2171                         tso_desc = TRUE;
2172                 adapter->tx_tso = FALSE;
2173         }
2174
2175         if (nsegs > (adapter->num_tx_desc_avail - 2)) {
2176                 adapter->no_tx_desc_avail2++;
2177                 bus_dmamap_unload(adapter->txtag, map);
2178                 return (ENOBUFS);
2179         }
2180         m_head = *m_headp;
2181
2182         /* Do hardware assists */
2183 #if __FreeBSD_version >= 700000
2184         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2185                 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
2186                 if (error != TRUE)
2187                         return (ENXIO); /* something foobar */
2188                 /* we need to make a final sentinel transmit desc */
2189                 tso_desc = TRUE;
2190         } else
2191 #endif
2192         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2193                 em_transmit_checksum_setup(adapter,  m_head,
2194                     &txd_upper, &txd_lower);
2195
2196         i = adapter->next_avail_tx_desc;
2197         if (adapter->pcix_82544)
2198                 txd_saved = i;
2199
2200         /* Set up our transmit descriptors */
2201         for (j = 0; j < nsegs; j++) {
2202                 bus_size_t seg_len;
2203                 bus_addr_t seg_addr;
2204                 /* If adapter is 82544 and on PCIX bus */
2205                 if(adapter->pcix_82544) {
2206                         DESC_ARRAY      desc_array;
2207                         u32             array_elements, counter;
2208                         /*
2209                          * Check the Address and Length combination and
2210                          * split the data accordingly
2211                          */
2212                         array_elements = em_fill_descriptors(segs[j].ds_addr,
2213                             segs[j].ds_len, &desc_array);
2214                         for (counter = 0; counter < array_elements; counter++) {
2215                                 if (txd_used == adapter->num_tx_desc_avail) {
2216                                         adapter->next_avail_tx_desc = txd_saved;
2217                                         adapter->no_tx_desc_avail2++;
2218                                         bus_dmamap_unload(adapter->txtag, map);
2219                                         return (ENOBUFS);
2220                                 }
2221                                 tx_buffer = &adapter->tx_buffer_area[i];
2222                                 ctxd = &adapter->tx_desc_base[i];
2223                                 ctxd->buffer_addr = htole64(
2224                                     desc_array.descriptor[counter].address);
2225                                 ctxd->lower.data = htole32(
2226                                     (adapter->txd_cmd | txd_lower | (u16)
2227                                     desc_array.descriptor[counter].length));
2228                                 ctxd->upper.data =
2229                                     htole32((txd_upper));
2230                                 last = i;
2231                                 if (++i == adapter->num_tx_desc)
2232                                          i = 0;
2233                                 tx_buffer->m_head = NULL;
2234                                 tx_buffer->next_eop = -1;
2235                                 txd_used++;
2236                         }
2237                 } else {
2238                         tx_buffer = &adapter->tx_buffer_area[i];
2239                         ctxd = &adapter->tx_desc_base[i];
2240                         seg_addr = segs[j].ds_addr;
2241                         seg_len  = segs[j].ds_len;
2242                         /*
2243                         ** TSO Workaround:
2244                         ** If this is the last descriptor, we want to
2245                         ** split it so we have a small final sentinel
2246                         */
2247                         if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2248                                 seg_len -= 4;
2249                                 ctxd->buffer_addr = htole64(seg_addr);
2250                                 ctxd->lower.data = htole32(
2251                                 adapter->txd_cmd | txd_lower | seg_len);
2252                                 ctxd->upper.data =
2253                                     htole32(txd_upper);
2254                                 if (++i == adapter->num_tx_desc)
2255                                         i = 0;
2256                                 /* Now make the sentinel */
2257                                 ++txd_used; /* using an extra txd */
2258                                 ctxd = &adapter->tx_desc_base[i];
2259                                 tx_buffer = &adapter->tx_buffer_area[i];
2260                                 ctxd->buffer_addr =
2261                                     htole64(seg_addr + seg_len);
2262                                 ctxd->lower.data = htole32(
2263                                 adapter->txd_cmd | txd_lower | 4);
2264                                 ctxd->upper.data =
2265                                     htole32(txd_upper);
2266                                 last = i;
2267                                 if (++i == adapter->num_tx_desc)
2268                                         i = 0;
2269                         } else {
2270                                 ctxd->buffer_addr = htole64(seg_addr);
2271                                 ctxd->lower.data = htole32(
2272                                 adapter->txd_cmd | txd_lower | seg_len);
2273                                 ctxd->upper.data =
2274                                     htole32(txd_upper);
2275                                 last = i;
2276                                 if (++i == adapter->num_tx_desc)
2277                                         i = 0;
2278                         }
2279                         tx_buffer->m_head = NULL;
2280                         tx_buffer->next_eop = -1;
2281                 }
2282         }
2283
2284         adapter->next_avail_tx_desc = i;
2285         if (adapter->pcix_82544)
2286                 adapter->num_tx_desc_avail -= txd_used;
2287         else {
2288                 adapter->num_tx_desc_avail -= nsegs;
2289                 if (tso_desc) /* TSO used an extra for sentinel */
2290                         adapter->num_tx_desc_avail -= txd_used;
2291         }
2292
2293         /*
2294         ** Handle VLAN tag, this is the
2295         ** biggest difference between
2296         ** 6.x and 7
2297         */
2298 #if __FreeBSD_version < 700000
2299         /* Find out if we are in vlan mode. */
2300         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
2301         if (mtag != NULL) {
2302                 ctxd->upper.fields.special =
2303                     htole16(VLAN_TAG_VALUE(mtag));
2304 #else /* FreeBSD 7 */
2305         if (m_head->m_flags & M_VLANTAG) {
2306                 /* Set the vlan id. */
2307                 ctxd->upper.fields.special =
2308                     htole16(m_head->m_pkthdr.ether_vtag);
2309 #endif
2310                 /* Tell hardware to add tag */
2311                 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
2312         }
2313
2314         tx_buffer->m_head = m_head;
2315         tx_buffer_mapped->map = tx_buffer->map;
2316         tx_buffer->map = map;
2317         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
2318
2319         /*
2320          * Last Descriptor of Packet
2321          * needs End Of Packet (EOP)
2322          * and Report Status (RS)
2323          */
2324         ctxd->lower.data |=
2325             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2326         /*
2327          * Keep track in the first buffer which
2328          * descriptor will be written back
2329          */
2330         tx_buffer = &adapter->tx_buffer_area[first];
2331         tx_buffer->next_eop = last;
2332
2333         /*
2334          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2335          * that this frame is available to transmit.
2336          */
2337         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2338             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2339         if (adapter->hw.mac.type == e1000_82547 &&
2340             adapter->link_duplex == HALF_DUPLEX)
2341                 em_82547_move_tail(adapter);
2342         else {
2343                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
2344                 if (adapter->hw.mac.type == e1000_82547)
2345                         em_82547_update_fifo_head(adapter,
2346                             m_head->m_pkthdr.len);
2347         }
2348
2349         return (0);
2350 }
2351
2352 /*********************************************************************
2353  *
2354  * 82547 workaround to avoid controller hang in half-duplex environment.
2355  * The workaround is to avoid queuing a large packet that would span
2356  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
2357  * in this case. We do that only when FIFO is quiescent.
2358  *
2359  **********************************************************************/
2360 static void
2361 em_82547_move_tail(void *arg)
2362 {
2363         struct adapter *adapter = arg;
2364         struct e1000_tx_desc *tx_desc;
2365         u16     hw_tdt, sw_tdt, length = 0;
2366         bool    eop = 0;
2367
2368         EM_TX_LOCK_ASSERT(adapter);
2369
2370         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2371         sw_tdt = adapter->next_avail_tx_desc;
2372
2373         while (hw_tdt != sw_tdt) {
2374                 tx_desc = &adapter->tx_desc_base[hw_tdt];
2375                 length += tx_desc->lower.flags.length;
2376                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2377                 if (++hw_tdt == adapter->num_tx_desc)
2378                         hw_tdt = 0;
2379
2380                 if (eop) {
2381                         if (em_82547_fifo_workaround(adapter, length)) {
2382                                 adapter->tx_fifo_wrk_cnt++;
2383                                 callout_reset(&adapter->tx_fifo_timer, 1,
2384                                         em_82547_move_tail, adapter);
2385                                 break;
2386                         }
2387                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2388                         em_82547_update_fifo_head(adapter, length);
2389                         length = 0;
2390                 }
2391         }
2392 }
2393
2394 static int
2395 em_82547_fifo_workaround(struct adapter *adapter, int len)
2396 {
2397         int fifo_space, fifo_pkt_len;
2398
2399         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2400
2401         if (adapter->link_duplex == HALF_DUPLEX) {
2402                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2403
2404                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2405                         if (em_82547_tx_fifo_reset(adapter))
2406                                 return (0);
2407                         else
2408                                 return (1);
2409                 }
2410         }
2411
2412         return (0);
2413 }
2414
2415 static void
2416 em_82547_update_fifo_head(struct adapter *adapter, int len)
2417 {
2418         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2419
2420         /* tx_fifo_head is always 16 byte aligned */
2421         adapter->tx_fifo_head += fifo_pkt_len;
2422         if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
2423                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2424         }
2425 }
2426
2427
2428 static int
2429 em_82547_tx_fifo_reset(struct adapter *adapter)
2430 {
2431         u32 tctl;
2432
2433         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2434             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2435             (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
2436             E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2437             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2438             E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2439             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2440                 /* Disable TX unit */
2441                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2442                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2443                     tctl & ~E1000_TCTL_EN);
2444
2445                 /* Reset FIFO pointers */
2446                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2447                     adapter->tx_head_addr);
2448                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2449                     adapter->tx_head_addr);
2450                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2451                     adapter->tx_head_addr);
2452                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2453                     adapter->tx_head_addr);
2454
2455                 /* Re-enable TX unit */
2456                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2457                 E1000_WRITE_FLUSH(&adapter->hw);
2458
2459                 adapter->tx_fifo_head = 0;
2460                 adapter->tx_fifo_reset_cnt++;
2461
2462                 return (TRUE);
2463         }
2464         else {
2465                 return (FALSE);
2466         }
2467 }
2468
2469 static void
2470 em_set_promisc(struct adapter *adapter)
2471 {
2472         struct ifnet    *ifp = adapter->ifp;
2473         u32             reg_rctl;
2474
2475         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2476
2477         if (ifp->if_flags & IFF_PROMISC) {
2478                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2479                 /* Turn this on if you want to see bad packets */
2480                 if (em_debug_sbp)
2481                         reg_rctl |= E1000_RCTL_SBP;
2482                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2483         } else if (ifp->if_flags & IFF_ALLMULTI) {
2484                 reg_rctl |= E1000_RCTL_MPE;
2485                 reg_rctl &= ~E1000_RCTL_UPE;
2486                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2487         }
2488 }
2489
2490 static void
2491 em_disable_promisc(struct adapter *adapter)
2492 {
2493         u32     reg_rctl;
2494
2495         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2496
2497         reg_rctl &=  (~E1000_RCTL_UPE);
2498         reg_rctl &=  (~E1000_RCTL_MPE);
2499         reg_rctl &=  (~E1000_RCTL_SBP);
2500         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2501 }
2502
2503
2504 /*********************************************************************
2505  *  Multicast Update
2506  *
2507  *  This routine is called whenever multicast address list is updated.
2508  *
2509  **********************************************************************/
2510
2511 static void
2512 em_set_multi(struct adapter *adapter)
2513 {
2514         struct ifnet    *ifp = adapter->ifp;
2515         struct ifmultiaddr *ifma;
2516         u32 reg_rctl = 0;
2517         u8  *mta; /* Multicast array memory */
2518         int mcnt = 0;
2519
2520         IOCTL_DEBUGOUT("em_set_multi: begin");
2521
2522         if (adapter->hw.mac.type == e1000_82542 &&
2523             adapter->hw.revision_id == E1000_REVISION_2) {
2524                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2525                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2526                         e1000_pci_clear_mwi(&adapter->hw);
2527                 reg_rctl |= E1000_RCTL_RST;
2528                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2529                 msec_delay(5);
2530         }
2531
2532         /* Allocate temporary memory to setup array */
2533         mta = malloc(sizeof(u8) *
2534             (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
2535             M_DEVBUF, M_NOWAIT | M_ZERO);
2536         if (mta == NULL)
2537                 panic("em_set_multi memory failure\n");
2538
2539         if_maddr_rlock(ifp);
2540         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2541                 if (ifma->ifma_addr->sa_family != AF_LINK)
2542                         continue;
2543
2544                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2545                         break;
2546
2547                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2548                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2549                 mcnt++;
2550         }
2551         if_maddr_runlock(ifp);
2552
2553         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2554                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2555                 reg_rctl |= E1000_RCTL_MPE;
2556                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2557         } else
2558                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2559
2560         if (adapter->hw.mac.type == e1000_82542 &&
2561             adapter->hw.revision_id == E1000_REVISION_2) {
2562                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2563                 reg_rctl &= ~E1000_RCTL_RST;
2564                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2565                 msec_delay(5);
2566                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2567                         e1000_pci_set_mwi(&adapter->hw);
2568         }
2569         free(mta, M_DEVBUF);
2570 }
2571
2572
2573 /*********************************************************************
2574  *  Timer routine
2575  *
2576  *  This routine checks for link status and updates statistics.
2577  *
2578  **********************************************************************/
2579
2580 static void
2581 em_local_timer(void *arg)
2582 {
2583         struct adapter  *adapter = arg;
2584         struct ifnet    *ifp = adapter->ifp;
2585
2586         EM_CORE_LOCK_ASSERT(adapter);
2587
2588         taskqueue_enqueue(adapter->tq,
2589             &adapter->rxtx_task);
2590         em_update_link_status(adapter);
2591         em_update_stats_counters(adapter);
2592
2593         /* Reset LAA into RAR[0] on 82571 */
2594         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2595                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2596
2597         if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
2598                 em_print_hw_stats(adapter);
2599
2600         em_smartspeed(adapter);
2601
2602         /*
2603          * Each second we check the watchdog to
2604          * protect against hardware hangs.
2605          */
2606         em_watchdog(adapter);
2607
2608         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2609
2610 }
2611
2612 static void
2613 em_update_link_status(struct adapter *adapter)
2614 {
2615         struct e1000_hw *hw = &adapter->hw;
2616         struct ifnet *ifp = adapter->ifp;
2617         device_t dev = adapter->dev;
2618         u32 link_check = 0;
2619
2620         /* Get the cached link value or read phy for real */
2621         switch (hw->phy.media_type) {
2622         case e1000_media_type_copper:
2623                 if (hw->mac.get_link_status) {
2624                         /* Do the work to read phy */
2625                         e1000_check_for_link(hw);
2626                         link_check = !hw->mac.get_link_status;
2627                         if (link_check) /* ESB2 fix */
2628                                 e1000_cfg_on_link_up(hw);
2629                 } else
2630                         link_check = TRUE;
2631                 break;
2632         case e1000_media_type_fiber:
2633                 e1000_check_for_link(hw);
2634                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2635                                  E1000_STATUS_LU);
2636                 break;
2637         case e1000_media_type_internal_serdes:
2638                 e1000_check_for_link(hw);
2639                 link_check = adapter->hw.mac.serdes_has_link;
2640                 break;
2641         default:
2642         case e1000_media_type_unknown:
2643                 break;
2644         }
2645
2646         /* Now check for a transition */
2647         if (link_check && (adapter->link_active == 0)) {
2648                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2649                     &adapter->link_duplex);
2650                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2651                 if ((adapter->link_speed != SPEED_1000) &&
2652                     ((hw->mac.type == e1000_82571) ||
2653                     (hw->mac.type == e1000_82572))) {
2654                         int tarc0;
2655                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2656                         tarc0 &= ~SPEED_MODE_BIT;
2657                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2658                 }
2659                 if (bootverbose)
2660                         device_printf(dev, "Link is up %d Mbps %s\n",
2661                             adapter->link_speed,
2662                             ((adapter->link_duplex == FULL_DUPLEX) ?
2663                             "Full Duplex" : "Half Duplex"));
2664                 adapter->link_active = 1;
2665                 adapter->smartspeed = 0;
2666                 ifp->if_baudrate = adapter->link_speed * 1000000;
2667                 if_link_state_change(ifp, LINK_STATE_UP);
2668         } else if (!link_check && (adapter->link_active == 1)) {
2669                 ifp->if_baudrate = adapter->link_speed = 0;
2670                 adapter->link_duplex = 0;
2671                 if (bootverbose)
2672                         device_printf(dev, "Link is Down\n");
2673                 adapter->link_active = 0;
2674                 /* Link down, disable watchdog */
2675                 adapter->watchdog_timer = FALSE;
2676                 if_link_state_change(ifp, LINK_STATE_DOWN);
2677         }
2678 }
2679
2680 /*********************************************************************
2681  *
2682  *  This routine disables all traffic on the adapter by issuing a
2683  *  global reset on the MAC and deallocates TX/RX buffers.
2684  *
2685  *  This routine should always be called with BOTH the CORE
2686  *  and TX locks.
2687  **********************************************************************/
2688
2689 static void
2690 em_stop(void *arg)
2691 {
2692         struct adapter  *adapter = arg;
2693         struct ifnet    *ifp = adapter->ifp;
2694
2695         EM_CORE_LOCK_ASSERT(adapter);
2696         EM_TX_LOCK_ASSERT(adapter);
2697
2698         INIT_DEBUGOUT("em_stop: begin");
2699
2700         em_disable_intr(adapter);
2701         callout_stop(&adapter->timer);
2702         callout_stop(&adapter->tx_fifo_timer);
2703
2704         /* Tell the stack that the interface is no longer active */
2705         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2706
2707         e1000_reset_hw(&adapter->hw);
2708         if (adapter->hw.mac.type >= e1000_82544)
2709                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2710 }
2711
2712
2713 /*********************************************************************
2714  *
2715  *  Determine hardware revision.
2716  *
2717  **********************************************************************/
2718 static void
2719 em_identify_hardware(struct adapter *adapter)
2720 {
2721         device_t dev = adapter->dev;
2722
2723         /* Make sure our PCI config space has the necessary stuff set */
2724         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2725         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2726             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2727                 device_printf(dev, "Memory Access and/or Bus Master bits "
2728                     "were not set!\n");
2729                 adapter->hw.bus.pci_cmd_word |=
2730                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2731                 pci_write_config(dev, PCIR_COMMAND,
2732                     adapter->hw.bus.pci_cmd_word, 2);
2733         }
2734
2735         /* Save off the information about this board */
2736         adapter->hw.vendor_id = pci_get_vendor(dev);
2737         adapter->hw.device_id = pci_get_device(dev);
2738         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2739         adapter->hw.subsystem_vendor_id =
2740             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2741         adapter->hw.subsystem_device_id =
2742             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2743
2744         /* Do Shared Code Init and Setup */
2745         if (e1000_set_mac_type(&adapter->hw)) {
2746                 device_printf(dev, "Setup init failure\n");
2747                 return;
2748         }
2749 }
2750
2751 static int
2752 em_allocate_pci_resources(struct adapter *adapter)
2753 {
2754         device_t        dev = adapter->dev;
2755         int             val, rid, error = E1000_SUCCESS;
2756
2757         rid = PCIR_BAR(0);
2758         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2759             &rid, RF_ACTIVE);
2760         if (adapter->memory == NULL) {
2761                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2762                 return (ENXIO);
2763         }
2764         adapter->osdep.mem_bus_space_tag =
2765             rman_get_bustag(adapter->memory);
2766         adapter->osdep.mem_bus_space_handle =
2767             rman_get_bushandle(adapter->memory);
2768         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2769
2770         /* Only older adapters use IO mapping */
2771         if ((adapter->hw.mac.type > e1000_82543) &&
2772             (adapter->hw.mac.type < e1000_82571)) {
2773                 /* Figure our where our IO BAR is ? */
2774                 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2775                         val = pci_read_config(dev, rid, 4);
2776                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2777                                 adapter->io_rid = rid;
2778                                 break;
2779                         }
2780                         rid += 4;
2781                         /* check for 64bit BAR */
2782                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2783                                 rid += 4;
2784                 }
2785                 if (rid >= PCIR_CIS) {
2786                         device_printf(dev, "Unable to locate IO BAR\n");
2787                         return (ENXIO);
2788                 }
2789                 adapter->ioport = bus_alloc_resource_any(dev,
2790                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2791                 if (adapter->ioport == NULL) {
2792                         device_printf(dev, "Unable to allocate bus resource: "
2793                             "ioport\n");
2794                         return (ENXIO);
2795                 }
2796                 adapter->hw.io_base = 0;
2797                 adapter->osdep.io_bus_space_tag =
2798                     rman_get_bustag(adapter->ioport);
2799                 adapter->osdep.io_bus_space_handle =
2800                     rman_get_bushandle(adapter->ioport);
2801         }
2802
2803         /*
2804         ** Init the resource arrays
2805         **  used by MSIX setup
2806         */
2807         for (int i = 0; i < 3; i++) {
2808                 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2809                 adapter->tag[i] = NULL;
2810                 adapter->res[i] = NULL;
2811         }
2812
2813         /*
2814          * Setup MSI/X or MSI if PCI Express
2815          */
2816         if (em_enable_msi)
2817                 adapter->msi = em_setup_msix(adapter);
2818
2819         adapter->hw.back = &adapter->osdep;
2820
2821         return (error);
2822 }
2823
2824 /*********************************************************************
2825  *
2826  *  Setup the Legacy or MSI Interrupt handler
2827  *
2828  **********************************************************************/
2829 int
2830 em_allocate_legacy(struct adapter *adapter)
2831 {
2832         device_t dev = adapter->dev;
2833         int error;
2834
2835         /* Manually turn off all interrupts */
2836         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2837
2838         /* Legacy RID is 0 */
2839         if (adapter->msi == 0)
2840                 adapter->rid[0] = 0;
2841
2842         /* We allocate a single interrupt resource */
2843         adapter->res[0] = bus_alloc_resource_any(dev,
2844             SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2845         if (adapter->res[0] == NULL) {
2846                 device_printf(dev, "Unable to allocate bus resource: "
2847                     "interrupt\n");
2848                 return (ENXIO);
2849         }
2850
2851 #ifdef EM_LEGACY_IRQ
2852         /* We do Legacy setup */
2853         if ((error = bus_setup_intr(dev, adapter->res[0],
2854 #if __FreeBSD_version > 700000
2855             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter,
2856 #else /* 6.X */
2857             INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter,
2858 #endif
2859             &adapter->tag[0])) != 0) {
2860                 device_printf(dev, "Failed to register interrupt handler");
2861                 return (error);
2862         }
2863
2864 #else /* FAST_IRQ */
2865         /*
2866          * Try allocating a fast interrupt and the associated deferred
2867          * processing contexts.
2868          */
2869         TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2870         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2871         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2872             taskqueue_thread_enqueue, &adapter->tq);
2873         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2874             device_get_nameunit(adapter->dev));
2875 #if __FreeBSD_version < 700000
2876         if ((error = bus_setup_intr(dev, adapter->res[0],
2877             INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter,
2878 #else
2879         if ((error = bus_setup_intr(dev, adapter->res[0],
2880             INTR_TYPE_NET, em_irq_fast, NULL, adapter,
2881 #endif
2882             &adapter->tag[0])) != 0) {
2883                 device_printf(dev, "Failed to register fast interrupt "
2884                             "handler: %d\n", error);
2885                 taskqueue_free(adapter->tq);
2886                 adapter->tq = NULL;
2887                 return (error);
2888         }
2889 #endif  /* EM_LEGACY_IRQ */
2890
2891         return (0);
2892 }
2893
2894 /*********************************************************************
2895  *
2896  *  Setup the MSIX Interrupt handlers
2897  *   This is not really Multiqueue, rather
2898  *   its just multiple interrupt vectors.
2899  *
2900  **********************************************************************/
2901 int
2902 em_allocate_msix(struct adapter *adapter)
2903 {
2904         device_t dev = adapter->dev;
2905         int error;
2906
2907         /* Make sure all interrupts are disabled */
2908         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2909
2910         /* First get the resources */
2911         for (int i = 0; i < adapter->msi; i++) {
2912                 adapter->res[i] = bus_alloc_resource_any(dev,
2913                     SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE);
2914                 if (adapter->res[i] == NULL) {
2915                         device_printf(dev,
2916                             "Unable to allocate bus resource: "
2917                             "MSIX Interrupt\n");
2918                         return (ENXIO);
2919                 }
2920         }
2921
2922         /*
2923          * Now allocate deferred processing contexts.
2924          */
2925         TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter);
2926         TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter);
2927         /*
2928          * Handle compatibility for msi case for deferral due to
2929          * trylock failure
2930          */
2931         TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter);
2932         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2933         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2934             taskqueue_thread_enqueue, &adapter->tq);
2935         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2936             device_get_nameunit(adapter->dev));
2937
2938         /*
2939          * And setup the interrupt handlers
2940          */
2941
2942         /* First slot to RX */
2943         if ((error = bus_setup_intr(dev, adapter->res[0],
2944 #if __FreeBSD_version > 700000
2945             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter,
2946 #else /* 6.X */
2947             INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter,
2948 #endif
2949             &adapter->tag[0])) != 0) {
2950                 device_printf(dev, "Failed to register RX handler");
2951                 return (error);
2952         }
2953
2954         /* Next TX */
2955         if ((error = bus_setup_intr(dev, adapter->res[1],
2956 #if __FreeBSD_version > 700000
2957             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter,
2958 #else /* 6.X */
2959             INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter,
2960 #endif
2961             &adapter->tag[1])) != 0) {
2962                 device_printf(dev, "Failed to register TX handler");
2963                 return (error);
2964         }
2965
2966         /* And Link */
2967         if ((error = bus_setup_intr(dev, adapter->res[2],
2968 #if __FreeBSD_version > 700000
2969             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter,
2970 #else /* 6.X */
2971             INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter,
2972 #endif
2973             &adapter->tag[2])) != 0) {
2974                 device_printf(dev, "Failed to register TX handler");
2975                 return (error);
2976         }
2977
2978         return (0);
2979 }
2980
2981
2982 static void
2983 em_free_pci_resources(struct adapter *adapter)
2984 {
2985         device_t dev = adapter->dev;
2986
2987         /* Make sure the for loop below runs once */
2988         if (adapter->msi == 0)
2989                 adapter->msi = 1;
2990
2991         /*
2992          * First release all the interrupt resources:
2993          *      notice that since these are just kept
2994          *      in an array we can do the same logic
2995          *      whether its MSIX or just legacy.
2996          */
2997         for (int i = 0; i < adapter->msi; i++) {
2998                 if (adapter->tag[i] != NULL) {
2999                         bus_teardown_intr(dev, adapter->res[i],
3000                             adapter->tag[i]);
3001                         adapter->tag[i] = NULL;
3002                 }
3003                 if (adapter->res[i] != NULL) {
3004                         bus_release_resource(dev, SYS_RES_IRQ,
3005                             adapter->rid[i], adapter->res[i]);
3006                 }
3007         }
3008
3009         if (adapter->msi)
3010                 pci_release_msi(dev);
3011
3012         if (adapter->msix != NULL)
3013                 bus_release_resource(dev, SYS_RES_MEMORY,
3014                     PCIR_BAR(EM_MSIX_BAR), adapter->msix);
3015
3016         if (adapter->memory != NULL)
3017                 bus_release_resource(dev, SYS_RES_MEMORY,
3018                     PCIR_BAR(0), adapter->memory);
3019
3020         if (adapter->flash != NULL)
3021                 bus_release_resource(dev, SYS_RES_MEMORY,
3022                     EM_FLASH, adapter->flash);
3023
3024         if (adapter->ioport != NULL)
3025                 bus_release_resource(dev, SYS_RES_IOPORT,
3026                     adapter->io_rid, adapter->ioport);
3027 }
3028
3029 /*
3030  * Setup MSI or MSI/X
3031  */
3032 static int
3033 em_setup_msix(struct adapter *adapter)
3034 {
3035         device_t dev = adapter->dev;
3036         int val = 0;
3037
3038         if (adapter->hw.mac.type < e1000_82571)
3039                 return (0);
3040
3041         /* Setup MSI/X for Hartwell */
3042         if (adapter->hw.mac.type == e1000_82574) {
3043                 /* Map the MSIX BAR */
3044                 int rid = PCIR_BAR(EM_MSIX_BAR);
3045                 adapter->msix = bus_alloc_resource_any(dev,
3046                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
3047                 if (!adapter->msix) {
3048                         /* May not be enabled */
3049                         device_printf(adapter->dev,
3050                             "Unable to map MSIX table \n");
3051                         goto msi;
3052                 }
3053                 val = pci_msix_count(dev);
3054                 /*
3055                 ** 82574 can be configured for 5 but
3056                 ** we limit use to 3.
3057                 */
3058                 if (val > 3) val = 3;
3059                 if ((val) && pci_alloc_msix(dev, &val) == 0) {
3060                         device_printf(adapter->dev,"Using MSIX interrupts\n");
3061                         return (val);
3062                 }
3063         }
3064 msi:
3065         val = pci_msi_count(dev);
3066         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
3067                 adapter->msi = 1;
3068                 device_printf(adapter->dev,"Using MSI interrupt\n");
3069                 return (val);
3070         }
3071         return (0);
3072 }
3073
3074 /*********************************************************************
3075  *
3076  *  Initialize the hardware to a configuration
3077  *  as specified by the adapter structure.
3078  *
3079  **********************************************************************/
3080 static int
3081 em_hardware_init(struct adapter *adapter)
3082 {
3083         device_t dev = adapter->dev;
3084         u16     rx_buffer_size;
3085
3086         INIT_DEBUGOUT("em_hardware_init: begin");
3087
3088         /* Issue a global reset */
3089         e1000_reset_hw(&adapter->hw);
3090
3091         /* Get control from any management/hw control */
3092         if (((adapter->hw.mac.type == e1000_82573) ||
3093             (adapter->hw.mac.type == e1000_82583) ||
3094             (adapter->hw.mac.type == e1000_ich8lan) ||
3095             (adapter->hw.mac.type == e1000_ich10lan) ||
3096             (adapter->hw.mac.type == e1000_ich9lan)) &&
3097             e1000_check_mng_mode(&adapter->hw))
3098                 em_get_hw_control(adapter);
3099
3100         /* When hardware is reset, fifo_head is also reset */
3101         adapter->tx_fifo_head = 0;
3102
3103         /* Set up smart power down as default off on newer adapters. */
3104         if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 ||
3105             adapter->hw.mac.type == e1000_82572)) {
3106                 u16 phy_tmp = 0;
3107
3108                 /* Speed up time to link by disabling smart power down. */
3109                 e1000_read_phy_reg(&adapter->hw,
3110                     IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3111                 phy_tmp &= ~IGP02E1000_PM_SPD;
3112                 e1000_write_phy_reg(&adapter->hw,
3113                     IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3114         }
3115
3116         /*
3117          * These parameters control the automatic generation (Tx) and
3118          * response (Rx) to Ethernet PAUSE frames.
3119          * - High water mark should allow for at least two frames to be
3120          *   received after sending an XOFF.
3121          * - Low water mark works best when it is very near the high water mark.
3122          *   This allows the receiver to restart by sending XON when it has
3123          *   drained a bit. Here we use an arbitary value of 1500 which will
3124          *   restart after one full frame is pulled from the buffer. There
3125          *   could be several smaller frames in the buffer and if so they will
3126          *   not trigger the XON until their total number reduces the buffer
3127          *   by 1500.
3128          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3129          */
3130         rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
3131             0xffff) << 10 );
3132
3133         adapter->hw.fc.high_water = rx_buffer_size -
3134             roundup2(adapter->max_frame_size, 1024);
3135         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
3136
3137         if (adapter->hw.mac.type == e1000_80003es2lan)
3138                 adapter->hw.fc.pause_time = 0xFFFF;
3139         else
3140                 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
3141         adapter->hw.fc.send_xon = TRUE;
3142
3143         /* Set Flow control, use the tunable location if sane */
3144         if ((em_fc_setting >= 0) || (em_fc_setting < 4))
3145                 adapter->hw.fc.requested_mode = em_fc_setting;
3146         else
3147                 adapter->hw.fc.requested_mode = e1000_fc_none;
3148
3149
3150         if (e1000_init_hw(&adapter->hw) < 0) {
3151                 device_printf(dev, "Hardware Initialization Failed\n");
3152                 return (EIO);
3153         }
3154
3155         e1000_check_for_link(&adapter->hw);
3156
3157         return (0);
3158 }
3159
3160 /*********************************************************************
3161  *
3162  *  Setup networking device structure and register an interface.
3163  *
3164  **********************************************************************/
3165 static void
3166 em_setup_interface(device_t dev, struct adapter *adapter)
3167 {
3168         struct ifnet   *ifp;
3169
3170         INIT_DEBUGOUT("em_setup_interface: begin");
3171
3172         ifp = adapter->ifp = if_alloc(IFT_ETHER);
3173         if (ifp == NULL)
3174                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
3175         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3176         ifp->if_mtu = ETHERMTU;
3177         ifp->if_init =  em_init;
3178         ifp->if_softc = adapter;
3179         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3180         ifp->if_ioctl = em_ioctl;
3181         ifp->if_start = em_start;
3182         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3183         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3184         IFQ_SET_READY(&ifp->if_snd);
3185
3186         ether_ifattach(ifp, adapter->hw.mac.addr);
3187
3188         ifp->if_capabilities = ifp->if_capenable = 0;
3189
3190 #if __FreeBSD_version >= 800000
3191         /* Multiqueue tx functions */
3192         ifp->if_transmit = em_mq_start;
3193         ifp->if_qflush = em_qflush;
3194         adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx);
3195 #endif
3196         if (adapter->hw.mac.type >= e1000_82543) {
3197                 int version_cap;
3198 #if __FreeBSD_version < 700000
3199                 version_cap = IFCAP_HWCSUM;
3200 #else
3201                 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3202 #endif
3203                 ifp->if_capabilities |= version_cap;
3204                 ifp->if_capenable |= version_cap;
3205         }
3206
3207 #if __FreeBSD_version >= 700000
3208         /* Identify TSO capable adapters */
3209         if ((adapter->hw.mac.type > e1000_82544) &&
3210             (adapter->hw.mac.type != e1000_82547))
3211                 ifp->if_capabilities |= IFCAP_TSO4;
3212         /*
3213          * By default only enable on PCI-E, this
3214          * can be overriden by ifconfig.
3215          */
3216         if (adapter->hw.mac.type >= e1000_82571)
3217                 ifp->if_capenable |= IFCAP_TSO4;
3218 #endif
3219
3220         /*
3221          * Tell the upper layer(s) we support long frames.
3222          */
3223         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3224         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3225         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3226
3227 #ifdef DEVICE_POLLING
3228         ifp->if_capabilities |= IFCAP_POLLING;
3229 #endif
3230
3231         /*
3232          * Specify the media types supported by this adapter and register
3233          * callbacks to update media and link information
3234          */
3235         ifmedia_init(&adapter->media, IFM_IMASK,
3236             em_media_change, em_media_status);
3237         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3238             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3239                 u_char fiber_type = IFM_1000_SX;        /* default type */
3240
3241                 if (adapter->hw.mac.type == e1000_82545)
3242                         fiber_type = IFM_1000_LX;
3243                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3244                             0, NULL);
3245                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3246         } else {
3247                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3248                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3249                             0, NULL);
3250                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3251                             0, NULL);
3252                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3253                             0, NULL);
3254                 if (adapter->hw.phy.type != e1000_phy_ife) {
3255                         ifmedia_add(&adapter->media,
3256                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3257                         ifmedia_add(&adapter->media,
3258                                 IFM_ETHER | IFM_1000_T, 0, NULL);
3259                 }
3260         }
3261         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3262         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3263 }
3264
3265
3266 /*********************************************************************
3267  *
3268  *  Workaround for SmartSpeed on 82541 and 82547 controllers
3269  *
3270  **********************************************************************/
3271 static void
3272 em_smartspeed(struct adapter *adapter)
3273 {
3274         u16 phy_tmp;
3275
3276         if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
3277             adapter->hw.mac.autoneg == 0 ||
3278             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
3279                 return;
3280
3281         if (adapter->smartspeed == 0) {
3282                 /* If Master/Slave config fault is asserted twice,
3283                  * we assume back-to-back */
3284                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3285                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
3286                         return;
3287                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3288                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
3289                         e1000_read_phy_reg(&adapter->hw,
3290                             PHY_1000T_CTRL, &phy_tmp);
3291                         if(phy_tmp & CR_1000T_MS_ENABLE) {
3292                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
3293                                 e1000_write_phy_reg(&adapter->hw,
3294                                     PHY_1000T_CTRL, phy_tmp);
3295                                 adapter->smartspeed++;
3296                                 if(adapter->hw.mac.autoneg &&
3297                                    !e1000_phy_setup_autoneg(&adapter->hw) &&
3298                                    !e1000_read_phy_reg(&adapter->hw,
3299                                     PHY_CONTROL, &phy_tmp)) {
3300                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3301                                                     MII_CR_RESTART_AUTO_NEG);
3302                                         e1000_write_phy_reg(&adapter->hw,
3303                                             PHY_CONTROL, phy_tmp);
3304                                 }
3305                         }
3306                 }
3307                 return;
3308         } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
3309                 /* If still no link, perhaps using 2/3 pair cable */
3310                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
3311                 phy_tmp |= CR_1000T_MS_ENABLE;
3312                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
3313                 if(adapter->hw.mac.autoneg &&
3314                    !e1000_phy_setup_autoneg(&adapter->hw) &&
3315                    !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
3316                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3317                                     MII_CR_RESTART_AUTO_NEG);
3318                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
3319                 }
3320         }
3321         /* Restart process after EM_SMARTSPEED_MAX iterations */
3322         if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
3323                 adapter->smartspeed = 0;
3324 }
3325
3326
3327 /*
3328  * Manage DMA'able memory.
3329  */
3330 static void
3331 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3332 {
3333         if (error)
3334                 return;
3335         *(bus_addr_t *) arg = segs[0].ds_addr;
3336 }
3337
3338 static int
3339 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3340         struct em_dma_alloc *dma, int mapflags)
3341 {
3342         int error;
3343
3344 #if __FreeBSD_version >= 700000
3345         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3346 #else
3347         error = bus_dma_tag_create(NULL,                 /* parent */
3348 #endif
3349                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
3350                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3351                                 BUS_SPACE_MAXADDR,      /* highaddr */
3352                                 NULL, NULL,             /* filter, filterarg */
3353                                 size,                   /* maxsize */
3354                                 1,                      /* nsegments */
3355                                 size,                   /* maxsegsize */
3356                                 0,                      /* flags */
3357                                 NULL,                   /* lockfunc */
3358                                 NULL,                   /* lockarg */
3359                                 &dma->dma_tag);
3360         if (error) {
3361                 device_printf(adapter->dev,
3362                     "%s: bus_dma_tag_create failed: %d\n",
3363                     __func__, error);
3364                 goto fail_0;
3365         }
3366
3367         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3368             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3369         if (error) {
3370                 device_printf(adapter->dev,
3371                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3372                     __func__, (uintmax_t)size, error);
3373                 goto fail_2;
3374         }
3375
3376         dma->dma_paddr = 0;
3377         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3378             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3379         if (error || dma->dma_paddr == 0) {
3380                 device_printf(adapter->dev,
3381                     "%s: bus_dmamap_load failed: %d\n",
3382                     __func__, error);
3383                 goto fail_3;
3384         }
3385
3386         return (0);
3387
3388 fail_3:
3389         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3390 fail_2:
3391         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3392         bus_dma_tag_destroy(dma->dma_tag);
3393 fail_0:
3394         dma->dma_map = NULL;
3395         dma->dma_tag = NULL;
3396
3397         return (error);
3398 }
3399
3400 static void
3401 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3402 {
3403         if (dma->dma_tag == NULL)
3404                 return;
3405         if (dma->dma_map != NULL) {
3406                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3407                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3408                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3409                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3410                 dma->dma_map = NULL;
3411         }
3412         bus_dma_tag_destroy(dma->dma_tag);
3413         dma->dma_tag = NULL;
3414 }
3415
3416
3417 /*********************************************************************
3418  *
3419  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3420  *  the information needed to transmit a packet on the wire.
3421  *
3422  **********************************************************************/
3423 static int
3424 em_allocate_transmit_structures(struct adapter *adapter)
3425 {
3426         device_t dev = adapter->dev;
3427         struct em_buffer *tx_buffer;
3428         int error;
3429
3430         /*
3431          * Create DMA tags for tx descriptors
3432          */
3433 #if __FreeBSD_version >= 700000
3434         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3435 #else
3436         if ((error = bus_dma_tag_create(NULL,            /* parent */
3437 #endif
3438                                 1, 0,                   /* alignment, bounds */
3439                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3440                                 BUS_SPACE_MAXADDR,      /* highaddr */
3441                                 NULL, NULL,             /* filter, filterarg */
3442                                 EM_TSO_SIZE,            /* maxsize */
3443                                 EM_MAX_SCATTER,         /* nsegments */
3444                                 EM_TSO_SEG_SIZE,        /* maxsegsize */
3445                                 0,                      /* flags */
3446                                 NULL,           /* lockfunc */
3447                                 NULL,           /* lockarg */
3448                                 &adapter->txtag)) != 0) {
3449                 device_printf(dev, "Unable to allocate TX DMA tag\n");
3450                 goto fail;
3451         }
3452
3453         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
3454             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3455         if (adapter->tx_buffer_area == NULL) {
3456                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3457                 error = ENOMEM;
3458                 goto fail;
3459         }
3460
3461         /* Create the descriptor buffer dma maps */
3462         for (int i = 0; i < adapter->num_tx_desc; i++) {
3463                 tx_buffer = &adapter->tx_buffer_area[i];
3464                 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
3465                 if (error != 0) {
3466                         device_printf(dev, "Unable to create TX DMA map\n");
3467                         goto fail;
3468                 }
3469                 tx_buffer->next_eop = -1;
3470         }
3471
3472         return (0);
3473 fail:
3474         em_free_transmit_structures(adapter);
3475         return (error);
3476 }
3477
3478 /*********************************************************************
3479  *
3480  *  (Re)Initialize transmit structures.
3481  *
3482  **********************************************************************/
3483 static void
3484 em_setup_transmit_structures(struct adapter *adapter)
3485 {
3486         struct em_buffer *tx_buffer;
3487
3488         /* Clear the old ring contents */
3489         bzero(adapter->tx_desc_base,
3490             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3491
3492         /* Free any existing TX buffers */
3493         for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3494                 tx_buffer = &adapter->tx_buffer_area[i];
3495                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3496                     BUS_DMASYNC_POSTWRITE);
3497                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
3498                 m_freem(tx_buffer->m_head);
3499                 tx_buffer->m_head = NULL;
3500                 tx_buffer->next_eop = -1;
3501         }
3502
3503         /* Reset state */
3504         adapter->next_avail_tx_desc = 0;
3505         adapter->next_tx_to_clean = 0;
3506         adapter->num_tx_desc_avail = adapter->num_tx_desc;
3507
3508         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3509             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3510
3511         return;
3512 }
3513
3514 /*********************************************************************
3515  *
3516  *  Enable transmit unit.
3517  *
3518  **********************************************************************/
3519 static void
3520 em_initialize_transmit_unit(struct adapter *adapter)
3521 {
3522         u32     tctl, tarc, tipg = 0;
3523         u64     bus_addr;
3524
3525          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3526         /* Setup the Base and Length of the Tx Descriptor Ring */
3527         bus_addr = adapter->txdma.dma_paddr;
3528         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
3529             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3530         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
3531             (u32)(bus_addr >> 32));
3532         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
3533             (u32)bus_addr);
3534         /* Setup the HW Tx Head and Tail descriptor pointers */
3535         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
3536         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
3537
3538         HW_DEBUGOUT2("Base = %x, Length = %x\n",
3539             E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
3540             E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
3541
3542         /* Set the default values for the Tx Inter Packet Gap timer */
3543         switch (adapter->hw.mac.type) {
3544         case e1000_82542:
3545                 tipg = DEFAULT_82542_TIPG_IPGT;
3546                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3547                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3548                 break;
3549         case e1000_80003es2lan:
3550                 tipg = DEFAULT_82543_TIPG_IPGR1;
3551                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3552                     E1000_TIPG_IPGR2_SHIFT;
3553                 break;
3554         default:
3555                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3556                     (adapter->hw.phy.media_type ==
3557                     e1000_media_type_internal_serdes))
3558                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3559                 else
3560                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3561                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3562                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3563         }
3564
3565         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3566         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3567         if(adapter->hw.mac.type >= e1000_82540)
3568                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3569                     adapter->tx_abs_int_delay.value);
3570
3571         if ((adapter->hw.mac.type == e1000_82571) ||
3572             (adapter->hw.mac.type == e1000_82572)) {
3573                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3574                 tarc |= SPEED_MODE_BIT;
3575                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3576         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3577                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3578                 tarc |= 1;
3579                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3580                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3581                 tarc |= 1;
3582                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3583         }
3584
3585         /* Program the Transmit Control Register */
3586         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3587         tctl &= ~E1000_TCTL_CT;
3588         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3589                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3590
3591         if (adapter->hw.mac.type >= e1000_82571)
3592                 tctl |= E1000_TCTL_MULR;
3593
3594         /* This write will effectively turn on the transmit unit. */
3595         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3596
3597         /* Setup Transmit Descriptor Base Settings */
3598         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3599
3600         if (adapter->tx_int_delay.value > 0)
3601                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3602 }
3603
3604 /*********************************************************************
3605  *
3606  *  Free all transmit related data structures.
3607  *
3608  **********************************************************************/
3609 static void
3610 em_free_transmit_structures(struct adapter *adapter)
3611 {
3612         struct em_buffer *tx_buffer;
3613
3614         INIT_DEBUGOUT("free_transmit_structures: begin");
3615
3616         if (adapter->tx_buffer_area != NULL) {
3617                 for (int i = 0; i < adapter->num_tx_desc; i++) {
3618                         tx_buffer = &adapter->tx_buffer_area[i];
3619                         if (tx_buffer->m_head != NULL) {
3620                                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3621                                     BUS_DMASYNC_POSTWRITE);
3622                                 bus_dmamap_unload(adapter->txtag,
3623                                     tx_buffer->map);
3624                                 m_freem(tx_buffer->m_head);
3625                                 tx_buffer->m_head = NULL;
3626                         } else if (tx_buffer->map != NULL)
3627                                 bus_dmamap_unload(adapter->txtag,
3628                                     tx_buffer->map);
3629                         if (tx_buffer->map != NULL) {
3630                                 bus_dmamap_destroy(adapter->txtag,
3631                                     tx_buffer->map);
3632                                 tx_buffer->map = NULL;
3633                         }
3634                 }
3635         }
3636         if (adapter->tx_buffer_area != NULL) {
3637                 free(adapter->tx_buffer_area, M_DEVBUF);
3638                 adapter->tx_buffer_area = NULL;
3639         }
3640         if (adapter->txtag != NULL) {
3641                 bus_dma_tag_destroy(adapter->txtag);
3642                 adapter->txtag = NULL;
3643         }
3644 #if __FreeBSD_version >= 800000
3645         if (adapter->br != NULL)
3646                 buf_ring_free(adapter->br, M_DEVBUF);
3647 #endif
3648 }
3649
3650 /*********************************************************************
3651  *
3652  *  The offload context needs to be set when we transfer the first
3653  *  packet of a particular protocol (TCP/UDP). This routine has been
3654  *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3655  *
3656  *  Added back the old method of keeping the current context type
3657  *  and not setting if unnecessary, as this is reported to be a
3658  *  big performance win.  -jfv
3659  **********************************************************************/
3660 static void
3661 em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
3662     u32 *txd_upper, u32 *txd_lower)
3663 {
3664         struct e1000_context_desc *TXD = NULL;
3665         struct em_buffer *tx_buffer;
3666         struct ether_vlan_header *eh;
3667         struct ip *ip = NULL;
3668         struct ip6_hdr *ip6;
3669         int curr_txd, ehdrlen;
3670         u32 cmd, hdr_len, ip_hlen;
3671         u16 etype;
3672         u8 ipproto;
3673
3674
3675         cmd = hdr_len = ipproto = 0;
3676         curr_txd = adapter->next_avail_tx_desc;
3677
3678         /*
3679          * Determine where frame payload starts.
3680          * Jump over vlan headers if already present,
3681          * helpful for QinQ too.
3682          */
3683         eh = mtod(mp, struct ether_vlan_header *);
3684         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3685                 etype = ntohs(eh->evl_proto);
3686                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3687         } else {
3688                 etype = ntohs(eh->evl_encap_proto);
3689                 ehdrlen = ETHER_HDR_LEN;
3690         }
3691
3692         /*
3693          * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3694          * TODO: Support SCTP too when it hits the tree.
3695          */
3696         switch (etype) {
3697         case ETHERTYPE_IP:
3698                 ip = (struct ip *)(mp->m_data + ehdrlen);
3699                 ip_hlen = ip->ip_hl << 2;
3700
3701                 /* Setup of IP header checksum. */
3702                 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3703                         /*
3704                          * Start offset for header checksum calculation.
3705                          * End offset for header checksum calculation.
3706                          * Offset of place to put the checksum.
3707                          */
3708                         TXD = (struct e1000_context_desc *)
3709                             &adapter->tx_desc_base[curr_txd];
3710                         TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3711                         TXD->lower_setup.ip_fields.ipcse =
3712                             htole16(ehdrlen + ip_hlen);
3713                         TXD->lower_setup.ip_fields.ipcso =
3714                             ehdrlen + offsetof(struct ip, ip_sum);
3715                         cmd |= E1000_TXD_CMD_IP;
3716                         *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3717                 }
3718
3719                 if (mp->m_len < ehdrlen + ip_hlen)
3720                         return; /* failure */
3721
3722                 hdr_len = ehdrlen + ip_hlen;
3723                 ipproto = ip->ip_p;
3724
3725                 break;
3726         case ETHERTYPE_IPV6:
3727                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3728                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3729
3730                 if (mp->m_len < ehdrlen + ip_hlen)
3731                         return; /* failure */
3732
3733                 /* IPv6 doesn't have a header checksum. */
3734
3735                 hdr_len = ehdrlen + ip_hlen;
3736                 ipproto = ip6->ip6_nxt;
3737
3738                 break;
3739         default:
3740                 *txd_upper = 0;
3741                 *txd_lower = 0;
3742                 return;
3743         }
3744
3745         switch (ipproto) {
3746         case IPPROTO_TCP:
3747                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3748                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3749                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3750                         /* no need for context if already set */
3751                         if (adapter->last_hw_offload == CSUM_TCP)
3752                                 return;
3753                         adapter->last_hw_offload = CSUM_TCP;
3754                         /*
3755                          * Start offset for payload checksum calculation.
3756                          * End offset for payload checksum calculation.
3757                          * Offset of place to put the checksum.
3758                          */
3759                         TXD = (struct e1000_context_desc *)
3760                             &adapter->tx_desc_base[curr_txd];
3761                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3762                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3763                         TXD->upper_setup.tcp_fields.tucso =
3764                             hdr_len + offsetof(struct tcphdr, th_sum);
3765                         cmd |= E1000_TXD_CMD_TCP;
3766                 }
3767                 break;
3768         case IPPROTO_UDP:
3769         {
3770                 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3771                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3772                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3773                         /* no need for context if already set */
3774                         if (adapter->last_hw_offload == CSUM_UDP)
3775                                 return;
3776                         adapter->last_hw_offload = CSUM_UDP;
3777                         /*
3778                          * Start offset for header checksum calculation.
3779                          * End offset for header checksum calculation.
3780                          * Offset of place to put the checksum.
3781                          */
3782                         TXD = (struct e1000_context_desc *)
3783                             &adapter->tx_desc_base[curr_txd];
3784                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3785                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3786                         TXD->upper_setup.tcp_fields.tucso =
3787                             hdr_len + offsetof(struct udphdr, uh_sum);
3788                 }
3789                 /* Fall Thru */
3790         }
3791         default:
3792                 break;
3793         }
3794
3795         TXD->tcp_seg_setup.data = htole32(0);
3796         TXD->cmd_and_length =
3797             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3798         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3799         tx_buffer->m_head = NULL;
3800         tx_buffer->next_eop = -1;
3801
3802         if (++curr_txd == adapter->num_tx_desc)
3803                 curr_txd = 0;
3804
3805         adapter->num_tx_desc_avail--;
3806         adapter->next_avail_tx_desc = curr_txd;
3807 }
3808
3809
3810 #if __FreeBSD_version >= 700000
3811 /**********************************************************************
3812  *
3813  *  Setup work for hardware segmentation offload (TSO)
3814  *
3815  **********************************************************************/
3816 static bool
3817 em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper,
3818    u32 *txd_lower)
3819 {
3820         struct e1000_context_desc *TXD;
3821         struct em_buffer *tx_buffer;
3822         struct ether_vlan_header *eh;
3823         struct ip *ip;
3824         struct ip6_hdr *ip6;
3825         struct tcphdr *th;
3826         int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6;
3827         u16 etype;
3828
3829         /*
3830          * This function could/should be extended to support IP/IPv6
3831          * fragmentation as well.  But as they say, one step at a time.
3832          */
3833
3834         /*
3835          * Determine where frame payload starts.
3836          * Jump over vlan headers if already present,
3837          * helpful for QinQ too.
3838          */
3839         eh = mtod(mp, struct ether_vlan_header *);
3840         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3841                 etype = ntohs(eh->evl_proto);
3842                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3843         } else {
3844                 etype = ntohs(eh->evl_encap_proto);
3845                 ehdrlen = ETHER_HDR_LEN;
3846         }
3847
3848         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3849         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3850                 return FALSE;   /* -1 */
3851
3852         /*
3853          * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3854          * TODO: Support SCTP too when it hits the tree.
3855          */
3856         switch (etype) {
3857         case ETHERTYPE_IP:
3858                 isip6 = 0;
3859                 ip = (struct ip *)(mp->m_data + ehdrlen);
3860                 if (ip->ip_p != IPPROTO_TCP)
3861                         return FALSE;   /* 0 */
3862                 ip->ip_len = 0;
3863                 ip->ip_sum = 0;
3864                 ip_hlen = ip->ip_hl << 2;
3865                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3866                         return FALSE;   /* -1 */
3867                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3868 #if 1
3869                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3870                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3871 #else
3872                 th->th_sum = mp->m_pkthdr.csum_data;
3873 #endif
3874                 break;
3875         case ETHERTYPE_IPV6:
3876                 isip6 = 1;
3877                 return FALSE;                   /* Not supported yet. */
3878                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3879                 if (ip6->ip6_nxt != IPPROTO_TCP)
3880                         return FALSE;   /* 0 */
3881                 ip6->ip6_plen = 0;
3882                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3883                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3884                         return FALSE;   /* -1 */
3885                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3886 #if 0
3887                 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3888                     htons(IPPROTO_TCP));        /* XXX: function notyet. */
3889 #else
3890                 th->th_sum = mp->m_pkthdr.csum_data;
3891 #endif
3892                 break;
3893         default:
3894                 return FALSE;
3895         }
3896         hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3897
3898         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3899                       E1000_TXD_DTYP_D |        /* Data descr type */
3900                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3901
3902         /* IP and/or TCP header checksum calculation and insertion. */
3903         *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3904                       E1000_TXD_POPTS_TXSM) << 8;
3905
3906         curr_txd = adapter->next_avail_tx_desc;
3907         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3908         TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3909
3910         /* IPv6 doesn't have a header checksum. */
3911         if (!isip6) {
3912                 /*
3913                  * Start offset for header checksum calculation.
3914                  * End offset for header checksum calculation.
3915                  * Offset of place put the checksum.
3916                  */
3917                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3918                 TXD->lower_setup.ip_fields.ipcse =
3919                     htole16(ehdrlen + ip_hlen - 1);
3920                 TXD->lower_setup.ip_fields.ipcso =
3921                     ehdrlen + offsetof(struct ip, ip_sum);
3922         }
3923         /*
3924          * Start offset for payload checksum calculation.
3925          * End offset for payload checksum calculation.
3926          * Offset of place to put the checksum.
3927          */
3928         TXD->upper_setup.tcp_fields.tucss =
3929             ehdrlen + ip_hlen;
3930         TXD->upper_setup.tcp_fields.tucse = 0;
3931         TXD->upper_setup.tcp_fields.tucso =
3932             ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3933         /*
3934          * Payload size per packet w/o any headers.
3935          * Length of all headers up to payload.
3936          */
3937         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3938         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3939
3940         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3941                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3942                                 E1000_TXD_CMD_TSE |     /* TSE context */
3943                                 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */
3944                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3945                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3946
3947         tx_buffer->m_head = NULL;
3948         tx_buffer->next_eop = -1;
3949
3950         if (++curr_txd == adapter->num_tx_desc)
3951                 curr_txd = 0;
3952
3953         adapter->num_tx_desc_avail--;
3954         adapter->next_avail_tx_desc = curr_txd;
3955         adapter->tx_tso = TRUE;
3956
3957         return TRUE;
3958 }
3959
3960 #endif /* __FreeBSD_version >= 700000 */
3961
3962 /**********************************************************************
3963  *
3964  *  Examine each tx_buffer in the used queue. If the hardware is done
3965  *  processing the packet then free associated resources. The
3966  *  tx_buffer is put back on the free queue.
3967  *
3968  **********************************************************************/
3969 static void
3970 em_txeof(struct adapter *adapter)
3971 {
3972         int first, last, done, num_avail;
3973         u32 cleaned = 0;
3974         struct em_buffer *tx_buffer;
3975         struct e1000_tx_desc   *tx_desc, *eop_desc;
3976         struct ifnet   *ifp = adapter->ifp;
3977
3978         EM_TX_LOCK_ASSERT(adapter);
3979
3980         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3981                 return;
3982
3983         num_avail = adapter->num_tx_desc_avail;
3984         first = adapter->next_tx_to_clean;
3985         tx_desc = &adapter->tx_desc_base[first];
3986         tx_buffer = &adapter->tx_buffer_area[first];
3987         last = tx_buffer->next_eop;
3988         eop_desc = &adapter->tx_desc_base[last];
3989
3990         /*
3991          * What this does is get the index of the
3992          * first descriptor AFTER the EOP of the
3993          * first packet, that way we can do the
3994          * simple comparison on the inner while loop.
3995          */
3996         if (++last == adapter->num_tx_desc)
3997                 last = 0;
3998         done = last;
3999
4000         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
4001             BUS_DMASYNC_POSTREAD);
4002
4003         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
4004                 /* We clean the range of the packet */
4005                 while (first != done) {
4006                         tx_desc->upper.data = 0;
4007                         tx_desc->lower.data = 0;
4008                         tx_desc->buffer_addr = 0;
4009                         ++num_avail; ++cleaned;
4010
4011                         if (tx_buffer->m_head) {
4012                                 ifp->if_opackets++;
4013                                 bus_dmamap_sync(adapter->txtag,
4014                                     tx_buffer->map,
4015                                     BUS_DMASYNC_POSTWRITE);
4016                                 bus_dmamap_unload(adapter->txtag,
4017                                     tx_buffer->map);
4018
4019                                 m_freem(tx_buffer->m_head);
4020                                 tx_buffer->m_head = NULL;
4021                         }
4022                         tx_buffer->next_eop = -1;
4023
4024                         if (++first == adapter->num_tx_desc)
4025                                 first = 0;
4026
4027                         tx_buffer = &adapter->tx_buffer_area[first];
4028                         tx_desc = &adapter->tx_desc_base[first];
4029                 }
4030                 /* See if we can continue to the next packet */
4031                 last = tx_buffer->next_eop;
4032                 if (last != -1) {
4033                         eop_desc = &adapter->tx_desc_base[last];
4034                         /* Get new done point */
4035                         if (++last == adapter->num_tx_desc) last = 0;
4036                         done = last;
4037                 } else
4038                         break;
4039         }
4040         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
4041             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4042
4043         adapter->next_tx_to_clean = first;
4044
4045         /*
4046          * If we have enough room, clear IFF_DRV_OACTIVE to
4047          * tell the stack that it is OK to send packets.
4048          * If there are no pending descriptors, clear the timeout.
4049          */
4050         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
4051                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
4052                 if (num_avail == adapter->num_tx_desc) {
4053                         adapter->watchdog_timer = 0;
4054                         adapter->num_tx_desc_avail = num_avail;
4055                         return;
4056                 }
4057         }
4058
4059         /* If any descriptors cleaned, reset the watchdog */
4060         if (cleaned)
4061                 adapter->watchdog_timer = EM_TX_TIMEOUT;
4062         adapter->num_tx_desc_avail = num_avail;
4063         return;
4064 }
4065
4066 /*********************************************************************
4067  *
4068  *  When Link is lost sometimes there is work still in the TX ring
4069  *  which will result in a watchdog, rather than allow that do an
4070  *  attempted cleanup and then reinit here. Note that this has been
4071  *  seens mostly with fiber adapters.
4072  *
4073  **********************************************************************/
4074 static void
4075 em_tx_purge(struct adapter *adapter)
4076 {
4077         if ((!adapter->link_active) && (adapter->watchdog_timer)) {
4078                 EM_TX_LOCK(adapter);
4079                 em_txeof(adapter);
4080                 EM_TX_UNLOCK(adapter);
4081                 if (adapter->watchdog_timer) { /* Still not clean? */
4082                         adapter->watchdog_timer = 0;
4083                         em_init_locked(adapter);
4084                 }
4085         }
4086 }
4087
4088 /*********************************************************************
4089  *
4090  *  Get a buffer from system mbuf buffer pool.
4091  *
4092  **********************************************************************/
4093 static int
4094 em_get_buf(struct adapter *adapter, int i)
4095 {
4096         struct mbuf             *m;
4097         bus_dma_segment_t       segs[1];
4098         bus_dmamap_t            map;
4099         struct em_buffer        *rx_buffer;
4100         int                     error, nsegs;
4101
4102         m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
4103         if (m == NULL) {
4104                 adapter->mbuf_cluster_failed++;
4105                 return (ENOBUFS);
4106         }
4107         m->m_len = m->m_pkthdr.len = MCLBYTES;
4108
4109         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
4110                 m_adj(m, ETHER_ALIGN);
4111
4112         /*
4113          * Using memory from the mbuf cluster pool, invoke the
4114          * bus_dma machinery to arrange the memory mapping.
4115          */
4116         error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
4117             adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
4118         if (error != 0) {
4119                 m_free(m);
4120                 return (error);
4121         }
4122
4123         /* If nsegs is wrong then the stack is corrupt. */
4124         KASSERT(nsegs == 1, ("Too many segments returned!"));
4125
4126         rx_buffer = &adapter->rx_buffer_area[i];
4127         if (rx_buffer->m_head != NULL)
4128                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4129
4130         map = rx_buffer->map;
4131         rx_buffer->map = adapter->rx_sparemap;
4132         adapter->rx_sparemap = map;
4133         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
4134         rx_buffer->m_head = m;
4135
4136         adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
4137         return (0);
4138 }
4139
4140 /*********************************************************************
4141  *
4142  *  Allocate memory for rx_buffer structures. Since we use one
4143  *  rx_buffer per received packet, the maximum number of rx_buffer's
4144  *  that we'll need is equal to the number of receive descriptors
4145  *  that we've allocated.
4146  *
4147  **********************************************************************/
4148 static int
4149 em_allocate_receive_structures(struct adapter *adapter)
4150 {
4151         device_t dev = adapter->dev;
4152         struct em_buffer *rx_buffer;
4153         int i, error;
4154
4155         adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
4156             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
4157         if (adapter->rx_buffer_area == NULL) {
4158                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4159                 return (ENOMEM);
4160         }
4161
4162 #if __FreeBSD_version >= 700000
4163         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
4164 #else
4165         error = bus_dma_tag_create(NULL,                 /* parent */
4166 #endif
4167                                 1, 0,                   /* alignment, bounds */
4168                                 BUS_SPACE_MAXADDR,      /* lowaddr */
4169                                 BUS_SPACE_MAXADDR,      /* highaddr */
4170                                 NULL, NULL,             /* filter, filterarg */
4171                                 MCLBYTES,               /* maxsize */
4172                                 1,                      /* nsegments */
4173                                 MCLBYTES,               /* maxsegsize */
4174                                 0,                      /* flags */
4175                                 NULL,                   /* lockfunc */
4176                                 NULL,                   /* lockarg */
4177                                 &adapter->rxtag);
4178         if (error) {
4179                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
4180                     __func__, error);
4181                 goto fail;
4182         }
4183
4184         /* Create the spare map (used by getbuf) */
4185         error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4186              &adapter->rx_sparemap);
4187         if (error) {
4188                 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4189                     __func__, error);
4190                 goto fail;
4191         }
4192
4193         rx_buffer = adapter->rx_buffer_area;
4194         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4195                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4196                     &rx_buffer->map);
4197                 if (error) {
4198                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4199                             __func__, error);
4200                         goto fail;
4201                 }
4202         }
4203
4204         return (0);
4205
4206 fail:
4207         em_free_receive_structures(adapter);
4208         return (error);
4209 }
4210
4211 /*********************************************************************
4212  *
4213  *  (Re)initialize receive structures.
4214  *
4215  **********************************************************************/
4216 static int
4217 em_setup_receive_structures(struct adapter *adapter)
4218 {
4219         struct em_buffer *rx_buffer;
4220         int i, error;
4221
4222         /* Reset descriptor ring */
4223         bzero(adapter->rx_desc_base,
4224             (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
4225
4226         /* Free current RX buffers. */
4227         rx_buffer = adapter->rx_buffer_area;
4228         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4229                 if (rx_buffer->m_head != NULL) {
4230                         bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4231                             BUS_DMASYNC_POSTREAD);
4232                         bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4233                         m_freem(rx_buffer->m_head);
4234                         rx_buffer->m_head = NULL;
4235                 }
4236         }
4237
4238         /* Allocate new ones. */
4239         for (i = 0; i < adapter->num_rx_desc; i++) {
4240                 error = em_get_buf(adapter, i);
4241                 if (error)
4242                         return (error);
4243         }
4244
4245         /* Setup our descriptor pointers */
4246         adapter->next_rx_desc_to_check = 0;
4247         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4248             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4249
4250         return (0);
4251 }
4252
4253 /*********************************************************************
4254  *
4255  *  Enable receive unit.
4256  *
4257  **********************************************************************/
4258 #define MAX_INTS_PER_SEC        8000
4259 #define DEFAULT_ITR          1000000000/(MAX_INTS_PER_SEC * 256)
4260
4261 static void
4262 em_initialize_receive_unit(struct adapter *adapter)
4263 {
4264         struct ifnet    *ifp = adapter->ifp;
4265         u64     bus_addr;
4266         u32     rctl, rxcsum;
4267
4268         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
4269
4270         /*
4271          * Make sure receives are disabled while setting
4272          * up the descriptor ring
4273          */
4274         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4275         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4276
4277         if (adapter->hw.mac.type >= e1000_82540) {
4278                 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4279                     adapter->rx_abs_int_delay.value);
4280                 /*
4281                  * Set the interrupt throttling rate. Value is calculated
4282                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4283                  */
4284                 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
4285         }
4286
4287         /*
4288         ** When using MSIX interrupts we need to throttle
4289         ** using the EITR register (82574 only)
4290         */
4291         if (adapter->msix)
4292                 for (int i = 0; i < 4; i++)
4293                         E1000_WRITE_REG(&adapter->hw,
4294                             E1000_EITR_82574(i), DEFAULT_ITR);
4295
4296         /* Disable accelerated ackknowledge */
4297         if (adapter->hw.mac.type == e1000_82574)
4298                 E1000_WRITE_REG(&adapter->hw,
4299                     E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4300
4301         /* Setup the Base and Length of the Rx Descriptor Ring */
4302         bus_addr = adapter->rxdma.dma_paddr;
4303         E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
4304             adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4305         E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
4306             (u32)(bus_addr >> 32));
4307         E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
4308             (u32)bus_addr);
4309
4310         /* Setup the Receive Control Register */
4311         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4312         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4313                    E1000_RCTL_RDMTS_HALF |
4314                    (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4315
4316         /* Make sure VLAN Filters are off */
4317         rctl &= ~E1000_RCTL_VFE;
4318
4319         if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
4320                 rctl |= E1000_RCTL_SBP;
4321         else
4322                 rctl &= ~E1000_RCTL_SBP;
4323
4324         switch (adapter->rx_buffer_len) {
4325         default:
4326         case 2048:
4327                 rctl |= E1000_RCTL_SZ_2048;
4328                 break;
4329         case 4096:
4330                 rctl |= E1000_RCTL_SZ_4096 |
4331                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4332                 break;
4333         case 8192:
4334                 rctl |= E1000_RCTL_SZ_8192 |
4335                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4336                 break;
4337         case 16384:
4338                 rctl |= E1000_RCTL_SZ_16384 |
4339                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4340                 break;
4341         }
4342
4343         if (ifp->if_mtu > ETHERMTU)
4344                 rctl |= E1000_RCTL_LPE;
4345         else
4346                 rctl &= ~E1000_RCTL_LPE;
4347
4348         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
4349         if ((adapter->hw.mac.type >= e1000_82543) &&
4350             (ifp->if_capenable & IFCAP_RXCSUM)) {
4351                 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
4352                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4353                 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
4354         }
4355
4356         /*
4357         ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4358         ** long latencies are observed, like Lenovo X60. This
4359         ** change eliminates the problem, but since having positive
4360         ** values in RDTR is a known source of problems on other
4361         ** platforms another solution is being sought.
4362         */
4363         if (adapter->hw.mac.type == e1000_82573)
4364                 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20);
4365
4366         /* Enable Receives */
4367         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4368
4369         /*
4370          * Setup the HW Rx Head and
4371          * Tail Descriptor Pointers
4372          */
4373         E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
4374         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
4375
4376         return;
4377 }
4378
4379 /*********************************************************************
4380  *
4381  *  Free receive related data structures.
4382  *
4383  **********************************************************************/
4384 static void
4385 em_free_receive_structures(struct adapter *adapter)
4386 {
4387         struct em_buffer *rx_buffer;
4388         int i;
4389
4390         INIT_DEBUGOUT("free_receive_structures: begin");
4391
4392         if (adapter->rx_sparemap) {
4393                 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
4394                 adapter->rx_sparemap = NULL;
4395         }
4396
4397         /* Cleanup any existing buffers */
4398         if (adapter->rx_buffer_area != NULL) {
4399                 rx_buffer = adapter->rx_buffer_area;
4400                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4401                         if (rx_buffer->m_head != NULL) {
4402                                 bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4403                                     BUS_DMASYNC_POSTREAD);
4404                                 bus_dmamap_unload(adapter->rxtag,
4405                                     rx_buffer->map);
4406                                 m_freem(rx_buffer->m_head);
4407                                 rx_buffer->m_head = NULL;
4408                         } else if (rx_buffer->map != NULL)
4409                                 bus_dmamap_unload(adapter->rxtag,
4410                                     rx_buffer->map);
4411                         if (rx_buffer->map != NULL) {
4412                                 bus_dmamap_destroy(adapter->rxtag,
4413                                     rx_buffer->map);
4414                                 rx_buffer->map = NULL;
4415                         }
4416                 }
4417         }
4418
4419         if (adapter->rx_buffer_area != NULL) {
4420                 free(adapter->rx_buffer_area, M_DEVBUF);
4421                 adapter->rx_buffer_area = NULL;
4422         }
4423
4424         if (adapter->rxtag != NULL) {
4425                 bus_dma_tag_destroy(adapter->rxtag);
4426                 adapter->rxtag = NULL;
4427         }
4428 }
4429
4430 /*********************************************************************
4431  *
4432  *  This routine executes in interrupt context. It replenishes
4433  *  the mbufs in the descriptor and sends data which has been
4434  *  dma'ed into host memory to upper layer.
4435  *
4436  *  We loop at most count times if count is > 0, or until done if
4437  *  count < 0.
4438  *
4439  *  For polling we also now return the number of cleaned packets
4440  *********************************************************************/
4441 static int
4442 em_rxeof(struct adapter *adapter, int count)
4443 {
4444         struct ifnet    *ifp = adapter->ifp;;
4445         struct mbuf     *mp;
4446         u8              status, accept_frame = 0, eop = 0;
4447         u16             len, desc_len, prev_len_adj;
4448         int             i, rx_sent = 0;
4449         struct e1000_rx_desc   *current_desc;
4450
4451         EM_RX_LOCK(adapter);
4452         i = adapter->next_rx_desc_to_check;
4453         current_desc = &adapter->rx_desc_base[i];
4454         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4455             BUS_DMASYNC_POSTREAD);
4456
4457         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
4458                 EM_RX_UNLOCK(adapter);
4459                 return (rx_sent);
4460         }
4461
4462         while ((current_desc->status & E1000_RXD_STAT_DD) &&
4463             (count != 0) &&
4464             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4465                 struct mbuf *m = NULL;
4466
4467                 mp = adapter->rx_buffer_area[i].m_head;
4468                 /*
4469                  * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
4470                  * needs to access the last received byte in the mbuf.
4471                  */
4472                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
4473                     BUS_DMASYNC_POSTREAD);
4474
4475                 accept_frame = 1;
4476                 prev_len_adj = 0;
4477                 desc_len = le16toh(current_desc->length);
4478                 status = current_desc->status;
4479                 if (status & E1000_RXD_STAT_EOP) {
4480                         count--;
4481                         eop = 1;
4482                         if (desc_len < ETHER_CRC_LEN) {
4483                                 len = 0;
4484                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
4485                         } else
4486                                 len = desc_len - ETHER_CRC_LEN;
4487                 } else {
4488                         eop = 0;
4489                         len = desc_len;
4490                 }
4491
4492                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
4493                         u8      last_byte;
4494                         u32     pkt_len = desc_len;
4495
4496                         if (adapter->fmp != NULL)
4497                                 pkt_len += adapter->fmp->m_pkthdr.len;
4498
4499                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
4500                         if (TBI_ACCEPT(&adapter->hw, status,
4501                             current_desc->errors, pkt_len, last_byte,
4502                             adapter->min_frame_size, adapter->max_frame_size)) {
4503                                 e1000_tbi_adjust_stats_82543(&adapter->hw,
4504                                     &adapter->stats, pkt_len,
4505                                     adapter->hw.mac.addr,
4506                                     adapter->max_frame_size);
4507                                 if (len > 0)
4508                                         len--;
4509                         } else
4510                                 accept_frame = 0;
4511                 }
4512
4513                 if (accept_frame) {
4514                         if (em_get_buf(adapter, i) != 0) {
4515                                 ifp->if_iqdrops++;
4516                                 goto discard;
4517                         }
4518
4519                         /* Assign correct length to the current fragment */
4520                         mp->m_len = len;
4521
4522                         if (adapter->fmp == NULL) {
4523                                 mp->m_pkthdr.len = len;
4524                                 adapter->fmp = mp; /* Store the first mbuf */
4525                                 adapter->lmp = mp;
4526                         } else {
4527                                 /* Chain mbuf's together */
4528                                 mp->m_flags &= ~M_PKTHDR;
4529                                 /*
4530                                  * Adjust length of previous mbuf in chain if
4531                                  * we received less than 4 bytes in the last
4532                                  * descriptor.
4533                                  */
4534                                 if (prev_len_adj > 0) {
4535                                         adapter->lmp->m_len -= prev_len_adj;
4536                                         adapter->fmp->m_pkthdr.len -=
4537                                             prev_len_adj;
4538                                 }
4539                                 adapter->lmp->m_next = mp;
4540                                 adapter->lmp = adapter->lmp->m_next;
4541                                 adapter->fmp->m_pkthdr.len += len;
4542                         }
4543
4544                         if (eop) {
4545                                 adapter->fmp->m_pkthdr.rcvif = ifp;
4546                                 ifp->if_ipackets++;
4547                                 em_receive_checksum(adapter, current_desc,
4548                                     adapter->fmp);
4549 #ifndef __NO_STRICT_ALIGNMENT
4550                                 if (adapter->max_frame_size >
4551                                     (MCLBYTES - ETHER_ALIGN) &&
4552                                     em_fixup_rx(adapter) != 0)
4553                                         goto skip;
4554 #endif
4555                                 if (status & E1000_RXD_STAT_VP) {
4556 #if __FreeBSD_version < 700000
4557                                         VLAN_INPUT_TAG_NEW(ifp, adapter->fmp,
4558                                             (le16toh(current_desc->special) &
4559                                             E1000_RXD_SPC_VLAN_MASK));
4560 #else
4561                                         adapter->fmp->m_pkthdr.ether_vtag =
4562                                             (le16toh(current_desc->special) &
4563                                             E1000_RXD_SPC_VLAN_MASK);
4564                                         adapter->fmp->m_flags |= M_VLANTAG;
4565 #endif
4566                                 }
4567 #ifndef __NO_STRICT_ALIGNMENT
4568 skip:
4569 #endif
4570                                 m = adapter->fmp;
4571                                 adapter->fmp = NULL;
4572                                 adapter->lmp = NULL;
4573                         }
4574                 } else {
4575                         ifp->if_ierrors++;
4576 discard:
4577                         /* Reuse loaded DMA map and just update mbuf chain */
4578                         mp = adapter->rx_buffer_area[i].m_head;
4579                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
4580                         mp->m_data = mp->m_ext.ext_buf;
4581                         mp->m_next = NULL;
4582                         if (adapter->max_frame_size <=
4583                             (MCLBYTES - ETHER_ALIGN))
4584                                 m_adj(mp, ETHER_ALIGN);
4585                         if (adapter->fmp != NULL) {
4586                                 m_freem(adapter->fmp);
4587                                 adapter->fmp = NULL;
4588                                 adapter->lmp = NULL;
4589                         }
4590                         m = NULL;
4591                 }
4592
4593                 /* Zero out the receive descriptors status. */
4594                 current_desc->status = 0;
4595                 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4596                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4597
4598                 /* Advance our pointers to the next descriptor. */
4599                 if (++i == adapter->num_rx_desc)
4600                         i = 0;
4601                 /* Call into the stack */
4602                 if (m != NULL) {
4603                         adapter->next_rx_desc_to_check = i;
4604                         EM_RX_UNLOCK(adapter);
4605                         (*ifp->if_input)(ifp, m);
4606                         EM_RX_LOCK(adapter);
4607                         rx_sent++;
4608                         i = adapter->next_rx_desc_to_check;
4609                 }
4610                 current_desc = &adapter->rx_desc_base[i];
4611         }
4612         adapter->next_rx_desc_to_check = i;
4613
4614         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
4615         if (--i < 0)
4616                 i = adapter->num_rx_desc - 1;
4617         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
4618         EM_RX_UNLOCK(adapter);
4619         return (rx_sent);
4620 }
4621
4622 #ifndef __NO_STRICT_ALIGNMENT
4623 /*
4624  * When jumbo frames are enabled we should realign entire payload on
4625  * architecures with strict alignment. This is serious design mistake of 8254x
4626  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4627  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4628  * payload. On architecures without strict alignment restrictions 8254x still
4629  * performs unaligned memory access which would reduce the performance too.
4630  * To avoid copying over an entire frame to align, we allocate a new mbuf and
4631  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4632  * existing mbuf chain.
4633  *
4634  * Be aware, best performance of the 8254x is achived only when jumbo frame is
4635  * not used at all on architectures with strict alignment.
4636  */
4637 static int
4638 em_fixup_rx(struct adapter *adapter)
4639 {
4640         struct mbuf *m, *n;
4641         int error;
4642
4643         error = 0;
4644         m = adapter->fmp;
4645         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4646                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4647                 m->m_data += ETHER_HDR_LEN;
4648         } else {
4649                 MGETHDR(n, M_DONTWAIT, MT_DATA);
4650                 if (n != NULL) {
4651                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4652                         m->m_data += ETHER_HDR_LEN;
4653                         m->m_len -= ETHER_HDR_LEN;
4654                         n->m_len = ETHER_HDR_LEN;
4655                         M_MOVE_PKTHDR(n, m);
4656                         n->m_next = m;
4657                         adapter->fmp = n;
4658                 } else {
4659                         adapter->dropped_pkts++;
4660                         m_freem(adapter->fmp);
4661                         adapter->fmp = NULL;
4662                         error = ENOMEM;
4663                 }
4664         }
4665
4666         return (error);
4667 }
4668 #endif
4669
4670 /*********************************************************************
4671  *
4672  *  Verify that the hardware indicated that the checksum is valid.
4673  *  Inform the stack about the status of checksum so that stack
4674  *  doesn't spend time verifying the checksum.
4675  *
4676  *********************************************************************/
4677 static void
4678 em_receive_checksum(struct adapter *adapter,
4679             struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4680 {
4681         /* 82543 or newer only */
4682         if ((adapter->hw.mac.type < e1000_82543) ||
4683             /* Ignore Checksum bit is set */
4684             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
4685                 mp->m_pkthdr.csum_flags = 0;
4686                 return;
4687         }
4688
4689         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4690                 /* Did it pass? */
4691                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4692                         /* IP Checksum Good */
4693                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4694                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4695
4696                 } else {
4697                         mp->m_pkthdr.csum_flags = 0;
4698                 }
4699         }
4700
4701         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4702                 /* Did it pass? */
4703                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4704                         mp->m_pkthdr.csum_flags |=
4705                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4706                         mp->m_pkthdr.csum_data = htons(0xffff);
4707                 }
4708         }
4709 }
4710
4711 #if __FreeBSD_version >= 700029
4712 /*
4713  * This routine is run via an vlan
4714  * config EVENT
4715  */
4716 static void
4717 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4718 {
4719         struct adapter  *adapter = ifp->if_softc;
4720         u32             index, bit;
4721
4722         if (ifp->if_softc !=  arg)   /* Not our event */
4723                 return;
4724
4725         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
4726                 return;
4727
4728         index = (vtag >> 5) & 0x7F;
4729         bit = vtag & 0x1F;
4730         em_shadow_vfta[index] |= (1 << bit);
4731         ++adapter->num_vlans;
4732         /* Re-init to load the changes */
4733         em_init(adapter);
4734 }
4735
4736 /*
4737  * This routine is run via an vlan
4738  * unconfig EVENT
4739  */
4740 static void
4741 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4742 {
4743         struct adapter  *adapter = ifp->if_softc;
4744         u32             index, bit;
4745
4746         if (ifp->if_softc !=  arg)
4747                 return;
4748
4749         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4750                 return;
4751
4752         index = (vtag >> 5) & 0x7F;
4753         bit = vtag & 0x1F;
4754         em_shadow_vfta[index] &= ~(1 << bit);
4755         --adapter->num_vlans;
4756         /* Re-init to load the changes */
4757         em_init(adapter);
4758 }
4759
4760 static void
4761 em_setup_vlan_hw_support(struct adapter *adapter)
4762 {
4763         struct e1000_hw *hw = &adapter->hw;
4764         u32             reg;
4765
4766         /*
4767         ** We get here thru init_locked, meaning
4768         ** a soft reset, this has already cleared
4769         ** the VFTA and other state, so if there
4770         ** have been no vlan's registered do nothing.
4771         */
4772         if (adapter->num_vlans == 0)
4773                 return;
4774
4775         /*
4776         ** A soft reset zero's out the VFTA, so
4777         ** we need to repopulate it now.
4778         */
4779         for (int i = 0; i < EM_VFTA_SIZE; i++)
4780                 if (em_shadow_vfta[i] != 0)
4781                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4782                             i, em_shadow_vfta[i]);
4783
4784         reg = E1000_READ_REG(hw, E1000_CTRL);
4785         reg |= E1000_CTRL_VME;
4786         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4787
4788         /* Enable the Filter Table */
4789         reg = E1000_READ_REG(hw, E1000_RCTL);
4790         reg &= ~E1000_RCTL_CFIEN;
4791         reg |= E1000_RCTL_VFE;
4792         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4793
4794         /* Update the frame size */
4795         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4796             adapter->max_frame_size + VLAN_TAG_SIZE);
4797 }
4798 #endif
4799
4800 static void
4801 em_enable_intr(struct adapter *adapter)
4802 {
4803         struct e1000_hw *hw = &adapter->hw;
4804         u32 ims_mask = IMS_ENABLE_MASK;
4805
4806         if (adapter->msix) {
4807                 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4808                 ims_mask |= EM_MSIX_MASK;
4809         }
4810         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4811 }
4812
4813 static void
4814 em_disable_intr(struct adapter *adapter)
4815 {
4816         struct e1000_hw *hw = &adapter->hw;
4817
4818         if (adapter->msix)
4819                 E1000_WRITE_REG(hw, EM_EIAC, 0);
4820         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4821 }
4822
4823 /*
4824  * Bit of a misnomer, what this really means is
4825  * to enable OS management of the system... aka
4826  * to disable special hardware management features
4827  */
4828 static void
4829 em_init_manageability(struct adapter *adapter)
4830 {
4831         /* A shared code workaround */
4832 #define E1000_82542_MANC2H E1000_MANC2H
4833         if (adapter->has_manage) {
4834                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4835                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4836
4837                 /* disable hardware interception of ARP */
4838                 manc &= ~(E1000_MANC_ARP_EN);
4839
4840                 /* enable receiving management packets to the host */
4841                 if (adapter->hw.mac.type >= e1000_82571) {
4842                         manc |= E1000_MANC_EN_MNG2HOST;
4843 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4844 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4845                         manc2h |= E1000_MNG2HOST_PORT_623;
4846                         manc2h |= E1000_MNG2HOST_PORT_664;
4847                         E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4848                 }
4849
4850                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4851         }
4852 }
4853
4854 /*
4855  * Give control back to hardware management
4856  * controller if there is one.
4857  */
4858 static void
4859 em_release_manageability(struct adapter *adapter)
4860 {
4861         if (adapter->has_manage) {
4862                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4863
4864                 /* re-enable hardware interception of ARP */
4865                 manc |= E1000_MANC_ARP_EN;
4866
4867                 if (adapter->hw.mac.type >= e1000_82571)
4868                         manc &= ~E1000_MANC_EN_MNG2HOST;
4869
4870                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4871         }
4872 }
4873
4874 /*
4875  * em_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4876  * For ASF and Pass Through versions of f/w this means that
4877  * the driver is loaded. For AMT version (only with 82573)
4878  * of the f/w this means that the network i/f is open.
4879  *
4880  */
4881 static void
4882 em_get_hw_control(struct adapter *adapter)
4883 {
4884         u32 ctrl_ext, swsm;
4885
4886         /* Let firmware know the driver has taken over */
4887         switch (adapter->hw.mac.type) {
4888         case e1000_82573:
4889                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4890                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4891                     swsm | E1000_SWSM_DRV_LOAD);
4892                 break;
4893         case e1000_82571:
4894         case e1000_82572:
4895         case e1000_80003es2lan:
4896         case e1000_ich8lan:
4897         case e1000_ich9lan:
4898         case e1000_ich10lan:
4899                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4900                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4901                     ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4902                 break;
4903         default:
4904                 break;
4905         }
4906 }
4907
4908 /*
4909  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4910  * For ASF and Pass Through versions of f/w this means that the
4911  * driver is no longer loaded. For AMT version (only with 82573) i
4912  * of the f/w this means that the network i/f is closed.
4913  *
4914  */
4915 static void
4916 em_release_hw_control(struct adapter *adapter)
4917 {
4918         u32 ctrl_ext, swsm;
4919
4920         /* Let firmware taken over control of h/w */
4921         switch (adapter->hw.mac.type) {
4922         case e1000_82573:
4923                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4924                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4925                     swsm & ~E1000_SWSM_DRV_LOAD);
4926                 break;
4927         case e1000_82571:
4928         case e1000_82572:
4929         case e1000_80003es2lan:
4930         case e1000_ich8lan:
4931         case e1000_ich9lan:
4932         case e1000_ich10lan:
4933                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4934                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4935                     ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4936                 break;
4937         default:
4938                 break;
4939
4940         }
4941 }
4942
4943 static int
4944 em_is_valid_ether_addr(u8 *addr)
4945 {
4946         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4947
4948         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4949                 return (FALSE);
4950         }
4951
4952         return (TRUE);
4953 }
4954
4955 /*
4956  * Enable PCI Wake On Lan capability
4957  */
4958 void
4959 em_enable_wakeup(device_t dev)
4960 {
4961         u16     cap, status;
4962         u8      id;
4963
4964         /* First find the capabilities pointer*/
4965         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4966         /* Read the PM Capabilities */
4967         id = pci_read_config(dev, cap, 1);
4968         if (id != PCIY_PMG)     /* Something wrong */
4969                 return;
4970         /* OK, we have the power capabilities, so
4971            now get the status register */
4972         cap += PCIR_POWER_STATUS;
4973         status = pci_read_config(dev, cap, 2);
4974         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4975         pci_write_config(dev, cap, status, 2);
4976         return;
4977 }
4978
4979
4980 /*********************************************************************
4981 * 82544 Coexistence issue workaround.
4982 *    There are 2 issues.
4983 *       1. Transmit Hang issue.
4984 *    To detect this issue, following equation can be used...
4985 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4986 *         If SUM[3:0] is in between 1 to 4, we will have this issue.
4987 *
4988 *       2. DAC issue.
4989 *    To detect this issue, following equation can be used...
4990 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4991 *         If SUM[3:0] is in between 9 to c, we will have this issue.
4992 *
4993 *
4994 *    WORKAROUND:
4995 *         Make sure we do not have ending address
4996 *         as 1,2,3,4(Hang) or 9,a,b,c (DAC)
4997 *
4998 *************************************************************************/
4999 static u32
5000 em_fill_descriptors (bus_addr_t address, u32 length,
5001                 PDESC_ARRAY desc_array)
5002 {
5003         u32 safe_terminator;
5004
5005         /* Since issue is sensitive to length and address.*/
5006         /* Let us first check the address...*/
5007         if (length <= 4) {
5008                 desc_array->descriptor[0].address = address;
5009                 desc_array->descriptor[0].length = length;
5010                 desc_array->elements = 1;
5011                 return (desc_array->elements);
5012         }
5013         safe_terminator = (u32)((((u32)address & 0x7) +
5014             (length & 0xF)) & 0xF);
5015         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
5016         if (safe_terminator == 0   ||
5017         (safe_terminator > 4   &&
5018         safe_terminator < 9)   ||
5019         (safe_terminator > 0xC &&
5020         safe_terminator <= 0xF)) {
5021                 desc_array->descriptor[0].address = address;
5022                 desc_array->descriptor[0].length = length;
5023                 desc_array->elements = 1;
5024                 return (desc_array->elements);
5025         }
5026
5027         desc_array->descriptor[0].address = address;
5028         desc_array->descriptor[0].length = length - 4;
5029         desc_array->descriptor[1].address = address + (length - 4);
5030         desc_array->descriptor[1].length = 4;
5031         desc_array->elements = 2;
5032         return (desc_array->elements);
5033 }
5034
5035 /**********************************************************************
5036  *
5037  *  Update the board statistics counters.
5038  *
5039  **********************************************************************/
5040 static void
5041 em_update_stats_counters(struct adapter *adapter)
5042 {
5043         struct ifnet   *ifp;
5044
5045         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5046            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5047                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5048                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5049         }
5050         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5051         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5052         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5053         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5054
5055         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5056         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5057         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5058         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5059         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5060         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5061         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5062         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5063         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5064         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5065         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5066         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5067         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5068         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5069         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5070         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5071         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5072         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5073         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5074         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5075
5076         /* For the 64-bit byte counters the low dword must be read first. */
5077         /* Both registers clear on the read of the high dword */
5078
5079         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
5080         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
5081
5082         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5083         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5084         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5085         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5086         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5087
5088         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5089         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5090
5091         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5092         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5093         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5094         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5095         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5096         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5097         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5098         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5099         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5100         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5101
5102         if (adapter->hw.mac.type >= e1000_82543) {
5103                 adapter->stats.algnerrc +=
5104                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5105                 adapter->stats.rxerrc +=
5106                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5107                 adapter->stats.tncrs +=
5108                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5109                 adapter->stats.cexterr +=
5110                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5111                 adapter->stats.tsctc +=
5112                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5113                 adapter->stats.tsctfc +=
5114                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5115         }
5116         ifp = adapter->ifp;
5117
5118         ifp->if_collisions = adapter->stats.colc;
5119
5120         /* Rx Errors */
5121         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5122             adapter->stats.crcerrs + adapter->stats.algnerrc +
5123             adapter->stats.ruc + adapter->stats.roc +
5124             adapter->stats.mpc + adapter->stats.cexterr;
5125
5126         /* Tx Errors */
5127         ifp->if_oerrors = adapter->stats.ecol +
5128             adapter->stats.latecol + adapter->watchdog_events;
5129 }
5130
5131
5132 /**********************************************************************
5133  *
5134  *  This routine is called only when em_display_debug_stats is enabled.
5135  *  This routine provides a way to take a look at important statistics
5136  *  maintained by the driver and hardware.
5137  *
5138  **********************************************************************/
5139 static void
5140 em_print_debug_info(struct adapter *adapter)
5141 {
5142         device_t dev = adapter->dev;
5143         u8 *hw_addr = adapter->hw.hw_addr;
5144
5145         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
5146         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
5147             E1000_READ_REG(&adapter->hw, E1000_CTRL),
5148             E1000_READ_REG(&adapter->hw, E1000_RCTL));
5149         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
5150             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
5151             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
5152         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
5153             adapter->hw.fc.high_water,
5154             adapter->hw.fc.low_water);
5155         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
5156             E1000_READ_REG(&adapter->hw, E1000_TIDV),
5157             E1000_READ_REG(&adapter->hw, E1000_TADV));
5158         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
5159             E1000_READ_REG(&adapter->hw, E1000_RDTR),
5160             E1000_READ_REG(&adapter->hw, E1000_RADV));
5161         device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
5162             (long long)adapter->tx_fifo_wrk_cnt,
5163             (long long)adapter->tx_fifo_reset_cnt);
5164         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5165             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5166             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5167         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5168             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5169             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5170         device_printf(dev, "Num Tx descriptors avail = %d\n",
5171             adapter->num_tx_desc_avail);
5172         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
5173             adapter->no_tx_desc_avail1);
5174         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
5175             adapter->no_tx_desc_avail2);
5176         device_printf(dev, "Std mbuf failed = %ld\n",
5177             adapter->mbuf_alloc_failed);
5178         device_printf(dev, "Std mbuf cluster failed = %ld\n",
5179             adapter->mbuf_cluster_failed);
5180         device_printf(dev, "Driver dropped packets = %ld\n",
5181             adapter->dropped_pkts);
5182         device_printf(dev, "Driver tx dma failure in encap = %ld\n",
5183                 adapter->no_tx_dma_setup);
5184 }
5185
5186 static void
5187 em_print_hw_stats(struct adapter *adapter)
5188 {
5189         device_t dev = adapter->dev;
5190
5191         device_printf(dev, "Excessive collisions = %lld\n",
5192             (long long)adapter->stats.ecol);
5193 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
5194         device_printf(dev, "Symbol errors = %lld\n",
5195             (long long)adapter->stats.symerrs);
5196 #endif
5197         device_printf(dev, "Sequence errors = %lld\n",
5198             (long long)adapter->stats.sec);
5199         device_printf(dev, "Defer count = %lld\n",
5200             (long long)adapter->stats.dc);
5201         device_printf(dev, "Missed Packets = %lld\n",
5202             (long long)adapter->stats.mpc);
5203         device_printf(dev, "Receive No Buffers = %lld\n",
5204             (long long)adapter->stats.rnbc);
5205         /* RLEC is inaccurate on some hardware, calculate our own. */
5206         device_printf(dev, "Receive Length Errors = %lld\n",
5207             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
5208         device_printf(dev, "Receive errors = %lld\n",
5209             (long long)adapter->stats.rxerrc);
5210         device_printf(dev, "Crc errors = %lld\n",
5211             (long long)adapter->stats.crcerrs);
5212         device_printf(dev, "Alignment errors = %lld\n",
5213             (long long)adapter->stats.algnerrc);
5214         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
5215             (long long)adapter->stats.cexterr);
5216         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
5217         device_printf(dev, "watchdog timeouts = %ld\n",
5218             adapter->watchdog_events);
5219         device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld"
5220             " LINK MSIX IRQ = %ld\n", adapter->rx_irq,
5221             adapter->tx_irq , adapter->link_irq);
5222         device_printf(dev, "XON Rcvd = %lld\n",
5223             (long long)adapter->stats.xonrxc);
5224         device_printf(dev, "XON Xmtd = %lld\n",
5225             (long long)adapter->stats.xontxc);
5226         device_printf(dev, "XOFF Rcvd = %lld\n",
5227             (long long)adapter->stats.xoffrxc);
5228         device_printf(dev, "XOFF Xmtd = %lld\n",
5229             (long long)adapter->stats.xofftxc);
5230         device_printf(dev, "Good Packets Rcvd = %lld\n",
5231             (long long)adapter->stats.gprc);
5232         device_printf(dev, "Good Packets Xmtd = %lld\n",
5233             (long long)adapter->stats.gptc);
5234         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
5235             (long long)adapter->stats.tsctc);
5236         device_printf(dev, "TSO Contexts Failed = %lld\n",
5237             (long long)adapter->stats.tsctfc);
5238 }
5239
5240 /**********************************************************************
5241  *
5242  *  This routine provides a way to dump out the adapter eeprom,
5243  *  often a useful debug/service tool. This only dumps the first
5244  *  32 words, stuff that matters is in that extent.
5245  *
5246  **********************************************************************/
5247 static void
5248 em_print_nvm_info(struct adapter *adapter)
5249 {
5250         u16     eeprom_data;
5251         int     i, j, row = 0;
5252
5253         /* Its a bit crude, but it gets the job done */
5254         printf("\nInterface EEPROM Dump:\n");
5255         printf("Offset\n0x0000  ");
5256         for (i = 0, j = 0; i < 32; i++, j++) {
5257                 if (j == 8) { /* Make the offset block */
5258                         j = 0; ++row;
5259                         printf("\n0x00%x0  ",row);
5260                 }
5261                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5262                 printf("%04x ", eeprom_data);
5263         }
5264         printf("\n");
5265 }
5266
5267 static int
5268 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5269 {
5270         struct adapter *adapter;
5271         int error;
5272         int result;
5273
5274         result = -1;
5275         error = sysctl_handle_int(oidp, &result, 0, req);
5276
5277         if (error || !req->newptr)
5278                 return (error);
5279
5280         if (result == 1) {
5281                 adapter = (struct adapter *)arg1;
5282                 em_print_debug_info(adapter);
5283         }
5284         /*
5285          * This value will cause a hex dump of the
5286          * first 32 16-bit words of the EEPROM to
5287          * the screen.
5288          */
5289         if (result == 2) {
5290                 adapter = (struct adapter *)arg1;
5291                 em_print_nvm_info(adapter);
5292         }
5293
5294         return (error);
5295 }
5296
5297
5298 static int
5299 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
5300 {
5301         struct adapter *adapter;
5302         int error;
5303         int result;
5304
5305         result = -1;
5306         error = sysctl_handle_int(oidp, &result, 0, req);
5307
5308         if (error || !req->newptr)
5309                 return (error);
5310
5311         if (result == 1) {
5312                 adapter = (struct adapter *)arg1;
5313                 em_print_hw_stats(adapter);
5314         }
5315
5316         return (error);
5317 }
5318
5319 static int
5320 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5321 {
5322         struct em_int_delay_info *info;
5323         struct adapter *adapter;
5324         u32 regval;
5325         int error;
5326         int usecs;
5327         int ticks;
5328
5329         info = (struct em_int_delay_info *)arg1;
5330         usecs = info->value;
5331         error = sysctl_handle_int(oidp, &usecs, 0, req);
5332         if (error != 0 || req->newptr == NULL)
5333                 return (error);
5334         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5335                 return (EINVAL);
5336         info->value = usecs;
5337         ticks = EM_USECS_TO_TICKS(usecs);
5338
5339         adapter = info->adapter;
5340
5341         EM_CORE_LOCK(adapter);
5342         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5343         regval = (regval & ~0xffff) | (ticks & 0xffff);
5344         /* Handle a few special cases. */
5345         switch (info->offset) {
5346         case E1000_RDTR:
5347                 break;
5348         case E1000_TIDV:
5349                 if (ticks == 0) {
5350                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5351                         /* Don't write 0 into the TIDV register. */
5352                         regval++;
5353                 } else
5354                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5355                 break;
5356         }
5357         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5358         EM_CORE_UNLOCK(adapter);
5359         return (0);
5360 }
5361
5362 static void
5363 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5364         const char *description, struct em_int_delay_info *info,
5365         int offset, int value)
5366 {
5367         info->adapter = adapter;
5368         info->offset = offset;
5369         info->value = value;
5370         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5371             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5372             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5373             info, 0, em_sysctl_int_delay, "I", description);
5374 }
5375
5376 #ifndef EM_LEGACY_IRQ
5377 static void
5378 em_add_rx_process_limit(struct adapter *adapter, const char *name,
5379         const char *description, int *limit, int value)
5380 {
5381         *limit = value;
5382         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5383             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5384             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5385 }
5386 #endif
5387
5388