sys/dev/e1000/if_em.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2009, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #endif
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #if __FreeBSD_version >= 800000
  43 #include <sys/buf_ring.h>
  44 #endif
  45 #include <sys/bus.h>
  46 #include <sys/endian.h>
  47 #include <sys/kernel.h>
  48 #include <sys/kthread.h>
  49 #include <sys/malloc.h>
  50 #include <sys/mbuf.h>
  51 #include <sys/module.h>
  52 #include <sys/rman.h>
  53 #include <sys/socket.h>
  54 #include <sys/sockio.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/taskqueue.h>
  57 #if __FreeBSD_version >= 700029
  58 #include <sys/eventhandler.h>
  59 #endif
  60 #include <machine/bus.h>
  61 #include <machine/resource.h>
  62
  63 #include <net/bpf.h>
  64 #include <net/ethernet.h>
  65 #include <net/if.h>
  66 #include <net/if_arp.h>
  67 #include <net/if_dl.h>
  68 #include <net/if_media.h>
  69
  70 #include <net/if_types.h>
  71 #include <net/if_vlan_var.h>
  72
  73 #include <netinet/in_systm.h>
  74 #include <netinet/in.h>
  75 #include <netinet/if_ether.h>
  76 #include <netinet/ip.h>
  77 #include <netinet/ip6.h>
  78 #include <netinet/tcp.h>
  79 #include <netinet/udp.h>
  80
  81 #include <machine/in_cksum.h>
  82 #include <dev/pci/pcivar.h>
  83 #include <dev/pci/pcireg.h>
  84
  85 #include "e1000_api.h"
  86 #include "e1000_82571.h"
  87 #include "if_em.h"
  88
  89 /*********************************************************************
  90  *  Set this to one to display debug statistics
  91  *********************************************************************/
  92 int     em_display_debug_stats = 0;
  93
  94 /*********************************************************************
  95  *  Driver version:
  96  *********************************************************************/
  97 char em_driver_version[] = "6.9.14";
  98
  99
 100 /*********************************************************************
 101  *  PCI Device ID Table
 102  *
 103  *  Used by probe to select devices to load on
 104  *  Last field stores an index into e1000_strings
 105  *  Last entry must be all 0s
 106  *
 107  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 108  *********************************************************************/
 109
 110 static em_vendor_info_t em_vendor_info_array[] =
 111 {
 112         /* Intel(R) PRO/1000 Network Connection */
 113         { 0x8086, E1000_DEV_ID_82540EM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 114         { 0x8086, E1000_DEV_ID_82540EM_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82540EP,         PCI_ANY_ID, PCI_ANY_ID, 0},
 116         { 0x8086, E1000_DEV_ID_82540EP_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82540EP_LP,      PCI_ANY_ID, PCI_ANY_ID, 0},
 118
 119         { 0x8086, E1000_DEV_ID_82541EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 120         { 0x8086, E1000_DEV_ID_82541ER,         PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82541ER_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 122         { 0x8086, E1000_DEV_ID_82541EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82541GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 124         { 0x8086, E1000_DEV_ID_82541GI_LF,      PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82541GI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 126
 127         { 0x8086, E1000_DEV_ID_82542,           PCI_ANY_ID, PCI_ANY_ID, 0},
 128
 129         { 0x8086, E1000_DEV_ID_82543GC_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82543GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 131
 132         { 0x8086, E1000_DEV_ID_82544EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82544EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82544GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82544GC_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 136
 137         { 0x8086, E1000_DEV_ID_82545EM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 138         { 0x8086, E1000_DEV_ID_82545EM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 139         { 0x8086, E1000_DEV_ID_82545GM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_82545GM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 141         { 0x8086, E1000_DEV_ID_82545GM_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 142
 143         { 0x8086, E1000_DEV_ID_82546EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 144         { 0x8086, E1000_DEV_ID_82546EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_82546GB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_82546GB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_82546GB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_82546GB_PCIE,    PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
 152                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 153
 154         { 0x8086, E1000_DEV_ID_82547EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 155         { 0x8086, E1000_DEV_ID_82547EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_82547GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 157
 158         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 159         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 160         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
 162                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 163         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
 164                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 165         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 166                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 167         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
 168                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 169         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
 170                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 171         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
 172                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 173         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 174         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 175         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 176         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 177
 178         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 179         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 180         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 181         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
 182         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 183                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 184         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 185                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 186         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 187                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 188         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 189                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 190         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 191         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 192         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 193         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 194         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 195         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 196         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 197
 198         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 199         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 200         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 201         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 202         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 203         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 204         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 205         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 206         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 207         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 208         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
 209         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 210         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 211         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 212         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 213         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 214         /* required last entry */
 215         { 0, 0, 0, 0, 0}
 216 };
 217
 218 /*********************************************************************
 219  *  Table of branding strings for all supported NICs.
 220  *********************************************************************/
 221
 222 static char *em_strings[] = {
 223         "Intel(R) PRO/1000 Network Connection"
 224 };
 225
 226 /*********************************************************************
 227  *  Function prototypes
 228  *********************************************************************/
 229 static int      em_probe(device_t);
 230 static int      em_attach(device_t);
 231 static int      em_detach(device_t);
 232 static int      em_shutdown(device_t);
 233 static int      em_suspend(device_t);
 234 static int      em_resume(device_t);
 235 static void     em_start(struct ifnet *);
 236 static void     em_start_locked(struct ifnet *ifp);
 237 #if __FreeBSD_version >= 800000
 238 static int      em_mq_start(struct ifnet *, struct mbuf *);
 239 static int      em_mq_start_locked(struct ifnet *, struct mbuf *);
 240 static void     em_qflush(struct ifnet *);
 241 #endif
 242 static int      em_ioctl(struct ifnet *, u_long, caddr_t);
 243 static void     em_watchdog(struct adapter *);
 244 static void     em_init(void *);
 245 static void     em_init_locked(struct adapter *);
 246 static void     em_stop(void *);
 247 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 248 static int      em_media_change(struct ifnet *);
 249 static void     em_identify_hardware(struct adapter *);
 250 static int      em_allocate_pci_resources(struct adapter *);
 251 static int      em_allocate_legacy(struct adapter *adapter);
 252 static int      em_allocate_msix(struct adapter *adapter);
 253 static int      em_setup_msix(struct adapter *);
 254 static void     em_free_pci_resources(struct adapter *);
 255 static void     em_local_timer(void *);
 256 static int      em_hardware_init(struct adapter *);
 257 static void     em_setup_interface(device_t, struct adapter *);
 258 static void     em_setup_transmit_structures(struct adapter *);
 259 static void     em_initialize_transmit_unit(struct adapter *);
 260 static int      em_setup_receive_structures(struct adapter *);
 261 static void     em_initialize_receive_unit(struct adapter *);
 262 static void     em_enable_intr(struct adapter *);
 263 static void     em_disable_intr(struct adapter *);
 264 static void     em_free_transmit_structures(struct adapter *);
 265 static void     em_free_receive_structures(struct adapter *);
 266 static void     em_update_stats_counters(struct adapter *);
 267 static void     em_txeof(struct adapter *);
 268 static void     em_tx_purge(struct adapter *);
 269 static int      em_allocate_receive_structures(struct adapter *);
 270 static int      em_allocate_transmit_structures(struct adapter *);
 271 static int      em_rxeof(struct adapter *, int);
 272 #ifndef __NO_STRICT_ALIGNMENT
 273 static int      em_fixup_rx(struct adapter *);
 274 #endif
 275 static void     em_receive_checksum(struct adapter *, struct e1000_rx_desc *,
 276                     struct mbuf *);
 277 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 278                     u32 *, u32 *);
 279 #if __FreeBSD_version >= 700000
 280 static bool     em_tso_setup(struct adapter *, struct mbuf *,
 281                     u32 *, u32 *);
 282 #endif /* FreeBSD_version >= 700000 */
 283 static void     em_set_promisc(struct adapter *);
 284 static void     em_disable_promisc(struct adapter *);
 285 static void     em_set_multi(struct adapter *);
 286 static void     em_print_hw_stats(struct adapter *);
 287 static void     em_update_link_status(struct adapter *);
 288 static int      em_get_buf(struct adapter *, int);
 289 #if __FreeBSD_version >= 700029
 290 static void     em_register_vlan(void *, struct ifnet *, u16);
 291 static void     em_unregister_vlan(void *, struct ifnet *, u16);
 292 static void     em_setup_vlan_hw_support(struct adapter *);
 293 #endif
 294 static int      em_xmit(struct adapter *, struct mbuf **);
 295 static void     em_smartspeed(struct adapter *);
 296 static int      em_82547_fifo_workaround(struct adapter *, int);
 297 static void     em_82547_update_fifo_head(struct adapter *, int);
 298 static int      em_82547_tx_fifo_reset(struct adapter *);
 299 static void     em_82547_move_tail(void *);
 300 static int      em_dma_malloc(struct adapter *, bus_size_t,
 301                     struct em_dma_alloc *, int);
 302 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 303 static void     em_print_debug_info(struct adapter *);
 304 static void     em_print_nvm_info(struct adapter *);
 305 static int      em_is_valid_ether_addr(u8 *);
 306 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 307 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 308 static u32      em_fill_descriptors (bus_addr_t address, u32 length,
 309                     PDESC_ARRAY desc_array);
 310 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 311 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 312                     const char *, struct em_int_delay_info *, int, int);
 313 /* Management and WOL Support */
 314 static void     em_init_manageability(struct adapter *);
 315 static void     em_release_manageability(struct adapter *);
 316 static void     em_get_hw_control(struct adapter *);
 317 static void     em_release_hw_control(struct adapter *);
 318 static void     em_enable_wakeup(device_t);
 319
 320 #ifdef EM_LEGACY_IRQ
 321 static void     em_intr(void *);
 322 #else /* FAST IRQ */
 323 #if __FreeBSD_version < 700000
 324 static void     em_irq_fast(void *);
 325 #else
 326 static int      em_irq_fast(void *);
 327 #endif
 328
 329 /* MSIX handlers */
 330 static void     em_msix_tx(void *);
 331 static void     em_msix_rx(void *);
 332 static void     em_msix_link(void *);
 333 static void     em_handle_rx(void *context, int pending);
 334 static void     em_handle_tx(void *context, int pending);
 335
 336 static void     em_handle_rxtx(void *context, int pending);
 337 static void     em_handle_link(void *context, int pending);
 338 static void     em_add_rx_process_limit(struct adapter *, const char *,
 339                     const char *, int *, int);
 340 #endif /* ~EM_LEGACY_IRQ */
 341
 342 #ifdef DEVICE_POLLING
 343 static poll_handler_t em_poll;
 344 #endif /* POLLING */
 345
 346 /*********************************************************************
 347  *  FreeBSD Device Interface Entry Points
 348  *********************************************************************/
 349
 350 static device_method_t em_methods[] = {
 351         /* Device interface */
 352         DEVMETHOD(device_probe, em_probe),
 353         DEVMETHOD(device_attach, em_attach),
 354         DEVMETHOD(device_detach, em_detach),
 355         DEVMETHOD(device_shutdown, em_shutdown),
 356         DEVMETHOD(device_suspend, em_suspend),
 357         DEVMETHOD(device_resume, em_resume),
 358         {0, 0}
 359 };
 360
 361 static driver_t em_driver = {
 362         "em", em_methods, sizeof(struct adapter),
 363 };
 364
 365 static devclass_t em_devclass;
 366 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 367 MODULE_DEPEND(em, pci, 1, 1, 1);
 368 MODULE_DEPEND(em, ether, 1, 1, 1);
 369
 370 /*********************************************************************
 371  *  Tunable default values.
 372  *********************************************************************/
 373
 374 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 375 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 376 #define M_TSO_LEN                       66
 377
 378 /* Allow common code without TSO */
 379 #ifndef CSUM_TSO
 380 #define CSUM_TSO        0
 381 #endif
 382
 383 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 384 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 385 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 386 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 387 static int em_rxd = EM_DEFAULT_RXD;
 388 static int em_txd = EM_DEFAULT_TXD;
 389 static int em_smart_pwr_down = FALSE;
 390 /* Controls whether promiscuous also shows bad packets */
 391 static int em_debug_sbp = FALSE;
 392 /* Local switch for MSI/MSIX */
 393 static int em_enable_msi = TRUE;
 394
 395 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 396 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 397 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 398 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 399 TUNABLE_INT("hw.em.rxd", &em_rxd);
 400 TUNABLE_INT("hw.em.txd", &em_txd);
 401 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 402 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 403 TUNABLE_INT("hw.em.enable_msi", &em_enable_msi);
 404
 405 #ifndef EM_LEGACY_IRQ
 406 /* How many packets rxeof tries to clean at a time */
 407 static int em_rx_process_limit = 100;
 408 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
 409 #endif
 410
 411 /* Flow control setting - default to FULL */
 412 static int em_fc_setting = e1000_fc_full;
 413 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting);
 414
 415 /*
 416 ** Shadow VFTA table, this is needed because
 417 ** the real vlan filter table gets cleared during
 418 ** a soft reset and the driver needs to be able
 419 ** to repopulate it.
 420 */
 421 static u32 em_shadow_vfta[EM_VFTA_SIZE];
 422
 423 /* Global used in WOL setup with multiport cards */
 424 static int global_quad_port_a = 0;
 425
 426 /*********************************************************************
 427  *  Device identification routine
 428  *
 429  *  em_probe determines if the driver should be loaded on
 430  *  adapter based on PCI vendor/device id of the adapter.
 431  *
 432  *  return BUS_PROBE_DEFAULT on success, positive on failure
 433  *********************************************************************/
 434
 435 static int
 436 em_probe(device_t dev)
 437 {
 438         char            adapter_name[60];
 439         u16             pci_vendor_id = 0;
 440         u16             pci_device_id = 0;
 441         u16             pci_subvendor_id = 0;
 442         u16             pci_subdevice_id = 0;
 443         em_vendor_info_t *ent;
 444
 445         INIT_DEBUGOUT("em_probe: begin");
 446
 447         pci_vendor_id = pci_get_vendor(dev);
 448         if (pci_vendor_id != EM_VENDOR_ID)
 449                 return (ENXIO);
 450
 451         pci_device_id = pci_get_device(dev);
 452         pci_subvendor_id = pci_get_subvendor(dev);
 453         pci_subdevice_id = pci_get_subdevice(dev);
 454
 455         ent = em_vendor_info_array;
 456         while (ent->vendor_id != 0) {
 457                 if ((pci_vendor_id == ent->vendor_id) &&
 458                     (pci_device_id == ent->device_id) &&
 459
 460                     ((pci_subvendor_id == ent->subvendor_id) ||
 461                     (ent->subvendor_id == PCI_ANY_ID)) &&
 462
 463                     ((pci_subdevice_id == ent->subdevice_id) ||
 464                     (ent->subdevice_id == PCI_ANY_ID))) {
 465                         sprintf(adapter_name, "%s %s",
 466                                 em_strings[ent->index],
 467                                 em_driver_version);
 468                         device_set_desc_copy(dev, adapter_name);
 469                         return (BUS_PROBE_DEFAULT);
 470                 }
 471                 ent++;
 472         }
 473
 474         return (ENXIO);
 475 }
 476
 477 /*********************************************************************
 478  *  Device initialization routine
 479  *
 480  *  The attach entry point is called when the driver is being loaded.
 481  *  This routine identifies the type of hardware, allocates all resources
 482  *  and initializes the hardware.
 483  *
 484  *  return 0 on success, positive on failure
 485  *********************************************************************/
 486
 487 static int
 488 em_attach(device_t dev)
 489 {
 490         struct adapter  *adapter;
 491         int             tsize, rsize;
 492         int             error = 0;
 493         u16             eeprom_data, device_id;
 494
 495         INIT_DEBUGOUT("em_attach: begin");
 496
 497         adapter = device_get_softc(dev);
 498         adapter->dev = adapter->osdep.dev = dev;
 499         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 500         EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
 501         EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
 502
 503         /* SYSCTL stuff */
 504         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 505             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 506             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 507             em_sysctl_debug_info, "I", "Debug Information");
 508
 509         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 510             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 511             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 512             em_sysctl_stats, "I", "Statistics");
 513
 514         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 515         callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
 516
 517         /* Determine hardware and mac info */
 518         em_identify_hardware(adapter);
 519
 520         /* Setup PCI resources */
 521         if (em_allocate_pci_resources(adapter)) {
 522                 device_printf(dev, "Allocation of PCI resources failed\n");
 523                 error = ENXIO;
 524                 goto err_pci;
 525         }
 526
 527         /*
 528         ** For ICH8 and family we need to
 529         ** map the flash memory, and this
 530         ** must happen after the MAC is
 531         ** identified
 532         */
 533         if ((adapter->hw.mac.type == e1000_ich8lan) ||
 534             (adapter->hw.mac.type == e1000_ich9lan) ||
 535             (adapter->hw.mac.type == e1000_ich10lan)) {
 536                 int rid = EM_BAR_TYPE_FLASH;
 537                 adapter->flash = bus_alloc_resource_any(dev,
 538                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 539                 if (adapter->flash == NULL) {
 540                         device_printf(dev, "Mapping of Flash failed\n");
 541                         error = ENXIO;
 542                         goto err_pci;
 543                 }
 544                 /* This is used in the shared code */
 545                 adapter->hw.flash_address = (u8 *)adapter->flash;
 546                 adapter->osdep.flash_bus_space_tag =
 547                     rman_get_bustag(adapter->flash);
 548                 adapter->osdep.flash_bus_space_handle =
 549                     rman_get_bushandle(adapter->flash);
 550         }
 551
 552         /* Do Shared Code initialization */
 553         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 554                 device_printf(dev, "Setup of Shared code failed\n");
 555                 error = ENXIO;
 556                 goto err_pci;
 557         }
 558
 559         e1000_get_bus_info(&adapter->hw);
 560
 561         /* Set up some sysctls for the tunable interrupt delays */
 562         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 563             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 564             E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
 565         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 566             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 567             E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
 568         if (adapter->hw.mac.type >= e1000_82540) {
 569                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 570                     "receive interrupt delay limit in usecs",
 571                     &adapter->rx_abs_int_delay,
 572                     E1000_REGISTER(&adapter->hw, E1000_RADV),
 573                     em_rx_abs_int_delay_dflt);
 574                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 575                     "transmit interrupt delay limit in usecs",
 576                     &adapter->tx_abs_int_delay,
 577                     E1000_REGISTER(&adapter->hw, E1000_TADV),
 578                     em_tx_abs_int_delay_dflt);
 579         }
 580
 581 #ifndef EM_LEGACY_IRQ
 582         /* Sysctls for limiting the amount of work done in the taskqueue */
 583         em_add_rx_process_limit(adapter, "rx_processing_limit",
 584             "max number of rx packets to process", &adapter->rx_process_limit,
 585             em_rx_process_limit);
 586 #endif
 587
 588         /*
 589          * Validate number of transmit and receive descriptors. It
 590          * must not exceed hardware maximum, and must be multiple
 591          * of E1000_DBA_ALIGN.
 592          */
 593         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 594             (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
 595             (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
 596             (em_txd < EM_MIN_TXD)) {
 597                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 598                     EM_DEFAULT_TXD, em_txd);
 599                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 600         } else
 601                 adapter->num_tx_desc = em_txd;
 602         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 603             (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
 604             (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
 605             (em_rxd < EM_MIN_RXD)) {
 606                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 607                     EM_DEFAULT_RXD, em_rxd);
 608                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 609         } else
 610                 adapter->num_rx_desc = em_rxd;
 611
 612         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 613         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 614         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 615         adapter->rx_buffer_len = 2048;
 616
 617         e1000_init_script_state_82541(&adapter->hw, TRUE);
 618         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 619
 620         /* Copper options */
 621         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 622                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 623                 adapter->hw.phy.disable_polarity_correction = FALSE;
 624                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 625         }
 626
 627         /*
 628          * Set the frame limits assuming
 629          * standard ethernet sized frames.
 630          */
 631         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 632         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 633
 634         /*
 635          * This controls when hardware reports transmit completion
 636          * status.
 637          */
 638         adapter->hw.mac.report_tx_early = 1;
 639
 640         tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
 641             EM_DBA_ALIGN);
 642
 643         /* Allocate Transmit Descriptor ring */
 644         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
 645                 device_printf(dev, "Unable to allocate tx_desc memory\n");
 646                 error = ENOMEM;
 647                 goto err_tx_desc;
 648         }
 649         adapter->tx_desc_base =
 650             (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
 651
 652         rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
 653             EM_DBA_ALIGN);
 654
 655         /* Allocate Receive Descriptor ring */
 656         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 657                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 658                 error = ENOMEM;
 659                 goto err_rx_desc;
 660         }
 661         adapter->rx_desc_base =
 662             (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
 663
 664         /*
 665         ** Start from a known state, this is
 666         ** important in reading the nvm and
 667         ** mac from that.
 668         */
 669         e1000_reset_hw(&adapter->hw);
 670
 671         /* Make sure we have a good EEPROM before we read from it */
 672         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 673                 /*
 674                 ** Some PCI-E parts fail the first check due to
 675                 ** the link being in sleep state, call it again,
 676                 ** if it fails a second time its a real issue.
 677                 */
 678                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 679                         device_printf(dev,
 680                             "The EEPROM Checksum Is Not Valid\n");
 681                         error = EIO;
 682                         goto err_hw_init;
 683                 }
 684         }
 685
 686         /* Copy the permanent MAC address out of the EEPROM */
 687         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 688                 device_printf(dev, "EEPROM read error while reading MAC"
 689                     " address\n");
 690                 error = EIO;
 691                 goto err_hw_init;
 692         }
 693
 694         if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
 695                 device_printf(dev, "Invalid MAC address\n");
 696                 error = EIO;
 697                 goto err_hw_init;
 698         }
 699
 700         /* Initialize the hardware */
 701         if (em_hardware_init(adapter)) {
 702                 device_printf(dev, "Unable to initialize the hardware\n");
 703                 error = EIO;
 704                 goto err_hw_init;
 705         }
 706
 707         /* Allocate transmit descriptors and buffers */
 708         if (em_allocate_transmit_structures(adapter)) {
 709                 device_printf(dev, "Could not setup transmit structures\n");
 710                 error = ENOMEM;
 711                 goto err_tx_struct;
 712         }
 713
 714         /* Allocate receive descriptors and buffers */
 715         if (em_allocate_receive_structures(adapter)) {
 716                 device_printf(dev, "Could not setup receive structures\n");
 717                 error = ENOMEM;
 718                 goto err_rx_struct;
 719         }
 720
 721         /*
 722         **  Do interrupt configuration
 723         */
 724         if (adapter->msi > 1) /* Do MSI/X */
 725                 error = em_allocate_msix(adapter);
 726         else  /* MSI or Legacy */
 727                 error = em_allocate_legacy(adapter);
 728         if (error)
 729                 goto err_rx_struct;
 730
 731         /* Setup OS specific network interface */
 732         em_setup_interface(dev, adapter);
 733
 734         /* Initialize statistics */
 735         em_update_stats_counters(adapter);
 736
 737         adapter->hw.mac.get_link_status = 1;
 738         em_update_link_status(adapter);
 739
 740         /* Indicate SOL/IDER usage */
 741         if (e1000_check_reset_block(&adapter->hw))
 742                 device_printf(dev,
 743                     "PHY reset is blocked due to SOL/IDER session.\n");
 744
 745         /* Determine if we have to control management hardware */
 746         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
 747
 748         /*
 749          * Setup Wake-on-Lan
 750          */
 751         switch (adapter->hw.mac.type) {
 752
 753         case e1000_82542:
 754         case e1000_82543:
 755                 break;
 756         case e1000_82546:
 757         case e1000_82546_rev_3:
 758         case e1000_82571:
 759         case e1000_80003es2lan:
 760                 if (adapter->hw.bus.func == 1)
 761                         e1000_read_nvm(&adapter->hw,
 762                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
 763                 else
 764                         e1000_read_nvm(&adapter->hw,
 765                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 766                 eeprom_data &= EM_EEPROM_APME;
 767                 break;
 768         default:
 769                 /* APME bit in EEPROM is mapped to WUC.APME */
 770                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) &
 771                     E1000_WUC_APME;
 772                 break;
 773         }
 774         if (eeprom_data)
 775                 adapter->wol = E1000_WUFC_MAG;
 776         /*
 777          * We have the eeprom settings, now apply the special cases
 778          * where the eeprom may be wrong or the board won't support
 779          * wake on lan on a particular port
 780          */
 781         device_id = pci_get_device(dev);
 782         switch (device_id) {
 783         case E1000_DEV_ID_82546GB_PCIE:
 784                 adapter->wol = 0;
 785                 break;
 786         case E1000_DEV_ID_82546EB_FIBER:
 787         case E1000_DEV_ID_82546GB_FIBER:
 788         case E1000_DEV_ID_82571EB_FIBER:
 789                 /* Wake events only supported on port A for dual fiber
 790                  * regardless of eeprom setting */
 791                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
 792                     E1000_STATUS_FUNC_1)
 793                         adapter->wol = 0;
 794                 break;
 795         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
 796         case E1000_DEV_ID_82571EB_QUAD_COPPER:
 797         case E1000_DEV_ID_82571EB_QUAD_FIBER:
 798         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
 799                 /* if quad port adapter, disable WoL on all but port A */
 800                 if (global_quad_port_a != 0)
 801                         adapter->wol = 0;
 802                 /* Reset for multiple quad port adapters */
 803                 if (++global_quad_port_a == 4)
 804                         global_quad_port_a = 0;
 805                 break;
 806         }
 807
 808         /* Do we need workaround for 82544 PCI-X adapter? */
 809         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
 810             adapter->hw.mac.type == e1000_82544)
 811                 adapter->pcix_82544 = TRUE;
 812         else
 813                 adapter->pcix_82544 = FALSE;
 814
 815 #if __FreeBSD_version >= 700029
 816         /* Register for VLAN events */
 817         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 818             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 819         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 820             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 821 #endif
 822
 823         /* Tell the stack that the interface is not active */
 824         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 825
 826         INIT_DEBUGOUT("em_attach: end");
 827
 828         return (0);
 829
 830 err_rx_struct:
 831         em_free_transmit_structures(adapter);
 832 err_tx_struct:
 833 err_hw_init:
 834         em_release_hw_control(adapter);
 835         em_dma_free(adapter, &adapter->rxdma);
 836 err_rx_desc:
 837         em_dma_free(adapter, &adapter->txdma);
 838 err_tx_desc:
 839 err_pci:
 840         em_free_pci_resources(adapter);
 841         EM_TX_LOCK_DESTROY(adapter);
 842         EM_RX_LOCK_DESTROY(adapter);
 843         EM_CORE_LOCK_DESTROY(adapter);
 844
 845         return (error);
 846 }
 847
 848 /*********************************************************************
 849  *  Device removal routine
 850  *
 851  *  The detach entry point is called when the driver is being removed.
 852  *  This routine stops the adapter and deallocates all the resources
 853  *  that were allocated for driver operation.
 854  *
 855  *  return 0 on success, positive on failure
 856  *********************************************************************/
 857
 858 static int
 859 em_detach(device_t dev)
 860 {
 861         struct adapter  *adapter = device_get_softc(dev);
 862         struct ifnet    *ifp = adapter->ifp;
 863
 864         INIT_DEBUGOUT("em_detach: begin");
 865
 866         /* Make sure VLANS are not using driver */
 867 #if __FreeBSD_version >= 700000
 868         if (adapter->ifp->if_vlantrunk != NULL) {
 869 #else
 870         if (adapter->ifp->if_nvlans != 0) {
 871 #endif
 872                 device_printf(dev,"Vlan in use, detach first\n");
 873                 return (EBUSY);
 874         }
 875
 876 #ifdef DEVICE_POLLING
 877         if (ifp->if_capenable & IFCAP_POLLING)
 878                 ether_poll_deregister(ifp);
 879 #endif
 880
 881         EM_CORE_LOCK(adapter);
 882         EM_TX_LOCK(adapter);
 883         adapter->in_detach = 1;
 884         em_stop(adapter);
 885         e1000_phy_hw_reset(&adapter->hw);
 886
 887         em_release_manageability(adapter);
 888
 889         if (((adapter->hw.mac.type == e1000_82573) ||
 890             (adapter->hw.mac.type == e1000_82583) ||
 891             (adapter->hw.mac.type == e1000_ich8lan) ||
 892             (adapter->hw.mac.type == e1000_ich10lan) ||
 893             (adapter->hw.mac.type == e1000_ich9lan)) &&
 894             e1000_check_mng_mode(&adapter->hw))
 895                 em_release_hw_control(adapter);
 896
 897         if (adapter->wol) {
 898                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 899                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 900                 em_enable_wakeup(dev);
 901         }
 902
 903         EM_TX_UNLOCK(adapter);
 904         EM_CORE_UNLOCK(adapter);
 905
 906 #if __FreeBSD_version >= 700029
 907         /* Unregister VLAN events */
 908         if (adapter->vlan_attach != NULL)
 909                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 910         if (adapter->vlan_detach != NULL)
 911                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 912 #endif
 913
 914         ether_ifdetach(adapter->ifp);
 915         callout_drain(&adapter->timer);
 916         callout_drain(&adapter->tx_fifo_timer);
 917
 918         em_free_pci_resources(adapter);
 919         bus_generic_detach(dev);
 920         if_free(ifp);
 921
 922         em_free_transmit_structures(adapter);
 923         em_free_receive_structures(adapter);
 924
 925         /* Free Transmit Descriptor ring */
 926         if (adapter->tx_desc_base) {
 927                 em_dma_free(adapter, &adapter->txdma);
 928                 adapter->tx_desc_base = NULL;
 929         }
 930
 931         /* Free Receive Descriptor ring */
 932         if (adapter->rx_desc_base) {
 933                 em_dma_free(adapter, &adapter->rxdma);
 934                 adapter->rx_desc_base = NULL;
 935         }
 936
 937         EM_TX_LOCK_DESTROY(adapter);
 938         EM_RX_LOCK_DESTROY(adapter);
 939         EM_CORE_LOCK_DESTROY(adapter);
 940
 941         return (0);
 942 }
 943
 944 /*********************************************************************
 945  *
 946  *  Shutdown entry point
 947  *
 948  **********************************************************************/
 949
 950 static int
 951 em_shutdown(device_t dev)
 952 {
 953         return em_suspend(dev);
 954 }
 955
 956 /*
 957  * Suspend/resume device methods.
 958  */
 959 static int
 960 em_suspend(device_t dev)
 961 {
 962         struct adapter *adapter = device_get_softc(dev);
 963
 964         EM_CORE_LOCK(adapter);
 965
 966         EM_TX_LOCK(adapter);
 967         em_stop(adapter);
 968         EM_TX_UNLOCK(adapter);
 969
 970         em_release_manageability(adapter);
 971
 972         if (((adapter->hw.mac.type == e1000_82573) ||
 973             (adapter->hw.mac.type == e1000_82583) ||
 974             (adapter->hw.mac.type == e1000_ich8lan) ||
 975             (adapter->hw.mac.type == e1000_ich10lan) ||
 976             (adapter->hw.mac.type == e1000_ich9lan)) &&
 977             e1000_check_mng_mode(&adapter->hw))
 978                 em_release_hw_control(adapter);
 979
 980         if (adapter->wol) {
 981                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 982                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 983                 em_enable_wakeup(dev);
 984         }
 985
 986         EM_CORE_UNLOCK(adapter);
 987
 988         return bus_generic_suspend(dev);
 989 }
 990
 991 static int
 992 em_resume(device_t dev)
 993 {
 994         struct adapter *adapter = device_get_softc(dev);
 995         struct ifnet *ifp = adapter->ifp;
 996
 997         EM_CORE_LOCK(adapter);
 998         em_init_locked(adapter);
 999         em_init_manageability(adapter);
1000         EM_CORE_UNLOCK(adapter);
1001         em_start(ifp);
1002
1003         return bus_generic_resume(dev);
1004 }
1005
1006
1007 /*********************************************************************
1008  *  Transmit entry point
1009  *
1010  *  em_start is called by the stack to initiate a transmit.
1011  *  The driver will remain in this routine as long as there are
1012  *  packets to transmit and transmit resources are available.
1013  *  In case resources are not available stack is notified and
1014  *  the packet is requeued.
1015  **********************************************************************/
1016
1017 #if __FreeBSD_version >= 800000
1018 static int
1019 em_mq_start_locked(struct ifnet *ifp, struct mbuf *m)
1020 {
1021         struct adapter  *adapter = ifp->if_softc;
1022         struct mbuf     *next;
1023         int error = E1000_SUCCESS;
1024
1025         EM_TX_LOCK_ASSERT(adapter);
1026         /* To allow being called from a tasklet */
1027         if (m == NULL)
1028                 goto process;
1029
1030         if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
1031             IFF_DRV_RUNNING)
1032             || (!adapter->link_active)) {
1033                 error = drbr_enqueue(ifp, adapter->br, m);
1034                 return (error);
1035         } else if (drbr_empty(ifp, adapter->br) &&
1036             (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) {
1037                 if ((error = em_xmit(adapter, &m)) != 0) {
1038                         if (m != NULL)
1039                                 error = drbr_enqueue(ifp, adapter->br, m);
1040                         return (error);
1041                 } else {
1042                         /*
1043                          * We've bypassed the buf ring so we need to update
1044                          * ifp directly
1045                          */
1046                         drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
1047                         /*
1048                         ** Send a copy of the frame to the BPF
1049                         ** listener and set the watchdog on.
1050                         */
1051                         ETHER_BPF_MTAP(ifp, m);
1052                         adapter->watchdog_timer = EM_TX_TIMEOUT;
1053                 }
1054         } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0)
1055                 return (error);
1056
1057 process:
1058         if (drbr_empty(ifp, adapter->br))
1059                 return(error);
1060         /* Process the queue */
1061         while (TRUE) {
1062                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1063                         break;
1064                 next = drbr_dequeue(ifp, adapter->br);
1065                 if (next == NULL)
1066                         break;
1067                 if ((error = em_xmit(adapter, &next)) != 0) {
1068                         if (next != NULL)
1069                                 error = drbr_enqueue(ifp, adapter->br, next);
1070                         break;
1071                 }
1072                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
1073                 ETHER_BPF_MTAP(ifp, next);
1074                 /* Set the watchdog */
1075                 adapter->watchdog_timer = EM_TX_TIMEOUT;
1076         }
1077
1078         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1079                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1080
1081         return (error);
1082 }
1083
1084 /*
1085 ** Multiqueue capable stack interface, this is not
1086 ** yet truely multiqueue, but that is coming...
1087 */
1088 static int
1089 em_mq_start(struct ifnet *ifp, struct mbuf *m)
1090 {
1091
1092         struct adapter *adapter = ifp->if_softc;
1093         int error = 0;
1094
1095         if (EM_TX_TRYLOCK(adapter)) {
1096                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1097                         error = em_mq_start_locked(ifp, m);
1098                 EM_TX_UNLOCK(adapter);
1099         } else
1100                 error = drbr_enqueue(ifp, adapter->br, m);
1101
1102         return (error);
1103 }
1104
1105 static void
1106 em_qflush(struct ifnet *ifp)
1107 {
1108         struct mbuf *m;
1109         struct adapter *adapter = (struct adapter *)ifp->if_softc;
1110
1111         EM_TX_LOCK(adapter);
1112         while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL)
1113                 m_freem(m);
1114         if_qflush(ifp);
1115         EM_TX_UNLOCK(adapter);
1116 }
1117 #endif /* FreeBSD_version */
1118
1119 static void
1120 em_start_locked(struct ifnet *ifp)
1121 {
1122         struct adapter  *adapter = ifp->if_softc;
1123         struct mbuf     *m_head;
1124
1125         EM_TX_LOCK_ASSERT(adapter);
1126
1127         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
1128             IFF_DRV_RUNNING)
1129                 return;
1130         if (!adapter->link_active)
1131                 return;
1132
1133         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
1134
1135                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
1136                 if (m_head == NULL)
1137                         break;
1138                 /*
1139                  *  Encapsulation can modify our pointer, and or make it
1140                  *  NULL on failure.  In that event, we can't requeue.
1141                  */
1142                 if (em_xmit(adapter, &m_head)) {
1143                         if (m_head == NULL)
1144                                 break;
1145                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1146                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1147                         break;
1148                 }
1149
1150                 /* Send a copy of the frame to the BPF listener */
1151                 ETHER_BPF_MTAP(ifp, m_head);
1152
1153                 /* Set timeout in case hardware has problems transmitting. */
1154                 adapter->watchdog_timer = EM_TX_TIMEOUT;
1155         }
1156         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1157                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1158
1159         return;
1160 }
1161
1162 static void
1163 em_start(struct ifnet *ifp)
1164 {
1165         struct adapter *adapter = ifp->if_softc;
1166
1167         EM_TX_LOCK(adapter);
1168         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1169                 em_start_locked(ifp);
1170         EM_TX_UNLOCK(adapter);
1171 }
1172
1173 /*********************************************************************
1174  *  Ioctl entry point
1175  *
1176  *  em_ioctl is called when the user wants to configure the
1177  *  interface.
1178  *
1179  *  return 0 on success, positive on failure
1180  **********************************************************************/
1181
1182 static int
1183 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1184 {
1185         struct adapter  *adapter = ifp->if_softc;
1186         struct ifreq *ifr = (struct ifreq *)data;
1187 #ifdef INET
1188         struct ifaddr *ifa = (struct ifaddr *)data;
1189 #endif
1190         int error = 0;
1191
1192         if (adapter->in_detach)
1193                 return (error);
1194
1195         switch (command) {
1196         case SIOCSIFADDR:
1197 #ifdef INET
1198                 if (ifa->ifa_addr->sa_family == AF_INET) {
1199                         /*
1200                          * XXX
1201                          * Since resetting hardware takes a very long time
1202                          * and results in link renegotiation we only
1203                          * initialize the hardware only when it is absolutely
1204                          * required.
1205                          */
1206                         ifp->if_flags |= IFF_UP;
1207                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1208                                 EM_CORE_LOCK(adapter);
1209                                 em_init_locked(adapter);
1210                                 EM_CORE_UNLOCK(adapter);
1211                         }
1212                         if (!(ifp->if_flags & IFF_NOARP))
1213                                 arp_ifinit(ifp, ifa);
1214                 } else
1215 #endif
1216                         error = ether_ioctl(ifp, command, data);
1217                 break;
1218         case SIOCSIFMTU:
1219             {
1220                 int max_frame_size;
1221                 u16 eeprom_data = 0;
1222
1223                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1224
1225                 EM_CORE_LOCK(adapter);
1226                 switch (adapter->hw.mac.type) {
1227                 case e1000_82573:
1228                         /*
1229                          * 82573 only supports jumbo frames
1230                          * if ASPM is disabled.
1231                          */
1232                         e1000_read_nvm(&adapter->hw,
1233                             NVM_INIT_3GIO_3, 1, &eeprom_data);
1234                         if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1235                                 max_frame_size = ETHER_MAX_LEN;
1236                                 break;
1237                         }
1238                         /* Allow Jumbo frames - fall thru */
1239                 case e1000_82571:
1240                 case e1000_82572:
1241                 case e1000_ich9lan:
1242                 case e1000_ich10lan:
1243                 case e1000_82574:
1244                 case e1000_80003es2lan: /* Limit Jumbo Frame size */
1245                         max_frame_size = 9234;
1246                         break;
1247                         /* Adapters that do not support jumbo frames */
1248                 case e1000_82542:
1249                 case e1000_82583:
1250                 case e1000_ich8lan:
1251                         max_frame_size = ETHER_MAX_LEN;
1252                         break;
1253                 default:
1254                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1255                 }
1256                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1257                     ETHER_CRC_LEN) {
1258                         EM_CORE_UNLOCK(adapter);
1259                         error = EINVAL;
1260                         break;
1261                 }
1262
1263                 ifp->if_mtu = ifr->ifr_mtu;
1264                 adapter->max_frame_size =
1265                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1266                 em_init_locked(adapter);
1267                 EM_CORE_UNLOCK(adapter);
1268                 break;
1269             }
1270         case SIOCSIFFLAGS:
1271                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1272                     SIOCSIFFLAGS (Set Interface Flags)");
1273                 EM_CORE_LOCK(adapter);
1274                 if (ifp->if_flags & IFF_UP) {
1275                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1276                                 if ((ifp->if_flags ^ adapter->if_flags) &
1277                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1278                                         em_disable_promisc(adapter);
1279                                         em_set_promisc(adapter);
1280                                 }
1281                         } else
1282                                 em_init_locked(adapter);
1283                 } else
1284                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1285                                 EM_TX_LOCK(adapter);
1286                                 em_stop(adapter);
1287                                 EM_TX_UNLOCK(adapter);
1288                         }
1289                 adapter->if_flags = ifp->if_flags;
1290                 EM_CORE_UNLOCK(adapter);
1291                 break;
1292         case SIOCADDMULTI:
1293         case SIOCDELMULTI:
1294                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1295                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1296                         EM_CORE_LOCK(adapter);
1297                         em_disable_intr(adapter);
1298                         em_set_multi(adapter);
1299                         if (adapter->hw.mac.type == e1000_82542 &&
1300                             adapter->hw.revision_id == E1000_REVISION_2) {
1301                                 em_initialize_receive_unit(adapter);
1302                         }
1303 #ifdef DEVICE_POLLING
1304                         if (!(ifp->if_capenable & IFCAP_POLLING))
1305 #endif
1306                                 em_enable_intr(adapter);
1307                         EM_CORE_UNLOCK(adapter);
1308                 }
1309                 break;
1310         case SIOCSIFMEDIA:
1311                 /* Check SOL/IDER usage */
1312                 EM_CORE_LOCK(adapter);
1313                 if (e1000_check_reset_block(&adapter->hw)) {
1314                         EM_CORE_UNLOCK(adapter);
1315                         device_printf(adapter->dev, "Media change is"
1316                             " blocked due to SOL/IDER session.\n");
1317                         break;
1318                 }
1319                 EM_CORE_UNLOCK(adapter);
1320         case SIOCGIFMEDIA:
1321                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1322                     SIOCxIFMEDIA (Get/Set Interface Media)");
1323                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1324                 break;
1325         case SIOCSIFCAP:
1326             {
1327                 int mask, reinit;
1328
1329                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1330                 reinit = 0;
1331                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1332 #ifdef DEVICE_POLLING
1333                 if (mask & IFCAP_POLLING) {
1334                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1335                                 error = ether_poll_register(em_poll, ifp);
1336                                 if (error)
1337                                         return (error);
1338                                 EM_CORE_LOCK(adapter);
1339                                 em_disable_intr(adapter);
1340                                 ifp->if_capenable |= IFCAP_POLLING;
1341                                 EM_CORE_UNLOCK(adapter);
1342                         } else {
1343                                 error = ether_poll_deregister(ifp);
1344                                 /* Enable interrupt even in error case */
1345                                 EM_CORE_LOCK(adapter);
1346                                 em_enable_intr(adapter);
1347                                 ifp->if_capenable &= ~IFCAP_POLLING;
1348                                 EM_CORE_UNLOCK(adapter);
1349                         }
1350                 }
1351 #endif
1352                 if (mask & IFCAP_HWCSUM) {
1353                         ifp->if_capenable ^= IFCAP_HWCSUM;
1354                         reinit = 1;
1355                 }
1356 #if __FreeBSD_version >= 700000
1357                 if (mask & IFCAP_TSO4) {
1358                         ifp->if_capenable ^= IFCAP_TSO4;
1359                         reinit = 1;
1360                 }
1361 #endif
1362
1363                 if (mask & IFCAP_VLAN_HWTAGGING) {
1364                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1365                         reinit = 1;
1366                 }
1367                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1368                         em_init(adapter);
1369 #if __FreeBSD_version >= 700000
1370                 VLAN_CAPABILITIES(ifp);
1371 #endif
1372                 break;
1373             }
1374
1375         default:
1376                 error = ether_ioctl(ifp, command, data);
1377                 break;
1378         }
1379
1380         return (error);
1381 }
1382
1383 /*********************************************************************
1384  *  Watchdog timer:
1385  *
1386  *  This routine is called from the local timer every second.
1387  *  As long as transmit descriptors are being cleaned the value
1388  *  is non-zero and we do nothing. Reaching 0 indicates a tx hang
1389  *  and we then reset the device.
1390  *
1391  **********************************************************************/
1392
1393 static void
1394 em_watchdog(struct adapter *adapter)
1395 {
1396
1397         EM_CORE_LOCK_ASSERT(adapter);
1398
1399         /*
1400         ** The timer is set to 5 every time start queues a packet.
1401         ** Then txeof keeps resetting it as long as it cleans at
1402         ** least one descriptor.
1403         ** Finally, anytime all descriptors are clean the timer is
1404         ** set to 0.
1405         */
1406         EM_TX_LOCK(adapter);
1407         if ((adapter->watchdog_timer == 0) || (--adapter->watchdog_timer)) {
1408                 EM_TX_UNLOCK(adapter);
1409                 return;
1410         }
1411
1412         /* If we are in this routine because of pause frames, then
1413          * don't reset the hardware.
1414          */
1415         if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1416             E1000_STATUS_TXOFF) {
1417                 adapter->watchdog_timer = EM_TX_TIMEOUT;
1418                 EM_TX_UNLOCK(adapter);
1419                 return;
1420         }
1421
1422         if (e1000_check_for_link(&adapter->hw) == 0)
1423                 device_printf(adapter->dev, "watchdog timeout -- resetting\n");
1424         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1425         adapter->watchdog_events++;
1426         EM_TX_UNLOCK(adapter);
1427
1428         em_init_locked(adapter);
1429 }
1430
1431 /*********************************************************************
1432  *  Init entry point
1433  *
1434  *  This routine is used in two ways. It is used by the stack as
1435  *  init entry point in network interface structure. It is also used
1436  *  by the driver as a hw/sw initialization routine to get to a
1437  *  consistent state.
1438  *
1439  *  return 0 on success, positive on failure
1440  **********************************************************************/
1441
1442 static void
1443 em_init_locked(struct adapter *adapter)
1444 {
1445         struct ifnet    *ifp = adapter->ifp;
1446         device_t        dev = adapter->dev;
1447         u32             pba;
1448
1449         INIT_DEBUGOUT("em_init: begin");
1450
1451         EM_CORE_LOCK_ASSERT(adapter);
1452
1453         EM_TX_LOCK(adapter);
1454         em_stop(adapter);
1455         EM_TX_UNLOCK(adapter);
1456
1457         /*
1458          * Packet Buffer Allocation (PBA)
1459          * Writing PBA sets the receive portion of the buffer
1460          * the remainder is used for the transmit buffer.
1461          *
1462          * Devices before the 82547 had a Packet Buffer of 64K.
1463          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1464          * After the 82547 the buffer was reduced to 40K.
1465          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1466          *   Note: default does not leave enough room for Jumbo Frame >10k.
1467          */
1468         switch (adapter->hw.mac.type) {
1469         case e1000_82547:
1470         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1471                 if (adapter->max_frame_size > 8192)
1472                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1473                 else
1474                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1475                 adapter->tx_fifo_head = 0;
1476                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1477                 adapter->tx_fifo_size =
1478                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1479                 break;
1480         /* Total Packet Buffer on these is 48K */
1481         case e1000_82571:
1482         case e1000_82572:
1483         case e1000_80003es2lan:
1484                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1485                 break;
1486         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1487                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1488                 break;
1489         case e1000_82574:
1490         case e1000_82583:
1491                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1492                 break;
1493         case e1000_ich9lan:
1494         case e1000_ich10lan:
1495         case e1000_ich8lan:
1496                 pba = E1000_PBA_8K;
1497                 break;
1498         default:
1499                 /* Devices before 82547 had a Packet Buffer of 64K.   */
1500                 if (adapter->max_frame_size > 8192)
1501                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1502                 else
1503                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1504         }
1505
1506         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1507         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1508
1509         /* Get the latest mac address, User can use a LAA */
1510         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1511               ETHER_ADDR_LEN);
1512
1513         /* Put the address into the Receive Address Array */
1514         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1515
1516         /*
1517          * With the 82571 adapter, RAR[0] may be overwritten
1518          * when the other port is reset, we make a duplicate
1519          * in RAR[14] for that eventuality, this assures
1520          * the interface continues to function.
1521          */
1522         if (adapter->hw.mac.type == e1000_82571) {
1523                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1524                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1525                     E1000_RAR_ENTRIES - 1);
1526         }
1527
1528         /* Initialize the hardware */
1529         if (em_hardware_init(adapter)) {
1530                 device_printf(dev, "Unable to initialize the hardware\n");
1531                 return;
1532         }
1533         em_update_link_status(adapter);
1534
1535         /* Setup VLAN support, basic and offload if available */
1536         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1537
1538 #if __FreeBSD_version < 700029
1539         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1540                 u32 ctrl;
1541                 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1542                 ctrl |= E1000_CTRL_VME;
1543                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1544         }
1545 #else
1546         /* Use real VLAN Filter support */
1547         em_setup_vlan_hw_support(adapter);
1548 #endif
1549
1550         /* Set hardware offload abilities */
1551         ifp->if_hwassist = 0;
1552         if (adapter->hw.mac.type >= e1000_82543) {
1553                 if (ifp->if_capenable & IFCAP_TXCSUM)
1554                         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1555 #if __FreeBSD_version >= 700000
1556                 if (ifp->if_capenable & IFCAP_TSO4)
1557                         ifp->if_hwassist |= CSUM_TSO;
1558 #endif
1559         }
1560
1561         /* Configure for OS presence */
1562         em_init_manageability(adapter);
1563
1564         /* Prepare transmit descriptors and buffers */
1565         em_setup_transmit_structures(adapter);
1566         em_initialize_transmit_unit(adapter);
1567
1568         /* Setup Multicast table */
1569         em_set_multi(adapter);
1570
1571         /* Prepare receive descriptors and buffers */
1572         if (em_setup_receive_structures(adapter)) {
1573                 device_printf(dev, "Could not setup receive structures\n");
1574                 EM_TX_LOCK(adapter);
1575                 em_stop(adapter);
1576                 EM_TX_UNLOCK(adapter);
1577                 return;
1578         }
1579         em_initialize_receive_unit(adapter);
1580
1581         /* Don't lose promiscuous settings */
1582         em_set_promisc(adapter);
1583
1584         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1585         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1586
1587         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1588         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1589
1590         /* MSI/X configuration for 82574 */
1591         if (adapter->hw.mac.type == e1000_82574) {
1592                 int tmp;
1593                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1594                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1595                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1596                 /*
1597                 ** Set the IVAR - interrupt vector routing.
1598                 ** Each nibble represents a vector, high bit
1599                 ** is enable, other 3 bits are the MSIX table
1600                 ** entry, we map RXQ0 to 0, TXQ0 to 1, and
1601                 ** Link (other) to 2, hence the magic number.
1602                 */
1603                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1604         }
1605
1606 #ifdef DEVICE_POLLING
1607         /*
1608          * Only enable interrupts if we are not polling, make sure
1609          * they are off otherwise.
1610          */
1611         if (ifp->if_capenable & IFCAP_POLLING)
1612                 em_disable_intr(adapter);
1613         else
1614 #endif /* DEVICE_POLLING */
1615                 em_enable_intr(adapter);
1616
1617         /* Don't reset the phy next time init gets called */
1618         adapter->hw.phy.reset_disable = TRUE;
1619 }
1620
1621 static void
1622 em_init(void *arg)
1623 {
1624         struct adapter *adapter = arg;
1625
1626         EM_CORE_LOCK(adapter);
1627         em_init_locked(adapter);
1628         EM_CORE_UNLOCK(adapter);
1629 }
1630
1631
1632 #ifdef DEVICE_POLLING
1633 /*********************************************************************
1634  *
1635  *  Legacy polling routine
1636  *
1637  *********************************************************************/
1638 static int
1639 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1640 {
1641         struct adapter *adapter = ifp->if_softc;
1642         u32             reg_icr, rx_done = 0;
1643
1644         EM_CORE_LOCK(adapter);
1645         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1646                 EM_CORE_UNLOCK(adapter);
1647                 return (rx_done);
1648         }
1649
1650         if (cmd == POLL_AND_CHECK_STATUS) {
1651                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1652                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1653                         callout_stop(&adapter->timer);
1654                         adapter->hw.mac.get_link_status = 1;
1655                         em_update_link_status(adapter);
1656                         callout_reset(&adapter->timer, hz,
1657                             em_local_timer, adapter);
1658                 }
1659         }
1660         EM_CORE_UNLOCK(adapter);
1661
1662         rx_done = em_rxeof(adapter, count);
1663
1664         EM_TX_LOCK(adapter);
1665         em_txeof(adapter);
1666 #if __FreeBSD_version >= 800000
1667         if (!drbr_empty(ifp, adapter->br))
1668                 em_mq_start_locked(ifp, NULL);
1669 #else
1670         if (!IFQ_DRV_IS_EMPTY(&ifp->snd))
1671                 em_start_locked(ifp);
1672 #endif
1673         EM_TX_UNLOCK(adapter);
1674         return (rx_done);
1675 }
1676 #endif /* DEVICE_POLLING */
1677
1678 #ifdef EM_LEGACY_IRQ
1679 /*********************************************************************
1680  *
1681  *  Legacy Interrupt Service routine
1682  *
1683  *********************************************************************/
1684
1685 static void
1686 em_intr(void *arg)
1687 {
1688         struct adapter  *adapter = arg;
1689         struct ifnet    *ifp = adapter->ifp;
1690         u32             reg_icr;
1691
1692
1693         if (ifp->if_capenable & IFCAP_POLLING)
1694                 return;
1695
1696         EM_CORE_LOCK(adapter);
1697         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1698         if (reg_icr & E1000_ICR_RXO)
1699                 adapter->rx_overruns++;
1700         if ((reg_icr == 0xffffffff) || (reg_icr == 0)||
1701             (adapter->hw.mac.type >= e1000_82571 &&
1702             (reg_icr & E1000_ICR_INT_ASSERTED) == 0))
1703                         goto out;
1704
1705         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1706                         goto out;
1707
1708         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1709                 callout_stop(&adapter->timer);
1710                 adapter->hw.mac.get_link_status = 1;
1711                 em_update_link_status(adapter);
1712                 /* Deal with TX cruft when link lost */
1713                 em_tx_purge(adapter);
1714                 callout_reset(&adapter->timer, hz,
1715                     em_local_timer, adapter);
1716                 goto out;
1717         }
1718
1719         EM_TX_LOCK(adapter);
1720         em_txeof(adapter);
1721         em_rxeof(adapter, -1);
1722         em_txeof(adapter);
1723         if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
1724             !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1725                 em_start_locked(ifp);
1726         EM_TX_UNLOCK(adapter);
1727
1728 out:
1729         EM_CORE_UNLOCK(adapter);
1730         return;
1731 }
1732
1733 #else /* EM_FAST_IRQ, then fast interrupt routines only */
1734
1735 static void
1736 em_handle_link(void *context, int pending)
1737 {
1738         struct adapter  *adapter = context;
1739         struct ifnet *ifp = adapter->ifp;
1740
1741         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1742                 return;
1743
1744         EM_CORE_LOCK(adapter);
1745         callout_stop(&adapter->timer);
1746         em_update_link_status(adapter);
1747         /* Deal with TX cruft when link lost */
1748         em_tx_purge(adapter);
1749         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1750         EM_CORE_UNLOCK(adapter);
1751 }
1752
1753
1754 /* Combined RX/TX handler, used by Legacy and MSI */
1755 static void
1756 em_handle_rxtx(void *context, int pending)
1757 {
1758         struct adapter  *adapter = context;
1759         struct ifnet    *ifp = adapter->ifp;
1760
1761
1762         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1763                 if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1764                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1765                 EM_TX_LOCK(adapter);
1766                 em_txeof(adapter);
1767
1768 #if __FreeBSD_version >= 800000
1769                 if (!drbr_empty(ifp, adapter->br))
1770                         em_mq_start_locked(ifp, NULL);
1771 #else
1772                 if (!IFQ_DRV_IS_EMPTY(&ifp->snd))
1773                         em_start_locked(ifp);
1774 #endif
1775                 EM_TX_UNLOCK(adapter);
1776         }
1777
1778         em_enable_intr(adapter);
1779 }
1780
1781 /*********************************************************************
1782  *
1783  *  Fast Legacy/MSI Combined Interrupt Service routine
1784  *
1785  *********************************************************************/
1786 #if __FreeBSD_version < 700000
1787 #define FILTER_STRAY
1788 #define FILTER_HANDLED
1789 static void
1790 #else
1791 static int
1792 #endif
1793 em_irq_fast(void *arg)
1794 {
1795         struct adapter  *adapter = arg;
1796         struct ifnet    *ifp;
1797         u32             reg_icr;
1798
1799         ifp = adapter->ifp;
1800
1801         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1802
1803         /* Hot eject?  */
1804         if (reg_icr == 0xffffffff)
1805                 return FILTER_STRAY;
1806
1807         /* Definitely not our interrupt.  */
1808         if (reg_icr == 0x0)
1809                 return FILTER_STRAY;
1810
1811         /*
1812          * Starting with the 82571 chip, bit 31 should be used to
1813          * determine whether the interrupt belongs to us.
1814          */
1815         if (adapter->hw.mac.type >= e1000_82571 &&
1816             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1817                 return FILTER_STRAY;
1818
1819         /*
1820          * Mask interrupts until the taskqueue is finished running.  This is
1821          * cheap, just assume that it is needed.  This also works around the
1822          * MSI message reordering errata on certain systems.
1823          */
1824         em_disable_intr(adapter);
1825         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1826
1827         /* Link status change */
1828         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1829                 adapter->hw.mac.get_link_status = 1;
1830                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1831         }
1832
1833         if (reg_icr & E1000_ICR_RXO)
1834                 adapter->rx_overruns++;
1835         return FILTER_HANDLED;
1836 }
1837
1838 /*********************************************************************
1839  *
1840  *  MSIX Interrupt Service Routines
1841  *
1842  **********************************************************************/
1843 #define EM_MSIX_TX      0x00040000
1844 #define EM_MSIX_RX      0x00010000
1845 #define EM_MSIX_LINK    0x00100000
1846
1847 static void
1848 em_msix_tx(void *arg)
1849 {
1850         struct adapter *adapter = arg;
1851         struct ifnet    *ifp = adapter->ifp;
1852
1853         ++adapter->tx_irq;
1854         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1855                 EM_TX_LOCK(adapter);
1856                 em_txeof(adapter);
1857                 EM_TX_UNLOCK(adapter);
1858                 taskqueue_enqueue(adapter->tq, &adapter->tx_task);
1859         }
1860         /* Reenable this interrupt */
1861         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX);
1862         return;
1863 }
1864
1865 /*********************************************************************
1866  *
1867  *  MSIX RX Interrupt Service routine
1868  *
1869  **********************************************************************/
1870
1871 static void
1872 em_msix_rx(void *arg)
1873 {
1874         struct adapter *adapter = arg;
1875         struct ifnet    *ifp = adapter->ifp;
1876
1877         ++adapter->rx_irq;
1878         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1879             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1880                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1881         /* Reenable this interrupt */
1882         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX);
1883         return;
1884 }
1885
1886 /*********************************************************************
1887  *
1888  *  MSIX Link Fast Interrupt Service routine
1889  *
1890  **********************************************************************/
1891
1892 static void
1893 em_msix_link(void *arg)
1894 {
1895         struct adapter  *adapter = arg;
1896         u32             reg_icr;
1897
1898         ++adapter->link_irq;
1899         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1900
1901         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1902                 adapter->hw.mac.get_link_status = 1;
1903                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1904         }
1905         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1906             EM_MSIX_LINK | E1000_IMS_LSC);
1907         return;
1908 }
1909
1910 static void
1911 em_handle_rx(void *context, int pending)
1912 {
1913         struct adapter  *adapter = context;
1914         struct ifnet    *ifp = adapter->ifp;
1915
1916         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1917             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1918                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1919
1920 }
1921
1922 static void
1923 em_handle_tx(void *context, int pending)
1924 {
1925         struct adapter  *adapter = context;
1926         struct ifnet    *ifp = adapter->ifp;
1927
1928         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1929                 if (!EM_TX_TRYLOCK(adapter))
1930                         return;
1931                 em_txeof(adapter);
1932 #if __FreeBSD_version >= 800000
1933                 if (!drbr_empty(ifp, adapter->br))
1934                         em_mq_start_locked(ifp, NULL);
1935 #else
1936                 if (!IFQ_DRV_IS_EMPTY(&ifp->snd))
1937                         em_start_locked(ifp);
1938 #endif
1939                 EM_TX_UNLOCK(adapter);
1940         }
1941 }
1942 #endif /* EM_FAST_IRQ */
1943
1944 /*********************************************************************
1945  *
1946  *  Media Ioctl callback
1947  *
1948  *  This routine is called whenever the user queries the status of
1949  *  the interface using ifconfig.
1950  *
1951  **********************************************************************/
1952 static void
1953 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1954 {
1955         struct adapter *adapter = ifp->if_softc;
1956         u_char fiber_type = IFM_1000_SX;
1957
1958         INIT_DEBUGOUT("em_media_status: begin");
1959
1960         EM_CORE_LOCK(adapter);
1961         em_update_link_status(adapter);
1962
1963         ifmr->ifm_status = IFM_AVALID;
1964         ifmr->ifm_active = IFM_ETHER;
1965
1966         if (!adapter->link_active) {
1967                 EM_CORE_UNLOCK(adapter);
1968                 return;
1969         }
1970
1971         ifmr->ifm_status |= IFM_ACTIVE;
1972
1973         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1974             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1975                 if (adapter->hw.mac.type == e1000_82545)
1976                         fiber_type = IFM_1000_LX;
1977                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1978         } else {
1979                 switch (adapter->link_speed) {
1980                 case 10:
1981                         ifmr->ifm_active |= IFM_10_T;
1982                         break;
1983                 case 100:
1984                         ifmr->ifm_active |= IFM_100_TX;
1985                         break;
1986                 case 1000:
1987                         ifmr->ifm_active |= IFM_1000_T;
1988                         break;
1989                 }
1990                 if (adapter->link_duplex == FULL_DUPLEX)
1991                         ifmr->ifm_active |= IFM_FDX;
1992                 else
1993                         ifmr->ifm_active |= IFM_HDX;
1994         }
1995         EM_CORE_UNLOCK(adapter);
1996 }
1997
1998 /*********************************************************************
1999  *
2000  *  Media Ioctl callback
2001  *
2002  *  This routine is called when the user changes speed/duplex using
2003  *  media/mediopt option with ifconfig.
2004  *
2005  **********************************************************************/
2006 static int
2007 em_media_change(struct ifnet *ifp)
2008 {
2009         struct adapter *adapter = ifp->if_softc;
2010         struct ifmedia  *ifm = &adapter->media;
2011
2012         INIT_DEBUGOUT("em_media_change: begin");
2013
2014         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2015                 return (EINVAL);
2016
2017         EM_CORE_LOCK(adapter);
2018         switch (IFM_SUBTYPE(ifm->ifm_media)) {
2019         case IFM_AUTO:
2020                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
2021                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
2022                 break;
2023         case IFM_1000_LX:
2024         case IFM_1000_SX:
2025         case IFM_1000_T:
2026                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
2027                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
2028                 break;
2029         case IFM_100_TX:
2030                 adapter->hw.mac.autoneg = FALSE;
2031                 adapter->hw.phy.autoneg_advertised = 0;
2032                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
2033                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
2034                 else
2035                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
2036                 break;
2037         case IFM_10_T:
2038                 adapter->hw.mac.autoneg = FALSE;
2039                 adapter->hw.phy.autoneg_advertised = 0;
2040                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
2041                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
2042                 else
2043                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
2044                 break;
2045         default:
2046                 device_printf(adapter->dev, "Unsupported media type\n");
2047         }
2048
2049         /* As the speed/duplex settings my have changed we need to
2050          * reset the PHY.
2051          */
2052         adapter->hw.phy.reset_disable = FALSE;
2053
2054         em_init_locked(adapter);
2055         EM_CORE_UNLOCK(adapter);
2056
2057         return (0);
2058 }
2059
2060 /*********************************************************************
2061  *
2062  *  This routine maps the mbufs to tx descriptors.
2063  *
2064  *  return 0 on success, positive on failure
2065  **********************************************************************/
2066
2067 static int
2068 em_xmit(struct adapter *adapter, struct mbuf **m_headp)
2069 {
2070         bus_dma_segment_t       segs[EM_MAX_SCATTER];
2071         bus_dmamap_t            map;
2072         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
2073         struct e1000_tx_desc    *ctxd = NULL;
2074         struct mbuf             *m_head;
2075         u32                     txd_upper, txd_lower, txd_used, txd_saved;
2076         int                     nsegs, i, j, first, last = 0;
2077         int                     error, do_tso, tso_desc = 0;
2078 #if __FreeBSD_version < 700000
2079         struct m_tag            *mtag;
2080 #endif
2081         m_head = *m_headp;
2082         txd_upper = txd_lower = txd_used = txd_saved = 0;
2083
2084 #if __FreeBSD_version >= 700000
2085         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
2086 #else
2087         do_tso = 0;
2088 #endif
2089
2090         /*
2091          * Force a cleanup if number of TX descriptors
2092          * available hits the threshold
2093          */
2094         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
2095                 em_txeof(adapter);
2096                 /* Now do we at least have a minimal? */
2097                 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
2098                         adapter->no_tx_desc_avail1++;
2099                         return (ENOBUFS);
2100                 }
2101         }
2102
2103
2104         /*
2105          * TSO workaround:
2106          *  If an mbuf is only header we need
2107          *     to pull 4 bytes of data into it.
2108          */
2109         if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
2110                 m_head = m_pullup(m_head, M_TSO_LEN + 4);
2111                 *m_headp = m_head;
2112                 if (m_head == NULL)
2113                         return (ENOBUFS);
2114         }
2115
2116         /*
2117          * Map the packet for DMA
2118          *
2119          * Capture the first descriptor index,
2120          * this descriptor will have the index
2121          * of the EOP which is the only one that
2122          * now gets a DONE bit writeback.
2123          */
2124         first = adapter->next_avail_tx_desc;
2125         tx_buffer = &adapter->tx_buffer_area[first];
2126         tx_buffer_mapped = tx_buffer;
2127         map = tx_buffer->map;
2128
2129         error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2130             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2131
2132         /*
2133          * There are two types of errors we can (try) to handle:
2134          * - EFBIG means the mbuf chain was too long and bus_dma ran
2135          *   out of segments.  Defragment the mbuf chain and try again.
2136          * - ENOMEM means bus_dma could not obtain enough bounce buffers
2137          *   at this point in time.  Defer sending and try again later.
2138          * All other errors, in particular EINVAL, are fatal and prevent the
2139          * mbuf chain from ever going through.  Drop it and report error.
2140          */
2141         if (error == EFBIG) {
2142                 struct mbuf *m;
2143
2144                 m = m_defrag(*m_headp, M_DONTWAIT);
2145                 if (m == NULL) {
2146                         adapter->mbuf_alloc_failed++;
2147                         m_freem(*m_headp);
2148                         *m_headp = NULL;
2149                         return (ENOBUFS);
2150                 }
2151                 *m_headp = m;
2152
2153                 /* Try it again */
2154                 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2155                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2156
2157                 if (error) {
2158                         adapter->no_tx_dma_setup++;
2159                         m_freem(*m_headp);
2160                         *m_headp = NULL;
2161                         return (error);
2162                 }
2163         } else if (error != 0) {
2164                 adapter->no_tx_dma_setup++;
2165                 return (error);
2166         }
2167
2168         /*
2169          * TSO Hardware workaround, if this packet is not
2170          * TSO, and is only a single descriptor long, and
2171          * it follows a TSO burst, then we need to add a
2172          * sentinel descriptor to prevent premature writeback.
2173          */
2174         if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
2175                 if (nsegs == 1)
2176                         tso_desc = TRUE;
2177                 adapter->tx_tso = FALSE;
2178         }
2179
2180         if (nsegs > (adapter->num_tx_desc_avail - 2)) {
2181                 adapter->no_tx_desc_avail2++;
2182                 bus_dmamap_unload(adapter->txtag, map);
2183                 return (ENOBUFS);
2184         }
2185         m_head = *m_headp;
2186
2187         /* Do hardware assists */
2188 #if __FreeBSD_version >= 700000
2189         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2190                 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
2191                 if (error != TRUE)
2192                         return (ENXIO); /* something foobar */
2193                 /* we need to make a final sentinel transmit desc */
2194                 tso_desc = TRUE;
2195         } else
2196 #endif
2197         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2198                 em_transmit_checksum_setup(adapter,  m_head,
2199                     &txd_upper, &txd_lower);
2200
2201         i = adapter->next_avail_tx_desc;
2202         if (adapter->pcix_82544)
2203                 txd_saved = i;
2204
2205         /* Set up our transmit descriptors */
2206         for (j = 0; j < nsegs; j++) {
2207                 bus_size_t seg_len;
2208                 bus_addr_t seg_addr;
2209                 /* If adapter is 82544 and on PCIX bus */
2210                 if(adapter->pcix_82544) {
2211                         DESC_ARRAY      desc_array;
2212                         u32             array_elements, counter;
2213                         /*
2214                          * Check the Address and Length combination and
2215                          * split the data accordingly
2216                          */
2217                         array_elements = em_fill_descriptors(segs[j].ds_addr,
2218                             segs[j].ds_len, &desc_array);
2219                         for (counter = 0; counter < array_elements; counter++) {
2220                                 if (txd_used == adapter->num_tx_desc_avail) {
2221                                         adapter->next_avail_tx_desc = txd_saved;
2222                                         adapter->no_tx_desc_avail2++;
2223                                         bus_dmamap_unload(adapter->txtag, map);
2224                                         return (ENOBUFS);
2225                                 }
2226                                 tx_buffer = &adapter->tx_buffer_area[i];
2227                                 ctxd = &adapter->tx_desc_base[i];
2228                                 ctxd->buffer_addr = htole64(
2229                                     desc_array.descriptor[counter].address);
2230                                 ctxd->lower.data = htole32(
2231                                     (adapter->txd_cmd | txd_lower | (u16)
2232                                     desc_array.descriptor[counter].length));
2233                                 ctxd->upper.data =
2234                                     htole32((txd_upper));
2235                                 last = i;
2236                                 if (++i == adapter->num_tx_desc)
2237                                          i = 0;
2238                                 tx_buffer->m_head = NULL;
2239                                 tx_buffer->next_eop = -1;
2240                                 txd_used++;
2241                         }
2242                 } else {
2243                         tx_buffer = &adapter->tx_buffer_area[i];
2244                         ctxd = &adapter->tx_desc_base[i];
2245                         seg_addr = segs[j].ds_addr;
2246                         seg_len  = segs[j].ds_len;
2247                         /*
2248                         ** TSO Workaround:
2249                         ** If this is the last descriptor, we want to
2250                         ** split it so we have a small final sentinel
2251                         */
2252                         if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2253                                 seg_len -= 4;
2254                                 ctxd->buffer_addr = htole64(seg_addr);
2255                                 ctxd->lower.data = htole32(
2256                                 adapter->txd_cmd | txd_lower | seg_len);
2257                                 ctxd->upper.data =
2258                                     htole32(txd_upper);
2259                                 if (++i == adapter->num_tx_desc)
2260                                         i = 0;
2261                                 /* Now make the sentinel */
2262                                 ++txd_used; /* using an extra txd */
2263                                 ctxd = &adapter->tx_desc_base[i];
2264                                 tx_buffer = &adapter->tx_buffer_area[i];
2265                                 ctxd->buffer_addr =
2266                                     htole64(seg_addr + seg_len);
2267                                 ctxd->lower.data = htole32(
2268                                 adapter->txd_cmd | txd_lower | 4);
2269                                 ctxd->upper.data =
2270                                     htole32(txd_upper);
2271                                 last = i;
2272                                 if (++i == adapter->num_tx_desc)
2273                                         i = 0;
2274                         } else {
2275                                 ctxd->buffer_addr = htole64(seg_addr);
2276                                 ctxd->lower.data = htole32(
2277                                 adapter->txd_cmd | txd_lower | seg_len);
2278                                 ctxd->upper.data =
2279                                     htole32(txd_upper);
2280                                 last = i;
2281                                 if (++i == adapter->num_tx_desc)
2282                                         i = 0;
2283                         }
2284                         tx_buffer->m_head = NULL;
2285                         tx_buffer->next_eop = -1;
2286                 }
2287         }
2288
2289         adapter->next_avail_tx_desc = i;
2290         if (adapter->pcix_82544)
2291                 adapter->num_tx_desc_avail -= txd_used;
2292         else {
2293                 adapter->num_tx_desc_avail -= nsegs;
2294                 if (tso_desc) /* TSO used an extra for sentinel */
2295                         adapter->num_tx_desc_avail -= txd_used;
2296         }
2297
2298         /*
2299         ** Handle VLAN tag, this is the
2300         ** biggest difference between
2301         ** 6.x and 7
2302         */
2303 #if __FreeBSD_version < 700000
2304         /* Find out if we are in vlan mode. */
2305         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
2306         if (mtag != NULL) {
2307                 ctxd->upper.fields.special =
2308                     htole16(VLAN_TAG_VALUE(mtag));
2309 #else /* FreeBSD 7 */
2310         if (m_head->m_flags & M_VLANTAG) {
2311                 /* Set the vlan id. */
2312                 ctxd->upper.fields.special =
2313                     htole16(m_head->m_pkthdr.ether_vtag);
2314 #endif
2315                 /* Tell hardware to add tag */
2316                 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
2317         }
2318
2319         tx_buffer->m_head = m_head;
2320         tx_buffer_mapped->map = tx_buffer->map;
2321         tx_buffer->map = map;
2322         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
2323
2324         /*
2325          * Last Descriptor of Packet
2326          * needs End Of Packet (EOP)
2327          * and Report Status (RS)
2328          */
2329         ctxd->lower.data |=
2330             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2331         /*
2332          * Keep track in the first buffer which
2333          * descriptor will be written back
2334          */
2335         tx_buffer = &adapter->tx_buffer_area[first];
2336         tx_buffer->next_eop = last;
2337
2338         /*
2339          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2340          * that this frame is available to transmit.
2341          */
2342         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2343             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2344         if (adapter->hw.mac.type == e1000_82547 &&
2345             adapter->link_duplex == HALF_DUPLEX)
2346                 em_82547_move_tail(adapter);
2347         else {
2348                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
2349                 if (adapter->hw.mac.type == e1000_82547)
2350                         em_82547_update_fifo_head(adapter,
2351                             m_head->m_pkthdr.len);
2352         }
2353
2354         return (0);
2355 }
2356
2357 /*********************************************************************
2358  *
2359  * 82547 workaround to avoid controller hang in half-duplex environment.
2360  * The workaround is to avoid queuing a large packet that would span
2361  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
2362  * in this case. We do that only when FIFO is quiescent.
2363  *
2364  **********************************************************************/
2365 static void
2366 em_82547_move_tail(void *arg)
2367 {
2368         struct adapter *adapter = arg;
2369         struct e1000_tx_desc *tx_desc;
2370         u16     hw_tdt, sw_tdt, length = 0;
2371         bool    eop = 0;
2372
2373         EM_TX_LOCK_ASSERT(adapter);
2374
2375         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2376         sw_tdt = adapter->next_avail_tx_desc;
2377
2378         while (hw_tdt != sw_tdt) {
2379                 tx_desc = &adapter->tx_desc_base[hw_tdt];
2380                 length += tx_desc->lower.flags.length;
2381                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2382                 if (++hw_tdt == adapter->num_tx_desc)
2383                         hw_tdt = 0;
2384
2385                 if (eop) {
2386                         if (em_82547_fifo_workaround(adapter, length)) {
2387                                 adapter->tx_fifo_wrk_cnt++;
2388                                 callout_reset(&adapter->tx_fifo_timer, 1,
2389                                         em_82547_move_tail, adapter);
2390                                 break;
2391                         }
2392                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2393                         em_82547_update_fifo_head(adapter, length);
2394                         length = 0;
2395                 }
2396         }
2397 }
2398
2399 static int
2400 em_82547_fifo_workaround(struct adapter *adapter, int len)
2401 {
2402         int fifo_space, fifo_pkt_len;
2403
2404         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2405
2406         if (adapter->link_duplex == HALF_DUPLEX) {
2407                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2408
2409                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2410                         if (em_82547_tx_fifo_reset(adapter))
2411                                 return (0);
2412                         else
2413                                 return (1);
2414                 }
2415         }
2416
2417         return (0);
2418 }
2419
2420 static void
2421 em_82547_update_fifo_head(struct adapter *adapter, int len)
2422 {
2423         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2424
2425         /* tx_fifo_head is always 16 byte aligned */
2426         adapter->tx_fifo_head += fifo_pkt_len;
2427         if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
2428                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2429         }
2430 }
2431
2432
2433 static int
2434 em_82547_tx_fifo_reset(struct adapter *adapter)
2435 {
2436         u32 tctl;
2437
2438         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2439             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2440             (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
2441             E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2442             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2443             E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2444             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2445                 /* Disable TX unit */
2446                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2447                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2448                     tctl & ~E1000_TCTL_EN);
2449
2450                 /* Reset FIFO pointers */
2451                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2452                     adapter->tx_head_addr);
2453                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2454                     adapter->tx_head_addr);
2455                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2456                     adapter->tx_head_addr);
2457                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2458                     adapter->tx_head_addr);
2459
2460                 /* Re-enable TX unit */
2461                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2462                 E1000_WRITE_FLUSH(&adapter->hw);
2463
2464                 adapter->tx_fifo_head = 0;
2465                 adapter->tx_fifo_reset_cnt++;
2466
2467                 return (TRUE);
2468         }
2469         else {
2470                 return (FALSE);
2471         }
2472 }
2473
2474 static void
2475 em_set_promisc(struct adapter *adapter)
2476 {
2477         struct ifnet    *ifp = adapter->ifp;
2478         u32             reg_rctl;
2479
2480         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2481
2482         if (ifp->if_flags & IFF_PROMISC) {
2483                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2484                 /* Turn this on if you want to see bad packets */
2485                 if (em_debug_sbp)
2486                         reg_rctl |= E1000_RCTL_SBP;
2487                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2488         } else if (ifp->if_flags & IFF_ALLMULTI) {
2489                 reg_rctl |= E1000_RCTL_MPE;
2490                 reg_rctl &= ~E1000_RCTL_UPE;
2491                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2492         }
2493 }
2494
2495 static void
2496 em_disable_promisc(struct adapter *adapter)
2497 {
2498         u32     reg_rctl;
2499
2500         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2501
2502         reg_rctl &=  (~E1000_RCTL_UPE);
2503         reg_rctl &=  (~E1000_RCTL_MPE);
2504         reg_rctl &=  (~E1000_RCTL_SBP);
2505         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2506 }
2507
2508
2509 /*********************************************************************
2510  *  Multicast Update
2511  *
2512  *  This routine is called whenever multicast address list is updated.
2513  *
2514  **********************************************************************/
2515
2516 static void
2517 em_set_multi(struct adapter *adapter)
2518 {
2519         struct ifnet    *ifp = adapter->ifp;
2520         struct ifmultiaddr *ifma;
2521         u32 reg_rctl = 0;
2522         u8  *mta; /* Multicast array memory */
2523         int mcnt = 0;
2524
2525         IOCTL_DEBUGOUT("em_set_multi: begin");
2526
2527         if (adapter->hw.mac.type == e1000_82542 &&
2528             adapter->hw.revision_id == E1000_REVISION_2) {
2529                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2530                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2531                         e1000_pci_clear_mwi(&adapter->hw);
2532                 reg_rctl |= E1000_RCTL_RST;
2533                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2534                 msec_delay(5);
2535         }
2536
2537         /* Allocate temporary memory to setup array */
2538         mta = malloc(sizeof(u8) *
2539             (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
2540             M_DEVBUF, M_NOWAIT | M_ZERO);
2541         if (mta == NULL)
2542                 panic("em_set_multi memory failure\n");
2543
2544         if_maddr_rlock(ifp);
2545         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2546                 if (ifma->ifma_addr->sa_family != AF_LINK)
2547                         continue;
2548
2549                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2550                         break;
2551
2552                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2553                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2554                 mcnt++;
2555         }
2556         if_maddr_runlock(ifp);
2557
2558         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2559                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2560                 reg_rctl |= E1000_RCTL_MPE;
2561                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2562         } else
2563                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2564
2565         if (adapter->hw.mac.type == e1000_82542 &&
2566             adapter->hw.revision_id == E1000_REVISION_2) {
2567                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2568                 reg_rctl &= ~E1000_RCTL_RST;
2569                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2570                 msec_delay(5);
2571                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2572                         e1000_pci_set_mwi(&adapter->hw);
2573         }
2574         free(mta, M_DEVBUF);
2575 }
2576
2577
2578 /*********************************************************************
2579  *  Timer routine
2580  *
2581  *  This routine checks for link status and updates statistics.
2582  *
2583  **********************************************************************/
2584
2585 static void
2586 em_local_timer(void *arg)
2587 {
2588         struct adapter  *adapter = arg;
2589         struct ifnet    *ifp = adapter->ifp;
2590
2591         EM_CORE_LOCK_ASSERT(adapter);
2592
2593         taskqueue_enqueue(adapter->tq,
2594             &adapter->rxtx_task);
2595         em_update_link_status(adapter);
2596         em_update_stats_counters(adapter);
2597
2598         /* Reset LAA into RAR[0] on 82571 */
2599         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2600                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2601
2602         if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
2603                 em_print_hw_stats(adapter);
2604
2605         em_smartspeed(adapter);
2606
2607         /*
2608          * Each second we check the watchdog to
2609          * protect against hardware hangs.
2610          */
2611         em_watchdog(adapter);
2612
2613         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2614
2615 }
2616
2617 static void
2618 em_update_link_status(struct adapter *adapter)
2619 {
2620         struct e1000_hw *hw = &adapter->hw;
2621         struct ifnet *ifp = adapter->ifp;
2622         device_t dev = adapter->dev;
2623         u32 link_check = 0;
2624
2625         /* Get the cached link value or read phy for real */
2626         switch (hw->phy.media_type) {
2627         case e1000_media_type_copper:
2628                 if (hw->mac.get_link_status) {
2629                         /* Do the work to read phy */
2630                         e1000_check_for_link(hw);
2631                         link_check = !hw->mac.get_link_status;
2632                         if (link_check) /* ESB2 fix */
2633                                 e1000_cfg_on_link_up(hw);
2634                 } else
2635                         link_check = TRUE;
2636                 break;
2637         case e1000_media_type_fiber:
2638                 e1000_check_for_link(hw);
2639                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2640                                  E1000_STATUS_LU);
2641                 break;
2642         case e1000_media_type_internal_serdes:
2643                 e1000_check_for_link(hw);
2644                 link_check = adapter->hw.mac.serdes_has_link;
2645                 break;
2646         default:
2647         case e1000_media_type_unknown:
2648                 break;
2649         }
2650
2651         /* Now check for a transition */
2652         if (link_check && (adapter->link_active == 0)) {
2653                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2654                     &adapter->link_duplex);
2655                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2656                 if ((adapter->link_speed != SPEED_1000) &&
2657                     ((hw->mac.type == e1000_82571) ||
2658                     (hw->mac.type == e1000_82572))) {
2659                         int tarc0;
2660                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2661                         tarc0 &= ~SPEED_MODE_BIT;
2662                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2663                 }
2664                 if (bootverbose)
2665                         device_printf(dev, "Link is up %d Mbps %s\n",
2666                             adapter->link_speed,
2667                             ((adapter->link_duplex == FULL_DUPLEX) ?
2668                             "Full Duplex" : "Half Duplex"));
2669                 adapter->link_active = 1;
2670                 adapter->smartspeed = 0;
2671                 ifp->if_baudrate = adapter->link_speed * 1000000;
2672                 if_link_state_change(ifp, LINK_STATE_UP);
2673         } else if (!link_check && (adapter->link_active == 1)) {
2674                 ifp->if_baudrate = adapter->link_speed = 0;
2675                 adapter->link_duplex = 0;
2676                 if (bootverbose)
2677                         device_printf(dev, "Link is Down\n");
2678                 adapter->link_active = 0;
2679                 /* Link down, disable watchdog */
2680                 adapter->watchdog_timer = FALSE;
2681                 if_link_state_change(ifp, LINK_STATE_DOWN);
2682         }
2683 }
2684
2685 /*********************************************************************
2686  *
2687  *  This routine disables all traffic on the adapter by issuing a
2688  *  global reset on the MAC and deallocates TX/RX buffers.
2689  *
2690  *  This routine should always be called with BOTH the CORE
2691  *  and TX locks.
2692  **********************************************************************/
2693
2694 static void
2695 em_stop(void *arg)
2696 {
2697         struct adapter  *adapter = arg;
2698         struct ifnet    *ifp = adapter->ifp;
2699
2700         EM_CORE_LOCK_ASSERT(adapter);
2701         EM_TX_LOCK_ASSERT(adapter);
2702
2703         INIT_DEBUGOUT("em_stop: begin");
2704
2705         em_disable_intr(adapter);
2706         callout_stop(&adapter->timer);
2707         callout_stop(&adapter->tx_fifo_timer);
2708
2709         /* Tell the stack that the interface is no longer active */
2710         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2711
2712         e1000_reset_hw(&adapter->hw);
2713         if (adapter->hw.mac.type >= e1000_82544)
2714                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2715 }
2716
2717
2718 /*********************************************************************
2719  *
2720  *  Determine hardware revision.
2721  *
2722  **********************************************************************/
2723 static void
2724 em_identify_hardware(struct adapter *adapter)
2725 {
2726         device_t dev = adapter->dev;
2727
2728         /* Make sure our PCI config space has the necessary stuff set */
2729         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2730         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2731             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2732                 device_printf(dev, "Memory Access and/or Bus Master bits "
2733                     "were not set!\n");
2734                 adapter->hw.bus.pci_cmd_word |=
2735                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2736                 pci_write_config(dev, PCIR_COMMAND,
2737                     adapter->hw.bus.pci_cmd_word, 2);
2738         }
2739
2740         /* Save off the information about this board */
2741         adapter->hw.vendor_id = pci_get_vendor(dev);
2742         adapter->hw.device_id = pci_get_device(dev);
2743         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2744         adapter->hw.subsystem_vendor_id =
2745             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2746         adapter->hw.subsystem_device_id =
2747             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2748
2749         /* Do Shared Code Init and Setup */
2750         if (e1000_set_mac_type(&adapter->hw)) {
2751                 device_printf(dev, "Setup init failure\n");
2752                 return;
2753         }
2754 }
2755
2756 static int
2757 em_allocate_pci_resources(struct adapter *adapter)
2758 {
2759         device_t        dev = adapter->dev;
2760         int             val, rid, error = E1000_SUCCESS;
2761
2762         rid = PCIR_BAR(0);
2763         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2764             &rid, RF_ACTIVE);
2765         if (adapter->memory == NULL) {
2766                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2767                 return (ENXIO);
2768         }
2769         adapter->osdep.mem_bus_space_tag =
2770             rman_get_bustag(adapter->memory);
2771         adapter->osdep.mem_bus_space_handle =
2772             rman_get_bushandle(adapter->memory);
2773         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2774
2775         /* Only older adapters use IO mapping */
2776         if ((adapter->hw.mac.type > e1000_82543) &&
2777             (adapter->hw.mac.type < e1000_82571)) {
2778                 /* Figure our where our IO BAR is ? */
2779                 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2780                         val = pci_read_config(dev, rid, 4);
2781                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2782                                 adapter->io_rid = rid;
2783                                 break;
2784                         }
2785                         rid += 4;
2786                         /* check for 64bit BAR */
2787                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2788                                 rid += 4;
2789                 }
2790                 if (rid >= PCIR_CIS) {
2791                         device_printf(dev, "Unable to locate IO BAR\n");
2792                         return (ENXIO);
2793                 }
2794                 adapter->ioport = bus_alloc_resource_any(dev,
2795                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2796                 if (adapter->ioport == NULL) {
2797                         device_printf(dev, "Unable to allocate bus resource: "
2798                             "ioport\n");
2799                         return (ENXIO);
2800                 }
2801                 adapter->hw.io_base = 0;
2802                 adapter->osdep.io_bus_space_tag =
2803                     rman_get_bustag(adapter->ioport);
2804                 adapter->osdep.io_bus_space_handle =
2805                     rman_get_bushandle(adapter->ioport);
2806         }
2807
2808         /*
2809         ** Init the resource arrays
2810         **  used by MSIX setup
2811         */
2812         for (int i = 0; i < 3; i++) {
2813                 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2814                 adapter->tag[i] = NULL;
2815                 adapter->res[i] = NULL;
2816         }
2817
2818         /*
2819          * Setup MSI/X or MSI if PCI Express
2820          */
2821         if (em_enable_msi)
2822                 adapter->msi = em_setup_msix(adapter);
2823
2824         adapter->hw.back = &adapter->osdep;
2825
2826         return (error);
2827 }
2828
2829 /*********************************************************************
2830  *
2831  *  Setup the Legacy or MSI Interrupt handler
2832  *
2833  **********************************************************************/
2834 int
2835 em_allocate_legacy(struct adapter *adapter)
2836 {
2837         device_t dev = adapter->dev;
2838         int error;
2839
2840         /* Manually turn off all interrupts */
2841         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2842
2843         /* Legacy RID is 0 */
2844         if (adapter->msi == 0)
2845                 adapter->rid[0] = 0;
2846
2847         /* We allocate a single interrupt resource */
2848         adapter->res[0] = bus_alloc_resource_any(dev,
2849             SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2850         if (adapter->res[0] == NULL) {
2851                 device_printf(dev, "Unable to allocate bus resource: "
2852                     "interrupt\n");
2853                 return (ENXIO);
2854         }
2855
2856 #ifdef EM_LEGACY_IRQ
2857         /* We do Legacy setup */
2858         if ((error = bus_setup_intr(dev, adapter->res[0],
2859 #if __FreeBSD_version > 700000
2860             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter,
2861 #else /* 6.X */
2862             INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter,
2863 #endif
2864             &adapter->tag[0])) != 0) {
2865                 device_printf(dev, "Failed to register interrupt handler");
2866                 return (error);
2867         }
2868
2869 #else /* FAST_IRQ */
2870         /*
2871          * Try allocating a fast interrupt and the associated deferred
2872          * processing contexts.
2873          */
2874         TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2875         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2876         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2877             taskqueue_thread_enqueue, &adapter->tq);
2878         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2879             device_get_nameunit(adapter->dev));
2880 #if __FreeBSD_version < 700000
2881         if ((error = bus_setup_intr(dev, adapter->res[0],
2882             INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter,
2883 #else
2884         if ((error = bus_setup_intr(dev, adapter->res[0],
2885             INTR_TYPE_NET, em_irq_fast, NULL, adapter,
2886 #endif
2887             &adapter->tag[0])) != 0) {
2888                 device_printf(dev, "Failed to register fast interrupt "
2889                             "handler: %d\n", error);
2890                 taskqueue_free(adapter->tq);
2891                 adapter->tq = NULL;
2892                 return (error);
2893         }
2894 #endif  /* EM_LEGACY_IRQ */
2895
2896         return (0);
2897 }
2898
2899 /*********************************************************************
2900  *
2901  *  Setup the MSIX Interrupt handlers
2902  *   This is not really Multiqueue, rather
2903  *   its just multiple interrupt vectors.
2904  *
2905  **********************************************************************/
2906 int
2907 em_allocate_msix(struct adapter *adapter)
2908 {
2909         device_t dev = adapter->dev;
2910         int error;
2911
2912         /* Make sure all interrupts are disabled */
2913         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2914
2915         /* First get the resources */
2916         for (int i = 0; i < adapter->msi; i++) {
2917                 adapter->res[i] = bus_alloc_resource_any(dev,
2918                     SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE);
2919                 if (adapter->res[i] == NULL) {
2920                         device_printf(dev,
2921                             "Unable to allocate bus resource: "
2922                             "MSIX Interrupt\n");
2923                         return (ENXIO);
2924                 }
2925         }
2926
2927         /*
2928          * Now allocate deferred processing contexts.
2929          */
2930         TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter);
2931         TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter);
2932         /*
2933          * Handle compatibility for msi case for deferral due to
2934          * trylock failure
2935          */
2936         TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter);
2937         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2938         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2939             taskqueue_thread_enqueue, &adapter->tq);
2940         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2941             device_get_nameunit(adapter->dev));
2942
2943         /*
2944          * And setup the interrupt handlers
2945          */
2946
2947         /* First slot to RX */
2948         if ((error = bus_setup_intr(dev, adapter->res[0],
2949 #if __FreeBSD_version > 700000
2950             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter,
2951 #else /* 6.X */
2952             INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter,
2953 #endif
2954             &adapter->tag[0])) != 0) {
2955                 device_printf(dev, "Failed to register RX handler");
2956                 return (error);
2957         }
2958
2959         /* Next TX */
2960         if ((error = bus_setup_intr(dev, adapter->res[1],
2961 #if __FreeBSD_version > 700000
2962             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter,
2963 #else /* 6.X */
2964             INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter,
2965 #endif
2966             &adapter->tag[1])) != 0) {
2967                 device_printf(dev, "Failed to register TX handler");
2968                 return (error);
2969         }
2970
2971         /* And Link */
2972         if ((error = bus_setup_intr(dev, adapter->res[2],
2973 #if __FreeBSD_version > 700000
2974             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter,
2975 #else /* 6.X */
2976             INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter,
2977 #endif
2978             &adapter->tag[2])) != 0) {
2979                 device_printf(dev, "Failed to register TX handler");
2980                 return (error);
2981         }
2982
2983         return (0);
2984 }
2985
2986
2987 static void
2988 em_free_pci_resources(struct adapter *adapter)
2989 {
2990         device_t dev = adapter->dev;
2991
2992         /* Make sure the for loop below runs once */
2993         if (adapter->msi == 0)
2994                 adapter->msi = 1;
2995
2996         /*
2997          * First release all the interrupt resources:
2998          *      notice that since these are just kept
2999          *      in an array we can do the same logic
3000          *      whether its MSIX or just legacy.
3001          */
3002         for (int i = 0; i < adapter->msi; i++) {
3003                 if (adapter->tag[i] != NULL) {
3004                         bus_teardown_intr(dev, adapter->res[i],
3005                             adapter->tag[i]);
3006                         adapter->tag[i] = NULL;
3007                 }
3008                 if (adapter->res[i] != NULL) {
3009                         bus_release_resource(dev, SYS_RES_IRQ,
3010                             adapter->rid[i], adapter->res[i]);
3011                 }
3012         }
3013
3014         if (adapter->msi)
3015                 pci_release_msi(dev);
3016
3017         if (adapter->msix != NULL)
3018                 bus_release_resource(dev, SYS_RES_MEMORY,
3019                     PCIR_BAR(EM_MSIX_BAR), adapter->msix);
3020
3021         if (adapter->memory != NULL)
3022                 bus_release_resource(dev, SYS_RES_MEMORY,
3023                     PCIR_BAR(0), adapter->memory);
3024
3025         if (adapter->flash != NULL)
3026                 bus_release_resource(dev, SYS_RES_MEMORY,
3027                     EM_FLASH, adapter->flash);
3028
3029         if (adapter->ioport != NULL)
3030                 bus_release_resource(dev, SYS_RES_IOPORT,
3031                     adapter->io_rid, adapter->ioport);
3032 }
3033
3034 /*
3035  * Setup MSI or MSI/X
3036  */
3037 static int
3038 em_setup_msix(struct adapter *adapter)
3039 {
3040         device_t dev = adapter->dev;
3041         int val = 0;
3042
3043         if (adapter->hw.mac.type < e1000_82571)
3044                 return (0);
3045
3046         /* Setup MSI/X for Hartwell */
3047         if (adapter->hw.mac.type == e1000_82574) {
3048                 /* Map the MSIX BAR */
3049                 int rid = PCIR_BAR(EM_MSIX_BAR);
3050                 adapter->msix = bus_alloc_resource_any(dev,
3051                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
3052                 if (!adapter->msix) {
3053                         /* May not be enabled */
3054                         device_printf(adapter->dev,
3055                             "Unable to map MSIX table \n");
3056                         goto msi;
3057                 }
3058                 val = pci_msix_count(dev);
3059                 /*
3060                 ** 82574 can be configured for 5 but
3061                 ** we limit use to 3.
3062                 */
3063                 if (val > 3) val = 3;
3064                 if ((val) && pci_alloc_msix(dev, &val) == 0) {
3065                         device_printf(adapter->dev,"Using MSIX interrupts\n");
3066                         return (val);
3067                 }
3068         }
3069 msi:
3070         val = pci_msi_count(dev);
3071         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
3072                 adapter->msi = 1;
3073                 device_printf(adapter->dev,"Using MSI interrupt\n");
3074                 return (val);
3075         }
3076         return (0);
3077 }
3078
3079 /*********************************************************************
3080  *
3081  *  Initialize the hardware to a configuration
3082  *  as specified by the adapter structure.
3083  *
3084  **********************************************************************/
3085 static int
3086 em_hardware_init(struct adapter *adapter)
3087 {
3088         device_t dev = adapter->dev;
3089         u16     rx_buffer_size;
3090
3091         INIT_DEBUGOUT("em_hardware_init: begin");
3092
3093         /* Issue a global reset */
3094         e1000_reset_hw(&adapter->hw);
3095
3096         /* Get control from any management/hw control */
3097         if (((adapter->hw.mac.type == e1000_82573) ||
3098             (adapter->hw.mac.type == e1000_82583) ||
3099             (adapter->hw.mac.type == e1000_ich8lan) ||
3100             (adapter->hw.mac.type == e1000_ich10lan) ||
3101             (adapter->hw.mac.type == e1000_ich9lan)) &&
3102             e1000_check_mng_mode(&adapter->hw))
3103                 em_get_hw_control(adapter);
3104
3105         /* When hardware is reset, fifo_head is also reset */
3106         adapter->tx_fifo_head = 0;
3107
3108         /* Set up smart power down as default off on newer adapters. */
3109         if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 ||
3110             adapter->hw.mac.type == e1000_82572)) {
3111                 u16 phy_tmp = 0;
3112
3113                 /* Speed up time to link by disabling smart power down. */
3114                 e1000_read_phy_reg(&adapter->hw,
3115                     IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3116                 phy_tmp &= ~IGP02E1000_PM_SPD;
3117                 e1000_write_phy_reg(&adapter->hw,
3118                     IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3119         }
3120
3121         /*
3122          * These parameters control the automatic generation (Tx) and
3123          * response (Rx) to Ethernet PAUSE frames.
3124          * - High water mark should allow for at least two frames to be
3125          *   received after sending an XOFF.
3126          * - Low water mark works best when it is very near the high water mark.
3127          *   This allows the receiver to restart by sending XON when it has
3128          *   drained a bit. Here we use an arbitary value of 1500 which will
3129          *   restart after one full frame is pulled from the buffer. There
3130          *   could be several smaller frames in the buffer and if so they will
3131          *   not trigger the XON until their total number reduces the buffer
3132          *   by 1500.
3133          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3134          */
3135         rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
3136             0xffff) << 10 );
3137
3138         adapter->hw.fc.high_water = rx_buffer_size -
3139             roundup2(adapter->max_frame_size, 1024);
3140         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
3141
3142         if (adapter->hw.mac.type == e1000_80003es2lan)
3143                 adapter->hw.fc.pause_time = 0xFFFF;
3144         else
3145                 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
3146         adapter->hw.fc.send_xon = TRUE;
3147
3148         /* Set Flow control, use the tunable location if sane */
3149         if ((em_fc_setting >= 0) || (em_fc_setting < 4))
3150                 adapter->hw.fc.requested_mode = em_fc_setting;
3151         else
3152                 adapter->hw.fc.requested_mode = e1000_fc_none;
3153
3154
3155         if (e1000_init_hw(&adapter->hw) < 0) {
3156                 device_printf(dev, "Hardware Initialization Failed\n");
3157                 return (EIO);
3158         }
3159
3160         e1000_check_for_link(&adapter->hw);
3161
3162         return (0);
3163 }
3164
3165 /*********************************************************************
3166  *
3167  *  Setup networking device structure and register an interface.
3168  *
3169  **********************************************************************/
3170 static void
3171 em_setup_interface(device_t dev, struct adapter *adapter)
3172 {
3173         struct ifnet   *ifp;
3174
3175         INIT_DEBUGOUT("em_setup_interface: begin");
3176
3177         ifp = adapter->ifp = if_alloc(IFT_ETHER);
3178         if (ifp == NULL)
3179                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
3180         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3181         ifp->if_mtu = ETHERMTU;
3182         ifp->if_init =  em_init;
3183         ifp->if_softc = adapter;
3184         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3185         ifp->if_ioctl = em_ioctl;
3186         ifp->if_start = em_start;
3187         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3188         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3189         IFQ_SET_READY(&ifp->if_snd);
3190
3191         ether_ifattach(ifp, adapter->hw.mac.addr);
3192
3193         ifp->if_capabilities = ifp->if_capenable = 0;
3194
3195 #if __FreeBSD_version >= 800000
3196         /* Multiqueue tx functions */
3197         ifp->if_transmit = em_mq_start;
3198         ifp->if_qflush = em_qflush;
3199         adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx);
3200 #endif
3201         if (adapter->hw.mac.type >= e1000_82543) {
3202                 int version_cap;
3203 #if __FreeBSD_version < 700000
3204                 version_cap = IFCAP_HWCSUM;
3205 #else
3206                 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3207 #endif
3208                 ifp->if_capabilities |= version_cap;
3209                 ifp->if_capenable |= version_cap;
3210         }
3211
3212 #if __FreeBSD_version >= 700000
3213         /* Identify TSO capable adapters */
3214         if ((adapter->hw.mac.type > e1000_82544) &&
3215             (adapter->hw.mac.type != e1000_82547))
3216                 ifp->if_capabilities |= IFCAP_TSO4;
3217         /*
3218          * By default only enable on PCI-E, this
3219          * can be overriden by ifconfig.
3220          */
3221         if (adapter->hw.mac.type >= e1000_82571)
3222                 ifp->if_capenable |= IFCAP_TSO4;
3223 #endif
3224
3225         /*
3226          * Tell the upper layer(s) we support long frames.
3227          */
3228         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3229         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3230         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3231
3232 #ifdef DEVICE_POLLING
3233         ifp->if_capabilities |= IFCAP_POLLING;
3234 #endif
3235
3236         /*
3237          * Specify the media types supported by this adapter and register
3238          * callbacks to update media and link information
3239          */
3240         ifmedia_init(&adapter->media, IFM_IMASK,
3241             em_media_change, em_media_status);
3242         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3243             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3244                 u_char fiber_type = IFM_1000_SX;        /* default type */
3245
3246                 if (adapter->hw.mac.type == e1000_82545)
3247                         fiber_type = IFM_1000_LX;
3248                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3249                             0, NULL);
3250                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3251         } else {
3252                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3253                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3254                             0, NULL);
3255                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3256                             0, NULL);
3257                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3258                             0, NULL);
3259                 if (adapter->hw.phy.type != e1000_phy_ife) {
3260                         ifmedia_add(&adapter->media,
3261                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3262                         ifmedia_add(&adapter->media,
3263                                 IFM_ETHER | IFM_1000_T, 0, NULL);
3264                 }
3265         }
3266         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3267         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3268 }
3269
3270
3271 /*********************************************************************
3272  *
3273  *  Workaround for SmartSpeed on 82541 and 82547 controllers
3274  *
3275  **********************************************************************/
3276 static void
3277 em_smartspeed(struct adapter *adapter)
3278 {
3279         u16 phy_tmp;
3280
3281         if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
3282             adapter->hw.mac.autoneg == 0 ||
3283             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
3284                 return;
3285
3286         if (adapter->smartspeed == 0) {
3287                 /* If Master/Slave config fault is asserted twice,
3288                  * we assume back-to-back */
3289                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3290                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
3291                         return;
3292                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3293                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
3294                         e1000_read_phy_reg(&adapter->hw,
3295                             PHY_1000T_CTRL, &phy_tmp);
3296                         if(phy_tmp & CR_1000T_MS_ENABLE) {
3297                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
3298                                 e1000_write_phy_reg(&adapter->hw,
3299                                     PHY_1000T_CTRL, phy_tmp);
3300                                 adapter->smartspeed++;
3301                                 if(adapter->hw.mac.autoneg &&
3302                                    !e1000_phy_setup_autoneg(&adapter->hw) &&
3303                                    !e1000_read_phy_reg(&adapter->hw,
3304                                     PHY_CONTROL, &phy_tmp)) {
3305                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3306                                                     MII_CR_RESTART_AUTO_NEG);
3307                                         e1000_write_phy_reg(&adapter->hw,
3308                                             PHY_CONTROL, phy_tmp);
3309                                 }
3310                         }
3311                 }
3312                 return;
3313         } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
3314                 /* If still no link, perhaps using 2/3 pair cable */
3315                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
3316                 phy_tmp |= CR_1000T_MS_ENABLE;
3317                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
3318                 if(adapter->hw.mac.autoneg &&
3319                    !e1000_phy_setup_autoneg(&adapter->hw) &&
3320                    !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
3321                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3322                                     MII_CR_RESTART_AUTO_NEG);
3323                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
3324                 }
3325         }
3326         /* Restart process after EM_SMARTSPEED_MAX iterations */
3327         if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
3328                 adapter->smartspeed = 0;
3329 }
3330
3331
3332 /*
3333  * Manage DMA'able memory.
3334  */
3335 static void
3336 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3337 {
3338         if (error)
3339                 return;
3340         *(bus_addr_t *) arg = segs[0].ds_addr;
3341 }
3342
3343 static int
3344 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3345         struct em_dma_alloc *dma, int mapflags)
3346 {
3347         int error;
3348
3349 #if __FreeBSD_version >= 700000
3350         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3351 #else
3352         error = bus_dma_tag_create(NULL,                 /* parent */
3353 #endif
3354                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
3355                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3356                                 BUS_SPACE_MAXADDR,      /* highaddr */
3357                                 NULL, NULL,             /* filter, filterarg */
3358                                 size,                   /* maxsize */
3359                                 1,                      /* nsegments */
3360                                 size,                   /* maxsegsize */
3361                                 0,                      /* flags */
3362                                 NULL,                   /* lockfunc */
3363                                 NULL,                   /* lockarg */
3364                                 &dma->dma_tag);
3365         if (error) {
3366                 device_printf(adapter->dev,
3367                     "%s: bus_dma_tag_create failed: %d\n",
3368                     __func__, error);
3369                 goto fail_0;
3370         }
3371
3372         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3373             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3374         if (error) {
3375                 device_printf(adapter->dev,
3376                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3377                     __func__, (uintmax_t)size, error);
3378                 goto fail_2;
3379         }
3380
3381         dma->dma_paddr = 0;
3382         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3383             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3384         if (error || dma->dma_paddr == 0) {
3385                 device_printf(adapter->dev,
3386                     "%s: bus_dmamap_load failed: %d\n",
3387                     __func__, error);
3388                 goto fail_3;
3389         }
3390
3391         return (0);
3392
3393 fail_3:
3394         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3395 fail_2:
3396         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3397         bus_dma_tag_destroy(dma->dma_tag);
3398 fail_0:
3399         dma->dma_map = NULL;
3400         dma->dma_tag = NULL;
3401
3402         return (error);
3403 }
3404
3405 static void
3406 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3407 {
3408         if (dma->dma_tag == NULL)
3409                 return;
3410         if (dma->dma_map != NULL) {
3411                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3412                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3413                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3414                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3415                 dma->dma_map = NULL;
3416         }
3417         bus_dma_tag_destroy(dma->dma_tag);
3418         dma->dma_tag = NULL;
3419 }
3420
3421
3422 /*********************************************************************
3423  *
3424  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3425  *  the information needed to transmit a packet on the wire.
3426  *
3427  **********************************************************************/
3428 static int
3429 em_allocate_transmit_structures(struct adapter *adapter)
3430 {
3431         device_t dev = adapter->dev;
3432         struct em_buffer *tx_buffer;
3433         int error;
3434
3435         /*
3436          * Create DMA tags for tx descriptors
3437          */
3438 #if __FreeBSD_version >= 700000
3439         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3440 #else
3441         if ((error = bus_dma_tag_create(NULL,            /* parent */
3442 #endif
3443                                 1, 0,                   /* alignment, bounds */
3444                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3445                                 BUS_SPACE_MAXADDR,      /* highaddr */
3446                                 NULL, NULL,             /* filter, filterarg */
3447                                 EM_TSO_SIZE,            /* maxsize */
3448                                 EM_MAX_SCATTER,         /* nsegments */
3449                                 EM_TSO_SEG_SIZE,        /* maxsegsize */
3450                                 0,                      /* flags */
3451                                 NULL,           /* lockfunc */
3452                                 NULL,           /* lockarg */
3453                                 &adapter->txtag)) != 0) {
3454                 device_printf(dev, "Unable to allocate TX DMA tag\n");
3455                 goto fail;
3456         }
3457
3458         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
3459             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3460         if (adapter->tx_buffer_area == NULL) {
3461                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3462                 error = ENOMEM;
3463                 goto fail;
3464         }
3465
3466         /* Create the descriptor buffer dma maps */
3467         for (int i = 0; i < adapter->num_tx_desc; i++) {
3468                 tx_buffer = &adapter->tx_buffer_area[i];
3469                 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
3470                 if (error != 0) {
3471                         device_printf(dev, "Unable to create TX DMA map\n");
3472                         goto fail;
3473                 }
3474                 tx_buffer->next_eop = -1;
3475         }
3476
3477         return (0);
3478 fail:
3479         em_free_transmit_structures(adapter);
3480         return (error);
3481 }
3482
3483 /*********************************************************************
3484  *
3485  *  (Re)Initialize transmit structures.
3486  *
3487  **********************************************************************/
3488 static void
3489 em_setup_transmit_structures(struct adapter *adapter)
3490 {
3491         struct em_buffer *tx_buffer;
3492
3493         /* Clear the old ring contents */
3494         bzero(adapter->tx_desc_base,
3495             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3496
3497         /* Free any existing TX buffers */
3498         for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3499                 tx_buffer = &adapter->tx_buffer_area[i];
3500                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3501                     BUS_DMASYNC_POSTWRITE);
3502                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
3503                 m_freem(tx_buffer->m_head);
3504                 tx_buffer->m_head = NULL;
3505                 tx_buffer->next_eop = -1;
3506         }
3507
3508         /* Reset state */
3509         adapter->next_avail_tx_desc = 0;
3510         adapter->next_tx_to_clean = 0;
3511         adapter->num_tx_desc_avail = adapter->num_tx_desc;
3512
3513         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3514             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3515
3516         return;
3517 }
3518
3519 /*********************************************************************
3520  *
3521  *  Enable transmit unit.
3522  *
3523  **********************************************************************/
3524 static void
3525 em_initialize_transmit_unit(struct adapter *adapter)
3526 {
3527         u32     tctl, tarc, tipg = 0;
3528         u64     bus_addr;
3529
3530          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3531         /* Setup the Base and Length of the Tx Descriptor Ring */
3532         bus_addr = adapter->txdma.dma_paddr;
3533         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
3534             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3535         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
3536             (u32)(bus_addr >> 32));
3537         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
3538             (u32)bus_addr);
3539         /* Setup the HW Tx Head and Tail descriptor pointers */
3540         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
3541         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
3542
3543         HW_DEBUGOUT2("Base = %x, Length = %x\n",
3544             E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
3545             E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
3546
3547         /* Set the default values for the Tx Inter Packet Gap timer */
3548         switch (adapter->hw.mac.type) {
3549         case e1000_82542:
3550                 tipg = DEFAULT_82542_TIPG_IPGT;
3551                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3552                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3553                 break;
3554         case e1000_80003es2lan:
3555                 tipg = DEFAULT_82543_TIPG_IPGR1;
3556                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3557                     E1000_TIPG_IPGR2_SHIFT;
3558                 break;
3559         default:
3560                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3561                     (adapter->hw.phy.media_type ==
3562                     e1000_media_type_internal_serdes))
3563                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3564                 else
3565                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3566                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3567                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3568         }
3569
3570         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3571         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3572         if(adapter->hw.mac.type >= e1000_82540)
3573                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3574                     adapter->tx_abs_int_delay.value);
3575
3576         if ((adapter->hw.mac.type == e1000_82571) ||
3577             (adapter->hw.mac.type == e1000_82572)) {
3578                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3579                 tarc |= SPEED_MODE_BIT;
3580                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3581         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3582                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3583                 tarc |= 1;
3584                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3585                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3586                 tarc |= 1;
3587                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3588         }
3589
3590         /* Program the Transmit Control Register */
3591         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3592         tctl &= ~E1000_TCTL_CT;
3593         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3594                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3595
3596         if (adapter->hw.mac.type >= e1000_82571)
3597                 tctl |= E1000_TCTL_MULR;
3598
3599         /* This write will effectively turn on the transmit unit. */
3600         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3601
3602         /* Setup Transmit Descriptor Base Settings */
3603         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3604
3605         if (adapter->tx_int_delay.value > 0)
3606                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3607 }
3608
3609 /*********************************************************************
3610  *
3611  *  Free all transmit related data structures.
3612  *
3613  **********************************************************************/
3614 static void
3615 em_free_transmit_structures(struct adapter *adapter)
3616 {
3617         struct em_buffer *tx_buffer;
3618
3619         INIT_DEBUGOUT("free_transmit_structures: begin");
3620
3621         if (adapter->tx_buffer_area != NULL) {
3622                 for (int i = 0; i < adapter->num_tx_desc; i++) {
3623                         tx_buffer = &adapter->tx_buffer_area[i];
3624                         if (tx_buffer->m_head != NULL) {
3625                                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3626                                     BUS_DMASYNC_POSTWRITE);
3627                                 bus_dmamap_unload(adapter->txtag,
3628                                     tx_buffer->map);
3629                                 m_freem(tx_buffer->m_head);
3630                                 tx_buffer->m_head = NULL;
3631                         } else if (tx_buffer->map != NULL)
3632                                 bus_dmamap_unload(adapter->txtag,
3633                                     tx_buffer->map);
3634                         if (tx_buffer->map != NULL) {
3635                                 bus_dmamap_destroy(adapter->txtag,
3636                                     tx_buffer->map);
3637                                 tx_buffer->map = NULL;
3638                         }
3639                 }
3640         }
3641         if (adapter->tx_buffer_area != NULL) {
3642                 free(adapter->tx_buffer_area, M_DEVBUF);
3643                 adapter->tx_buffer_area = NULL;
3644         }
3645         if (adapter->txtag != NULL) {
3646                 bus_dma_tag_destroy(adapter->txtag);
3647                 adapter->txtag = NULL;
3648         }
3649 #if __FreeBSD_version >= 800000
3650         if (adapter->br != NULL)
3651                 buf_ring_free(adapter->br, M_DEVBUF);
3652 #endif
3653 }
3654
3655 /*********************************************************************
3656  *
3657  *  The offload context needs to be set when we transfer the first
3658  *  packet of a particular protocol (TCP/UDP). This routine has been
3659  *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3660  *
3661  *  Added back the old method of keeping the current context type
3662  *  and not setting if unnecessary, as this is reported to be a
3663  *  big performance win.  -jfv
3664  **********************************************************************/
3665 static void
3666 em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
3667     u32 *txd_upper, u32 *txd_lower)
3668 {
3669         struct e1000_context_desc *TXD = NULL;
3670         struct em_buffer *tx_buffer;
3671         struct ether_vlan_header *eh;
3672         struct ip *ip = NULL;
3673         struct ip6_hdr *ip6;
3674         int curr_txd, ehdrlen;
3675         u32 cmd, hdr_len, ip_hlen;
3676         u16 etype;
3677         u8 ipproto;
3678
3679
3680         cmd = hdr_len = ipproto = 0;
3681         curr_txd = adapter->next_avail_tx_desc;
3682
3683         /*
3684          * Determine where frame payload starts.
3685          * Jump over vlan headers if already present,
3686          * helpful for QinQ too.
3687          */
3688         eh = mtod(mp, struct ether_vlan_header *);
3689         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3690                 etype = ntohs(eh->evl_proto);
3691                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3692         } else {
3693                 etype = ntohs(eh->evl_encap_proto);
3694                 ehdrlen = ETHER_HDR_LEN;
3695         }
3696
3697         /*
3698          * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3699          * TODO: Support SCTP too when it hits the tree.
3700          */
3701         switch (etype) {
3702         case ETHERTYPE_IP:
3703                 ip = (struct ip *)(mp->m_data + ehdrlen);
3704                 ip_hlen = ip->ip_hl << 2;
3705
3706                 /* Setup of IP header checksum. */
3707                 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3708                         /*
3709                          * Start offset for header checksum calculation.
3710                          * End offset for header checksum calculation.
3711                          * Offset of place to put the checksum.
3712                          */
3713                         TXD = (struct e1000_context_desc *)
3714                             &adapter->tx_desc_base[curr_txd];
3715                         TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3716                         TXD->lower_setup.ip_fields.ipcse =
3717                             htole16(ehdrlen + ip_hlen);
3718                         TXD->lower_setup.ip_fields.ipcso =
3719                             ehdrlen + offsetof(struct ip, ip_sum);
3720                         cmd |= E1000_TXD_CMD_IP;
3721                         *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3722                 }
3723
3724                 if (mp->m_len < ehdrlen + ip_hlen)
3725                         return; /* failure */
3726
3727                 hdr_len = ehdrlen + ip_hlen;
3728                 ipproto = ip->ip_p;
3729
3730                 break;
3731         case ETHERTYPE_IPV6:
3732                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3733                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3734
3735                 if (mp->m_len < ehdrlen + ip_hlen)
3736                         return; /* failure */
3737
3738                 /* IPv6 doesn't have a header checksum. */
3739
3740                 hdr_len = ehdrlen + ip_hlen;
3741                 ipproto = ip6->ip6_nxt;
3742
3743                 break;
3744         default:
3745                 *txd_upper = 0;
3746                 *txd_lower = 0;
3747                 return;
3748         }
3749
3750         switch (ipproto) {
3751         case IPPROTO_TCP:
3752                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3753                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3754                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3755                         /* no need for context if already set */
3756                         if (adapter->last_hw_offload == CSUM_TCP)
3757                                 return;
3758                         adapter->last_hw_offload = CSUM_TCP;
3759                         /*
3760                          * Start offset for payload checksum calculation.
3761                          * End offset for payload checksum calculation.
3762                          * Offset of place to put the checksum.
3763                          */
3764                         TXD = (struct e1000_context_desc *)
3765                             &adapter->tx_desc_base[curr_txd];
3766                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3767                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3768                         TXD->upper_setup.tcp_fields.tucso =
3769                             hdr_len + offsetof(struct tcphdr, th_sum);
3770                         cmd |= E1000_TXD_CMD_TCP;
3771                 }
3772                 break;
3773         case IPPROTO_UDP:
3774         {
3775                 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3776                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3777                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3778                         /* no need for context if already set */
3779                         if (adapter->last_hw_offload == CSUM_UDP)
3780                                 return;
3781                         adapter->last_hw_offload = CSUM_UDP;
3782                         /*
3783                          * Start offset for header checksum calculation.
3784                          * End offset for header checksum calculation.
3785                          * Offset of place to put the checksum.
3786                          */
3787                         TXD = (struct e1000_context_desc *)
3788                             &adapter->tx_desc_base[curr_txd];
3789                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3790                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3791                         TXD->upper_setup.tcp_fields.tucso =
3792                             hdr_len + offsetof(struct udphdr, uh_sum);
3793                 }
3794                 /* Fall Thru */
3795         }
3796         default:
3797                 break;
3798         }
3799
3800         TXD->tcp_seg_setup.data = htole32(0);
3801         TXD->cmd_and_length =
3802             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3803         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3804         tx_buffer->m_head = NULL;
3805         tx_buffer->next_eop = -1;
3806
3807         if (++curr_txd == adapter->num_tx_desc)
3808                 curr_txd = 0;
3809
3810         adapter->num_tx_desc_avail--;
3811         adapter->next_avail_tx_desc = curr_txd;
3812 }
3813
3814
3815 #if __FreeBSD_version >= 700000
3816 /**********************************************************************
3817  *
3818  *  Setup work for hardware segmentation offload (TSO)
3819  *
3820  **********************************************************************/
3821 static bool
3822 em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper,
3823    u32 *txd_lower)
3824 {
3825         struct e1000_context_desc *TXD;
3826         struct em_buffer *tx_buffer;
3827         struct ether_vlan_header *eh;
3828         struct ip *ip;
3829         struct ip6_hdr *ip6;
3830         struct tcphdr *th;
3831         int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6;
3832         u16 etype;
3833
3834         /*
3835          * This function could/should be extended to support IP/IPv6
3836          * fragmentation as well.  But as they say, one step at a time.
3837          */
3838
3839         /*
3840          * Determine where frame payload starts.
3841          * Jump over vlan headers if already present,
3842          * helpful for QinQ too.
3843          */
3844         eh = mtod(mp, struct ether_vlan_header *);
3845         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3846                 etype = ntohs(eh->evl_proto);
3847                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3848         } else {
3849                 etype = ntohs(eh->evl_encap_proto);
3850                 ehdrlen = ETHER_HDR_LEN;
3851         }
3852
3853         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3854         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3855                 return FALSE;   /* -1 */
3856
3857         /*
3858          * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3859          * TODO: Support SCTP too when it hits the tree.
3860          */
3861         switch (etype) {
3862         case ETHERTYPE_IP:
3863                 isip6 = 0;
3864                 ip = (struct ip *)(mp->m_data + ehdrlen);
3865                 if (ip->ip_p != IPPROTO_TCP)
3866                         return FALSE;   /* 0 */
3867                 ip->ip_len = 0;
3868                 ip->ip_sum = 0;
3869                 ip_hlen = ip->ip_hl << 2;
3870                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3871                         return FALSE;   /* -1 */
3872                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3873 #if 1
3874                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3875                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3876 #else
3877                 th->th_sum = mp->m_pkthdr.csum_data;
3878 #endif
3879                 break;
3880         case ETHERTYPE_IPV6:
3881                 isip6 = 1;
3882                 return FALSE;                   /* Not supported yet. */
3883                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3884                 if (ip6->ip6_nxt != IPPROTO_TCP)
3885                         return FALSE;   /* 0 */
3886                 ip6->ip6_plen = 0;
3887                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3888                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3889                         return FALSE;   /* -1 */
3890                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3891 #if 0
3892                 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3893                     htons(IPPROTO_TCP));        /* XXX: function notyet. */
3894 #else
3895                 th->th_sum = mp->m_pkthdr.csum_data;
3896 #endif
3897                 break;
3898         default:
3899                 return FALSE;
3900         }
3901         hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3902
3903         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3904                       E1000_TXD_DTYP_D |        /* Data descr type */
3905                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3906
3907         /* IP and/or TCP header checksum calculation and insertion. */
3908         *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3909                       E1000_TXD_POPTS_TXSM) << 8;
3910
3911         curr_txd = adapter->next_avail_tx_desc;
3912         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3913         TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3914
3915         /* IPv6 doesn't have a header checksum. */
3916         if (!isip6) {
3917                 /*
3918                  * Start offset for header checksum calculation.
3919                  * End offset for header checksum calculation.
3920                  * Offset of place put the checksum.
3921                  */
3922                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3923                 TXD->lower_setup.ip_fields.ipcse =
3924                     htole16(ehdrlen + ip_hlen - 1);
3925                 TXD->lower_setup.ip_fields.ipcso =
3926                     ehdrlen + offsetof(struct ip, ip_sum);
3927         }
3928         /*
3929          * Start offset for payload checksum calculation.
3930          * End offset for payload checksum calculation.
3931          * Offset of place to put the checksum.
3932          */
3933         TXD->upper_setup.tcp_fields.tucss =
3934             ehdrlen + ip_hlen;
3935         TXD->upper_setup.tcp_fields.tucse = 0;
3936         TXD->upper_setup.tcp_fields.tucso =
3937             ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3938         /*
3939          * Payload size per packet w/o any headers.
3940          * Length of all headers up to payload.
3941          */
3942         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3943         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3944
3945         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3946                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3947                                 E1000_TXD_CMD_TSE |     /* TSE context */
3948                                 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */
3949                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3950                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3951
3952         tx_buffer->m_head = NULL;
3953         tx_buffer->next_eop = -1;
3954
3955         if (++curr_txd == adapter->num_tx_desc)
3956                 curr_txd = 0;
3957
3958         adapter->num_tx_desc_avail--;
3959         adapter->next_avail_tx_desc = curr_txd;
3960         adapter->tx_tso = TRUE;
3961
3962         return TRUE;
3963 }
3964
3965 #endif /* __FreeBSD_version >= 700000 */
3966
3967 /**********************************************************************
3968  *
3969  *  Examine each tx_buffer in the used queue. If the hardware is done
3970  *  processing the packet then free associated resources. The
3971  *  tx_buffer is put back on the free queue.
3972  *
3973  **********************************************************************/
3974 static void
3975 em_txeof(struct adapter *adapter)
3976 {
3977         int first, last, done, num_avail;
3978         u32 cleaned = 0;
3979         struct em_buffer *tx_buffer;
3980         struct e1000_tx_desc   *tx_desc, *eop_desc;
3981         struct ifnet   *ifp = adapter->ifp;
3982
3983         EM_TX_LOCK_ASSERT(adapter);
3984
3985         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3986                 return;
3987
3988         num_avail = adapter->num_tx_desc_avail;
3989         first = adapter->next_tx_to_clean;
3990         tx_desc = &adapter->tx_desc_base[first];
3991         tx_buffer = &adapter->tx_buffer_area[first];
3992         last = tx_buffer->next_eop;
3993         eop_desc = &adapter->tx_desc_base[last];
3994
3995         /*
3996          * What this does is get the index of the
3997          * first descriptor AFTER the EOP of the
3998          * first packet, that way we can do the
3999          * simple comparison on the inner while loop.
4000          */
4001         if (++last == adapter->num_tx_desc)
4002                 last = 0;
4003         done = last;
4004
4005         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
4006             BUS_DMASYNC_POSTREAD);
4007
4008         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
4009                 /* We clean the range of the packet */
4010                 while (first != done) {
4011                         tx_desc->upper.data = 0;
4012                         tx_desc->lower.data = 0;
4013                         tx_desc->buffer_addr = 0;
4014                         ++num_avail; ++cleaned;
4015
4016                         if (tx_buffer->m_head) {
4017                                 ifp->if_opackets++;
4018                                 bus_dmamap_sync(adapter->txtag,
4019                                     tx_buffer->map,
4020                                     BUS_DMASYNC_POSTWRITE);
4021                                 bus_dmamap_unload(adapter->txtag,
4022                                     tx_buffer->map);
4023
4024                                 m_freem(tx_buffer->m_head);
4025                                 tx_buffer->m_head = NULL;
4026                         }
4027                         tx_buffer->next_eop = -1;
4028
4029                         if (++first == adapter->num_tx_desc)
4030                                 first = 0;
4031
4032                         tx_buffer = &adapter->tx_buffer_area[first];
4033                         tx_desc = &adapter->tx_desc_base[first];
4034                 }
4035                 /* See if we can continue to the next packet */
4036                 last = tx_buffer->next_eop;
4037                 if (last != -1) {
4038                         eop_desc = &adapter->tx_desc_base[last];
4039                         /* Get new done point */
4040                         if (++last == adapter->num_tx_desc) last = 0;
4041                         done = last;
4042                 } else
4043                         break;
4044         }
4045         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
4046             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4047
4048         adapter->next_tx_to_clean = first;
4049
4050         /*
4051          * If we have enough room, clear IFF_DRV_OACTIVE to
4052          * tell the stack that it is OK to send packets.
4053          * If there are no pending descriptors, clear the timeout.
4054          */
4055         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
4056                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
4057                 if (num_avail == adapter->num_tx_desc) {
4058                         adapter->watchdog_timer = 0;
4059                         adapter->num_tx_desc_avail = num_avail;
4060                         return;
4061                 }
4062         }
4063
4064         /* If any descriptors cleaned, reset the watchdog */
4065         if (cleaned)
4066                 adapter->watchdog_timer = EM_TX_TIMEOUT;
4067         adapter->num_tx_desc_avail = num_avail;
4068         return;
4069 }
4070
4071 /*********************************************************************
4072  *
4073  *  When Link is lost sometimes there is work still in the TX ring
4074  *  which will result in a watchdog, rather than allow that do an
4075  *  attempted cleanup and then reinit here. Note that this has been
4076  *  seens mostly with fiber adapters.
4077  *
4078  **********************************************************************/
4079 static void
4080 em_tx_purge(struct adapter *adapter)
4081 {
4082         if ((!adapter->link_active) && (adapter->watchdog_timer)) {
4083                 EM_TX_LOCK(adapter);
4084                 em_txeof(adapter);
4085                 EM_TX_UNLOCK(adapter);
4086                 if (adapter->watchdog_timer) { /* Still not clean? */
4087                         adapter->watchdog_timer = 0;
4088                         em_init_locked(adapter);
4089                 }
4090         }
4091 }
4092
4093 /*********************************************************************
4094  *
4095  *  Get a buffer from system mbuf buffer pool.
4096  *
4097  **********************************************************************/
4098 static int
4099 em_get_buf(struct adapter *adapter, int i)
4100 {
4101         struct mbuf             *m;
4102         bus_dma_segment_t       segs[1];
4103         bus_dmamap_t            map;
4104         struct em_buffer        *rx_buffer;
4105         int                     error, nsegs;
4106
4107         m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
4108         if (m == NULL) {
4109                 adapter->mbuf_cluster_failed++;
4110                 return (ENOBUFS);
4111         }
4112         m->m_len = m->m_pkthdr.len = MCLBYTES;
4113
4114         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
4115                 m_adj(m, ETHER_ALIGN);
4116
4117         /*
4118          * Using memory from the mbuf cluster pool, invoke the
4119          * bus_dma machinery to arrange the memory mapping.
4120          */
4121         error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
4122             adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
4123         if (error != 0) {
4124                 m_free(m);
4125                 return (error);
4126         }
4127
4128         /* If nsegs is wrong then the stack is corrupt. */
4129         KASSERT(nsegs == 1, ("Too many segments returned!"));
4130
4131         rx_buffer = &adapter->rx_buffer_area[i];
4132         if (rx_buffer->m_head != NULL)
4133                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4134
4135         map = rx_buffer->map;
4136         rx_buffer->map = adapter->rx_sparemap;
4137         adapter->rx_sparemap = map;
4138         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
4139         rx_buffer->m_head = m;
4140
4141         adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
4142         return (0);
4143 }
4144
4145 /*********************************************************************
4146  *
4147  *  Allocate memory for rx_buffer structures. Since we use one
4148  *  rx_buffer per received packet, the maximum number of rx_buffer's
4149  *  that we'll need is equal to the number of receive descriptors
4150  *  that we've allocated.
4151  *
4152  **********************************************************************/
4153 static int
4154 em_allocate_receive_structures(struct adapter *adapter)
4155 {
4156         device_t dev = adapter->dev;
4157         struct em_buffer *rx_buffer;
4158         int i, error;
4159
4160         adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
4161             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
4162         if (adapter->rx_buffer_area == NULL) {
4163                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4164                 return (ENOMEM);
4165         }
4166
4167 #if __FreeBSD_version >= 700000
4168         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
4169 #else
4170         error = bus_dma_tag_create(NULL,                 /* parent */
4171 #endif
4172                                 1, 0,                   /* alignment, bounds */
4173                                 BUS_SPACE_MAXADDR,      /* lowaddr */
4174                                 BUS_SPACE_MAXADDR,      /* highaddr */
4175                                 NULL, NULL,             /* filter, filterarg */
4176                                 MCLBYTES,               /* maxsize */
4177                                 1,                      /* nsegments */
4178                                 MCLBYTES,               /* maxsegsize */
4179                                 0,                      /* flags */
4180                                 NULL,                   /* lockfunc */
4181                                 NULL,                   /* lockarg */
4182                                 &adapter->rxtag);
4183         if (error) {
4184                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
4185                     __func__, error);
4186                 goto fail;
4187         }
4188
4189         /* Create the spare map (used by getbuf) */
4190         error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4191              &adapter->rx_sparemap);
4192         if (error) {
4193                 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4194                     __func__, error);
4195                 goto fail;
4196         }
4197
4198         rx_buffer = adapter->rx_buffer_area;
4199         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4200                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4201                     &rx_buffer->map);
4202                 if (error) {
4203                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4204                             __func__, error);
4205                         goto fail;
4206                 }
4207         }
4208
4209         return (0);
4210
4211 fail:
4212         em_free_receive_structures(adapter);
4213         return (error);
4214 }
4215
4216 /*********************************************************************
4217  *
4218  *  (Re)initialize receive structures.
4219  *
4220  **********************************************************************/
4221 static int
4222 em_setup_receive_structures(struct adapter *adapter)
4223 {
4224         struct em_buffer *rx_buffer;
4225         int i, error;
4226
4227         /* Reset descriptor ring */
4228         bzero(adapter->rx_desc_base,
4229             (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
4230
4231         /* Free current RX buffers. */
4232         rx_buffer = adapter->rx_buffer_area;
4233         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4234                 if (rx_buffer->m_head != NULL) {
4235                         bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4236                             BUS_DMASYNC_POSTREAD);
4237                         bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4238                         m_freem(rx_buffer->m_head);
4239                         rx_buffer->m_head = NULL;
4240                 }
4241         }
4242
4243         /* Allocate new ones. */
4244         for (i = 0; i < adapter->num_rx_desc; i++) {
4245                 error = em_get_buf(adapter, i);
4246                 if (error)
4247                         return (error);
4248         }
4249
4250         /* Setup our descriptor pointers */
4251         adapter->next_rx_desc_to_check = 0;
4252         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4253             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4254
4255         return (0);
4256 }
4257
4258 /*********************************************************************
4259  *
4260  *  Enable receive unit.
4261  *
4262  **********************************************************************/
4263 #define MAX_INTS_PER_SEC        8000
4264 #define DEFAULT_ITR          1000000000/(MAX_INTS_PER_SEC * 256)
4265
4266 static void
4267 em_initialize_receive_unit(struct adapter *adapter)
4268 {
4269         struct ifnet    *ifp = adapter->ifp;
4270         u64     bus_addr;
4271         u32     rctl, rxcsum;
4272
4273         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
4274
4275         /*
4276          * Make sure receives are disabled while setting
4277          * up the descriptor ring
4278          */
4279         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4280         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4281
4282         if (adapter->hw.mac.type >= e1000_82540) {
4283                 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4284                     adapter->rx_abs_int_delay.value);
4285                 /*
4286                  * Set the interrupt throttling rate. Value is calculated
4287                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4288                  */
4289                 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
4290         }
4291
4292         /*
4293         ** When using MSIX interrupts we need to throttle
4294         ** using the EITR register (82574 only)
4295         */
4296         if (adapter->msix)
4297                 for (int i = 0; i < 4; i++)
4298                         E1000_WRITE_REG(&adapter->hw,
4299                             E1000_EITR_82574(i), DEFAULT_ITR);
4300
4301         /* Disable accelerated ackknowledge */
4302         if (adapter->hw.mac.type == e1000_82574)
4303                 E1000_WRITE_REG(&adapter->hw,
4304                     E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4305
4306         /* Setup the Base and Length of the Rx Descriptor Ring */
4307         bus_addr = adapter->rxdma.dma_paddr;
4308         E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
4309             adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4310         E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
4311             (u32)(bus_addr >> 32));
4312         E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
4313             (u32)bus_addr);
4314
4315         /* Setup the Receive Control Register */
4316         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4317         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4318                    E1000_RCTL_RDMTS_HALF |
4319                    (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4320
4321         /* Make sure VLAN Filters are off */
4322         rctl &= ~E1000_RCTL_VFE;
4323
4324         if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
4325                 rctl |= E1000_RCTL_SBP;
4326         else
4327                 rctl &= ~E1000_RCTL_SBP;
4328
4329         switch (adapter->rx_buffer_len) {
4330         default:
4331         case 2048:
4332                 rctl |= E1000_RCTL_SZ_2048;
4333                 break;
4334         case 4096:
4335                 rctl |= E1000_RCTL_SZ_4096 |
4336                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4337                 break;
4338         case 8192:
4339                 rctl |= E1000_RCTL_SZ_8192 |
4340                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4341                 break;
4342         case 16384:
4343                 rctl |= E1000_RCTL_SZ_16384 |
4344                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4345                 break;
4346         }
4347
4348         if (ifp->if_mtu > ETHERMTU)
4349                 rctl |= E1000_RCTL_LPE;
4350         else
4351                 rctl &= ~E1000_RCTL_LPE;
4352
4353         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
4354         if ((adapter->hw.mac.type >= e1000_82543) &&
4355             (ifp->if_capenable & IFCAP_RXCSUM)) {
4356                 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
4357                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4358                 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
4359         }
4360
4361         /*
4362         ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4363         ** long latencies are observed, like Lenovo X60. This
4364         ** change eliminates the problem, but since having positive
4365         ** values in RDTR is a known source of problems on other
4366         ** platforms another solution is being sought.
4367         */
4368         if (adapter->hw.mac.type == e1000_82573)
4369                 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20);
4370
4371         /* Enable Receives */
4372         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4373
4374         /*
4375          * Setup the HW Rx Head and
4376          * Tail Descriptor Pointers
4377          */
4378         E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
4379         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
4380
4381         return;
4382 }
4383
4384 /*********************************************************************
4385  *
4386  *  Free receive related data structures.
4387  *
4388  **********************************************************************/
4389 static void
4390 em_free_receive_structures(struct adapter *adapter)
4391 {
4392         struct em_buffer *rx_buffer;
4393         int i;
4394
4395         INIT_DEBUGOUT("free_receive_structures: begin");
4396
4397         if (adapter->rx_sparemap) {
4398                 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
4399                 adapter->rx_sparemap = NULL;
4400         }
4401
4402         /* Cleanup any existing buffers */
4403         if (adapter->rx_buffer_area != NULL) {
4404                 rx_buffer = adapter->rx_buffer_area;
4405                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4406                         if (rx_buffer->m_head != NULL) {
4407                                 bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4408                                     BUS_DMASYNC_POSTREAD);
4409                                 bus_dmamap_unload(adapter->rxtag,
4410                                     rx_buffer->map);
4411                                 m_freem(rx_buffer->m_head);
4412                                 rx_buffer->m_head = NULL;
4413                         } else if (rx_buffer->map != NULL)
4414                                 bus_dmamap_unload(adapter->rxtag,
4415                                     rx_buffer->map);
4416                         if (rx_buffer->map != NULL) {
4417                                 bus_dmamap_destroy(adapter->rxtag,
4418                                     rx_buffer->map);
4419                                 rx_buffer->map = NULL;
4420                         }
4421                 }
4422         }
4423
4424         if (adapter->rx_buffer_area != NULL) {
4425                 free(adapter->rx_buffer_area, M_DEVBUF);
4426                 adapter->rx_buffer_area = NULL;
4427         }
4428
4429         if (adapter->rxtag != NULL) {
4430                 bus_dma_tag_destroy(adapter->rxtag);
4431                 adapter->rxtag = NULL;
4432         }
4433 }
4434
4435 /*********************************************************************
4436  *
4437  *  This routine executes in interrupt context. It replenishes
4438  *  the mbufs in the descriptor and sends data which has been
4439  *  dma'ed into host memory to upper layer.
4440  *
4441  *  We loop at most count times if count is > 0, or until done if
4442  *  count < 0.
4443  *
4444  *  For polling we also now return the number of cleaned packets
4445  *********************************************************************/
4446 static int
4447 em_rxeof(struct adapter *adapter, int count)
4448 {
4449         struct ifnet    *ifp = adapter->ifp;;
4450         struct mbuf     *mp;
4451         u8              status, accept_frame = 0, eop = 0;
4452         u16             len, desc_len, prev_len_adj;
4453         int             i, rx_sent = 0;
4454         struct e1000_rx_desc   *current_desc;
4455
4456         EM_RX_LOCK(adapter);
4457         i = adapter->next_rx_desc_to_check;
4458         current_desc = &adapter->rx_desc_base[i];
4459         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4460             BUS_DMASYNC_POSTREAD);
4461
4462         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
4463                 EM_RX_UNLOCK(adapter);
4464                 return (rx_sent);
4465         }
4466
4467         while ((current_desc->status & E1000_RXD_STAT_DD) &&
4468             (count != 0) &&
4469             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4470                 struct mbuf *m = NULL;
4471
4472                 mp = adapter->rx_buffer_area[i].m_head;
4473                 /*
4474                  * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
4475                  * needs to access the last received byte in the mbuf.
4476                  */
4477                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
4478                     BUS_DMASYNC_POSTREAD);
4479
4480                 accept_frame = 1;
4481                 prev_len_adj = 0;
4482                 desc_len = le16toh(current_desc->length);
4483                 status = current_desc->status;
4484                 if (status & E1000_RXD_STAT_EOP) {
4485                         count--;
4486                         eop = 1;
4487                         if (desc_len < ETHER_CRC_LEN) {
4488                                 len = 0;
4489                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
4490                         } else
4491                                 len = desc_len - ETHER_CRC_LEN;
4492                 } else {
4493                         eop = 0;
4494                         len = desc_len;
4495                 }
4496
4497                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
4498                         u8      last_byte;
4499                         u32     pkt_len = desc_len;
4500
4501                         if (adapter->fmp != NULL)
4502                                 pkt_len += adapter->fmp->m_pkthdr.len;
4503
4504                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
4505                         if (TBI_ACCEPT(&adapter->hw, status,
4506                             current_desc->errors, pkt_len, last_byte,
4507                             adapter->min_frame_size, adapter->max_frame_size)) {
4508                                 e1000_tbi_adjust_stats_82543(&adapter->hw,
4509                                     &adapter->stats, pkt_len,
4510                                     adapter->hw.mac.addr,
4511                                     adapter->max_frame_size);
4512                                 if (len > 0)
4513                                         len--;
4514                         } else
4515                                 accept_frame = 0;
4516                 }
4517
4518                 if (accept_frame) {
4519                         if (em_get_buf(adapter, i) != 0) {
4520                                 ifp->if_iqdrops++;
4521                                 goto discard;
4522                         }
4523
4524                         /* Assign correct length to the current fragment */
4525                         mp->m_len = len;
4526
4527                         if (adapter->fmp == NULL) {
4528                                 mp->m_pkthdr.len = len;
4529                                 adapter->fmp = mp; /* Store the first mbuf */
4530                                 adapter->lmp = mp;
4531                         } else {
4532                                 /* Chain mbuf's together */
4533                                 mp->m_flags &= ~M_PKTHDR;
4534                                 /*
4535                                  * Adjust length of previous mbuf in chain if
4536                                  * we received less than 4 bytes in the last
4537                                  * descriptor.
4538                                  */
4539                                 if (prev_len_adj > 0) {
4540                                         adapter->lmp->m_len -= prev_len_adj;
4541                                         adapter->fmp->m_pkthdr.len -=
4542                                             prev_len_adj;
4543                                 }
4544                                 adapter->lmp->m_next = mp;
4545                                 adapter->lmp = adapter->lmp->m_next;
4546                                 adapter->fmp->m_pkthdr.len += len;
4547                         }
4548
4549                         if (eop) {
4550                                 adapter->fmp->m_pkthdr.rcvif = ifp;
4551                                 ifp->if_ipackets++;
4552                                 em_receive_checksum(adapter, current_desc,
4553                                     adapter->fmp);
4554 #ifndef __NO_STRICT_ALIGNMENT
4555                                 if (adapter->max_frame_size >
4556                                     (MCLBYTES - ETHER_ALIGN) &&
4557                                     em_fixup_rx(adapter) != 0)
4558                                         goto skip;
4559 #endif
4560                                 if (status & E1000_RXD_STAT_VP) {
4561 #if __FreeBSD_version < 700000
4562                                         VLAN_INPUT_TAG_NEW(ifp, adapter->fmp,
4563                                             (le16toh(current_desc->special) &
4564                                             E1000_RXD_SPC_VLAN_MASK));
4565 #else
4566                                         adapter->fmp->m_pkthdr.ether_vtag =
4567                                             (le16toh(current_desc->special) &
4568                                             E1000_RXD_SPC_VLAN_MASK);
4569                                         adapter->fmp->m_flags |= M_VLANTAG;
4570 #endif
4571                                 }
4572 #ifndef __NO_STRICT_ALIGNMENT
4573 skip:
4574 #endif
4575                                 m = adapter->fmp;
4576                                 adapter->fmp = NULL;
4577                                 adapter->lmp = NULL;
4578                         }
4579                 } else {
4580                         ifp->if_ierrors++;
4581 discard:
4582                         /* Reuse loaded DMA map and just update mbuf chain */
4583                         mp = adapter->rx_buffer_area[i].m_head;
4584                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
4585                         mp->m_data = mp->m_ext.ext_buf;
4586                         mp->m_next = NULL;
4587                         if (adapter->max_frame_size <=
4588                             (MCLBYTES - ETHER_ALIGN))
4589                                 m_adj(mp, ETHER_ALIGN);
4590                         if (adapter->fmp != NULL) {
4591                                 m_freem(adapter->fmp);
4592                                 adapter->fmp = NULL;
4593                                 adapter->lmp = NULL;
4594                         }
4595                         m = NULL;
4596                 }
4597
4598                 /* Zero out the receive descriptors status. */
4599                 current_desc->status = 0;
4600                 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4601                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4602
4603                 /* Advance our pointers to the next descriptor. */
4604                 if (++i == adapter->num_rx_desc)
4605                         i = 0;
4606                 /* Call into the stack */
4607                 if (m != NULL) {
4608                         adapter->next_rx_desc_to_check = i;
4609                         EM_RX_UNLOCK(adapter);
4610                         (*ifp->if_input)(ifp, m);
4611                         EM_RX_LOCK(adapter);
4612                         rx_sent++;
4613                         i = adapter->next_rx_desc_to_check;
4614                 }
4615                 current_desc = &adapter->rx_desc_base[i];
4616         }
4617         adapter->next_rx_desc_to_check = i;
4618
4619         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
4620         if (--i < 0)
4621                 i = adapter->num_rx_desc - 1;
4622         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
4623         EM_RX_UNLOCK(adapter);
4624         return (rx_sent);
4625 }
4626
4627 #ifndef __NO_STRICT_ALIGNMENT
4628 /*
4629  * When jumbo frames are enabled we should realign entire payload on
4630  * architecures with strict alignment. This is serious design mistake of 8254x
4631  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4632  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4633  * payload. On architecures without strict alignment restrictions 8254x still
4634  * performs unaligned memory access which would reduce the performance too.
4635  * To avoid copying over an entire frame to align, we allocate a new mbuf and
4636  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4637  * existing mbuf chain.
4638  *
4639  * Be aware, best performance of the 8254x is achived only when jumbo frame is
4640  * not used at all on architectures with strict alignment.
4641  */
4642 static int
4643 em_fixup_rx(struct adapter *adapter)
4644 {
4645         struct mbuf *m, *n;
4646         int error;
4647
4648         error = 0;
4649         m = adapter->fmp;
4650         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4651                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4652                 m->m_data += ETHER_HDR_LEN;
4653         } else {
4654                 MGETHDR(n, M_DONTWAIT, MT_DATA);
4655                 if (n != NULL) {
4656                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4657                         m->m_data += ETHER_HDR_LEN;
4658                         m->m_len -= ETHER_HDR_LEN;
4659                         n->m_len = ETHER_HDR_LEN;
4660                         M_MOVE_PKTHDR(n, m);
4661                         n->m_next = m;
4662                         adapter->fmp = n;
4663                 } else {
4664                         adapter->dropped_pkts++;
4665                         m_freem(adapter->fmp);
4666                         adapter->fmp = NULL;
4667                         error = ENOMEM;
4668                 }
4669         }
4670
4671         return (error);
4672 }
4673 #endif
4674
4675 /*********************************************************************
4676  *
4677  *  Verify that the hardware indicated that the checksum is valid.
4678  *  Inform the stack about the status of checksum so that stack
4679  *  doesn't spend time verifying the checksum.
4680  *
4681  *********************************************************************/
4682 static void
4683 em_receive_checksum(struct adapter *adapter,
4684             struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4685 {
4686         /* 82543 or newer only */
4687         if ((adapter->hw.mac.type < e1000_82543) ||
4688             /* Ignore Checksum bit is set */
4689             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
4690                 mp->m_pkthdr.csum_flags = 0;
4691                 return;
4692         }
4693
4694         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4695                 /* Did it pass? */
4696                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4697                         /* IP Checksum Good */
4698                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4699                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4700
4701                 } else {
4702                         mp->m_pkthdr.csum_flags = 0;
4703                 }
4704         }
4705
4706         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4707                 /* Did it pass? */
4708                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4709                         mp->m_pkthdr.csum_flags |=
4710                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4711                         mp->m_pkthdr.csum_data = htons(0xffff);
4712                 }
4713         }
4714 }
4715
4716 #if __FreeBSD_version >= 700029
4717 /*
4718  * This routine is run via an vlan
4719  * config EVENT
4720  */
4721 static void
4722 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4723 {
4724         struct adapter  *adapter = ifp->if_softc;
4725         u32             index, bit;
4726
4727         if (ifp->if_softc !=  arg)   /* Not our event */
4728                 return;
4729
4730         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
4731                 return;
4732
4733         index = (vtag >> 5) & 0x7F;
4734         bit = vtag & 0x1F;
4735         em_shadow_vfta[index] |= (1 << bit);
4736         ++adapter->num_vlans;
4737         /* Re-init to load the changes */
4738         em_init(adapter);
4739 }
4740
4741 /*
4742  * This routine is run via an vlan
4743  * unconfig EVENT
4744  */
4745 static void
4746 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4747 {
4748         struct adapter  *adapter = ifp->if_softc;
4749         u32             index, bit;
4750
4751         if (ifp->if_softc !=  arg)
4752                 return;
4753
4754         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4755                 return;
4756
4757         index = (vtag >> 5) & 0x7F;
4758         bit = vtag & 0x1F;
4759         em_shadow_vfta[index] &= ~(1 << bit);
4760         --adapter->num_vlans;
4761         /* Re-init to load the changes */
4762         em_init(adapter);
4763 }
4764
4765 static void
4766 em_setup_vlan_hw_support(struct adapter *adapter)
4767 {
4768         struct e1000_hw *hw = &adapter->hw;
4769         u32             reg;
4770
4771         /*
4772         ** We get here thru init_locked, meaning
4773         ** a soft reset, this has already cleared
4774         ** the VFTA and other state, so if there
4775         ** have been no vlan's registered do nothing.
4776         */
4777         if (adapter->num_vlans == 0)
4778                 return;
4779
4780         /*
4781         ** A soft reset zero's out the VFTA, so
4782         ** we need to repopulate it now.
4783         */
4784         for (int i = 0; i < EM_VFTA_SIZE; i++)
4785                 if (em_shadow_vfta[i] != 0)
4786                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4787                             i, em_shadow_vfta[i]);
4788
4789         reg = E1000_READ_REG(hw, E1000_CTRL);
4790         reg |= E1000_CTRL_VME;
4791         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4792
4793         /* Enable the Filter Table */
4794         reg = E1000_READ_REG(hw, E1000_RCTL);
4795         reg &= ~E1000_RCTL_CFIEN;
4796         reg |= E1000_RCTL_VFE;
4797         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4798
4799         /* Update the frame size */
4800         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4801             adapter->max_frame_size + VLAN_TAG_SIZE);
4802 }
4803 #endif
4804
4805 static void
4806 em_enable_intr(struct adapter *adapter)
4807 {
4808         struct e1000_hw *hw = &adapter->hw;
4809         u32 ims_mask = IMS_ENABLE_MASK;
4810
4811         if (adapter->msix) {
4812                 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4813                 ims_mask |= EM_MSIX_MASK;
4814         }
4815         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4816 }
4817
4818 static void
4819 em_disable_intr(struct adapter *adapter)
4820 {
4821         struct e1000_hw *hw = &adapter->hw;
4822
4823         if (adapter->msix)
4824                 E1000_WRITE_REG(hw, EM_EIAC, 0);
4825         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4826 }
4827
4828 /*
4829  * Bit of a misnomer, what this really means is
4830  * to enable OS management of the system... aka
4831  * to disable special hardware management features
4832  */
4833 static void
4834 em_init_manageability(struct adapter *adapter)
4835 {
4836         /* A shared code workaround */
4837 #define E1000_82542_MANC2H E1000_MANC2H
4838         if (adapter->has_manage) {
4839                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4840                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4841
4842                 /* disable hardware interception of ARP */
4843                 manc &= ~(E1000_MANC_ARP_EN);
4844
4845                 /* enable receiving management packets to the host */
4846                 if (adapter->hw.mac.type >= e1000_82571) {
4847                         manc |= E1000_MANC_EN_MNG2HOST;
4848 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4849 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4850                         manc2h |= E1000_MNG2HOST_PORT_623;
4851                         manc2h |= E1000_MNG2HOST_PORT_664;
4852                         E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4853                 }
4854
4855                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4856         }
4857 }
4858
4859 /*
4860  * Give control back to hardware management
4861  * controller if there is one.
4862  */
4863 static void
4864 em_release_manageability(struct adapter *adapter)
4865 {
4866         if (adapter->has_manage) {
4867                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4868
4869                 /* re-enable hardware interception of ARP */
4870                 manc |= E1000_MANC_ARP_EN;
4871
4872                 if (adapter->hw.mac.type >= e1000_82571)
4873                         manc &= ~E1000_MANC_EN_MNG2HOST;
4874
4875                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4876         }
4877 }
4878
4879 /*
4880  * em_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4881  * For ASF and Pass Through versions of f/w this means that
4882  * the driver is loaded. For AMT version (only with 82573)
4883  * of the f/w this means that the network i/f is open.
4884  *
4885  */
4886 static void
4887 em_get_hw_control(struct adapter *adapter)
4888 {
4889         u32 ctrl_ext, swsm;
4890
4891         /* Let firmware know the driver has taken over */
4892         switch (adapter->hw.mac.type) {
4893         case e1000_82573:
4894                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4895                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4896                     swsm | E1000_SWSM_DRV_LOAD);
4897                 break;
4898         case e1000_82571:
4899         case e1000_82572:
4900         case e1000_80003es2lan:
4901         case e1000_ich8lan:
4902         case e1000_ich9lan:
4903         case e1000_ich10lan:
4904                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4905                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4906                     ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4907                 break;
4908         default:
4909                 break;
4910         }
4911 }
4912
4913 /*
4914  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4915  * For ASF and Pass Through versions of f/w this means that the
4916  * driver is no longer loaded. For AMT version (only with 82573) i
4917  * of the f/w this means that the network i/f is closed.
4918  *
4919  */
4920 static void
4921 em_release_hw_control(struct adapter *adapter)
4922 {
4923         u32 ctrl_ext, swsm;
4924
4925         /* Let firmware taken over control of h/w */
4926         switch (adapter->hw.mac.type) {
4927         case e1000_82573:
4928                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4929                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4930                     swsm & ~E1000_SWSM_DRV_LOAD);
4931                 break;
4932         case e1000_82571:
4933         case e1000_82572:
4934         case e1000_80003es2lan:
4935         case e1000_ich8lan:
4936         case e1000_ich9lan:
4937         case e1000_ich10lan:
4938                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4939                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4940                     ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4941                 break;
4942         default:
4943                 break;
4944
4945         }
4946 }
4947
4948 static int
4949 em_is_valid_ether_addr(u8 *addr)
4950 {
4951         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4952
4953         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4954                 return (FALSE);
4955         }
4956
4957         return (TRUE);
4958 }
4959
4960 /*
4961  * Enable PCI Wake On Lan capability
4962  */
4963 void
4964 em_enable_wakeup(device_t dev)
4965 {
4966         u16     cap, status;
4967         u8      id;
4968
4969         /* First find the capabilities pointer*/
4970         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4971         /* Read the PM Capabilities */
4972         id = pci_read_config(dev, cap, 1);
4973         if (id != PCIY_PMG)     /* Something wrong */
4974                 return;
4975         /* OK, we have the power capabilities, so
4976            now get the status register */
4977         cap += PCIR_POWER_STATUS;
4978         status = pci_read_config(dev, cap, 2);
4979         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4980         pci_write_config(dev, cap, status, 2);
4981         return;
4982 }
4983
4984
4985 /*********************************************************************
4986 * 82544 Coexistence issue workaround.
4987 *    There are 2 issues.
4988 *       1. Transmit Hang issue.
4989 *    To detect this issue, following equation can be used...
4990 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4991 *         If SUM[3:0] is in between 1 to 4, we will have this issue.
4992 *
4993 *       2. DAC issue.
4994 *    To detect this issue, following equation can be used...
4995 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4996 *         If SUM[3:0] is in between 9 to c, we will have this issue.
4997 *
4998 *
4999 *    WORKAROUND:
5000 *         Make sure we do not have ending address
5001 *         as 1,2,3,4(Hang) or 9,a,b,c (DAC)
5002 *
5003 *************************************************************************/
5004 static u32
5005 em_fill_descriptors (bus_addr_t address, u32 length,
5006                 PDESC_ARRAY desc_array)
5007 {
5008         u32 safe_terminator;
5009
5010         /* Since issue is sensitive to length and address.*/
5011         /* Let us first check the address...*/
5012         if (length <= 4) {
5013                 desc_array->descriptor[0].address = address;
5014                 desc_array->descriptor[0].length = length;
5015                 desc_array->elements = 1;
5016                 return (desc_array->elements);
5017         }
5018         safe_terminator = (u32)((((u32)address & 0x7) +
5019             (length & 0xF)) & 0xF);
5020         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
5021         if (safe_terminator == 0   ||
5022         (safe_terminator > 4   &&
5023         safe_terminator < 9)   ||
5024         (safe_terminator > 0xC &&
5025         safe_terminator <= 0xF)) {
5026                 desc_array->descriptor[0].address = address;
5027                 desc_array->descriptor[0].length = length;
5028                 desc_array->elements = 1;
5029                 return (desc_array->elements);
5030         }
5031
5032         desc_array->descriptor[0].address = address;
5033         desc_array->descriptor[0].length = length - 4;
5034         desc_array->descriptor[1].address = address + (length - 4);
5035         desc_array->descriptor[1].length = 4;
5036         desc_array->elements = 2;
5037         return (desc_array->elements);
5038 }
5039
5040 /**********************************************************************
5041  *
5042  *  Update the board statistics counters.
5043  *
5044  **********************************************************************/
5045 static void
5046 em_update_stats_counters(struct adapter *adapter)
5047 {
5048         struct ifnet   *ifp;
5049
5050         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5051            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5052                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5053                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5054         }
5055         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5056         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5057         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5058         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5059
5060         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5061         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5062         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5063         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5064         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5065         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5066         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5067         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5068         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5069         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5070         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5071         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5072         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5073         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5074         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5075         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5076         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5077         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5078         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5079         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5080
5081         /* For the 64-bit byte counters the low dword must be read first. */
5082         /* Both registers clear on the read of the high dword */
5083
5084         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
5085         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
5086
5087         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5088         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5089         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5090         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5091         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5092
5093         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5094         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5095
5096         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5097         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5098         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5099         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5100         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5101         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5102         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5103         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5104         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5105         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5106
5107         if (adapter->hw.mac.type >= e1000_82543) {
5108                 adapter->stats.algnerrc +=
5109                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5110                 adapter->stats.rxerrc +=
5111                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5112                 adapter->stats.tncrs +=
5113                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5114                 adapter->stats.cexterr +=
5115                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5116                 adapter->stats.tsctc +=
5117                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5118                 adapter->stats.tsctfc +=
5119                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5120         }
5121         ifp = adapter->ifp;
5122
5123         ifp->if_collisions = adapter->stats.colc;
5124
5125         /* Rx Errors */
5126         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5127             adapter->stats.crcerrs + adapter->stats.algnerrc +
5128             adapter->stats.ruc + adapter->stats.roc +
5129             adapter->stats.mpc + adapter->stats.cexterr;
5130
5131         /* Tx Errors */
5132         ifp->if_oerrors = adapter->stats.ecol +
5133             adapter->stats.latecol + adapter->watchdog_events;
5134 }
5135
5136
5137 /**********************************************************************
5138  *
5139  *  This routine is called only when em_display_debug_stats is enabled.
5140  *  This routine provides a way to take a look at important statistics
5141  *  maintained by the driver and hardware.
5142  *
5143  **********************************************************************/
5144 static void
5145 em_print_debug_info(struct adapter *adapter)
5146 {
5147         device_t dev = adapter->dev;
5148         u8 *hw_addr = adapter->hw.hw_addr;
5149
5150         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
5151         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
5152             E1000_READ_REG(&adapter->hw, E1000_CTRL),
5153             E1000_READ_REG(&adapter->hw, E1000_RCTL));
5154         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
5155             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
5156             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
5157         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
5158             adapter->hw.fc.high_water,
5159             adapter->hw.fc.low_water);
5160         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
5161             E1000_READ_REG(&adapter->hw, E1000_TIDV),
5162             E1000_READ_REG(&adapter->hw, E1000_TADV));
5163         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
5164             E1000_READ_REG(&adapter->hw, E1000_RDTR),
5165             E1000_READ_REG(&adapter->hw, E1000_RADV));
5166         device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
5167             (long long)adapter->tx_fifo_wrk_cnt,
5168             (long long)adapter->tx_fifo_reset_cnt);
5169         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5170             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5171             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5172         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5173             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5174             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5175         device_printf(dev, "Num Tx descriptors avail = %d\n",
5176             adapter->num_tx_desc_avail);
5177         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
5178             adapter->no_tx_desc_avail1);
5179         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
5180             adapter->no_tx_desc_avail2);
5181         device_printf(dev, "Std mbuf failed = %ld\n",
5182             adapter->mbuf_alloc_failed);
5183         device_printf(dev, "Std mbuf cluster failed = %ld\n",
5184             adapter->mbuf_cluster_failed);
5185         device_printf(dev, "Driver dropped packets = %ld\n",
5186             adapter->dropped_pkts);
5187         device_printf(dev, "Driver tx dma failure in encap = %ld\n",
5188                 adapter->no_tx_dma_setup);
5189 }
5190
5191 static void
5192 em_print_hw_stats(struct adapter *adapter)
5193 {
5194         device_t dev = adapter->dev;
5195
5196         device_printf(dev, "Excessive collisions = %lld\n",
5197             (long long)adapter->stats.ecol);
5198 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
5199         device_printf(dev, "Symbol errors = %lld\n",
5200             (long long)adapter->stats.symerrs);
5201 #endif
5202         device_printf(dev, "Sequence errors = %lld\n",
5203             (long long)adapter->stats.sec);
5204         device_printf(dev, "Defer count = %lld\n",
5205             (long long)adapter->stats.dc);
5206         device_printf(dev, "Missed Packets = %lld\n",
5207             (long long)adapter->stats.mpc);
5208         device_printf(dev, "Receive No Buffers = %lld\n",
5209             (long long)adapter->stats.rnbc);
5210         /* RLEC is inaccurate on some hardware, calculate our own. */
5211         device_printf(dev, "Receive Length Errors = %lld\n",
5212             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
5213         device_printf(dev, "Receive errors = %lld\n",
5214             (long long)adapter->stats.rxerrc);
5215         device_printf(dev, "Crc errors = %lld\n",
5216             (long long)adapter->stats.crcerrs);
5217         device_printf(dev, "Alignment errors = %lld\n",
5218             (long long)adapter->stats.algnerrc);
5219         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
5220             (long long)adapter->stats.cexterr);
5221         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
5222         device_printf(dev, "watchdog timeouts = %ld\n",
5223             adapter->watchdog_events);
5224         device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld"
5225             " LINK MSIX IRQ = %ld\n", adapter->rx_irq,
5226             adapter->tx_irq , adapter->link_irq);
5227         device_printf(dev, "XON Rcvd = %lld\n",
5228             (long long)adapter->stats.xonrxc);
5229         device_printf(dev, "XON Xmtd = %lld\n",
5230             (long long)adapter->stats.xontxc);
5231         device_printf(dev, "XOFF Rcvd = %lld\n",
5232             (long long)adapter->stats.xoffrxc);
5233         device_printf(dev, "XOFF Xmtd = %lld\n",
5234             (long long)adapter->stats.xofftxc);
5235         device_printf(dev, "Good Packets Rcvd = %lld\n",
5236             (long long)adapter->stats.gprc);
5237         device_printf(dev, "Good Packets Xmtd = %lld\n",
5238             (long long)adapter->stats.gptc);
5239         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
5240             (long long)adapter->stats.tsctc);
5241         device_printf(dev, "TSO Contexts Failed = %lld\n",
5242             (long long)adapter->stats.tsctfc);
5243 }
5244
5245 /**********************************************************************
5246  *
5247  *  This routine provides a way to dump out the adapter eeprom,
5248  *  often a useful debug/service tool. This only dumps the first
5249  *  32 words, stuff that matters is in that extent.
5250  *
5251  **********************************************************************/
5252 static void
5253 em_print_nvm_info(struct adapter *adapter)
5254 {
5255         u16     eeprom_data;
5256         int     i, j, row = 0;
5257
5258         /* Its a bit crude, but it gets the job done */
5259         printf("\nInterface EEPROM Dump:\n");
5260         printf("Offset\n0x0000  ");
5261         for (i = 0, j = 0; i < 32; i++, j++) {
5262                 if (j == 8) { /* Make the offset block */
5263                         j = 0; ++row;
5264                         printf("\n0x00%x0  ",row);
5265                 }
5266                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5267                 printf("%04x ", eeprom_data);
5268         }
5269         printf("\n");
5270 }
5271
5272 static int
5273 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5274 {
5275         struct adapter *adapter;
5276         int error;
5277         int result;
5278
5279         result = -1;
5280         error = sysctl_handle_int(oidp, &result, 0, req);
5281
5282         if (error || !req->newptr)
5283                 return (error);
5284
5285         if (result == 1) {
5286                 adapter = (struct adapter *)arg1;
5287                 em_print_debug_info(adapter);
5288         }
5289         /*
5290          * This value will cause a hex dump of the
5291          * first 32 16-bit words of the EEPROM to
5292          * the screen.
5293          */
5294         if (result == 2) {
5295                 adapter = (struct adapter *)arg1;
5296                 em_print_nvm_info(adapter);
5297         }
5298
5299         return (error);
5300 }
5301
5302
5303 static int
5304 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
5305 {
5306         struct adapter *adapter;
5307         int error;
5308         int result;
5309
5310         result = -1;
5311         error = sysctl_handle_int(oidp, &result, 0, req);
5312
5313         if (error || !req->newptr)
5314                 return (error);
5315
5316         if (result == 1) {
5317                 adapter = (struct adapter *)arg1;
5318                 em_print_hw_stats(adapter);
5319         }
5320
5321         return (error);
5322 }
5323
5324 static int
5325 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5326 {
5327         struct em_int_delay_info *info;
5328         struct adapter *adapter;
5329         u32 regval;
5330         int error;
5331         int usecs;
5332         int ticks;
5333
5334         info = (struct em_int_delay_info *)arg1;
5335         usecs = info->value;
5336         error = sysctl_handle_int(oidp, &usecs, 0, req);
5337         if (error != 0 || req->newptr == NULL)
5338                 return (error);
5339         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5340                 return (EINVAL);
5341         info->value = usecs;
5342         ticks = EM_USECS_TO_TICKS(usecs);
5343
5344         adapter = info->adapter;
5345
5346         EM_CORE_LOCK(adapter);
5347         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5348         regval = (regval & ~0xffff) | (ticks & 0xffff);
5349         /* Handle a few special cases. */
5350         switch (info->offset) {
5351         case E1000_RDTR:
5352                 break;
5353         case E1000_TIDV:
5354                 if (ticks == 0) {
5355                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5356                         /* Don't write 0 into the TIDV register. */
5357                         regval++;
5358                 } else
5359                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5360                 break;
5361         }
5362         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5363         EM_CORE_UNLOCK(adapter);
5364         return (0);
5365 }
5366
5367 static void
5368 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5369         const char *description, struct em_int_delay_info *info,
5370         int offset, int value)
5371 {
5372         info->adapter = adapter;
5373         info->offset = offset;
5374         info->value = value;
5375         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5376             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5377             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5378             info, 0, em_sysctl_int_delay, "I", description);
5379 }
5380
5381 #ifndef EM_LEGACY_IRQ
5382 static void
5383 em_add_rx_process_limit(struct adapter *adapter, const char *name,
5384         const char *description, int *limit, int value)
5385 {
5386         *limit = value;
5387         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5388             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5389             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5390 }
5391 #endif
5392
5393