sys/dev/e1000/if_em.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2009, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #endif
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #if __FreeBSD_version >= 800000
  43 #include <sys/buf_ring.h>
  44 #endif
  45 #include <sys/bus.h>
  46 #include <sys/endian.h>
  47 #include <sys/kernel.h>
  48 #include <sys/kthread.h>
  49 #include <sys/malloc.h>
  50 #include <sys/mbuf.h>
  51 #include <sys/module.h>
  52 #include <sys/rman.h>
  53 #include <sys/socket.h>
  54 #include <sys/sockio.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/taskqueue.h>
  57 #if __FreeBSD_version >= 700029
  58 #include <sys/eventhandler.h>
  59 #endif
  60 #include <machine/bus.h>
  61 #include <machine/resource.h>
  62
  63 #include <net/bpf.h>
  64 #include <net/ethernet.h>
  65 #include <net/if.h>
  66 #include <net/if_arp.h>
  67 #include <net/if_dl.h>
  68 #include <net/if_media.h>
  69
  70 #include <net/if_types.h>
  71 #include <net/if_vlan_var.h>
  72
  73 #include <netinet/in_systm.h>
  74 #include <netinet/in.h>
  75 #include <netinet/if_ether.h>
  76 #include <netinet/ip.h>
  77 #include <netinet/ip6.h>
  78 #include <netinet/tcp.h>
  79 #include <netinet/udp.h>
  80
  81 #include <machine/in_cksum.h>
  82 #include <dev/pci/pcivar.h>
  83 #include <dev/pci/pcireg.h>
  84
  85 #include "e1000_api.h"
  86 #include "e1000_82571.h"
  87 #include "if_em.h"
  88
  89 /*********************************************************************
  90  *  Set this to one to display debug statistics
  91  *********************************************************************/
  92 int     em_display_debug_stats = 0;
  93
  94 /*********************************************************************
  95  *  Driver version:
  96  *********************************************************************/
  97 char em_driver_version[] = "6.9.24";
  98
  99
 100 /*********************************************************************
 101  *  PCI Device ID Table
 102  *
 103  *  Used by probe to select devices to load on
 104  *  Last field stores an index into e1000_strings
 105  *  Last entry must be all 0s
 106  *
 107  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 108  *********************************************************************/
 109
 110 static em_vendor_info_t em_vendor_info_array[] =
 111 {
 112         /* Intel(R) PRO/1000 Network Connection */
 113         { 0x8086, E1000_DEV_ID_82540EM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 114         { 0x8086, E1000_DEV_ID_82540EM_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82540EP,         PCI_ANY_ID, PCI_ANY_ID, 0},
 116         { 0x8086, E1000_DEV_ID_82540EP_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82540EP_LP,      PCI_ANY_ID, PCI_ANY_ID, 0},
 118
 119         { 0x8086, E1000_DEV_ID_82541EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 120         { 0x8086, E1000_DEV_ID_82541ER,         PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82541ER_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 122         { 0x8086, E1000_DEV_ID_82541EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82541GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 124         { 0x8086, E1000_DEV_ID_82541GI_LF,      PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82541GI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 126
 127         { 0x8086, E1000_DEV_ID_82542,           PCI_ANY_ID, PCI_ANY_ID, 0},
 128
 129         { 0x8086, E1000_DEV_ID_82543GC_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82543GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 131
 132         { 0x8086, E1000_DEV_ID_82544EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82544EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82544GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82544GC_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 136
 137         { 0x8086, E1000_DEV_ID_82545EM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 138         { 0x8086, E1000_DEV_ID_82545EM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 139         { 0x8086, E1000_DEV_ID_82545GM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_82545GM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 141         { 0x8086, E1000_DEV_ID_82545GM_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 142
 143         { 0x8086, E1000_DEV_ID_82546EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 144         { 0x8086, E1000_DEV_ID_82546EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_82546GB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_82546GB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_82546GB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_82546GB_PCIE,    PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
 152                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 153
 154         { 0x8086, E1000_DEV_ID_82547EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 155         { 0x8086, E1000_DEV_ID_82547EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_82547GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 157
 158         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 159         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 160         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
 162                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 163         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
 164                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 165         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 166                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 167         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
 168                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 169         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
 170                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 171         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
 172                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 173         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 174         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 175         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 176         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 177
 178         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 179         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 180         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 181         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
 182         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 183                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 184         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 185                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 186         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 187                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 188         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 189                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 190         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 191         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 192         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 193         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 194         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 195         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 196         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 197         { 0x8086, E1000_DEV_ID_ICH8_82567V_3,   PCI_ANY_ID, PCI_ANY_ID, 0},
 198         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 199         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 200         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 201         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 202         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 203         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 204         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 205         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 206         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 207         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 208         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
 209         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 210         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 211         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 212         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 213         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 214         { 0x8086, E1000_DEV_ID_PCH_M_HV_LM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 215         { 0x8086, E1000_DEV_ID_PCH_M_HV_LC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 216         { 0x8086, E1000_DEV_ID_PCH_D_HV_DM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 217         { 0x8086, E1000_DEV_ID_PCH_D_HV_DC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 218         /* required last entry */
 219         { 0, 0, 0, 0, 0}
 220 };
 221
 222 /*********************************************************************
 223  *  Table of branding strings for all supported NICs.
 224  *********************************************************************/
 225
 226 static char *em_strings[] = {
 227         "Intel(R) PRO/1000 Network Connection"
 228 };
 229
 230 /*********************************************************************
 231  *  Function prototypes
 232  *********************************************************************/
 233 static int      em_probe(device_t);
 234 static int      em_attach(device_t);
 235 static int      em_detach(device_t);
 236 static int      em_shutdown(device_t);
 237 static int      em_suspend(device_t);
 238 static int      em_resume(device_t);
 239 static void     em_start(struct ifnet *);
 240 static void     em_start_locked(struct ifnet *ifp);
 241 #if __FreeBSD_version >= 800000
 242 static int      em_mq_start(struct ifnet *, struct mbuf *);
 243 static int      em_mq_start_locked(struct ifnet *, struct mbuf *);
 244 static void     em_qflush(struct ifnet *);
 245 #endif
 246 static int      em_ioctl(struct ifnet *, u_long, caddr_t);
 247 static void     em_init(void *);
 248 static void     em_init_locked(struct adapter *);
 249 static void     em_stop(void *);
 250 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 251 static int      em_media_change(struct ifnet *);
 252 static void     em_identify_hardware(struct adapter *);
 253 static int      em_allocate_pci_resources(struct adapter *);
 254 static int      em_allocate_legacy(struct adapter *adapter);
 255 static int      em_allocate_msix(struct adapter *adapter);
 256 static int      em_setup_msix(struct adapter *);
 257 static void     em_free_pci_resources(struct adapter *);
 258 static void     em_local_timer(void *);
 259 static int      em_hardware_init(struct adapter *);
 260 static void     em_setup_interface(device_t, struct adapter *);
 261 static void     em_setup_transmit_structures(struct adapter *);
 262 static void     em_initialize_transmit_unit(struct adapter *);
 263 static int      em_setup_receive_structures(struct adapter *);
 264 static void     em_initialize_receive_unit(struct adapter *);
 265 static void     em_enable_intr(struct adapter *);
 266 static void     em_disable_intr(struct adapter *);
 267 static void     em_free_transmit_structures(struct adapter *);
 268 static void     em_free_receive_structures(struct adapter *);
 269 static void     em_update_stats_counters(struct adapter *);
 270 static void     em_txeof(struct adapter *);
 271 static void     em_tx_purge(struct adapter *);
 272 static int      em_allocate_receive_structures(struct adapter *);
 273 static int      em_allocate_transmit_structures(struct adapter *);
 274 static int      em_rxeof(struct adapter *, int);
 275 #ifndef __NO_STRICT_ALIGNMENT
 276 static int      em_fixup_rx(struct adapter *);
 277 #endif
 278 static void     em_receive_checksum(struct adapter *, struct e1000_rx_desc *,
 279                     struct mbuf *);
 280 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 281                     u32 *, u32 *);
 282 #if __FreeBSD_version >= 700000
 283 static bool     em_tso_setup(struct adapter *, struct mbuf *,
 284                     u32 *, u32 *);
 285 #endif /* FreeBSD_version >= 700000 */
 286 static void     em_set_promisc(struct adapter *);
 287 static void     em_disable_promisc(struct adapter *);
 288 static void     em_set_multi(struct adapter *);
 289 static void     em_print_hw_stats(struct adapter *);
 290 static void     em_update_link_status(struct adapter *);
 291 static int      em_get_buf(struct adapter *, int);
 292 #if __FreeBSD_version >= 700029
 293 static void     em_register_vlan(void *, struct ifnet *, u16);
 294 static void     em_unregister_vlan(void *, struct ifnet *, u16);
 295 static void     em_setup_vlan_hw_support(struct adapter *);
 296 #endif
 297 static int      em_xmit(struct adapter *, struct mbuf **);
 298 static void     em_smartspeed(struct adapter *);
 299 static int      em_82547_fifo_workaround(struct adapter *, int);
 300 static void     em_82547_update_fifo_head(struct adapter *, int);
 301 static int      em_82547_tx_fifo_reset(struct adapter *);
 302 static void     em_82547_move_tail(void *);
 303 static int      em_dma_malloc(struct adapter *, bus_size_t,
 304                     struct em_dma_alloc *, int);
 305 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 306 static void     em_print_debug_info(struct adapter *);
 307 static void     em_print_nvm_info(struct adapter *);
 308 static int      em_is_valid_ether_addr(u8 *);
 309 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 310 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 311 static u32      em_fill_descriptors (bus_addr_t address, u32 length,
 312                     PDESC_ARRAY desc_array);
 313 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 314 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 315                     const char *, struct em_int_delay_info *, int, int);
 316 /* Management and WOL Support */
 317 static void     em_init_manageability(struct adapter *);
 318 static void     em_release_manageability(struct adapter *);
 319 static void     em_get_hw_control(struct adapter *);
 320 static void     em_release_hw_control(struct adapter *);
 321 static void     em_get_wakeup(device_t);
 322 static void     em_enable_wakeup(device_t);
 323 static int      em_enable_phy_wakeup(struct adapter *);
 324
 325 #ifdef EM_LEGACY_IRQ
 326 static void     em_intr(void *);
 327 #else /* FAST IRQ */
 328 #if __FreeBSD_version < 700000
 329 static void     em_irq_fast(void *);
 330 #else
 331 static int      em_irq_fast(void *);
 332 #endif
 333
 334 /* MSIX handlers */
 335 static void     em_msix_tx(void *);
 336 static void     em_msix_rx(void *);
 337 static void     em_msix_link(void *);
 338 static void     em_handle_rx(void *context, int pending);
 339 static void     em_handle_tx(void *context, int pending);
 340
 341 static void     em_handle_rxtx(void *context, int pending);
 342 static void     em_handle_link(void *context, int pending);
 343 static void     em_add_rx_process_limit(struct adapter *, const char *,
 344                     const char *, int *, int);
 345 #endif /* ~EM_LEGACY_IRQ */
 346
 347 #ifdef DEVICE_POLLING
 348 static poll_handler_t em_poll;
 349 #endif /* POLLING */
 350
 351 /*********************************************************************
 352  *  FreeBSD Device Interface Entry Points
 353  *********************************************************************/
 354
 355 static device_method_t em_methods[] = {
 356         /* Device interface */
 357         DEVMETHOD(device_probe, em_probe),
 358         DEVMETHOD(device_attach, em_attach),
 359         DEVMETHOD(device_detach, em_detach),
 360         DEVMETHOD(device_shutdown, em_shutdown),
 361         DEVMETHOD(device_suspend, em_suspend),
 362         DEVMETHOD(device_resume, em_resume),
 363         {0, 0}
 364 };
 365
 366 static driver_t em_driver = {
 367         "em", em_methods, sizeof(struct adapter),
 368 };
 369
 370 static devclass_t em_devclass;
 371 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 372 MODULE_DEPEND(em, pci, 1, 1, 1);
 373 MODULE_DEPEND(em, ether, 1, 1, 1);
 374
 375 /*********************************************************************
 376  *  Tunable default values.
 377  *********************************************************************/
 378
 379 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 380 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 381 #define M_TSO_LEN                       66
 382
 383 /* Allow common code without TSO */
 384 #ifndef CSUM_TSO
 385 #define CSUM_TSO        0
 386 #endif
 387
 388 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 389 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 390 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 391 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 392 static int em_rxd = EM_DEFAULT_RXD;
 393 static int em_txd = EM_DEFAULT_TXD;
 394 static int em_smart_pwr_down = FALSE;
 395 /* Controls whether promiscuous also shows bad packets */
 396 static int em_debug_sbp = FALSE;
 397 /* Local switch for MSI/MSIX */
 398 static int em_enable_msi = TRUE;
 399
 400 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 401 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 402 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 403 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 404 TUNABLE_INT("hw.em.rxd", &em_rxd);
 405 TUNABLE_INT("hw.em.txd", &em_txd);
 406 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 407 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 408 TUNABLE_INT("hw.em.enable_msi", &em_enable_msi);
 409
 410 #ifndef EM_LEGACY_IRQ
 411 /* How many packets rxeof tries to clean at a time */
 412 static int em_rx_process_limit = 100;
 413 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
 414 #endif
 415
 416 /* Flow control setting - default to FULL */
 417 static int em_fc_setting = e1000_fc_full;
 418 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting);
 419
 420 /*
 421 ** Shadow VFTA table, this is needed because
 422 ** the real vlan filter table gets cleared during
 423 ** a soft reset and the driver needs to be able
 424 ** to repopulate it.
 425 */
 426 static u32 em_shadow_vfta[EM_VFTA_SIZE];
 427
 428 /* Global used in WOL setup with multiport cards */
 429 static int global_quad_port_a = 0;
 430
 431 /*********************************************************************
 432  *  Device identification routine
 433  *
 434  *  em_probe determines if the driver should be loaded on
 435  *  adapter based on PCI vendor/device id of the adapter.
 436  *
 437  *  return BUS_PROBE_DEFAULT on success, positive on failure
 438  *********************************************************************/
 439
 440 static int
 441 em_probe(device_t dev)
 442 {
 443         char            adapter_name[60];
 444         u16             pci_vendor_id = 0;
 445         u16             pci_device_id = 0;
 446         u16             pci_subvendor_id = 0;
 447         u16             pci_subdevice_id = 0;
 448         em_vendor_info_t *ent;
 449
 450         INIT_DEBUGOUT("em_probe: begin");
 451
 452         pci_vendor_id = pci_get_vendor(dev);
 453         if (pci_vendor_id != EM_VENDOR_ID)
 454                 return (ENXIO);
 455
 456         pci_device_id = pci_get_device(dev);
 457         pci_subvendor_id = pci_get_subvendor(dev);
 458         pci_subdevice_id = pci_get_subdevice(dev);
 459
 460         ent = em_vendor_info_array;
 461         while (ent->vendor_id != 0) {
 462                 if ((pci_vendor_id == ent->vendor_id) &&
 463                     (pci_device_id == ent->device_id) &&
 464
 465                     ((pci_subvendor_id == ent->subvendor_id) ||
 466                     (ent->subvendor_id == PCI_ANY_ID)) &&
 467
 468                     ((pci_subdevice_id == ent->subdevice_id) ||
 469                     (ent->subdevice_id == PCI_ANY_ID))) {
 470                         sprintf(adapter_name, "%s %s",
 471                                 em_strings[ent->index],
 472                                 em_driver_version);
 473                         device_set_desc_copy(dev, adapter_name);
 474                         return (BUS_PROBE_DEFAULT);
 475                 }
 476                 ent++;
 477         }
 478
 479         return (ENXIO);
 480 }
 481
 482 /*********************************************************************
 483  *  Device initialization routine
 484  *
 485  *  The attach entry point is called when the driver is being loaded.
 486  *  This routine identifies the type of hardware, allocates all resources
 487  *  and initializes the hardware.
 488  *
 489  *  return 0 on success, positive on failure
 490  *********************************************************************/
 491
 492 static int
 493 em_attach(device_t dev)
 494 {
 495         struct adapter  *adapter;
 496         int             tsize, rsize;
 497         int             error = 0;
 498
 499         INIT_DEBUGOUT("em_attach: begin");
 500
 501         adapter = device_get_softc(dev);
 502         adapter->dev = adapter->osdep.dev = dev;
 503         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 504         EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
 505         EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
 506
 507         /* SYSCTL stuff */
 508         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 509             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 510             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 511             em_sysctl_debug_info, "I", "Debug Information");
 512
 513         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 514             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 515             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 516             em_sysctl_stats, "I", "Statistics");
 517
 518         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 519         callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
 520
 521         /* Determine hardware and mac info */
 522         em_identify_hardware(adapter);
 523
 524         /* Setup PCI resources */
 525         if (em_allocate_pci_resources(adapter)) {
 526                 device_printf(dev, "Allocation of PCI resources failed\n");
 527                 error = ENXIO;
 528                 goto err_pci;
 529         }
 530
 531         /*
 532         ** For ICH8 and family we need to
 533         ** map the flash memory, and this
 534         ** must happen after the MAC is
 535         ** identified
 536         */
 537         if ((adapter->hw.mac.type == e1000_ich8lan) ||
 538             (adapter->hw.mac.type == e1000_pchlan) ||
 539             (adapter->hw.mac.type == e1000_ich9lan) ||
 540             (adapter->hw.mac.type == e1000_ich10lan)) {
 541                 int rid = EM_BAR_TYPE_FLASH;
 542                 adapter->flash = bus_alloc_resource_any(dev,
 543                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 544                 if (adapter->flash == NULL) {
 545                         device_printf(dev, "Mapping of Flash failed\n");
 546                         error = ENXIO;
 547                         goto err_pci;
 548                 }
 549                 /* This is used in the shared code */
 550                 adapter->hw.flash_address = (u8 *)adapter->flash;
 551                 adapter->osdep.flash_bus_space_tag =
 552                     rman_get_bustag(adapter->flash);
 553                 adapter->osdep.flash_bus_space_handle =
 554                     rman_get_bushandle(adapter->flash);
 555         }
 556
 557         /* Do Shared Code initialization */
 558         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 559                 device_printf(dev, "Setup of Shared code failed\n");
 560                 error = ENXIO;
 561                 goto err_pci;
 562         }
 563
 564         e1000_get_bus_info(&adapter->hw);
 565
 566         /* Set up some sysctls for the tunable interrupt delays */
 567         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 568             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 569             E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
 570         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 571             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 572             E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
 573         if (adapter->hw.mac.type >= e1000_82540) {
 574                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 575                     "receive interrupt delay limit in usecs",
 576                     &adapter->rx_abs_int_delay,
 577                     E1000_REGISTER(&adapter->hw, E1000_RADV),
 578                     em_rx_abs_int_delay_dflt);
 579                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 580                     "transmit interrupt delay limit in usecs",
 581                     &adapter->tx_abs_int_delay,
 582                     E1000_REGISTER(&adapter->hw, E1000_TADV),
 583                     em_tx_abs_int_delay_dflt);
 584         }
 585
 586 #ifndef EM_LEGACY_IRQ
 587         /* Sysctls for limiting the amount of work done in the taskqueue */
 588         em_add_rx_process_limit(adapter, "rx_processing_limit",
 589             "max number of rx packets to process", &adapter->rx_process_limit,
 590             em_rx_process_limit);
 591 #endif
 592
 593         /*
 594          * Validate number of transmit and receive descriptors. It
 595          * must not exceed hardware maximum, and must be multiple
 596          * of E1000_DBA_ALIGN.
 597          */
 598         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 599             (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
 600             (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
 601             (em_txd < EM_MIN_TXD)) {
 602                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 603                     EM_DEFAULT_TXD, em_txd);
 604                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 605         } else
 606                 adapter->num_tx_desc = em_txd;
 607         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 608             (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
 609             (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
 610             (em_rxd < EM_MIN_RXD)) {
 611                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 612                     EM_DEFAULT_RXD, em_rxd);
 613                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 614         } else
 615                 adapter->num_rx_desc = em_rxd;
 616
 617         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 618         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 619         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 620         adapter->rx_buffer_len = 2048;
 621
 622         e1000_init_script_state_82541(&adapter->hw, TRUE);
 623         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 624
 625         /* Copper options */
 626         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 627                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 628                 adapter->hw.phy.disable_polarity_correction = FALSE;
 629                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 630         }
 631
 632         /*
 633          * Set the frame limits assuming
 634          * standard ethernet sized frames.
 635          */
 636         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 637         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 638
 639         /*
 640          * This controls when hardware reports transmit completion
 641          * status.
 642          */
 643         adapter->hw.mac.report_tx_early = 1;
 644
 645         tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
 646             EM_DBA_ALIGN);
 647
 648         /* Allocate Transmit Descriptor ring */
 649         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
 650                 device_printf(dev, "Unable to allocate tx_desc memory\n");
 651                 error = ENOMEM;
 652                 goto err_tx_desc;
 653         }
 654         adapter->tx_desc_base =
 655             (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
 656
 657         rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
 658             EM_DBA_ALIGN);
 659
 660         /* Allocate Receive Descriptor ring */
 661         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 662                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 663                 error = ENOMEM;
 664                 goto err_rx_desc;
 665         }
 666         adapter->rx_desc_base =
 667             (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
 668
 669         /*
 670         ** Start from a known state, this is
 671         ** important in reading the nvm and
 672         ** mac from that.
 673         */
 674         e1000_reset_hw(&adapter->hw);
 675
 676         /* Make sure we have a good EEPROM before we read from it */
 677         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 678                 /*
 679                 ** Some PCI-E parts fail the first check due to
 680                 ** the link being in sleep state, call it again,
 681                 ** if it fails a second time its a real issue.
 682                 */
 683                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 684                         device_printf(dev,
 685                             "The EEPROM Checksum Is Not Valid\n");
 686                         error = EIO;
 687                         goto err_hw_init;
 688                 }
 689         }
 690
 691         /* Copy the permanent MAC address out of the EEPROM */
 692         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 693                 device_printf(dev, "EEPROM read error while reading MAC"
 694                     " address\n");
 695                 error = EIO;
 696                 goto err_hw_init;
 697         }
 698
 699         if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
 700                 device_printf(dev, "Invalid MAC address\n");
 701                 error = EIO;
 702                 goto err_hw_init;
 703         }
 704
 705         /* Initialize the hardware */
 706         if (em_hardware_init(adapter)) {
 707                 device_printf(dev, "Unable to initialize the hardware\n");
 708                 error = EIO;
 709                 goto err_hw_init;
 710         }
 711
 712         /* Allocate transmit descriptors and buffers */
 713         if (em_allocate_transmit_structures(adapter)) {
 714                 device_printf(dev, "Could not setup transmit structures\n");
 715                 error = ENOMEM;
 716                 goto err_tx_struct;
 717         }
 718
 719         /* Allocate receive descriptors and buffers */
 720         if (em_allocate_receive_structures(adapter)) {
 721                 device_printf(dev, "Could not setup receive structures\n");
 722                 error = ENOMEM;
 723                 goto err_rx_struct;
 724         }
 725
 726         /*
 727         **  Do interrupt configuration
 728         */
 729         if (adapter->msi > 1) /* Do MSI/X */
 730                 error = em_allocate_msix(adapter);
 731         else  /* MSI or Legacy */
 732                 error = em_allocate_legacy(adapter);
 733         if (error)
 734                 goto err_rx_struct;
 735
 736         /*
 737          * Get Wake-on-Lan and Management info for later use
 738          */
 739         em_get_wakeup(dev);
 740
 741         /* Setup OS specific network interface */
 742         em_setup_interface(dev, adapter);
 743
 744         /* Initialize statistics */
 745         em_update_stats_counters(adapter);
 746
 747         adapter->hw.mac.get_link_status = 1;
 748         em_update_link_status(adapter);
 749
 750         /* Indicate SOL/IDER usage */
 751         if (e1000_check_reset_block(&adapter->hw))
 752                 device_printf(dev,
 753                     "PHY reset is blocked due to SOL/IDER session.\n");
 754
 755         /* Do we need workaround for 82544 PCI-X adapter? */
 756         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
 757             adapter->hw.mac.type == e1000_82544)
 758                 adapter->pcix_82544 = TRUE;
 759         else
 760                 adapter->pcix_82544 = FALSE;
 761
 762 #if __FreeBSD_version >= 700029
 763         /* Register for VLAN events */
 764         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 765             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 766         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 767             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 768 #endif
 769
 770         /* Non-AMT based hardware can now take control from firmware */
 771         if (adapter->has_manage && !adapter->has_amt)
 772                 em_get_hw_control(adapter);
 773
 774         /* Tell the stack that the interface is not active */
 775         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 776
 777         INIT_DEBUGOUT("em_attach: end");
 778
 779         return (0);
 780
 781 err_rx_struct:
 782         em_free_transmit_structures(adapter);
 783 err_tx_struct:
 784 err_hw_init:
 785         em_release_hw_control(adapter);
 786         em_dma_free(adapter, &adapter->rxdma);
 787 err_rx_desc:
 788         em_dma_free(adapter, &adapter->txdma);
 789 err_tx_desc:
 790 err_pci:
 791         em_free_pci_resources(adapter);
 792         EM_TX_LOCK_DESTROY(adapter);
 793         EM_RX_LOCK_DESTROY(adapter);
 794         EM_CORE_LOCK_DESTROY(adapter);
 795
 796         return (error);
 797 }
 798
 799 /*********************************************************************
 800  *  Device removal routine
 801  *
 802  *  The detach entry point is called when the driver is being removed.
 803  *  This routine stops the adapter and deallocates all the resources
 804  *  that were allocated for driver operation.
 805  *
 806  *  return 0 on success, positive on failure
 807  *********************************************************************/
 808
 809 static int
 810 em_detach(device_t dev)
 811 {
 812         struct adapter  *adapter = device_get_softc(dev);
 813         struct ifnet    *ifp = adapter->ifp;
 814
 815         INIT_DEBUGOUT("em_detach: begin");
 816
 817         /* Make sure VLANS are not using driver */
 818 #if __FreeBSD_version >= 700000
 819         if (adapter->ifp->if_vlantrunk != NULL) {
 820 #else
 821         if (adapter->ifp->if_nvlans != 0) {
 822 #endif
 823                 device_printf(dev,"Vlan in use, detach first\n");
 824                 return (EBUSY);
 825         }
 826
 827 #ifdef DEVICE_POLLING
 828         if (ifp->if_capenable & IFCAP_POLLING)
 829                 ether_poll_deregister(ifp);
 830 #endif
 831
 832         EM_CORE_LOCK(adapter);
 833         EM_TX_LOCK(adapter);
 834         adapter->in_detach = 1;
 835         em_stop(adapter);
 836         e1000_phy_hw_reset(&adapter->hw);
 837
 838         em_release_manageability(adapter);
 839
 840         EM_TX_UNLOCK(adapter);
 841         EM_CORE_UNLOCK(adapter);
 842
 843 #if __FreeBSD_version >= 700029
 844         /* Unregister VLAN events */
 845         if (adapter->vlan_attach != NULL)
 846                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 847         if (adapter->vlan_detach != NULL)
 848                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 849 #endif
 850
 851         ether_ifdetach(adapter->ifp);
 852         callout_drain(&adapter->timer);
 853         callout_drain(&adapter->tx_fifo_timer);
 854
 855         em_free_pci_resources(adapter);
 856         bus_generic_detach(dev);
 857         if_free(ifp);
 858
 859         em_free_transmit_structures(adapter);
 860         em_free_receive_structures(adapter);
 861
 862         /* Free Transmit Descriptor ring */
 863         if (adapter->tx_desc_base) {
 864                 em_dma_free(adapter, &adapter->txdma);
 865                 adapter->tx_desc_base = NULL;
 866         }
 867
 868         /* Free Receive Descriptor ring */
 869         if (adapter->rx_desc_base) {
 870                 em_dma_free(adapter, &adapter->rxdma);
 871                 adapter->rx_desc_base = NULL;
 872         }
 873
 874         em_release_hw_control(adapter);
 875         EM_TX_LOCK_DESTROY(adapter);
 876         EM_RX_LOCK_DESTROY(adapter);
 877         EM_CORE_LOCK_DESTROY(adapter);
 878
 879         return (0);
 880 }
 881
 882 /*********************************************************************
 883  *
 884  *  Shutdown entry point
 885  *
 886  **********************************************************************/
 887
 888 static int
 889 em_shutdown(device_t dev)
 890 {
 891         return em_suspend(dev);
 892 }
 893
 894 /*
 895  * Suspend/resume device methods.
 896  */
 897 static int
 898 em_suspend(device_t dev)
 899 {
 900         struct adapter *adapter = device_get_softc(dev);
 901
 902         EM_CORE_LOCK(adapter);
 903
 904         em_release_manageability(adapter);
 905         em_release_hw_control(adapter);
 906         em_enable_wakeup(dev);
 907
 908         EM_CORE_UNLOCK(adapter);
 909
 910         return bus_generic_suspend(dev);
 911 }
 912
 913 static int
 914 em_resume(device_t dev)
 915 {
 916         struct adapter *adapter = device_get_softc(dev);
 917         struct ifnet *ifp = adapter->ifp;
 918
 919         EM_CORE_LOCK(adapter);
 920         em_init_locked(adapter);
 921         em_init_manageability(adapter);
 922         EM_CORE_UNLOCK(adapter);
 923         em_start(ifp);
 924
 925         return bus_generic_resume(dev);
 926 }
 927
 928
 929 /*********************************************************************
 930  *  Transmit entry point
 931  *
 932  *  em_start is called by the stack to initiate a transmit.
 933  *  The driver will remain in this routine as long as there are
 934  *  packets to transmit and transmit resources are available.
 935  *  In case resources are not available stack is notified and
 936  *  the packet is requeued.
 937  **********************************************************************/
 938
 939 #if __FreeBSD_version >= 800000
 940 static int
 941 em_mq_start_locked(struct ifnet *ifp, struct mbuf *m)
 942 {
 943         struct adapter  *adapter = ifp->if_softc;
 944         struct mbuf     *next;
 945         int error = E1000_SUCCESS;
 946
 947         EM_TX_LOCK_ASSERT(adapter);
 948         /* To allow being called from a tasklet */
 949         if (m == NULL)
 950                 goto process;
 951
 952         if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 953             IFF_DRV_RUNNING)
 954             || (!adapter->link_active)) {
 955                 error = drbr_enqueue(ifp, adapter->br, m);
 956                 return (error);
 957         } else if (drbr_empty(ifp, adapter->br) &&
 958             (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) {
 959                 if ((error = em_xmit(adapter, &m)) != 0) {
 960                         if (m)
 961                                 error = drbr_enqueue(ifp, adapter->br, m);
 962                         return (error);
 963                 } else {
 964                         /*
 965                          * We've bypassed the buf ring so we need to update
 966                          * ifp directly
 967                          */
 968                         drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
 969                         /*
 970                         ** Send a copy of the frame to the BPF
 971                         ** listener and set the watchdog on.
 972                         */
 973                         ETHER_BPF_MTAP(ifp, m);
 974                         adapter->watchdog_check = TRUE;
 975                 }
 976         } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0)
 977                 return (error);
 978
 979 process:
 980         if (drbr_empty(ifp, adapter->br))
 981                 return(error);
 982         /* Process the queue */
 983         while (TRUE) {
 984                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 985                         break;
 986                 next = drbr_dequeue(ifp, adapter->br);
 987                 if (next == NULL)
 988                         break;
 989                 if ((error = em_xmit(adapter, &next)) != 0) {
 990                         if (next != NULL)
 991                                 error = drbr_enqueue(ifp, adapter->br, next);
 992                         break;
 993                 }
 994                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 995                 ETHER_BPF_MTAP(ifp, next);
 996                 /* Set the watchdog */
 997                 adapter->watchdog_check = TRUE;
 998         }
 999
1000         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1001                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1002
1003         return (error);
1004 }
1005
1006 /*
1007 ** Multiqueue capable stack interface, this is not
1008 ** yet truely multiqueue, but that is coming...
1009 */
1010 static int
1011 em_mq_start(struct ifnet *ifp, struct mbuf *m)
1012 {
1013
1014         struct adapter *adapter = ifp->if_softc;
1015         int error = 0;
1016
1017         if (EM_TX_TRYLOCK(adapter)) {
1018                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1019                         error = em_mq_start_locked(ifp, m);
1020                 EM_TX_UNLOCK(adapter);
1021         } else
1022                 error = drbr_enqueue(ifp, adapter->br, m);
1023
1024         return (error);
1025 }
1026
1027 static void
1028 em_qflush(struct ifnet *ifp)
1029 {
1030         struct mbuf *m;
1031         struct adapter *adapter = (struct adapter *)ifp->if_softc;
1032
1033         EM_TX_LOCK(adapter);
1034         while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL)
1035                 m_freem(m);
1036         if_qflush(ifp);
1037         EM_TX_UNLOCK(adapter);
1038 }
1039 #endif /* FreeBSD_version */
1040
1041 static void
1042 em_start_locked(struct ifnet *ifp)
1043 {
1044         struct adapter  *adapter = ifp->if_softc;
1045         struct mbuf     *m_head;
1046
1047         EM_TX_LOCK_ASSERT(adapter);
1048
1049         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
1050             IFF_DRV_RUNNING)
1051                 return;
1052         if (!adapter->link_active)
1053                 return;
1054
1055         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
1056
1057                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
1058                 if (m_head == NULL)
1059                         break;
1060                 /*
1061                  *  Encapsulation can modify our pointer, and or make it
1062                  *  NULL on failure.  In that event, we can't requeue.
1063                  */
1064                 if (em_xmit(adapter, &m_head)) {
1065                         if (m_head == NULL)
1066                                 break;
1067                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1068                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1069                         break;
1070                 }
1071
1072                 /* Send a copy of the frame to the BPF listener */
1073                 ETHER_BPF_MTAP(ifp, m_head);
1074
1075                 /* Set timeout in case hardware has problems transmitting. */
1076                 adapter->watchdog_check = TRUE;
1077         }
1078         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1079                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1080
1081         return;
1082 }
1083
1084 static void
1085 em_start(struct ifnet *ifp)
1086 {
1087         struct adapter *adapter = ifp->if_softc;
1088
1089         EM_TX_LOCK(adapter);
1090         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1091                 em_start_locked(ifp);
1092         EM_TX_UNLOCK(adapter);
1093 }
1094
1095 /*********************************************************************
1096  *  Ioctl entry point
1097  *
1098  *  em_ioctl is called when the user wants to configure the
1099  *  interface.
1100  *
1101  *  return 0 on success, positive on failure
1102  **********************************************************************/
1103
1104 static int
1105 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1106 {
1107         struct adapter  *adapter = ifp->if_softc;
1108         struct ifreq *ifr = (struct ifreq *)data;
1109 #ifdef INET
1110         struct ifaddr *ifa = (struct ifaddr *)data;
1111 #endif
1112         int error = 0;
1113
1114         if (adapter->in_detach)
1115                 return (error);
1116
1117         switch (command) {
1118         case SIOCSIFADDR:
1119 #ifdef INET
1120                 if (ifa->ifa_addr->sa_family == AF_INET) {
1121                         /*
1122                          * XXX
1123                          * Since resetting hardware takes a very long time
1124                          * and results in link renegotiation we only
1125                          * initialize the hardware only when it is absolutely
1126                          * required.
1127                          */
1128                         ifp->if_flags |= IFF_UP;
1129                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1130                                 EM_CORE_LOCK(adapter);
1131                                 em_init_locked(adapter);
1132                                 EM_CORE_UNLOCK(adapter);
1133                         }
1134                         arp_ifinit(ifp, ifa);
1135                 } else
1136 #endif
1137                         error = ether_ioctl(ifp, command, data);
1138                 break;
1139         case SIOCSIFMTU:
1140             {
1141                 int max_frame_size;
1142                 u16 eeprom_data = 0;
1143
1144                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1145
1146                 EM_CORE_LOCK(adapter);
1147                 switch (adapter->hw.mac.type) {
1148                 case e1000_82573:
1149                         /*
1150                          * 82573 only supports jumbo frames
1151                          * if ASPM is disabled.
1152                          */
1153                         e1000_read_nvm(&adapter->hw,
1154                             NVM_INIT_3GIO_3, 1, &eeprom_data);
1155                         if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1156                                 max_frame_size = ETHER_MAX_LEN;
1157                                 break;
1158                         }
1159                         /* Allow Jumbo frames - fall thru */
1160                 case e1000_82571:
1161                 case e1000_82572:
1162                 case e1000_ich9lan:
1163                 case e1000_ich10lan:
1164                 case e1000_82574:
1165                 case e1000_80003es2lan: /* Limit Jumbo Frame size */
1166                         max_frame_size = 9234;
1167                         break;
1168                 case e1000_pchlan:
1169                         max_frame_size = 4096;
1170                         break;
1171                         /* Adapters that do not support jumbo frames */
1172                 case e1000_82542:
1173                 case e1000_82583:
1174                 case e1000_ich8lan:
1175                         max_frame_size = ETHER_MAX_LEN;
1176                         break;
1177                 default:
1178                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1179                 }
1180                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1181                     ETHER_CRC_LEN) {
1182                         EM_CORE_UNLOCK(adapter);
1183                         error = EINVAL;
1184                         break;
1185                 }
1186
1187                 ifp->if_mtu = ifr->ifr_mtu;
1188                 adapter->max_frame_size =
1189                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1190                 em_init_locked(adapter);
1191                 EM_CORE_UNLOCK(adapter);
1192                 break;
1193             }
1194         case SIOCSIFFLAGS:
1195                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1196                     SIOCSIFFLAGS (Set Interface Flags)");
1197                 EM_CORE_LOCK(adapter);
1198                 if (ifp->if_flags & IFF_UP) {
1199                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1200                                 if ((ifp->if_flags ^ adapter->if_flags) &
1201                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1202                                         em_disable_promisc(adapter);
1203                                         em_set_promisc(adapter);
1204                                 }
1205                         } else
1206                                 em_init_locked(adapter);
1207                 } else
1208                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1209                                 EM_TX_LOCK(adapter);
1210                                 em_stop(adapter);
1211                                 EM_TX_UNLOCK(adapter);
1212                         }
1213                 adapter->if_flags = ifp->if_flags;
1214                 EM_CORE_UNLOCK(adapter);
1215                 break;
1216         case SIOCADDMULTI:
1217         case SIOCDELMULTI:
1218                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1219                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1220                         EM_CORE_LOCK(adapter);
1221                         em_disable_intr(adapter);
1222                         em_set_multi(adapter);
1223                         if (adapter->hw.mac.type == e1000_82542 &&
1224                             adapter->hw.revision_id == E1000_REVISION_2) {
1225                                 em_initialize_receive_unit(adapter);
1226                         }
1227 #ifdef DEVICE_POLLING
1228                         if (!(ifp->if_capenable & IFCAP_POLLING))
1229 #endif
1230                                 em_enable_intr(adapter);
1231                         EM_CORE_UNLOCK(adapter);
1232                 }
1233                 break;
1234         case SIOCSIFMEDIA:
1235                 /* Check SOL/IDER usage */
1236                 EM_CORE_LOCK(adapter);
1237                 if (e1000_check_reset_block(&adapter->hw)) {
1238                         EM_CORE_UNLOCK(adapter);
1239                         device_printf(adapter->dev, "Media change is"
1240                             " blocked due to SOL/IDER session.\n");
1241                         break;
1242                 }
1243                 EM_CORE_UNLOCK(adapter);
1244         case SIOCGIFMEDIA:
1245                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1246                     SIOCxIFMEDIA (Get/Set Interface Media)");
1247                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1248                 break;
1249         case SIOCSIFCAP:
1250             {
1251                 int mask, reinit;
1252
1253                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1254                 reinit = 0;
1255                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1256 #ifdef DEVICE_POLLING
1257                 if (mask & IFCAP_POLLING) {
1258                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1259                                 error = ether_poll_register(em_poll, ifp);
1260                                 if (error)
1261                                         return (error);
1262                                 EM_CORE_LOCK(adapter);
1263                                 em_disable_intr(adapter);
1264                                 ifp->if_capenable |= IFCAP_POLLING;
1265                                 EM_CORE_UNLOCK(adapter);
1266                         } else {
1267                                 error = ether_poll_deregister(ifp);
1268                                 /* Enable interrupt even in error case */
1269                                 EM_CORE_LOCK(adapter);
1270                                 em_enable_intr(adapter);
1271                                 ifp->if_capenable &= ~IFCAP_POLLING;
1272                                 EM_CORE_UNLOCK(adapter);
1273                         }
1274                 }
1275 #endif
1276                 if (mask & IFCAP_HWCSUM) {
1277                         ifp->if_capenable ^= IFCAP_HWCSUM;
1278                         reinit = 1;
1279                 }
1280 #if __FreeBSD_version >= 700000
1281                 if (mask & IFCAP_TSO4) {
1282                         ifp->if_capenable ^= IFCAP_TSO4;
1283                         reinit = 1;
1284                 }
1285 #endif
1286                 if (mask & IFCAP_VLAN_HWTAGGING) {
1287                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1288                         reinit = 1;
1289                 }
1290                 if ((mask & IFCAP_WOL) &&
1291                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
1292                         if (mask & IFCAP_WOL_MCAST)
1293                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1294                         if (mask & IFCAP_WOL_MAGIC)
1295                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1296                 }
1297                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1298                         em_init(adapter);
1299 #if __FreeBSD_version >= 700000
1300                 VLAN_CAPABILITIES(ifp);
1301 #endif
1302                 break;
1303             }
1304
1305         default:
1306                 error = ether_ioctl(ifp, command, data);
1307                 break;
1308         }
1309
1310         return (error);
1311 }
1312
1313
1314 /*********************************************************************
1315  *  Init entry point
1316  *
1317  *  This routine is used in two ways. It is used by the stack as
1318  *  init entry point in network interface structure. It is also used
1319  *  by the driver as a hw/sw initialization routine to get to a
1320  *  consistent state.
1321  *
1322  *  return 0 on success, positive on failure
1323  **********************************************************************/
1324
1325 static void
1326 em_init_locked(struct adapter *adapter)
1327 {
1328         struct ifnet    *ifp = adapter->ifp;
1329         device_t        dev = adapter->dev;
1330         u32             pba;
1331
1332         INIT_DEBUGOUT("em_init: begin");
1333
1334         EM_CORE_LOCK_ASSERT(adapter);
1335
1336         EM_TX_LOCK(adapter);
1337         em_stop(adapter);
1338         EM_TX_UNLOCK(adapter);
1339
1340         /*
1341          * Packet Buffer Allocation (PBA)
1342          * Writing PBA sets the receive portion of the buffer
1343          * the remainder is used for the transmit buffer.
1344          *
1345          * Devices before the 82547 had a Packet Buffer of 64K.
1346          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1347          * After the 82547 the buffer was reduced to 40K.
1348          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1349          *   Note: default does not leave enough room for Jumbo Frame >10k.
1350          */
1351         switch (adapter->hw.mac.type) {
1352         case e1000_82547:
1353         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1354                 if (adapter->max_frame_size > 8192)
1355                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1356                 else
1357                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1358                 adapter->tx_fifo_head = 0;
1359                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1360                 adapter->tx_fifo_size =
1361                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1362                 break;
1363         /* Total Packet Buffer on these is 48K */
1364         case e1000_82571:
1365         case e1000_82572:
1366         case e1000_80003es2lan:
1367                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1368                 break;
1369         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1370                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1371                 break;
1372         case e1000_82574:
1373         case e1000_82583:
1374                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1375                 break;
1376         case e1000_ich9lan:
1377         case e1000_ich10lan:
1378         case e1000_pchlan:
1379                 pba = E1000_PBA_10K;
1380                 break;
1381         case e1000_ich8lan:
1382                 pba = E1000_PBA_8K;
1383                 break;
1384         default:
1385                 /* Devices before 82547 had a Packet Buffer of 64K.   */
1386                 if (adapter->max_frame_size > 8192)
1387                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1388                 else
1389                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1390         }
1391
1392         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1393         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1394
1395         /* Get the latest mac address, User can use a LAA */
1396         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1397               ETHER_ADDR_LEN);
1398
1399         /* Put the address into the Receive Address Array */
1400         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1401
1402         /*
1403          * With the 82571 adapter, RAR[0] may be overwritten
1404          * when the other port is reset, we make a duplicate
1405          * in RAR[14] for that eventuality, this assures
1406          * the interface continues to function.
1407          */
1408         if (adapter->hw.mac.type == e1000_82571) {
1409                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1410                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1411                     E1000_RAR_ENTRIES - 1);
1412         }
1413
1414         /* Initialize the hardware */
1415         if (em_hardware_init(adapter)) {
1416                 device_printf(dev, "Unable to initialize the hardware\n");
1417                 return;
1418         }
1419         em_update_link_status(adapter);
1420
1421         /* Setup VLAN support, basic and offload if available */
1422         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1423
1424 #if __FreeBSD_version < 700029
1425         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1426                 u32 ctrl;
1427                 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1428                 ctrl |= E1000_CTRL_VME;
1429                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1430         }
1431 #else
1432         /* Use real VLAN Filter support */
1433         em_setup_vlan_hw_support(adapter);
1434 #endif
1435
1436         /* Set hardware offload abilities */
1437         ifp->if_hwassist = 0;
1438         if (adapter->hw.mac.type >= e1000_82543) {
1439                 if (ifp->if_capenable & IFCAP_TXCSUM)
1440                         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1441 #if __FreeBSD_version >= 700000
1442                 if (ifp->if_capenable & IFCAP_TSO4)
1443                         ifp->if_hwassist |= CSUM_TSO;
1444 #endif
1445         }
1446
1447         /* Configure for OS presence */
1448         em_init_manageability(adapter);
1449
1450         /* Prepare transmit descriptors and buffers */
1451         em_setup_transmit_structures(adapter);
1452         em_initialize_transmit_unit(adapter);
1453
1454         /* Setup Multicast table */
1455         em_set_multi(adapter);
1456
1457         /* Prepare receive descriptors and buffers */
1458         if (em_setup_receive_structures(adapter)) {
1459                 device_printf(dev, "Could not setup receive structures\n");
1460                 EM_TX_LOCK(adapter);
1461                 em_stop(adapter);
1462                 EM_TX_UNLOCK(adapter);
1463                 return;
1464         }
1465         em_initialize_receive_unit(adapter);
1466
1467         /* Don't lose promiscuous settings */
1468         em_set_promisc(adapter);
1469
1470         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1471         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1472
1473         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1474         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1475
1476         /* MSI/X configuration for 82574 */
1477         if (adapter->hw.mac.type == e1000_82574) {
1478                 int tmp;
1479                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1480                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1481                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1482                 /*
1483                 ** Set the IVAR - interrupt vector routing.
1484                 ** Each nibble represents a vector, high bit
1485                 ** is enable, other 3 bits are the MSIX table
1486                 ** entry, we map RXQ0 to 0, TXQ0 to 1, and
1487                 ** Link (other) to 2, hence the magic number.
1488                 */
1489                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1490         }
1491
1492 #ifdef DEVICE_POLLING
1493         /*
1494          * Only enable interrupts if we are not polling, make sure
1495          * they are off otherwise.
1496          */
1497         if (ifp->if_capenable & IFCAP_POLLING)
1498                 em_disable_intr(adapter);
1499         else
1500 #endif /* DEVICE_POLLING */
1501                 em_enable_intr(adapter);
1502
1503         /* AMT based hardware can now take control from firmware */
1504         if (adapter->has_manage && adapter->has_amt)
1505                 em_get_hw_control(adapter);
1506
1507         /* Don't reset the phy next time init gets called */
1508         adapter->hw.phy.reset_disable = TRUE;
1509 }
1510
1511 static void
1512 em_init(void *arg)
1513 {
1514         struct adapter *adapter = arg;
1515
1516         EM_CORE_LOCK(adapter);
1517         em_init_locked(adapter);
1518         EM_CORE_UNLOCK(adapter);
1519 }
1520
1521
1522 #ifdef DEVICE_POLLING
1523 /*********************************************************************
1524  *
1525  *  Legacy polling routine
1526  *
1527  *********************************************************************/
1528 static int
1529 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1530 {
1531         struct adapter *adapter = ifp->if_softc;
1532         u32             reg_icr, rx_done = 0;
1533
1534         EM_CORE_LOCK(adapter);
1535         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1536                 EM_CORE_UNLOCK(adapter);
1537                 return (rx_done);
1538         }
1539
1540         if (cmd == POLL_AND_CHECK_STATUS) {
1541                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1542                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1543                         callout_stop(&adapter->timer);
1544                         adapter->hw.mac.get_link_status = 1;
1545                         em_update_link_status(adapter);
1546                         callout_reset(&adapter->timer, hz,
1547                             em_local_timer, adapter);
1548                 }
1549         }
1550         EM_CORE_UNLOCK(adapter);
1551
1552         rx_done = em_rxeof(adapter, count);
1553
1554         EM_TX_LOCK(adapter);
1555         em_txeof(adapter);
1556 #if __FreeBSD_version >= 800000
1557         if (!drbr_empty(ifp, adapter->br))
1558                 em_mq_start_locked(ifp, NULL);
1559 #else
1560         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1561                 em_start_locked(ifp);
1562 #endif
1563         EM_TX_UNLOCK(adapter);
1564         return (rx_done);
1565 }
1566 #endif /* DEVICE_POLLING */
1567
1568 #ifdef EM_LEGACY_IRQ
1569 /*********************************************************************
1570  *
1571  *  Legacy Interrupt Service routine
1572  *
1573  *********************************************************************/
1574
1575 static void
1576 em_intr(void *arg)
1577 {
1578         struct adapter  *adapter = arg;
1579         struct ifnet    *ifp = adapter->ifp;
1580         u32             reg_icr;
1581
1582
1583         if (ifp->if_capenable & IFCAP_POLLING)
1584                 return;
1585
1586         EM_CORE_LOCK(adapter);
1587         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1588         if (reg_icr & E1000_ICR_RXO)
1589                 adapter->rx_overruns++;
1590         if ((reg_icr == 0xffffffff) || (reg_icr == 0)||
1591             (adapter->hw.mac.type >= e1000_82571 &&
1592             (reg_icr & E1000_ICR_INT_ASSERTED) == 0))
1593                         goto out;
1594
1595         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1596                         goto out;
1597
1598         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1599                 callout_stop(&adapter->timer);
1600                 adapter->hw.mac.get_link_status = 1;
1601                 em_update_link_status(adapter);
1602                 /* Deal with TX cruft when link lost */
1603                 em_tx_purge(adapter);
1604                 callout_reset(&adapter->timer, hz,
1605                     em_local_timer, adapter);
1606                 goto out;
1607         }
1608
1609         EM_TX_LOCK(adapter);
1610         em_txeof(adapter);
1611         em_rxeof(adapter, -1);
1612         em_txeof(adapter);
1613         if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
1614             !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1615                 em_start_locked(ifp);
1616         EM_TX_UNLOCK(adapter);
1617
1618 out:
1619         EM_CORE_UNLOCK(adapter);
1620         return;
1621 }
1622
1623 #else /* EM_FAST_IRQ, then fast interrupt routines only */
1624
1625 static void
1626 em_handle_link(void *context, int pending)
1627 {
1628         struct adapter  *adapter = context;
1629         struct ifnet *ifp = adapter->ifp;
1630
1631         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1632                 return;
1633
1634         EM_CORE_LOCK(adapter);
1635         callout_stop(&adapter->timer);
1636         em_update_link_status(adapter);
1637         /* Deal with TX cruft when link lost */
1638         em_tx_purge(adapter);
1639         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1640         EM_CORE_UNLOCK(adapter);
1641 }
1642
1643
1644 /* Combined RX/TX handler, used by Legacy and MSI */
1645 static void
1646 em_handle_rxtx(void *context, int pending)
1647 {
1648         struct adapter  *adapter = context;
1649         struct ifnet    *ifp = adapter->ifp;
1650
1651
1652         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1653                 if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1654                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1655                 EM_TX_LOCK(adapter);
1656                 em_txeof(adapter);
1657
1658 #if __FreeBSD_version >= 800000
1659                 if (!drbr_empty(ifp, adapter->br))
1660                         em_mq_start_locked(ifp, NULL);
1661 #else
1662                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1663                         em_start_locked(ifp);
1664 #endif
1665                 EM_TX_UNLOCK(adapter);
1666         }
1667
1668         em_enable_intr(adapter);
1669 }
1670
1671 /*********************************************************************
1672  *
1673  *  Fast Legacy/MSI Combined Interrupt Service routine
1674  *
1675  *********************************************************************/
1676 #if __FreeBSD_version < 700000
1677 #define FILTER_STRAY
1678 #define FILTER_HANDLED
1679 static void
1680 #else
1681 static int
1682 #endif
1683 em_irq_fast(void *arg)
1684 {
1685         struct adapter  *adapter = arg;
1686         struct ifnet    *ifp;
1687         u32             reg_icr;
1688
1689         ifp = adapter->ifp;
1690
1691         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1692
1693         /* Hot eject?  */
1694         if (reg_icr == 0xffffffff)
1695                 return FILTER_STRAY;
1696
1697         /* Definitely not our interrupt.  */
1698         if (reg_icr == 0x0)
1699                 return FILTER_STRAY;
1700
1701         /*
1702          * Starting with the 82571 chip, bit 31 should be used to
1703          * determine whether the interrupt belongs to us.
1704          */
1705         if (adapter->hw.mac.type >= e1000_82571 &&
1706             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1707                 return FILTER_STRAY;
1708
1709         /*
1710          * Mask interrupts until the taskqueue is finished running.  This is
1711          * cheap, just assume that it is needed.  This also works around the
1712          * MSI message reordering errata on certain systems.
1713          */
1714         em_disable_intr(adapter);
1715         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1716
1717         /* Link status change */
1718         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1719                 adapter->hw.mac.get_link_status = 1;
1720                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1721         }
1722
1723         if (reg_icr & E1000_ICR_RXO)
1724                 adapter->rx_overruns++;
1725         return FILTER_HANDLED;
1726 }
1727
1728 /*********************************************************************
1729  *
1730  *  MSIX Interrupt Service Routines
1731  *
1732  **********************************************************************/
1733 #define EM_MSIX_TX      0x00040000
1734 #define EM_MSIX_RX      0x00010000
1735 #define EM_MSIX_LINK    0x00100000
1736
1737 static void
1738 em_msix_tx(void *arg)
1739 {
1740         struct adapter *adapter = arg;
1741         struct ifnet    *ifp = adapter->ifp;
1742
1743         ++adapter->tx_irq;
1744         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1745                 EM_TX_LOCK(adapter);
1746                 em_txeof(adapter);
1747                 EM_TX_UNLOCK(adapter);
1748                 taskqueue_enqueue(adapter->tq, &adapter->tx_task);
1749         }
1750         /* Reenable this interrupt */
1751         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX);
1752         return;
1753 }
1754
1755 /*********************************************************************
1756  *
1757  *  MSIX RX Interrupt Service routine
1758  *
1759  **********************************************************************/
1760
1761 static void
1762 em_msix_rx(void *arg)
1763 {
1764         struct adapter *adapter = arg;
1765         struct ifnet    *ifp = adapter->ifp;
1766
1767         ++adapter->rx_irq;
1768         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1769             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1770                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1771         /* Reenable this interrupt */
1772         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX);
1773         return;
1774 }
1775
1776 /*********************************************************************
1777  *
1778  *  MSIX Link Fast Interrupt Service routine
1779  *
1780  **********************************************************************/
1781
1782 static void
1783 em_msix_link(void *arg)
1784 {
1785         struct adapter  *adapter = arg;
1786         u32             reg_icr;
1787
1788         ++adapter->link_irq;
1789         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1790
1791         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1792                 adapter->hw.mac.get_link_status = 1;
1793                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1794         }
1795         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1796             EM_MSIX_LINK | E1000_IMS_LSC);
1797         return;
1798 }
1799
1800 static void
1801 em_handle_rx(void *context, int pending)
1802 {
1803         struct adapter  *adapter = context;
1804         struct ifnet    *ifp = adapter->ifp;
1805
1806         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1807             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1808                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1809
1810 }
1811
1812 static void
1813 em_handle_tx(void *context, int pending)
1814 {
1815         struct adapter  *adapter = context;
1816         struct ifnet    *ifp = adapter->ifp;
1817
1818         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1819                 if (!EM_TX_TRYLOCK(adapter))
1820                         return;
1821                 em_txeof(adapter);
1822 #if __FreeBSD_version >= 800000
1823                 if (!drbr_empty(ifp, adapter->br))
1824                         em_mq_start_locked(ifp, NULL);
1825 #else
1826                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1827                         em_start_locked(ifp);
1828 #endif
1829                 EM_TX_UNLOCK(adapter);
1830         }
1831 }
1832 #endif /* EM_FAST_IRQ */
1833
1834 /*********************************************************************
1835  *
1836  *  Media Ioctl callback
1837  *
1838  *  This routine is called whenever the user queries the status of
1839  *  the interface using ifconfig.
1840  *
1841  **********************************************************************/
1842 static void
1843 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1844 {
1845         struct adapter *adapter = ifp->if_softc;
1846         u_char fiber_type = IFM_1000_SX;
1847
1848         INIT_DEBUGOUT("em_media_status: begin");
1849
1850         EM_CORE_LOCK(adapter);
1851         em_update_link_status(adapter);
1852
1853         ifmr->ifm_status = IFM_AVALID;
1854         ifmr->ifm_active = IFM_ETHER;
1855
1856         if (!adapter->link_active) {
1857                 EM_CORE_UNLOCK(adapter);
1858                 return;
1859         }
1860
1861         ifmr->ifm_status |= IFM_ACTIVE;
1862
1863         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1864             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1865                 if (adapter->hw.mac.type == e1000_82545)
1866                         fiber_type = IFM_1000_LX;
1867                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1868         } else {
1869                 switch (adapter->link_speed) {
1870                 case 10:
1871                         ifmr->ifm_active |= IFM_10_T;
1872                         break;
1873                 case 100:
1874                         ifmr->ifm_active |= IFM_100_TX;
1875                         break;
1876                 case 1000:
1877                         ifmr->ifm_active |= IFM_1000_T;
1878                         break;
1879                 }
1880                 if (adapter->link_duplex == FULL_DUPLEX)
1881                         ifmr->ifm_active |= IFM_FDX;
1882                 else
1883                         ifmr->ifm_active |= IFM_HDX;
1884         }
1885         EM_CORE_UNLOCK(adapter);
1886 }
1887
1888 /*********************************************************************
1889  *
1890  *  Media Ioctl callback
1891  *
1892  *  This routine is called when the user changes speed/duplex using
1893  *  media/mediopt option with ifconfig.
1894  *
1895  **********************************************************************/
1896 static int
1897 em_media_change(struct ifnet *ifp)
1898 {
1899         struct adapter *adapter = ifp->if_softc;
1900         struct ifmedia  *ifm = &adapter->media;
1901
1902         INIT_DEBUGOUT("em_media_change: begin");
1903
1904         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1905                 return (EINVAL);
1906
1907         EM_CORE_LOCK(adapter);
1908         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1909         case IFM_AUTO:
1910                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1911                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1912                 break;
1913         case IFM_1000_LX:
1914         case IFM_1000_SX:
1915         case IFM_1000_T:
1916                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1917                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1918                 break;
1919         case IFM_100_TX:
1920                 adapter->hw.mac.autoneg = FALSE;
1921                 adapter->hw.phy.autoneg_advertised = 0;
1922                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1923                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1924                 else
1925                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1926                 break;
1927         case IFM_10_T:
1928                 adapter->hw.mac.autoneg = FALSE;
1929                 adapter->hw.phy.autoneg_advertised = 0;
1930                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1931                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1932                 else
1933                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1934                 break;
1935         default:
1936                 device_printf(adapter->dev, "Unsupported media type\n");
1937         }
1938
1939         /* As the speed/duplex settings my have changed we need to
1940          * reset the PHY.
1941          */
1942         adapter->hw.phy.reset_disable = FALSE;
1943
1944         em_init_locked(adapter);
1945         EM_CORE_UNLOCK(adapter);
1946
1947         return (0);
1948 }
1949
1950 /*********************************************************************
1951  *
1952  *  This routine maps the mbufs to tx descriptors.
1953  *
1954  *  return 0 on success, positive on failure
1955  **********************************************************************/
1956
1957 static int
1958 em_xmit(struct adapter *adapter, struct mbuf **m_headp)
1959 {
1960         bus_dma_segment_t       segs[EM_MAX_SCATTER];
1961         bus_dmamap_t            map;
1962         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
1963         struct e1000_tx_desc    *ctxd = NULL;
1964         struct mbuf             *m_head;
1965         u32                     txd_upper, txd_lower, txd_used, txd_saved;
1966         int                     nsegs, i, j, first, last = 0;
1967         int                     error, do_tso, tso_desc = 0;
1968 #if __FreeBSD_version < 700000
1969         struct m_tag            *mtag;
1970 #endif
1971         m_head = *m_headp;
1972         txd_upper = txd_lower = txd_used = txd_saved = 0;
1973
1974 #if __FreeBSD_version >= 700000
1975         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1976 #else
1977         do_tso = 0;
1978 #endif
1979
1980         /*
1981          * Force a cleanup if number of TX descriptors
1982          * available hits the threshold
1983          */
1984         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1985                 em_txeof(adapter);
1986                 /* Now do we at least have a minimal? */
1987                 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
1988                         adapter->no_tx_desc_avail1++;
1989                         return (ENOBUFS);
1990                 }
1991         }
1992
1993
1994         /*
1995          * TSO workaround:
1996          *  If an mbuf is only header we need
1997          *     to pull 4 bytes of data into it.
1998          */
1999         if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
2000                 m_head = m_pullup(m_head, M_TSO_LEN + 4);
2001                 *m_headp = m_head;
2002                 if (m_head == NULL)
2003                         return (ENOBUFS);
2004         }
2005
2006         /*
2007          * Map the packet for DMA
2008          *
2009          * Capture the first descriptor index,
2010          * this descriptor will have the index
2011          * of the EOP which is the only one that
2012          * now gets a DONE bit writeback.
2013          */
2014         first = adapter->next_avail_tx_desc;
2015         tx_buffer = &adapter->tx_buffer_area[first];
2016         tx_buffer_mapped = tx_buffer;
2017         map = tx_buffer->map;
2018
2019         error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2020             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2021
2022         /*
2023          * There are two types of errors we can (try) to handle:
2024          * - EFBIG means the mbuf chain was too long and bus_dma ran
2025          *   out of segments.  Defragment the mbuf chain and try again.
2026          * - ENOMEM means bus_dma could not obtain enough bounce buffers
2027          *   at this point in time.  Defer sending and try again later.
2028          * All other errors, in particular EINVAL, are fatal and prevent the
2029          * mbuf chain from ever going through.  Drop it and report error.
2030          */
2031         if (error == EFBIG) {
2032                 struct mbuf *m;
2033
2034                 m = m_defrag(*m_headp, M_DONTWAIT);
2035                 if (m == NULL) {
2036                         adapter->mbuf_alloc_failed++;
2037                         m_freem(*m_headp);
2038                         *m_headp = NULL;
2039                         return (ENOBUFS);
2040                 }
2041                 *m_headp = m;
2042
2043                 /* Try it again */
2044                 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2045                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2046
2047                 if (error) {
2048                         adapter->no_tx_dma_setup++;
2049                         m_freem(*m_headp);
2050                         *m_headp = NULL;
2051                         return (error);
2052                 }
2053         } else if (error != 0) {
2054                 adapter->no_tx_dma_setup++;
2055                 return (error);
2056         }
2057
2058         /*
2059          * TSO Hardware workaround, if this packet is not
2060          * TSO, and is only a single descriptor long, and
2061          * it follows a TSO burst, then we need to add a
2062          * sentinel descriptor to prevent premature writeback.
2063          */
2064         if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
2065                 if (nsegs == 1)
2066                         tso_desc = TRUE;
2067                 adapter->tx_tso = FALSE;
2068         }
2069
2070         if (nsegs > (adapter->num_tx_desc_avail - 2)) {
2071                 adapter->no_tx_desc_avail2++;
2072                 bus_dmamap_unload(adapter->txtag, map);
2073                 return (ENOBUFS);
2074         }
2075         m_head = *m_headp;
2076
2077         /* Do hardware assists */
2078 #if __FreeBSD_version >= 700000
2079         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2080                 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
2081                 if (error != TRUE)
2082                         return (ENXIO); /* something foobar */
2083                 /* we need to make a final sentinel transmit desc */
2084                 tso_desc = TRUE;
2085         } else
2086 #endif
2087         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2088                 em_transmit_checksum_setup(adapter,  m_head,
2089                     &txd_upper, &txd_lower);
2090
2091         i = adapter->next_avail_tx_desc;
2092         if (adapter->pcix_82544)
2093                 txd_saved = i;
2094
2095         /* Set up our transmit descriptors */
2096         for (j = 0; j < nsegs; j++) {
2097                 bus_size_t seg_len;
2098                 bus_addr_t seg_addr;
2099                 /* If adapter is 82544 and on PCIX bus */
2100                 if(adapter->pcix_82544) {
2101                         DESC_ARRAY      desc_array;
2102                         u32             array_elements, counter;
2103                         /*
2104                          * Check the Address and Length combination and
2105                          * split the data accordingly
2106                          */
2107                         array_elements = em_fill_descriptors(segs[j].ds_addr,
2108                             segs[j].ds_len, &desc_array);
2109                         for (counter = 0; counter < array_elements; counter++) {
2110                                 if (txd_used == adapter->num_tx_desc_avail) {
2111                                         adapter->next_avail_tx_desc = txd_saved;
2112                                         adapter->no_tx_desc_avail2++;
2113                                         bus_dmamap_unload(adapter->txtag, map);
2114                                         return (ENOBUFS);
2115                                 }
2116                                 tx_buffer = &adapter->tx_buffer_area[i];
2117                                 ctxd = &adapter->tx_desc_base[i];
2118                                 ctxd->buffer_addr = htole64(
2119                                     desc_array.descriptor[counter].address);
2120                                 ctxd->lower.data = htole32(
2121                                     (adapter->txd_cmd | txd_lower | (u16)
2122                                     desc_array.descriptor[counter].length));
2123                                 ctxd->upper.data =
2124                                     htole32((txd_upper));
2125                                 last = i;
2126                                 if (++i == adapter->num_tx_desc)
2127                                          i = 0;
2128                                 tx_buffer->m_head = NULL;
2129                                 tx_buffer->next_eop = -1;
2130                                 txd_used++;
2131                         }
2132                 } else {
2133                         tx_buffer = &adapter->tx_buffer_area[i];
2134                         ctxd = &adapter->tx_desc_base[i];
2135                         seg_addr = segs[j].ds_addr;
2136                         seg_len  = segs[j].ds_len;
2137                         /*
2138                         ** TSO Workaround:
2139                         ** If this is the last descriptor, we want to
2140                         ** split it so we have a small final sentinel
2141                         */
2142                         if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2143                                 seg_len -= 4;
2144                                 ctxd->buffer_addr = htole64(seg_addr);
2145                                 ctxd->lower.data = htole32(
2146                                 adapter->txd_cmd | txd_lower | seg_len);
2147                                 ctxd->upper.data =
2148                                     htole32(txd_upper);
2149                                 if (++i == adapter->num_tx_desc)
2150                                         i = 0;
2151                                 /* Now make the sentinel */
2152                                 ++txd_used; /* using an extra txd */
2153                                 ctxd = &adapter->tx_desc_base[i];
2154                                 tx_buffer = &adapter->tx_buffer_area[i];
2155                                 ctxd->buffer_addr =
2156                                     htole64(seg_addr + seg_len);
2157                                 ctxd->lower.data = htole32(
2158                                 adapter->txd_cmd | txd_lower | 4);
2159                                 ctxd->upper.data =
2160                                     htole32(txd_upper);
2161                                 last = i;
2162                                 if (++i == adapter->num_tx_desc)
2163                                         i = 0;
2164                         } else {
2165                                 ctxd->buffer_addr = htole64(seg_addr);
2166                                 ctxd->lower.data = htole32(
2167                                 adapter->txd_cmd | txd_lower | seg_len);
2168                                 ctxd->upper.data =
2169                                     htole32(txd_upper);
2170                                 last = i;
2171                                 if (++i == adapter->num_tx_desc)
2172                                         i = 0;
2173                         }
2174                         tx_buffer->m_head = NULL;
2175                         tx_buffer->next_eop = -1;
2176                 }
2177         }
2178
2179         adapter->next_avail_tx_desc = i;
2180         if (adapter->pcix_82544)
2181                 adapter->num_tx_desc_avail -= txd_used;
2182         else {
2183                 adapter->num_tx_desc_avail -= nsegs;
2184                 if (tso_desc) /* TSO used an extra for sentinel */
2185                         adapter->num_tx_desc_avail -= txd_used;
2186         }
2187
2188         /*
2189         ** Handle VLAN tag, this is the
2190         ** biggest difference between
2191         ** 6.x and 7
2192         */
2193 #if __FreeBSD_version < 700000
2194         /* Find out if we are in vlan mode. */
2195         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
2196         if (mtag != NULL) {
2197                 ctxd->upper.fields.special =
2198                     htole16(VLAN_TAG_VALUE(mtag));
2199 #else /* FreeBSD 7 */
2200         if (m_head->m_flags & M_VLANTAG) {
2201                 /* Set the vlan id. */
2202                 ctxd->upper.fields.special =
2203                     htole16(m_head->m_pkthdr.ether_vtag);
2204 #endif
2205                 /* Tell hardware to add tag */
2206                 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
2207         }
2208
2209         tx_buffer->m_head = m_head;
2210         tx_buffer_mapped->map = tx_buffer->map;
2211         tx_buffer->map = map;
2212         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
2213
2214         /*
2215          * Last Descriptor of Packet
2216          * needs End Of Packet (EOP)
2217          * and Report Status (RS)
2218          */
2219         ctxd->lower.data |=
2220             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2221         /*
2222          * Keep track in the first buffer which
2223          * descriptor will be written back
2224          */
2225         tx_buffer = &adapter->tx_buffer_area[first];
2226         tx_buffer->next_eop = last;
2227
2228         /*
2229          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2230          * that this frame is available to transmit.
2231          */
2232         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2233             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2234         if (adapter->hw.mac.type == e1000_82547 &&
2235             adapter->link_duplex == HALF_DUPLEX)
2236                 em_82547_move_tail(adapter);
2237         else {
2238                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
2239                 if (adapter->hw.mac.type == e1000_82547)
2240                         em_82547_update_fifo_head(adapter,
2241                             m_head->m_pkthdr.len);
2242         }
2243
2244         return (0);
2245 }
2246
2247 /*********************************************************************
2248  *
2249  * 82547 workaround to avoid controller hang in half-duplex environment.
2250  * The workaround is to avoid queuing a large packet that would span
2251  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
2252  * in this case. We do that only when FIFO is quiescent.
2253  *
2254  **********************************************************************/
2255 static void
2256 em_82547_move_tail(void *arg)
2257 {
2258         struct adapter *adapter = arg;
2259         struct e1000_tx_desc *tx_desc;
2260         u16     hw_tdt, sw_tdt, length = 0;
2261         bool    eop = 0;
2262
2263         EM_TX_LOCK_ASSERT(adapter);
2264
2265         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2266         sw_tdt = adapter->next_avail_tx_desc;
2267
2268         while (hw_tdt != sw_tdt) {
2269                 tx_desc = &adapter->tx_desc_base[hw_tdt];
2270                 length += tx_desc->lower.flags.length;
2271                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2272                 if (++hw_tdt == adapter->num_tx_desc)
2273                         hw_tdt = 0;
2274
2275                 if (eop) {
2276                         if (em_82547_fifo_workaround(adapter, length)) {
2277                                 adapter->tx_fifo_wrk_cnt++;
2278                                 callout_reset(&adapter->tx_fifo_timer, 1,
2279                                         em_82547_move_tail, adapter);
2280                                 break;
2281                         }
2282                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2283                         em_82547_update_fifo_head(adapter, length);
2284                         length = 0;
2285                 }
2286         }
2287 }
2288
2289 static int
2290 em_82547_fifo_workaround(struct adapter *adapter, int len)
2291 {
2292         int fifo_space, fifo_pkt_len;
2293
2294         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2295
2296         if (adapter->link_duplex == HALF_DUPLEX) {
2297                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2298
2299                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2300                         if (em_82547_tx_fifo_reset(adapter))
2301                                 return (0);
2302                         else
2303                                 return (1);
2304                 }
2305         }
2306
2307         return (0);
2308 }
2309
2310 static void
2311 em_82547_update_fifo_head(struct adapter *adapter, int len)
2312 {
2313         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2314
2315         /* tx_fifo_head is always 16 byte aligned */
2316         adapter->tx_fifo_head += fifo_pkt_len;
2317         if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
2318                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2319         }
2320 }
2321
2322
2323 static int
2324 em_82547_tx_fifo_reset(struct adapter *adapter)
2325 {
2326         u32 tctl;
2327
2328         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2329             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2330             (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
2331             E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2332             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2333             E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2334             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2335                 /* Disable TX unit */
2336                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2337                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2338                     tctl & ~E1000_TCTL_EN);
2339
2340                 /* Reset FIFO pointers */
2341                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2342                     adapter->tx_head_addr);
2343                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2344                     adapter->tx_head_addr);
2345                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2346                     adapter->tx_head_addr);
2347                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2348                     adapter->tx_head_addr);
2349
2350                 /* Re-enable TX unit */
2351                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2352                 E1000_WRITE_FLUSH(&adapter->hw);
2353
2354                 adapter->tx_fifo_head = 0;
2355                 adapter->tx_fifo_reset_cnt++;
2356
2357                 return (TRUE);
2358         }
2359         else {
2360                 return (FALSE);
2361         }
2362 }
2363
2364 static void
2365 em_set_promisc(struct adapter *adapter)
2366 {
2367         struct ifnet    *ifp = adapter->ifp;
2368         u32             reg_rctl;
2369
2370         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2371
2372         if (ifp->if_flags & IFF_PROMISC) {
2373                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2374                 /* Turn this on if you want to see bad packets */
2375                 if (em_debug_sbp)
2376                         reg_rctl |= E1000_RCTL_SBP;
2377                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2378         } else if (ifp->if_flags & IFF_ALLMULTI) {
2379                 reg_rctl |= E1000_RCTL_MPE;
2380                 reg_rctl &= ~E1000_RCTL_UPE;
2381                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2382         }
2383 }
2384
2385 static void
2386 em_disable_promisc(struct adapter *adapter)
2387 {
2388         u32     reg_rctl;
2389
2390         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2391
2392         reg_rctl &=  (~E1000_RCTL_UPE);
2393         reg_rctl &=  (~E1000_RCTL_MPE);
2394         reg_rctl &=  (~E1000_RCTL_SBP);
2395         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2396 }
2397
2398
2399 /*********************************************************************
2400  *  Multicast Update
2401  *
2402  *  This routine is called whenever multicast address list is updated.
2403  *
2404  **********************************************************************/
2405
2406 static void
2407 em_set_multi(struct adapter *adapter)
2408 {
2409         struct ifnet    *ifp = adapter->ifp;
2410         struct ifmultiaddr *ifma;
2411         u32 reg_rctl = 0;
2412         u8  *mta; /* Multicast array memory */
2413         int mcnt = 0;
2414
2415         IOCTL_DEBUGOUT("em_set_multi: begin");
2416
2417         if (adapter->hw.mac.type == e1000_82542 &&
2418             adapter->hw.revision_id == E1000_REVISION_2) {
2419                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2420                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2421                         e1000_pci_clear_mwi(&adapter->hw);
2422                 reg_rctl |= E1000_RCTL_RST;
2423                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2424                 msec_delay(5);
2425         }
2426
2427         /* Allocate temporary memory to setup array */
2428         mta = malloc(sizeof(u8) *
2429             (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
2430             M_DEVBUF, M_NOWAIT | M_ZERO);
2431         if (mta == NULL)
2432                 panic("em_set_multi memory failure\n");
2433
2434 #if __FreeBSD_version < 800000
2435         IF_ADDR_LOCK(ifp);
2436 #else
2437         if_maddr_rlock(ifp);
2438 #endif
2439         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2440                 if (ifma->ifma_addr->sa_family != AF_LINK)
2441                         continue;
2442
2443                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2444                         break;
2445
2446                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2447                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2448                 mcnt++;
2449         }
2450 #if __FreeBSD_version < 800000
2451         IF_ADDR_UNLOCK(ifp);
2452 #else
2453         if_maddr_runlock(ifp);
2454 #endif
2455         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2456                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2457                 reg_rctl |= E1000_RCTL_MPE;
2458                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2459         } else
2460                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2461
2462         if (adapter->hw.mac.type == e1000_82542 &&
2463             adapter->hw.revision_id == E1000_REVISION_2) {
2464                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2465                 reg_rctl &= ~E1000_RCTL_RST;
2466                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2467                 msec_delay(5);
2468                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2469                         e1000_pci_set_mwi(&adapter->hw);
2470         }
2471         free(mta, M_DEVBUF);
2472 }
2473
2474
2475 /*********************************************************************
2476  *  Timer routine
2477  *
2478  *  This routine checks for link status and updates statistics.
2479  *
2480  **********************************************************************/
2481
2482 static void
2483 em_local_timer(void *arg)
2484 {
2485         struct adapter  *adapter = arg;
2486         struct ifnet    *ifp = adapter->ifp;
2487
2488         EM_CORE_LOCK_ASSERT(adapter);
2489
2490         taskqueue_enqueue(adapter->tq,
2491             &adapter->rxtx_task);
2492         em_update_link_status(adapter);
2493         em_update_stats_counters(adapter);
2494
2495         /* Reset LAA into RAR[0] on 82571 */
2496         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2497                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2498
2499         if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
2500                 em_print_hw_stats(adapter);
2501
2502         em_smartspeed(adapter);
2503
2504         /*
2505          * We check the watchdog: the time since
2506          * the last TX descriptor was cleaned.
2507          * This implies a functional TX engine.
2508          */
2509         if ((adapter->watchdog_check == TRUE) &&
2510             (ticks - adapter->watchdog_time > EM_WATCHDOG))
2511                 goto hung;
2512
2513         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2514         return;
2515 hung:
2516         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2517         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2518         adapter->watchdog_events++;
2519         em_init_locked(adapter);
2520 }
2521
2522 static void
2523 em_update_link_status(struct adapter *adapter)
2524 {
2525         struct e1000_hw *hw = &adapter->hw;
2526         struct ifnet *ifp = adapter->ifp;
2527         device_t dev = adapter->dev;
2528         u32 link_check = 0;
2529
2530         /* Get the cached link value or read phy for real */
2531         switch (hw->phy.media_type) {
2532         case e1000_media_type_copper:
2533                 if (hw->mac.get_link_status) {
2534                         /* Do the work to read phy */
2535                         e1000_check_for_link(hw);
2536                         link_check = !hw->mac.get_link_status;
2537                         if (link_check) /* ESB2 fix */
2538                                 e1000_cfg_on_link_up(hw);
2539                 } else
2540                         link_check = TRUE;
2541                 break;
2542         case e1000_media_type_fiber:
2543                 e1000_check_for_link(hw);
2544                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2545                                  E1000_STATUS_LU);
2546                 break;
2547         case e1000_media_type_internal_serdes:
2548                 e1000_check_for_link(hw);
2549                 link_check = adapter->hw.mac.serdes_has_link;
2550                 break;
2551         default:
2552         case e1000_media_type_unknown:
2553                 break;
2554         }
2555
2556         /* Now check for a transition */
2557         if (link_check && (adapter->link_active == 0)) {
2558                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2559                     &adapter->link_duplex);
2560                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2561                 if ((adapter->link_speed != SPEED_1000) &&
2562                     ((hw->mac.type == e1000_82571) ||
2563                     (hw->mac.type == e1000_82572))) {
2564                         int tarc0;
2565                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2566                         tarc0 &= ~SPEED_MODE_BIT;
2567                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2568                 }
2569                 if (bootverbose)
2570                         device_printf(dev, "Link is up %d Mbps %s\n",
2571                             adapter->link_speed,
2572                             ((adapter->link_duplex == FULL_DUPLEX) ?
2573                             "Full Duplex" : "Half Duplex"));
2574                 adapter->link_active = 1;
2575                 adapter->smartspeed = 0;
2576                 ifp->if_baudrate = adapter->link_speed * 1000000;
2577                 if_link_state_change(ifp, LINK_STATE_UP);
2578         } else if (!link_check && (adapter->link_active == 1)) {
2579                 ifp->if_baudrate = adapter->link_speed = 0;
2580                 adapter->link_duplex = 0;
2581                 if (bootverbose)
2582                         device_printf(dev, "Link is Down\n");
2583                 adapter->link_active = 0;
2584                 /* Link down, disable watchdog */
2585                 adapter->watchdog_check = FALSE;
2586                 if_link_state_change(ifp, LINK_STATE_DOWN);
2587         }
2588 }
2589
2590 /*********************************************************************
2591  *
2592  *  This routine disables all traffic on the adapter by issuing a
2593  *  global reset on the MAC and deallocates TX/RX buffers.
2594  *
2595  *  This routine should always be called with BOTH the CORE
2596  *  and TX locks.
2597  **********************************************************************/
2598
2599 static void
2600 em_stop(void *arg)
2601 {
2602         struct adapter  *adapter = arg;
2603         struct ifnet    *ifp = adapter->ifp;
2604
2605         EM_CORE_LOCK_ASSERT(adapter);
2606         EM_TX_LOCK_ASSERT(adapter);
2607
2608         INIT_DEBUGOUT("em_stop: begin");
2609
2610         em_disable_intr(adapter);
2611         callout_stop(&adapter->timer);
2612         callout_stop(&adapter->tx_fifo_timer);
2613
2614         /* Tell the stack that the interface is no longer active */
2615         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2616
2617         e1000_reset_hw(&adapter->hw);
2618         if (adapter->hw.mac.type >= e1000_82544)
2619                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2620 }
2621
2622
2623 /*********************************************************************
2624  *
2625  *  Determine hardware revision.
2626  *
2627  **********************************************************************/
2628 static void
2629 em_identify_hardware(struct adapter *adapter)
2630 {
2631         device_t dev = adapter->dev;
2632
2633         /* Make sure our PCI config space has the necessary stuff set */
2634         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2635         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2636             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2637                 device_printf(dev, "Memory Access and/or Bus Master bits "
2638                     "were not set!\n");
2639                 adapter->hw.bus.pci_cmd_word |=
2640                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2641                 pci_write_config(dev, PCIR_COMMAND,
2642                     adapter->hw.bus.pci_cmd_word, 2);
2643         }
2644
2645         /* Save off the information about this board */
2646         adapter->hw.vendor_id = pci_get_vendor(dev);
2647         adapter->hw.device_id = pci_get_device(dev);
2648         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2649         adapter->hw.subsystem_vendor_id =
2650             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2651         adapter->hw.subsystem_device_id =
2652             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2653
2654         /* Do Shared Code Init and Setup */
2655         if (e1000_set_mac_type(&adapter->hw)) {
2656                 device_printf(dev, "Setup init failure\n");
2657                 return;
2658         }
2659 }
2660
2661 static int
2662 em_allocate_pci_resources(struct adapter *adapter)
2663 {
2664         device_t        dev = adapter->dev;
2665         int             val, rid, error = E1000_SUCCESS;
2666
2667         rid = PCIR_BAR(0);
2668         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2669             &rid, RF_ACTIVE);
2670         if (adapter->memory == NULL) {
2671                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2672                 return (ENXIO);
2673         }
2674         adapter->osdep.mem_bus_space_tag =
2675             rman_get_bustag(adapter->memory);
2676         adapter->osdep.mem_bus_space_handle =
2677             rman_get_bushandle(adapter->memory);
2678         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2679
2680         /* Only older adapters use IO mapping */
2681         if ((adapter->hw.mac.type > e1000_82543) &&
2682             (adapter->hw.mac.type < e1000_82571)) {
2683                 /* Figure our where our IO BAR is ? */
2684                 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2685                         val = pci_read_config(dev, rid, 4);
2686                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2687                                 adapter->io_rid = rid;
2688                                 break;
2689                         }
2690                         rid += 4;
2691                         /* check for 64bit BAR */
2692                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2693                                 rid += 4;
2694                 }
2695                 if (rid >= PCIR_CIS) {
2696                         device_printf(dev, "Unable to locate IO BAR\n");
2697                         return (ENXIO);
2698                 }
2699                 adapter->ioport = bus_alloc_resource_any(dev,
2700                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2701                 if (adapter->ioport == NULL) {
2702                         device_printf(dev, "Unable to allocate bus resource: "
2703                             "ioport\n");
2704                         return (ENXIO);
2705                 }
2706                 adapter->hw.io_base = 0;
2707                 adapter->osdep.io_bus_space_tag =
2708                     rman_get_bustag(adapter->ioport);
2709                 adapter->osdep.io_bus_space_handle =
2710                     rman_get_bushandle(adapter->ioport);
2711         }
2712
2713         /*
2714         ** Init the resource arrays
2715         **  used by MSIX setup
2716         */
2717         for (int i = 0; i < 3; i++) {
2718                 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2719                 adapter->tag[i] = NULL;
2720                 adapter->res[i] = NULL;
2721         }
2722
2723         /*
2724          * Setup MSI/X or MSI if PCI Express
2725          */
2726         if (em_enable_msi)
2727                 adapter->msi = em_setup_msix(adapter);
2728
2729         adapter->hw.back = &adapter->osdep;
2730
2731         return (error);
2732 }
2733
2734 /*********************************************************************
2735  *
2736  *  Setup the Legacy or MSI Interrupt handler
2737  *
2738  **********************************************************************/
2739 int
2740 em_allocate_legacy(struct adapter *adapter)
2741 {
2742         device_t dev = adapter->dev;
2743         int error;
2744
2745         /* Manually turn off all interrupts */
2746         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2747
2748         /* Legacy RID is 0 */
2749         if (adapter->msi == 0)
2750                 adapter->rid[0] = 0;
2751
2752         /* We allocate a single interrupt resource */
2753         adapter->res[0] = bus_alloc_resource_any(dev,
2754             SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2755         if (adapter->res[0] == NULL) {
2756                 device_printf(dev, "Unable to allocate bus resource: "
2757                     "interrupt\n");
2758                 return (ENXIO);
2759         }
2760
2761 #ifdef EM_LEGACY_IRQ
2762         /* We do Legacy setup */
2763         if ((error = bus_setup_intr(dev, adapter->res[0],
2764 #if __FreeBSD_version > 700000
2765             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter,
2766 #else /* 6.X */
2767             INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter,
2768 #endif
2769             &adapter->tag[0])) != 0) {
2770                 device_printf(dev, "Failed to register interrupt handler");
2771                 return (error);
2772         }
2773
2774 #else /* FAST_IRQ */
2775         /*
2776          * Try allocating a fast interrupt and the associated deferred
2777          * processing contexts.
2778          */
2779         TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2780         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2781         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2782             taskqueue_thread_enqueue, &adapter->tq);
2783         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2784             device_get_nameunit(adapter->dev));
2785 #if __FreeBSD_version < 700000
2786         if ((error = bus_setup_intr(dev, adapter->res[0],
2787             INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter,
2788 #else
2789         if ((error = bus_setup_intr(dev, adapter->res[0],
2790             INTR_TYPE_NET, em_irq_fast, NULL, adapter,
2791 #endif
2792             &adapter->tag[0])) != 0) {
2793                 device_printf(dev, "Failed to register fast interrupt "
2794                             "handler: %d\n", error);
2795                 taskqueue_free(adapter->tq);
2796                 adapter->tq = NULL;
2797                 return (error);
2798         }
2799 #endif  /* EM_LEGACY_IRQ */
2800
2801         return (0);
2802 }
2803
2804 /*********************************************************************
2805  *
2806  *  Setup the MSIX Interrupt handlers
2807  *   This is not really Multiqueue, rather
2808  *   its just multiple interrupt vectors.
2809  *
2810  **********************************************************************/
2811 int
2812 em_allocate_msix(struct adapter *adapter)
2813 {
2814         device_t dev = adapter->dev;
2815         int error;
2816
2817         /* Make sure all interrupts are disabled */
2818         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2819
2820         /* First get the resources */
2821         for (int i = 0; i < adapter->msi; i++) {
2822                 adapter->res[i] = bus_alloc_resource_any(dev,
2823                     SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE);
2824                 if (adapter->res[i] == NULL) {
2825                         device_printf(dev,
2826                             "Unable to allocate bus resource: "
2827                             "MSIX Interrupt\n");
2828                         return (ENXIO);
2829                 }
2830         }
2831
2832         /*
2833          * Now allocate deferred processing contexts.
2834          */
2835         TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter);
2836         TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter);
2837         /*
2838          * Handle compatibility for msi case for deferral due to
2839          * trylock failure
2840          */
2841         TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter);
2842         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2843         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2844             taskqueue_thread_enqueue, &adapter->tq);
2845         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2846             device_get_nameunit(adapter->dev));
2847
2848         /*
2849          * And setup the interrupt handlers
2850          */
2851
2852         /* First slot to RX */
2853         if ((error = bus_setup_intr(dev, adapter->res[0],
2854 #if __FreeBSD_version > 700000
2855             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter,
2856 #else /* 6.X */
2857             INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter,
2858 #endif
2859             &adapter->tag[0])) != 0) {
2860                 device_printf(dev, "Failed to register RX handler");
2861                 return (error);
2862         }
2863
2864         /* Next TX */
2865         if ((error = bus_setup_intr(dev, adapter->res[1],
2866 #if __FreeBSD_version > 700000
2867             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter,
2868 #else /* 6.X */
2869             INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter,
2870 #endif
2871             &adapter->tag[1])) != 0) {
2872                 device_printf(dev, "Failed to register TX handler");
2873                 return (error);
2874         }
2875
2876         /* And Link */
2877         if ((error = bus_setup_intr(dev, adapter->res[2],
2878 #if __FreeBSD_version > 700000
2879             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter,
2880 #else /* 6.X */
2881             INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter,
2882 #endif
2883             &adapter->tag[2])) != 0) {
2884                 device_printf(dev, "Failed to register TX handler");
2885                 return (error);
2886         }
2887
2888         return (0);
2889 }
2890
2891
2892 static void
2893 em_free_pci_resources(struct adapter *adapter)
2894 {
2895         device_t dev = adapter->dev;
2896
2897         /* Make sure the for loop below runs once */
2898         if (adapter->msi == 0)
2899                 adapter->msi = 1;
2900
2901         /*
2902          * First release all the interrupt resources:
2903          *      notice that since these are just kept
2904          *      in an array we can do the same logic
2905          *      whether its MSIX or just legacy.
2906          */
2907         for (int i = 0; i < adapter->msi; i++) {
2908                 if (adapter->tag[i] != NULL) {
2909                         bus_teardown_intr(dev, adapter->res[i],
2910                             adapter->tag[i]);
2911                         adapter->tag[i] = NULL;
2912                 }
2913                 if (adapter->res[i] != NULL) {
2914                         bus_release_resource(dev, SYS_RES_IRQ,
2915                             adapter->rid[i], adapter->res[i]);
2916                 }
2917         }
2918
2919         if (adapter->msi)
2920                 pci_release_msi(dev);
2921
2922         if (adapter->msix != NULL)
2923                 bus_release_resource(dev, SYS_RES_MEMORY,
2924                     PCIR_BAR(EM_MSIX_BAR), adapter->msix);
2925
2926         if (adapter->memory != NULL)
2927                 bus_release_resource(dev, SYS_RES_MEMORY,
2928                     PCIR_BAR(0), adapter->memory);
2929
2930         if (adapter->flash != NULL)
2931                 bus_release_resource(dev, SYS_RES_MEMORY,
2932                     EM_FLASH, adapter->flash);
2933
2934         if (adapter->ioport != NULL)
2935                 bus_release_resource(dev, SYS_RES_IOPORT,
2936                     adapter->io_rid, adapter->ioport);
2937 }
2938
2939 /*
2940  * Setup MSI or MSI/X
2941  */
2942 static int
2943 em_setup_msix(struct adapter *adapter)
2944 {
2945         device_t dev = adapter->dev;
2946         int val = 0;
2947
2948         if (adapter->hw.mac.type < e1000_82571)
2949                 return (0);
2950
2951         /* Setup MSI/X for Hartwell */
2952         if (adapter->hw.mac.type == e1000_82574) {
2953                 /* Map the MSIX BAR */
2954                 int rid = PCIR_BAR(EM_MSIX_BAR);
2955                 adapter->msix = bus_alloc_resource_any(dev,
2956                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2957                 if (!adapter->msix) {
2958                         /* May not be enabled */
2959                         device_printf(adapter->dev,
2960                             "Unable to map MSIX table \n");
2961                         goto msi;
2962                 }
2963                 val = pci_msix_count(dev);
2964                 /*
2965                 ** 82574 can be configured for 5 but
2966                 ** we limit use to 3.
2967                 */
2968                 if (val > 3) val = 3;
2969                 if ((val) && pci_alloc_msix(dev, &val) == 0) {
2970                         device_printf(adapter->dev,"Using MSIX interrupts\n");
2971                         return (val);
2972                 }
2973         }
2974 msi:
2975         val = pci_msi_count(dev);
2976         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2977                 adapter->msi = 1;
2978                 device_printf(adapter->dev,"Using MSI interrupt\n");
2979                 return (val);
2980         }
2981         return (0);
2982 }
2983
2984 /*********************************************************************
2985  *
2986  *  Initialize the hardware to a configuration
2987  *  as specified by the adapter structure.
2988  *
2989  **********************************************************************/
2990 static int
2991 em_hardware_init(struct adapter *adapter)
2992 {
2993         device_t dev = adapter->dev;
2994         u16     rx_buffer_size;
2995
2996         INIT_DEBUGOUT("em_hardware_init: begin");
2997
2998         /* Issue a global reset */
2999         e1000_reset_hw(&adapter->hw);
3000
3001         /* When hardware is reset, fifo_head is also reset */
3002         adapter->tx_fifo_head = 0;
3003
3004         /* Set up smart power down as default off on newer adapters. */
3005         if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 ||
3006             adapter->hw.mac.type == e1000_82572)) {
3007                 u16 phy_tmp = 0;
3008
3009                 /* Speed up time to link by disabling smart power down. */
3010                 e1000_read_phy_reg(&adapter->hw,
3011                     IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3012                 phy_tmp &= ~IGP02E1000_PM_SPD;
3013                 e1000_write_phy_reg(&adapter->hw,
3014                     IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3015         }
3016
3017         /*
3018          * These parameters control the automatic generation (Tx) and
3019          * response (Rx) to Ethernet PAUSE frames.
3020          * - High water mark should allow for at least two frames to be
3021          *   received after sending an XOFF.
3022          * - Low water mark works best when it is very near the high water mark.
3023          *   This allows the receiver to restart by sending XON when it has
3024          *   drained a bit. Here we use an arbitary value of 1500 which will
3025          *   restart after one full frame is pulled from the buffer. There
3026          *   could be several smaller frames in the buffer and if so they will
3027          *   not trigger the XON until their total number reduces the buffer
3028          *   by 1500.
3029          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3030          */
3031         rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
3032             0xffff) << 10 );
3033
3034         adapter->hw.fc.high_water = rx_buffer_size -
3035             roundup2(adapter->max_frame_size, 1024);
3036         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
3037
3038         if (adapter->hw.mac.type == e1000_80003es2lan)
3039                 adapter->hw.fc.pause_time = 0xFFFF;
3040         else
3041                 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
3042         adapter->hw.fc.send_xon = TRUE;
3043
3044         /* Set Flow control, use the tunable location if sane */
3045         if ((em_fc_setting >= 0) || (em_fc_setting < 4))
3046                 adapter->hw.fc.requested_mode = em_fc_setting;
3047         else
3048                 adapter->hw.fc.requested_mode = e1000_fc_none;
3049
3050         /* Override - workaround for PCHLAN issue */
3051         if (adapter->hw.mac.type == e1000_pchlan)
3052                 adapter->hw.fc.requested_mode = e1000_fc_rx_pause;
3053
3054         if (e1000_init_hw(&adapter->hw) < 0) {
3055                 device_printf(dev, "Hardware Initialization Failed\n");
3056                 return (EIO);
3057         }
3058
3059         e1000_check_for_link(&adapter->hw);
3060
3061         return (0);
3062 }
3063
3064 /*********************************************************************
3065  *
3066  *  Setup networking device structure and register an interface.
3067  *
3068  **********************************************************************/
3069 static void
3070 em_setup_interface(device_t dev, struct adapter *adapter)
3071 {
3072         struct ifnet   *ifp;
3073
3074         INIT_DEBUGOUT("em_setup_interface: begin");
3075
3076         ifp = adapter->ifp = if_alloc(IFT_ETHER);
3077         if (ifp == NULL)
3078                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
3079         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3080         ifp->if_mtu = ETHERMTU;
3081         ifp->if_init =  em_init;
3082         ifp->if_softc = adapter;
3083         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3084         ifp->if_ioctl = em_ioctl;
3085         ifp->if_start = em_start;
3086         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3087         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3088         IFQ_SET_READY(&ifp->if_snd);
3089
3090         ether_ifattach(ifp, adapter->hw.mac.addr);
3091
3092         ifp->if_capabilities = ifp->if_capenable = 0;
3093
3094 #if __FreeBSD_version >= 800000
3095         /* Multiqueue tx functions */
3096         ifp->if_transmit = em_mq_start;
3097         ifp->if_qflush = em_qflush;
3098         adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx);
3099 #endif
3100         if (adapter->hw.mac.type >= e1000_82543) {
3101                 int version_cap;
3102 #if __FreeBSD_version < 700000
3103                 version_cap = IFCAP_HWCSUM;
3104 #else
3105                 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3106 #endif
3107                 ifp->if_capabilities |= version_cap;
3108                 ifp->if_capenable |= version_cap;
3109         }
3110
3111 #if __FreeBSD_version >= 700000
3112         /* Identify TSO capable adapters */
3113         if ((adapter->hw.mac.type > e1000_82544) &&
3114             (adapter->hw.mac.type != e1000_82547))
3115                 ifp->if_capabilities |= IFCAP_TSO4;
3116         /*
3117          * By default only enable on PCI-E, this
3118          * can be overriden by ifconfig.
3119          */
3120         if (adapter->hw.mac.type >= e1000_82571)
3121                 ifp->if_capenable |= IFCAP_TSO4;
3122 #endif
3123
3124         /*
3125          * Tell the upper layer(s) we support long frames.
3126          */
3127         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3128         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3129         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3130
3131 #ifdef DEVICE_POLLING
3132         ifp->if_capabilities |= IFCAP_POLLING;
3133 #endif
3134
3135         /* Enable All WOL methods by default */
3136         if (adapter->wol) {
3137                 ifp->if_capabilities |= IFCAP_WOL;
3138                 ifp->if_capenable |= IFCAP_WOL;
3139         }
3140
3141         /*
3142          * Specify the media types supported by this adapter and register
3143          * callbacks to update media and link information
3144          */
3145         ifmedia_init(&adapter->media, IFM_IMASK,
3146             em_media_change, em_media_status);
3147         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3148             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3149                 u_char fiber_type = IFM_1000_SX;        /* default type */
3150
3151                 if (adapter->hw.mac.type == e1000_82545)
3152                         fiber_type = IFM_1000_LX;
3153                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3154                             0, NULL);
3155                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3156         } else {
3157                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3158                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3159                             0, NULL);
3160                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3161                             0, NULL);
3162                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3163                             0, NULL);
3164                 if (adapter->hw.phy.type != e1000_phy_ife) {
3165                         ifmedia_add(&adapter->media,
3166                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3167                         ifmedia_add(&adapter->media,
3168                                 IFM_ETHER | IFM_1000_T, 0, NULL);
3169                 }
3170         }
3171         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3172         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3173 }
3174
3175
3176 /*********************************************************************
3177  *
3178  *  Workaround for SmartSpeed on 82541 and 82547 controllers
3179  *
3180  **********************************************************************/
3181 static void
3182 em_smartspeed(struct adapter *adapter)
3183 {
3184         u16 phy_tmp;
3185
3186         if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
3187             adapter->hw.mac.autoneg == 0 ||
3188             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
3189                 return;
3190
3191         if (adapter->smartspeed == 0) {
3192                 /* If Master/Slave config fault is asserted twice,
3193                  * we assume back-to-back */
3194                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3195                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
3196                         return;
3197                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3198                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
3199                         e1000_read_phy_reg(&adapter->hw,
3200                             PHY_1000T_CTRL, &phy_tmp);
3201                         if(phy_tmp & CR_1000T_MS_ENABLE) {
3202                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
3203                                 e1000_write_phy_reg(&adapter->hw,
3204                                     PHY_1000T_CTRL, phy_tmp);
3205                                 adapter->smartspeed++;
3206                                 if(adapter->hw.mac.autoneg &&
3207                                    !e1000_copper_link_autoneg(&adapter->hw) &&
3208                                    !e1000_read_phy_reg(&adapter->hw,
3209                                     PHY_CONTROL, &phy_tmp)) {
3210                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3211                                                     MII_CR_RESTART_AUTO_NEG);
3212                                         e1000_write_phy_reg(&adapter->hw,
3213                                             PHY_CONTROL, phy_tmp);
3214                                 }
3215                         }
3216                 }
3217                 return;
3218         } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
3219                 /* If still no link, perhaps using 2/3 pair cable */
3220                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
3221                 phy_tmp |= CR_1000T_MS_ENABLE;
3222                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
3223                 if(adapter->hw.mac.autoneg &&
3224                    !e1000_copper_link_autoneg(&adapter->hw) &&
3225                    !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
3226                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3227                                     MII_CR_RESTART_AUTO_NEG);
3228                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
3229                 }
3230         }
3231         /* Restart process after EM_SMARTSPEED_MAX iterations */
3232         if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
3233                 adapter->smartspeed = 0;
3234 }
3235
3236
3237 /*
3238  * Manage DMA'able memory.
3239  */
3240 static void
3241 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3242 {
3243         if (error)
3244                 return;
3245         *(bus_addr_t *) arg = segs[0].ds_addr;
3246 }
3247
3248 static int
3249 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3250         struct em_dma_alloc *dma, int mapflags)
3251 {
3252         int error;
3253
3254 #if __FreeBSD_version >= 700000
3255         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3256 #else
3257         error = bus_dma_tag_create(NULL,                 /* parent */
3258 #endif
3259                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
3260                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3261                                 BUS_SPACE_MAXADDR,      /* highaddr */
3262                                 NULL, NULL,             /* filter, filterarg */
3263                                 size,                   /* maxsize */
3264                                 1,                      /* nsegments */
3265                                 size,                   /* maxsegsize */
3266                                 0,                      /* flags */
3267                                 NULL,                   /* lockfunc */
3268                                 NULL,                   /* lockarg */
3269                                 &dma->dma_tag);
3270         if (error) {
3271                 device_printf(adapter->dev,
3272                     "%s: bus_dma_tag_create failed: %d\n",
3273                     __func__, error);
3274                 goto fail_0;
3275         }
3276
3277         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3278             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3279         if (error) {
3280                 device_printf(adapter->dev,
3281                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3282                     __func__, (uintmax_t)size, error);
3283                 goto fail_2;
3284         }
3285
3286         dma->dma_paddr = 0;
3287         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3288             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3289         if (error || dma->dma_paddr == 0) {
3290                 device_printf(adapter->dev,
3291                     "%s: bus_dmamap_load failed: %d\n",
3292                     __func__, error);
3293                 goto fail_3;
3294         }
3295
3296         return (0);
3297
3298 fail_3:
3299         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3300 fail_2:
3301         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3302         bus_dma_tag_destroy(dma->dma_tag);
3303 fail_0:
3304         dma->dma_map = NULL;
3305         dma->dma_tag = NULL;
3306
3307         return (error);
3308 }
3309
3310 static void
3311 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3312 {
3313         if (dma->dma_tag == NULL)
3314                 return;
3315         if (dma->dma_map != NULL) {
3316                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3317                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3318                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3319                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3320                 dma->dma_map = NULL;
3321         }
3322         bus_dma_tag_destroy(dma->dma_tag);
3323         dma->dma_tag = NULL;
3324 }
3325
3326
3327 /*********************************************************************
3328  *
3329  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3330  *  the information needed to transmit a packet on the wire.
3331  *
3332  **********************************************************************/
3333 static int
3334 em_allocate_transmit_structures(struct adapter *adapter)
3335 {
3336         device_t dev = adapter->dev;
3337         struct em_buffer *tx_buffer;
3338         int error;
3339
3340         /*
3341          * Create DMA tags for tx descriptors
3342          */
3343 #if __FreeBSD_version >= 700000
3344         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3345 #else
3346         if ((error = bus_dma_tag_create(NULL,            /* parent */
3347 #endif
3348                                 1, 0,                   /* alignment, bounds */
3349                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3350                                 BUS_SPACE_MAXADDR,      /* highaddr */
3351                                 NULL, NULL,             /* filter, filterarg */
3352                                 EM_TSO_SIZE,            /* maxsize */
3353                                 EM_MAX_SCATTER,         /* nsegments */
3354                                 EM_TSO_SEG_SIZE,        /* maxsegsize */
3355                                 0,                      /* flags */
3356                                 NULL,           /* lockfunc */
3357                                 NULL,           /* lockarg */
3358                                 &adapter->txtag)) != 0) {
3359                 device_printf(dev, "Unable to allocate TX DMA tag\n");
3360                 goto fail;
3361         }
3362
3363         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
3364             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3365         if (adapter->tx_buffer_area == NULL) {
3366                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3367                 error = ENOMEM;
3368                 goto fail;
3369         }
3370
3371         /* Create the descriptor buffer dma maps */
3372         for (int i = 0; i < adapter->num_tx_desc; i++) {
3373                 tx_buffer = &adapter->tx_buffer_area[i];
3374                 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
3375                 if (error != 0) {
3376                         device_printf(dev, "Unable to create TX DMA map\n");
3377                         goto fail;
3378                 }
3379                 tx_buffer->next_eop = -1;
3380         }
3381
3382         return (0);
3383 fail:
3384         em_free_transmit_structures(adapter);
3385         return (error);
3386 }
3387
3388 /*********************************************************************
3389  *
3390  *  (Re)Initialize transmit structures.
3391  *
3392  **********************************************************************/
3393 static void
3394 em_setup_transmit_structures(struct adapter *adapter)
3395 {
3396         struct em_buffer *tx_buffer;
3397
3398         /* Clear the old ring contents */
3399         bzero(adapter->tx_desc_base,
3400             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3401
3402         /* Free any existing TX buffers */
3403         for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3404                 tx_buffer = &adapter->tx_buffer_area[i];
3405                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3406                     BUS_DMASYNC_POSTWRITE);
3407                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
3408                 m_freem(tx_buffer->m_head);
3409                 tx_buffer->m_head = NULL;
3410                 tx_buffer->next_eop = -1;
3411         }
3412
3413         /* Reset state */
3414         adapter->next_avail_tx_desc = 0;
3415         adapter->next_tx_to_clean = 0;
3416         adapter->num_tx_desc_avail = adapter->num_tx_desc;
3417
3418         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3419             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3420
3421         return;
3422 }
3423
3424 /*********************************************************************
3425  *
3426  *  Enable transmit unit.
3427  *
3428  **********************************************************************/
3429 static void
3430 em_initialize_transmit_unit(struct adapter *adapter)
3431 {
3432         u32     tctl, tarc, tipg = 0;
3433         u64     bus_addr;
3434
3435          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3436         /* Setup the Base and Length of the Tx Descriptor Ring */
3437         bus_addr = adapter->txdma.dma_paddr;
3438         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
3439             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3440         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
3441             (u32)(bus_addr >> 32));
3442         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
3443             (u32)bus_addr);
3444         /* Setup the HW Tx Head and Tail descriptor pointers */
3445         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
3446         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
3447
3448         HW_DEBUGOUT2("Base = %x, Length = %x\n",
3449             E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
3450             E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
3451
3452         /* Set the default values for the Tx Inter Packet Gap timer */
3453         switch (adapter->hw.mac.type) {
3454         case e1000_82542:
3455                 tipg = DEFAULT_82542_TIPG_IPGT;
3456                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3457                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3458                 break;
3459         case e1000_80003es2lan:
3460                 tipg = DEFAULT_82543_TIPG_IPGR1;
3461                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3462                     E1000_TIPG_IPGR2_SHIFT;
3463                 break;
3464         default:
3465                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3466                     (adapter->hw.phy.media_type ==
3467                     e1000_media_type_internal_serdes))
3468                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3469                 else
3470                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3471                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3472                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3473         }
3474
3475         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3476         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3477         if(adapter->hw.mac.type >= e1000_82540)
3478                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3479                     adapter->tx_abs_int_delay.value);
3480
3481         if ((adapter->hw.mac.type == e1000_82571) ||
3482             (adapter->hw.mac.type == e1000_82572)) {
3483                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3484                 tarc |= SPEED_MODE_BIT;
3485                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3486         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3487                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3488                 tarc |= 1;
3489                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3490                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3491                 tarc |= 1;
3492                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3493         }
3494
3495         /* Program the Transmit Control Register */
3496         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3497         tctl &= ~E1000_TCTL_CT;
3498         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3499                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3500
3501         if (adapter->hw.mac.type >= e1000_82571)
3502                 tctl |= E1000_TCTL_MULR;
3503
3504         /* This write will effectively turn on the transmit unit. */
3505         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3506
3507         /* Setup Transmit Descriptor Base Settings */
3508         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3509
3510         if (adapter->tx_int_delay.value > 0)
3511                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3512 }
3513
3514 /*********************************************************************
3515  *
3516  *  Free all transmit related data structures.
3517  *
3518  **********************************************************************/
3519 static void
3520 em_free_transmit_structures(struct adapter *adapter)
3521 {
3522         struct em_buffer *tx_buffer;
3523
3524         INIT_DEBUGOUT("free_transmit_structures: begin");
3525
3526         if (adapter->tx_buffer_area != NULL) {
3527                 for (int i = 0; i < adapter->num_tx_desc; i++) {
3528                         tx_buffer = &adapter->tx_buffer_area[i];
3529                         if (tx_buffer->m_head != NULL) {
3530                                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3531                                     BUS_DMASYNC_POSTWRITE);
3532                                 bus_dmamap_unload(adapter->txtag,
3533                                     tx_buffer->map);
3534                                 m_freem(tx_buffer->m_head);
3535                                 tx_buffer->m_head = NULL;
3536                         } else if (tx_buffer->map != NULL)
3537                                 bus_dmamap_unload(adapter->txtag,
3538                                     tx_buffer->map);
3539                         if (tx_buffer->map != NULL) {
3540                                 bus_dmamap_destroy(adapter->txtag,
3541                                     tx_buffer->map);
3542                                 tx_buffer->map = NULL;
3543                         }
3544                 }
3545         }
3546         if (adapter->tx_buffer_area != NULL) {
3547                 free(adapter->tx_buffer_area, M_DEVBUF);
3548                 adapter->tx_buffer_area = NULL;
3549         }
3550         if (adapter->txtag != NULL) {
3551                 bus_dma_tag_destroy(adapter->txtag);
3552                 adapter->txtag = NULL;
3553         }
3554 #if __FreeBSD_version >= 800000
3555         if (adapter->br != NULL)
3556                 buf_ring_free(adapter->br, M_DEVBUF);
3557 #endif
3558 }
3559
3560 /*********************************************************************
3561  *
3562  *  The offload context needs to be set when we transfer the first
3563  *  packet of a particular protocol (TCP/UDP). This routine has been
3564  *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3565  *
3566  *  Added back the old method of keeping the current context type
3567  *  and not setting if unnecessary, as this is reported to be a
3568  *  big performance win.  -jfv
3569  **********************************************************************/
3570 static void
3571 em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
3572     u32 *txd_upper, u32 *txd_lower)
3573 {
3574         struct e1000_context_desc *TXD = NULL;
3575         struct em_buffer *tx_buffer;
3576         struct ether_vlan_header *eh;
3577         struct ip *ip = NULL;
3578         struct ip6_hdr *ip6;
3579         int curr_txd, ehdrlen;
3580         u32 cmd, hdr_len, ip_hlen;
3581         u16 etype;
3582         u8 ipproto;
3583
3584
3585         cmd = hdr_len = ipproto = 0;
3586         curr_txd = adapter->next_avail_tx_desc;
3587
3588         /*
3589          * Determine where frame payload starts.
3590          * Jump over vlan headers if already present,
3591          * helpful for QinQ too.
3592          */
3593         eh = mtod(mp, struct ether_vlan_header *);
3594         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3595                 etype = ntohs(eh->evl_proto);
3596                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3597         } else {
3598                 etype = ntohs(eh->evl_encap_proto);
3599                 ehdrlen = ETHER_HDR_LEN;
3600         }
3601
3602         /*
3603          * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3604          * TODO: Support SCTP too when it hits the tree.
3605          */
3606         switch (etype) {
3607         case ETHERTYPE_IP:
3608                 ip = (struct ip *)(mp->m_data + ehdrlen);
3609                 ip_hlen = ip->ip_hl << 2;
3610
3611                 /* Setup of IP header checksum. */
3612                 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3613                         /*
3614                          * Start offset for header checksum calculation.
3615                          * End offset for header checksum calculation.
3616                          * Offset of place to put the checksum.
3617                          */
3618                         TXD = (struct e1000_context_desc *)
3619                             &adapter->tx_desc_base[curr_txd];
3620                         TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3621                         TXD->lower_setup.ip_fields.ipcse =
3622                             htole16(ehdrlen + ip_hlen);
3623                         TXD->lower_setup.ip_fields.ipcso =
3624                             ehdrlen + offsetof(struct ip, ip_sum);
3625                         cmd |= E1000_TXD_CMD_IP;
3626                         *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3627                 }
3628
3629                 if (mp->m_len < ehdrlen + ip_hlen)
3630                         return; /* failure */
3631
3632                 hdr_len = ehdrlen + ip_hlen;
3633                 ipproto = ip->ip_p;
3634
3635                 break;
3636         case ETHERTYPE_IPV6:
3637                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3638                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3639
3640                 if (mp->m_len < ehdrlen + ip_hlen)
3641                         return; /* failure */
3642
3643                 /* IPv6 doesn't have a header checksum. */
3644
3645                 hdr_len = ehdrlen + ip_hlen;
3646                 ipproto = ip6->ip6_nxt;
3647
3648                 break;
3649         default:
3650                 *txd_upper = 0;
3651                 *txd_lower = 0;
3652                 return;
3653         }
3654
3655         switch (ipproto) {
3656         case IPPROTO_TCP:
3657                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3658                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3659                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3660                         /* no need for context if already set */
3661                         if (adapter->last_hw_offload == CSUM_TCP)
3662                                 return;
3663                         adapter->last_hw_offload = CSUM_TCP;
3664                         /*
3665                          * Start offset for payload checksum calculation.
3666                          * End offset for payload checksum calculation.
3667                          * Offset of place to put the checksum.
3668                          */
3669                         TXD = (struct e1000_context_desc *)
3670                             &adapter->tx_desc_base[curr_txd];
3671                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3672                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3673                         TXD->upper_setup.tcp_fields.tucso =
3674                             hdr_len + offsetof(struct tcphdr, th_sum);
3675                         cmd |= E1000_TXD_CMD_TCP;
3676                 }
3677                 break;
3678         case IPPROTO_UDP:
3679         {
3680                 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3681                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3682                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3683                         /* no need for context if already set */
3684                         if (adapter->last_hw_offload == CSUM_UDP)
3685                                 return;
3686                         adapter->last_hw_offload = CSUM_UDP;
3687                         /*
3688                          * Start offset for header checksum calculation.
3689                          * End offset for header checksum calculation.
3690                          * Offset of place to put the checksum.
3691                          */
3692                         TXD = (struct e1000_context_desc *)
3693                             &adapter->tx_desc_base[curr_txd];
3694                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3695                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3696                         TXD->upper_setup.tcp_fields.tucso =
3697                             hdr_len + offsetof(struct udphdr, uh_sum);
3698                 }
3699                 /* Fall Thru */
3700         }
3701         default:
3702                 break;
3703         }
3704
3705         TXD->tcp_seg_setup.data = htole32(0);
3706         TXD->cmd_and_length =
3707             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3708         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3709         tx_buffer->m_head = NULL;
3710         tx_buffer->next_eop = -1;
3711
3712         if (++curr_txd == adapter->num_tx_desc)
3713                 curr_txd = 0;
3714
3715         adapter->num_tx_desc_avail--;
3716         adapter->next_avail_tx_desc = curr_txd;
3717 }
3718
3719
3720 #if __FreeBSD_version >= 700000
3721 /**********************************************************************
3722  *
3723  *  Setup work for hardware segmentation offload (TSO)
3724  *
3725  **********************************************************************/
3726 static bool
3727 em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper,
3728    u32 *txd_lower)
3729 {
3730         struct e1000_context_desc *TXD;
3731         struct em_buffer *tx_buffer;
3732         struct ether_vlan_header *eh;
3733         struct ip *ip;
3734         struct ip6_hdr *ip6;
3735         struct tcphdr *th;
3736         int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6;
3737         u16 etype;
3738
3739         /*
3740          * This function could/should be extended to support IP/IPv6
3741          * fragmentation as well.  But as they say, one step at a time.
3742          */
3743
3744         /*
3745          * Determine where frame payload starts.
3746          * Jump over vlan headers if already present,
3747          * helpful for QinQ too.
3748          */
3749         eh = mtod(mp, struct ether_vlan_header *);
3750         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3751                 etype = ntohs(eh->evl_proto);
3752                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3753         } else {
3754                 etype = ntohs(eh->evl_encap_proto);
3755                 ehdrlen = ETHER_HDR_LEN;
3756         }
3757
3758         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3759         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3760                 return FALSE;   /* -1 */
3761
3762         /*
3763          * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3764          * TODO: Support SCTP too when it hits the tree.
3765          */
3766         switch (etype) {
3767         case ETHERTYPE_IP:
3768                 isip6 = 0;
3769                 ip = (struct ip *)(mp->m_data + ehdrlen);
3770                 if (ip->ip_p != IPPROTO_TCP)
3771                         return FALSE;   /* 0 */
3772                 ip->ip_len = 0;
3773                 ip->ip_sum = 0;
3774                 ip_hlen = ip->ip_hl << 2;
3775                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3776                         return FALSE;   /* -1 */
3777                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3778 #if 1
3779                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3780                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3781 #else
3782                 th->th_sum = mp->m_pkthdr.csum_data;
3783 #endif
3784                 break;
3785         case ETHERTYPE_IPV6:
3786                 isip6 = 1;
3787                 return FALSE;                   /* Not supported yet. */
3788                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3789                 if (ip6->ip6_nxt != IPPROTO_TCP)
3790                         return FALSE;   /* 0 */
3791                 ip6->ip6_plen = 0;
3792                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3793                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3794                         return FALSE;   /* -1 */
3795                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3796 #if 0
3797                 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3798                     htons(IPPROTO_TCP));        /* XXX: function notyet. */
3799 #else
3800                 th->th_sum = mp->m_pkthdr.csum_data;
3801 #endif
3802                 break;
3803         default:
3804                 return FALSE;
3805         }
3806         hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3807
3808         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3809                       E1000_TXD_DTYP_D |        /* Data descr type */
3810                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3811
3812         /* IP and/or TCP header checksum calculation and insertion. */
3813         *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3814                       E1000_TXD_POPTS_TXSM) << 8;
3815
3816         curr_txd = adapter->next_avail_tx_desc;
3817         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3818         TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3819
3820         /* IPv6 doesn't have a header checksum. */
3821         if (!isip6) {
3822                 /*
3823                  * Start offset for header checksum calculation.
3824                  * End offset for header checksum calculation.
3825                  * Offset of place put the checksum.
3826                  */
3827                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3828                 TXD->lower_setup.ip_fields.ipcse =
3829                     htole16(ehdrlen + ip_hlen - 1);
3830                 TXD->lower_setup.ip_fields.ipcso =
3831                     ehdrlen + offsetof(struct ip, ip_sum);
3832         }
3833         /*
3834          * Start offset for payload checksum calculation.
3835          * End offset for payload checksum calculation.
3836          * Offset of place to put the checksum.
3837          */
3838         TXD->upper_setup.tcp_fields.tucss =
3839             ehdrlen + ip_hlen;
3840         TXD->upper_setup.tcp_fields.tucse = 0;
3841         TXD->upper_setup.tcp_fields.tucso =
3842             ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3843         /*
3844          * Payload size per packet w/o any headers.
3845          * Length of all headers up to payload.
3846          */
3847         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3848         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3849
3850         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3851                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3852                                 E1000_TXD_CMD_TSE |     /* TSE context */
3853                                 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */
3854                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3855                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3856
3857         tx_buffer->m_head = NULL;
3858         tx_buffer->next_eop = -1;
3859
3860         if (++curr_txd == adapter->num_tx_desc)
3861                 curr_txd = 0;
3862
3863         adapter->num_tx_desc_avail--;
3864         adapter->next_avail_tx_desc = curr_txd;
3865         adapter->tx_tso = TRUE;
3866
3867         return TRUE;
3868 }
3869
3870 #endif /* __FreeBSD_version >= 700000 */
3871
3872 /**********************************************************************
3873  *
3874  *  Examine each tx_buffer in the used queue. If the hardware is done
3875  *  processing the packet then free associated resources. The
3876  *  tx_buffer is put back on the free queue.
3877  *
3878  **********************************************************************/
3879 static void
3880 em_txeof(struct adapter *adapter)
3881 {
3882         int first, last, done, num_avail;
3883         struct em_buffer *tx_buffer;
3884         struct e1000_tx_desc   *tx_desc, *eop_desc;
3885         struct ifnet   *ifp = adapter->ifp;
3886
3887         EM_TX_LOCK_ASSERT(adapter);
3888
3889         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3890                 return;
3891
3892         num_avail = adapter->num_tx_desc_avail;
3893         first = adapter->next_tx_to_clean;
3894         tx_desc = &adapter->tx_desc_base[first];
3895         tx_buffer = &adapter->tx_buffer_area[first];
3896         last = tx_buffer->next_eop;
3897         eop_desc = &adapter->tx_desc_base[last];
3898
3899         /*
3900          * What this does is get the index of the
3901          * first descriptor AFTER the EOP of the
3902          * first packet, that way we can do the
3903          * simple comparison on the inner while loop.
3904          */
3905         if (++last == adapter->num_tx_desc)
3906                 last = 0;
3907         done = last;
3908
3909         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3910             BUS_DMASYNC_POSTREAD);
3911
3912         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3913                 /* We clean the range of the packet */
3914                 while (first != done) {
3915                         tx_desc->upper.data = 0;
3916                         tx_desc->lower.data = 0;
3917                         tx_desc->buffer_addr = 0;
3918                         ++num_avail;
3919
3920                         if (tx_buffer->m_head) {
3921                                 ifp->if_opackets++;
3922                                 bus_dmamap_sync(adapter->txtag,
3923                                     tx_buffer->map,
3924                                     BUS_DMASYNC_POSTWRITE);
3925                                 bus_dmamap_unload(adapter->txtag,
3926                                     tx_buffer->map);
3927
3928                                 m_freem(tx_buffer->m_head);
3929                                 tx_buffer->m_head = NULL;
3930                         }
3931                         tx_buffer->next_eop = -1;
3932                         adapter->watchdog_time = ticks;
3933
3934                         if (++first == adapter->num_tx_desc)
3935                                 first = 0;
3936
3937                         tx_buffer = &adapter->tx_buffer_area[first];
3938                         tx_desc = &adapter->tx_desc_base[first];
3939                 }
3940                 /* See if we can continue to the next packet */
3941                 last = tx_buffer->next_eop;
3942                 if (last != -1) {
3943                         eop_desc = &adapter->tx_desc_base[last];
3944                         /* Get new done point */
3945                         if (++last == adapter->num_tx_desc) last = 0;
3946                         done = last;
3947                 } else
3948                         break;
3949         }
3950         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3951             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3952
3953         adapter->next_tx_to_clean = first;
3954
3955         /*
3956          * If we have enough room, clear IFF_DRV_OACTIVE to
3957          * tell the stack that it is OK to send packets.
3958          * If there are no pending descriptors, clear the watchdog.
3959          */
3960         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
3961                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3962                 if (num_avail == adapter->num_tx_desc) {
3963                         adapter->watchdog_check = FALSE;
3964                         adapter->num_tx_desc_avail = num_avail;
3965                         return;
3966                 }
3967         }
3968
3969         adapter->num_tx_desc_avail = num_avail;
3970         return;
3971 }
3972
3973 /*********************************************************************
3974  *
3975  *  When Link is lost sometimes there is work still in the TX ring
3976  *  which may result in a watchdog, rather than allow that we do an
3977  *  attempted cleanup and then reinit here. Note that this has been
3978  *  seens mostly with fiber adapters.
3979  *
3980  **********************************************************************/
3981 static void
3982 em_tx_purge(struct adapter *adapter)
3983 {
3984         if ((!adapter->link_active) && (adapter->watchdog_check)) {
3985                 EM_TX_LOCK(adapter);
3986                 em_txeof(adapter);
3987                 EM_TX_UNLOCK(adapter);
3988                 if (adapter->watchdog_check) /* Still outstanding? */
3989                         em_init_locked(adapter);
3990         }
3991 }
3992
3993 /*********************************************************************
3994  *
3995  *  Get a buffer from system mbuf buffer pool.
3996  *
3997  **********************************************************************/
3998 static int
3999 em_get_buf(struct adapter *adapter, int i)
4000 {
4001         struct mbuf             *m;
4002         bus_dma_segment_t       segs[1];
4003         bus_dmamap_t            map;
4004         struct em_buffer        *rx_buffer;
4005         int                     error, nsegs;
4006
4007         m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
4008         if (m == NULL) {
4009                 adapter->mbuf_cluster_failed++;
4010                 return (ENOBUFS);
4011         }
4012         m->m_len = m->m_pkthdr.len = MCLBYTES;
4013
4014         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
4015                 m_adj(m, ETHER_ALIGN);
4016
4017         /*
4018          * Using memory from the mbuf cluster pool, invoke the
4019          * bus_dma machinery to arrange the memory mapping.
4020          */
4021         error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
4022             adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
4023         if (error != 0) {
4024                 m_free(m);
4025                 return (error);
4026         }
4027
4028         /* If nsegs is wrong then the stack is corrupt. */
4029         KASSERT(nsegs == 1, ("Too many segments returned!"));
4030
4031         rx_buffer = &adapter->rx_buffer_area[i];
4032         if (rx_buffer->m_head != NULL)
4033                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4034
4035         map = rx_buffer->map;
4036         rx_buffer->map = adapter->rx_sparemap;
4037         adapter->rx_sparemap = map;
4038         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
4039         rx_buffer->m_head = m;
4040
4041         adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
4042         return (0);
4043 }
4044
4045 /*********************************************************************
4046  *
4047  *  Allocate memory for rx_buffer structures. Since we use one
4048  *  rx_buffer per received packet, the maximum number of rx_buffer's
4049  *  that we'll need is equal to the number of receive descriptors
4050  *  that we've allocated.
4051  *
4052  **********************************************************************/
4053 static int
4054 em_allocate_receive_structures(struct adapter *adapter)
4055 {
4056         device_t dev = adapter->dev;
4057         struct em_buffer *rx_buffer;
4058         int i, error;
4059
4060         adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
4061             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
4062         if (adapter->rx_buffer_area == NULL) {
4063                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4064                 return (ENOMEM);
4065         }
4066
4067 #if __FreeBSD_version >= 700000
4068         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
4069 #else
4070         error = bus_dma_tag_create(NULL,                 /* parent */
4071 #endif
4072                                 1, 0,                   /* alignment, bounds */
4073                                 BUS_SPACE_MAXADDR,      /* lowaddr */
4074                                 BUS_SPACE_MAXADDR,      /* highaddr */
4075                                 NULL, NULL,             /* filter, filterarg */
4076                                 MCLBYTES,               /* maxsize */
4077                                 1,                      /* nsegments */
4078                                 MCLBYTES,               /* maxsegsize */
4079                                 0,                      /* flags */
4080                                 NULL,                   /* lockfunc */
4081                                 NULL,                   /* lockarg */
4082                                 &adapter->rxtag);
4083         if (error) {
4084                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
4085                     __func__, error);
4086                 goto fail;
4087         }
4088
4089         /* Create the spare map (used by getbuf) */
4090         error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4091              &adapter->rx_sparemap);
4092         if (error) {
4093                 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4094                     __func__, error);
4095                 goto fail;
4096         }
4097
4098         rx_buffer = adapter->rx_buffer_area;
4099         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4100                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4101                     &rx_buffer->map);
4102                 if (error) {
4103                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4104                             __func__, error);
4105                         goto fail;
4106                 }
4107         }
4108
4109         return (0);
4110
4111 fail:
4112         em_free_receive_structures(adapter);
4113         return (error);
4114 }
4115
4116 /*********************************************************************
4117  *
4118  *  (Re)initialize receive structures.
4119  *
4120  **********************************************************************/
4121 static int
4122 em_setup_receive_structures(struct adapter *adapter)
4123 {
4124         struct em_buffer *rx_buffer;
4125         int i, error;
4126
4127         /* Reset descriptor ring */
4128         bzero(adapter->rx_desc_base,
4129             (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
4130
4131         /* Free current RX buffers. */
4132         rx_buffer = adapter->rx_buffer_area;
4133         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4134                 if (rx_buffer->m_head != NULL) {
4135                         bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4136                             BUS_DMASYNC_POSTREAD);
4137                         bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4138                         m_freem(rx_buffer->m_head);
4139                         rx_buffer->m_head = NULL;
4140                 }
4141         }
4142
4143         /* Allocate new ones. */
4144         for (i = 0; i < adapter->num_rx_desc; i++) {
4145                 error = em_get_buf(adapter, i);
4146                 if (error)
4147                         return (error);
4148         }
4149
4150         /* Setup our descriptor pointers */
4151         adapter->next_rx_desc_to_check = 0;
4152         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4153             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4154
4155         return (0);
4156 }
4157
4158 /*********************************************************************
4159  *
4160  *  Enable receive unit.
4161  *
4162  **********************************************************************/
4163 #define MAX_INTS_PER_SEC        8000
4164 #define DEFAULT_ITR          1000000000/(MAX_INTS_PER_SEC * 256)
4165
4166 static void
4167 em_initialize_receive_unit(struct adapter *adapter)
4168 {
4169         struct ifnet    *ifp = adapter->ifp;
4170         u64     bus_addr;
4171         u32     rctl, rxcsum;
4172
4173         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
4174
4175         /*
4176          * Make sure receives are disabled while setting
4177          * up the descriptor ring
4178          */
4179         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4180         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4181
4182         if (adapter->hw.mac.type >= e1000_82540) {
4183                 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4184                     adapter->rx_abs_int_delay.value);
4185                 /*
4186                  * Set the interrupt throttling rate. Value is calculated
4187                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4188                  */
4189                 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
4190         }
4191
4192         /*
4193         ** When using MSIX interrupts we need to throttle
4194         ** using the EITR register (82574 only)
4195         */
4196         if (adapter->msix)
4197                 for (int i = 0; i < 4; i++)
4198                         E1000_WRITE_REG(&adapter->hw,
4199                             E1000_EITR_82574(i), DEFAULT_ITR);
4200
4201         /* Disable accelerated ackknowledge */
4202         if (adapter->hw.mac.type == e1000_82574)
4203                 E1000_WRITE_REG(&adapter->hw,
4204                     E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4205
4206         /* Setup the Base and Length of the Rx Descriptor Ring */
4207         bus_addr = adapter->rxdma.dma_paddr;
4208         E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
4209             adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4210         E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
4211             (u32)(bus_addr >> 32));
4212         E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
4213             (u32)bus_addr);
4214
4215         /* Setup the Receive Control Register */
4216         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4217         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4218                    E1000_RCTL_RDMTS_HALF |
4219                    (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4220
4221         /* Make sure VLAN Filters are off */
4222         rctl &= ~E1000_RCTL_VFE;
4223
4224         if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
4225                 rctl |= E1000_RCTL_SBP;
4226         else
4227                 rctl &= ~E1000_RCTL_SBP;
4228
4229         switch (adapter->rx_buffer_len) {
4230         default:
4231         case 2048:
4232                 rctl |= E1000_RCTL_SZ_2048;
4233                 break;
4234         case 4096:
4235                 rctl |= E1000_RCTL_SZ_4096 |
4236                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4237                 break;
4238         case 8192:
4239                 rctl |= E1000_RCTL_SZ_8192 |
4240                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4241                 break;
4242         case 16384:
4243                 rctl |= E1000_RCTL_SZ_16384 |
4244                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4245                 break;
4246         }
4247
4248         if (ifp->if_mtu > ETHERMTU)
4249                 rctl |= E1000_RCTL_LPE;
4250         else
4251                 rctl &= ~E1000_RCTL_LPE;
4252
4253         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
4254         if ((adapter->hw.mac.type >= e1000_82543) &&
4255             (ifp->if_capenable & IFCAP_RXCSUM)) {
4256                 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
4257                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4258                 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
4259         }
4260
4261         /*
4262         ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4263         ** long latencies are observed, like Lenovo X60. This
4264         ** change eliminates the problem, but since having positive
4265         ** values in RDTR is a known source of problems on other
4266         ** platforms another solution is being sought.
4267         */
4268         if (adapter->hw.mac.type == e1000_82573)
4269                 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20);
4270
4271         /* Enable Receives */
4272         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4273
4274         /*
4275          * Setup the HW Rx Head and
4276          * Tail Descriptor Pointers
4277          */
4278         E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
4279         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
4280
4281         return;
4282 }
4283
4284 /*********************************************************************
4285  *
4286  *  Free receive related data structures.
4287  *
4288  **********************************************************************/
4289 static void
4290 em_free_receive_structures(struct adapter *adapter)
4291 {
4292         struct em_buffer *rx_buffer;
4293         int i;
4294
4295         INIT_DEBUGOUT("free_receive_structures: begin");
4296
4297         if (adapter->rx_sparemap) {
4298                 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
4299                 adapter->rx_sparemap = NULL;
4300         }
4301
4302         /* Cleanup any existing buffers */
4303         if (adapter->rx_buffer_area != NULL) {
4304                 rx_buffer = adapter->rx_buffer_area;
4305                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4306                         if (rx_buffer->m_head != NULL) {
4307                                 bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4308                                     BUS_DMASYNC_POSTREAD);
4309                                 bus_dmamap_unload(adapter->rxtag,
4310                                     rx_buffer->map);
4311                                 m_freem(rx_buffer->m_head);
4312                                 rx_buffer->m_head = NULL;
4313                         } else if (rx_buffer->map != NULL)
4314                                 bus_dmamap_unload(adapter->rxtag,
4315                                     rx_buffer->map);
4316                         if (rx_buffer->map != NULL) {
4317                                 bus_dmamap_destroy(adapter->rxtag,
4318                                     rx_buffer->map);
4319                                 rx_buffer->map = NULL;
4320                         }
4321                 }
4322         }
4323
4324         if (adapter->rx_buffer_area != NULL) {
4325                 free(adapter->rx_buffer_area, M_DEVBUF);
4326                 adapter->rx_buffer_area = NULL;
4327         }
4328
4329         if (adapter->rxtag != NULL) {
4330                 bus_dma_tag_destroy(adapter->rxtag);
4331                 adapter->rxtag = NULL;
4332         }
4333 }
4334
4335 /*********************************************************************
4336  *
4337  *  This routine executes in interrupt context. It replenishes
4338  *  the mbufs in the descriptor and sends data which has been
4339  *  dma'ed into host memory to upper layer.
4340  *
4341  *  We loop at most count times if count is > 0, or until done if
4342  *  count < 0.
4343  *
4344  *  For polling we also now return the number of cleaned packets
4345  *********************************************************************/
4346 static int
4347 em_rxeof(struct adapter *adapter, int count)
4348 {
4349         struct ifnet    *ifp = adapter->ifp;
4350         struct mbuf     *mp;
4351         u8              status, accept_frame = 0, eop = 0;
4352         u16             len, desc_len, prev_len_adj;
4353         int             i, rx_sent = 0;
4354         struct e1000_rx_desc   *current_desc;
4355
4356         EM_RX_LOCK(adapter);
4357         i = adapter->next_rx_desc_to_check;
4358         current_desc = &adapter->rx_desc_base[i];
4359         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4360             BUS_DMASYNC_POSTREAD);
4361
4362         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
4363                 EM_RX_UNLOCK(adapter);
4364                 return (rx_sent);
4365         }
4366
4367         while ((current_desc->status & E1000_RXD_STAT_DD) &&
4368             (count != 0) &&
4369             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4370                 struct mbuf *m = NULL;
4371
4372                 mp = adapter->rx_buffer_area[i].m_head;
4373                 /*
4374                  * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
4375                  * needs to access the last received byte in the mbuf.
4376                  */
4377                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
4378                     BUS_DMASYNC_POSTREAD);
4379
4380                 accept_frame = 1;
4381                 prev_len_adj = 0;
4382                 desc_len = le16toh(current_desc->length);
4383                 status = current_desc->status;
4384                 if (status & E1000_RXD_STAT_EOP) {
4385                         count--;
4386                         eop = 1;
4387                         if (desc_len < ETHER_CRC_LEN) {
4388                                 len = 0;
4389                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
4390                         } else
4391                                 len = desc_len - ETHER_CRC_LEN;
4392                 } else {
4393                         eop = 0;
4394                         len = desc_len;
4395                 }
4396
4397                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
4398                         u8      last_byte;
4399                         u32     pkt_len = desc_len;
4400
4401                         if (adapter->fmp != NULL)
4402                                 pkt_len += adapter->fmp->m_pkthdr.len;
4403
4404                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
4405                         if (TBI_ACCEPT(&adapter->hw, status,
4406                             current_desc->errors, pkt_len, last_byte,
4407                             adapter->min_frame_size, adapter->max_frame_size)) {
4408                                 e1000_tbi_adjust_stats_82543(&adapter->hw,
4409                                     &adapter->stats, pkt_len,
4410                                     adapter->hw.mac.addr,
4411                                     adapter->max_frame_size);
4412                                 if (len > 0)
4413                                         len--;
4414                         } else
4415                                 accept_frame = 0;
4416                 }
4417
4418                 if (accept_frame) {
4419                         if (em_get_buf(adapter, i) != 0) {
4420                                 ifp->if_iqdrops++;
4421                                 goto discard;
4422                         }
4423
4424                         /* Assign correct length to the current fragment */
4425                         mp->m_len = len;
4426
4427                         if (adapter->fmp == NULL) {
4428                                 mp->m_pkthdr.len = len;
4429                                 adapter->fmp = mp; /* Store the first mbuf */
4430                                 adapter->lmp = mp;
4431                         } else {
4432                                 /* Chain mbuf's together */
4433                                 mp->m_flags &= ~M_PKTHDR;
4434                                 /*
4435                                  * Adjust length of previous mbuf in chain if
4436                                  * we received less than 4 bytes in the last
4437                                  * descriptor.
4438                                  */
4439                                 if (prev_len_adj > 0) {
4440                                         adapter->lmp->m_len -= prev_len_adj;
4441                                         adapter->fmp->m_pkthdr.len -=
4442                                             prev_len_adj;
4443                                 }
4444                                 adapter->lmp->m_next = mp;
4445                                 adapter->lmp = adapter->lmp->m_next;
4446                                 adapter->fmp->m_pkthdr.len += len;
4447                         }
4448
4449                         if (eop) {
4450                                 adapter->fmp->m_pkthdr.rcvif = ifp;
4451                                 ifp->if_ipackets++;
4452                                 em_receive_checksum(adapter, current_desc,
4453                                     adapter->fmp);
4454 #ifndef __NO_STRICT_ALIGNMENT
4455                                 if (adapter->max_frame_size >
4456                                     (MCLBYTES - ETHER_ALIGN) &&
4457                                     em_fixup_rx(adapter) != 0)
4458                                         goto skip;
4459 #endif
4460                                 if (status & E1000_RXD_STAT_VP) {
4461 #if __FreeBSD_version < 700000
4462                                         VLAN_INPUT_TAG_NEW(ifp, adapter->fmp,
4463                                             (le16toh(current_desc->special) &
4464                                             E1000_RXD_SPC_VLAN_MASK));
4465 #else
4466                                         adapter->fmp->m_pkthdr.ether_vtag =
4467                                             (le16toh(current_desc->special) &
4468                                             E1000_RXD_SPC_VLAN_MASK);
4469                                         adapter->fmp->m_flags |= M_VLANTAG;
4470 #endif
4471                                 }
4472 #ifndef __NO_STRICT_ALIGNMENT
4473 skip:
4474 #endif
4475                                 m = adapter->fmp;
4476                                 adapter->fmp = NULL;
4477                                 adapter->lmp = NULL;
4478                         }
4479                 } else {
4480                         ifp->if_ierrors++;
4481 discard:
4482                         /* Reuse loaded DMA map and just update mbuf chain */
4483                         mp = adapter->rx_buffer_area[i].m_head;
4484                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
4485                         mp->m_data = mp->m_ext.ext_buf;
4486                         mp->m_next = NULL;
4487                         if (adapter->max_frame_size <=
4488                             (MCLBYTES - ETHER_ALIGN))
4489                                 m_adj(mp, ETHER_ALIGN);
4490                         if (adapter->fmp != NULL) {
4491                                 m_freem(adapter->fmp);
4492                                 adapter->fmp = NULL;
4493                                 adapter->lmp = NULL;
4494                         }
4495                         m = NULL;
4496                 }
4497
4498                 /* Zero out the receive descriptors status. */
4499                 current_desc->status = 0;
4500                 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4501                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4502
4503                 /* Advance our pointers to the next descriptor. */
4504                 if (++i == adapter->num_rx_desc)
4505                         i = 0;
4506                 /* Call into the stack */
4507                 if (m != NULL) {
4508                         adapter->next_rx_desc_to_check = i;
4509                         EM_RX_UNLOCK(adapter);
4510                         (*ifp->if_input)(ifp, m);
4511                         EM_RX_LOCK(adapter);
4512                         rx_sent++;
4513                         i = adapter->next_rx_desc_to_check;
4514                 }
4515                 current_desc = &adapter->rx_desc_base[i];
4516         }
4517         adapter->next_rx_desc_to_check = i;
4518
4519         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
4520         if (--i < 0)
4521                 i = adapter->num_rx_desc - 1;
4522         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
4523         EM_RX_UNLOCK(adapter);
4524         return (rx_sent);
4525 }
4526
4527 #ifndef __NO_STRICT_ALIGNMENT
4528 /*
4529  * When jumbo frames are enabled we should realign entire payload on
4530  * architecures with strict alignment. This is serious design mistake of 8254x
4531  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4532  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4533  * payload. On architecures without strict alignment restrictions 8254x still
4534  * performs unaligned memory access which would reduce the performance too.
4535  * To avoid copying over an entire frame to align, we allocate a new mbuf and
4536  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4537  * existing mbuf chain.
4538  *
4539  * Be aware, best performance of the 8254x is achived only when jumbo frame is
4540  * not used at all on architectures with strict alignment.
4541  */
4542 static int
4543 em_fixup_rx(struct adapter *adapter)
4544 {
4545         struct mbuf *m, *n;
4546         int error;
4547
4548         error = 0;
4549         m = adapter->fmp;
4550         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4551                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4552                 m->m_data += ETHER_HDR_LEN;
4553         } else {
4554                 MGETHDR(n, M_DONTWAIT, MT_DATA);
4555                 if (n != NULL) {
4556                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4557                         m->m_data += ETHER_HDR_LEN;
4558                         m->m_len -= ETHER_HDR_LEN;
4559                         n->m_len = ETHER_HDR_LEN;
4560                         M_MOVE_PKTHDR(n, m);
4561                         n->m_next = m;
4562                         adapter->fmp = n;
4563                 } else {
4564                         adapter->dropped_pkts++;
4565                         m_freem(adapter->fmp);
4566                         adapter->fmp = NULL;
4567                         error = ENOMEM;
4568                 }
4569         }
4570
4571         return (error);
4572 }
4573 #endif
4574
4575 /*********************************************************************
4576  *
4577  *  Verify that the hardware indicated that the checksum is valid.
4578  *  Inform the stack about the status of checksum so that stack
4579  *  doesn't spend time verifying the checksum.
4580  *
4581  *********************************************************************/
4582 static void
4583 em_receive_checksum(struct adapter *adapter,
4584             struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4585 {
4586         /* 82543 or newer only */
4587         if ((adapter->hw.mac.type < e1000_82543) ||
4588             /* Ignore Checksum bit is set */
4589             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
4590                 mp->m_pkthdr.csum_flags = 0;
4591                 return;
4592         }
4593
4594         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4595                 /* Did it pass? */
4596                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4597                         /* IP Checksum Good */
4598                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4599                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4600
4601                 } else {
4602                         mp->m_pkthdr.csum_flags = 0;
4603                 }
4604         }
4605
4606         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4607                 /* Did it pass? */
4608                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4609                         mp->m_pkthdr.csum_flags |=
4610                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4611                         mp->m_pkthdr.csum_data = htons(0xffff);
4612                 }
4613         }
4614 }
4615
4616 #if __FreeBSD_version >= 700029
4617 /*
4618  * This routine is run via an vlan
4619  * config EVENT
4620  */
4621 static void
4622 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4623 {
4624         struct adapter  *adapter = ifp->if_softc;
4625         u32             index, bit;
4626
4627         if (ifp->if_softc !=  arg)   /* Not our event */
4628                 return;
4629
4630         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
4631                 return;
4632
4633         index = (vtag >> 5) & 0x7F;
4634         bit = vtag & 0x1F;
4635         em_shadow_vfta[index] |= (1 << bit);
4636         ++adapter->num_vlans;
4637         /* Re-init to load the changes */
4638         em_init(adapter);
4639 }
4640
4641 /*
4642  * This routine is run via an vlan
4643  * unconfig EVENT
4644  */
4645 static void
4646 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4647 {
4648         struct adapter  *adapter = ifp->if_softc;
4649         u32             index, bit;
4650
4651         if (ifp->if_softc !=  arg)
4652                 return;
4653
4654         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4655                 return;
4656
4657         index = (vtag >> 5) & 0x7F;
4658         bit = vtag & 0x1F;
4659         em_shadow_vfta[index] &= ~(1 << bit);
4660         --adapter->num_vlans;
4661         /* Re-init to load the changes */
4662         em_init(adapter);
4663 }
4664
4665 static void
4666 em_setup_vlan_hw_support(struct adapter *adapter)
4667 {
4668         struct e1000_hw *hw = &adapter->hw;
4669         u32             reg;
4670
4671         /*
4672         ** We get here thru init_locked, meaning
4673         ** a soft reset, this has already cleared
4674         ** the VFTA and other state, so if there
4675         ** have been no vlan's registered do nothing.
4676         */
4677         if (adapter->num_vlans == 0)
4678                 return;
4679
4680         /*
4681         ** A soft reset zero's out the VFTA, so
4682         ** we need to repopulate it now.
4683         */
4684         for (int i = 0; i < EM_VFTA_SIZE; i++)
4685                 if (em_shadow_vfta[i] != 0)
4686                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4687                             i, em_shadow_vfta[i]);
4688
4689         reg = E1000_READ_REG(hw, E1000_CTRL);
4690         reg |= E1000_CTRL_VME;
4691         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4692
4693         /* Enable the Filter Table */
4694         reg = E1000_READ_REG(hw, E1000_RCTL);
4695         reg &= ~E1000_RCTL_CFIEN;
4696         reg |= E1000_RCTL_VFE;
4697         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4698
4699         /* Update the frame size */
4700         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4701             adapter->max_frame_size + VLAN_TAG_SIZE);
4702 }
4703 #endif
4704
4705 static void
4706 em_enable_intr(struct adapter *adapter)
4707 {
4708         struct e1000_hw *hw = &adapter->hw;
4709         u32 ims_mask = IMS_ENABLE_MASK;
4710
4711         if (adapter->msix) {
4712                 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4713                 ims_mask |= EM_MSIX_MASK;
4714         }
4715         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4716 }
4717
4718 static void
4719 em_disable_intr(struct adapter *adapter)
4720 {
4721         struct e1000_hw *hw = &adapter->hw;
4722
4723         if (adapter->msix)
4724                 E1000_WRITE_REG(hw, EM_EIAC, 0);
4725         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4726 }
4727
4728 /*
4729  * Bit of a misnomer, what this really means is
4730  * to enable OS management of the system... aka
4731  * to disable special hardware management features
4732  */
4733 static void
4734 em_init_manageability(struct adapter *adapter)
4735 {
4736         /* A shared code workaround */
4737 #define E1000_82542_MANC2H E1000_MANC2H
4738         if (adapter->has_manage) {
4739                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4740                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4741
4742                 /* disable hardware interception of ARP */
4743                 manc &= ~(E1000_MANC_ARP_EN);
4744
4745                 /* enable receiving management packets to the host */
4746                 if (adapter->hw.mac.type >= e1000_82571) {
4747                         manc |= E1000_MANC_EN_MNG2HOST;
4748 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4749 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4750                         manc2h |= E1000_MNG2HOST_PORT_623;
4751                         manc2h |= E1000_MNG2HOST_PORT_664;
4752                         E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4753                 }
4754
4755                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4756         }
4757 }
4758
4759 /*
4760  * Give control back to hardware management
4761  * controller if there is one.
4762  */
4763 static void
4764 em_release_manageability(struct adapter *adapter)
4765 {
4766         if (adapter->has_manage) {
4767                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4768
4769                 /* re-enable hardware interception of ARP */
4770                 manc |= E1000_MANC_ARP_EN;
4771
4772                 if (adapter->hw.mac.type >= e1000_82571)
4773                         manc &= ~E1000_MANC_EN_MNG2HOST;
4774
4775                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4776         }
4777 }
4778
4779 /*
4780  * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4781  * For ASF and Pass Through versions of f/w this means
4782  * that the driver is loaded. For AMT version type f/w
4783  * this means that the network i/f is open.
4784  */
4785 static void
4786 em_get_hw_control(struct adapter *adapter)
4787 {
4788         u32 ctrl_ext, swsm;
4789
4790         if (adapter->hw.mac.type == e1000_82573) {
4791                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4792                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4793                     swsm | E1000_SWSM_DRV_LOAD);
4794                 return;
4795         }
4796         /* else */
4797         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4798         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4799             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4800         return;
4801 }
4802
4803 /*
4804  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4805  * For ASF and Pass Through versions of f/w this means that
4806  * the driver is no longer loaded. For AMT versions of the
4807  * f/w this means that the network i/f is closed.
4808  */
4809 static void
4810 em_release_hw_control(struct adapter *adapter)
4811 {
4812         u32 ctrl_ext, swsm;
4813
4814         if (!adapter->has_manage)
4815                 return;
4816
4817         if (adapter->hw.mac.type == e1000_82573) {
4818                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4819                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4820                     swsm & ~E1000_SWSM_DRV_LOAD);
4821                 return;
4822         }
4823         /* else */
4824         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4825         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4826             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4827         return;
4828 }
4829
4830 static int
4831 em_is_valid_ether_addr(u8 *addr)
4832 {
4833         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4834
4835         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4836                 return (FALSE);
4837         }
4838
4839         return (TRUE);
4840 }
4841
4842 /*
4843 ** Parse the interface capabilities with regard
4844 ** to both system management and wake-on-lan for
4845 ** later use.
4846 */
4847 static void
4848 em_get_wakeup(device_t dev)
4849 {
4850         struct adapter  *adapter = device_get_softc(dev);
4851         u16             eeprom_data = 0, device_id, apme_mask;
4852
4853         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4854         apme_mask = EM_EEPROM_APME;
4855
4856         switch (adapter->hw.mac.type) {
4857         case e1000_82542:
4858         case e1000_82543:
4859                 break;
4860         case e1000_82544:
4861                 e1000_read_nvm(&adapter->hw,
4862                     NVM_INIT_CONTROL2_REG, 1, &eeprom_data);
4863                 apme_mask = EM_82544_APME;
4864                 break;
4865         case e1000_82573:
4866         case e1000_82583:
4867                 adapter->has_amt = TRUE;
4868                 /* Falls thru */
4869         case e1000_82546:
4870         case e1000_82546_rev_3:
4871         case e1000_82571:
4872         case e1000_82572:
4873         case e1000_80003es2lan:
4874                 if (adapter->hw.bus.func == 1) {
4875                         e1000_read_nvm(&adapter->hw,
4876                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4877                         break;
4878                 } else
4879                         e1000_read_nvm(&adapter->hw,
4880                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4881                 break;
4882         case e1000_ich8lan:
4883         case e1000_ich9lan:
4884         case e1000_ich10lan:
4885         case e1000_pchlan:
4886                 apme_mask = E1000_WUC_APME;
4887                 adapter->has_amt = TRUE;
4888                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
4889                 break;
4890         default:
4891                 e1000_read_nvm(&adapter->hw,
4892                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4893                 break;
4894         }
4895         if (eeprom_data & apme_mask)
4896                 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4897         /*
4898          * We have the eeprom settings, now apply the special cases
4899          * where the eeprom may be wrong or the board won't support
4900          * wake on lan on a particular port
4901          */
4902         device_id = pci_get_device(dev);
4903         switch (device_id) {
4904         case E1000_DEV_ID_82546GB_PCIE:
4905                 adapter->wol = 0;
4906                 break;
4907         case E1000_DEV_ID_82546EB_FIBER:
4908         case E1000_DEV_ID_82546GB_FIBER:
4909         case E1000_DEV_ID_82571EB_FIBER:
4910                 /* Wake events only supported on port A for dual fiber
4911                  * regardless of eeprom setting */
4912                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4913                     E1000_STATUS_FUNC_1)
4914                         adapter->wol = 0;
4915                 break;
4916         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
4917         case E1000_DEV_ID_82571EB_QUAD_COPPER:
4918         case E1000_DEV_ID_82571EB_QUAD_FIBER:
4919         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
4920                 /* if quad port adapter, disable WoL on all but port A */
4921                 if (global_quad_port_a != 0)
4922                         adapter->wol = 0;
4923                 /* Reset for multiple quad port adapters */
4924                 if (++global_quad_port_a == 4)
4925                         global_quad_port_a = 0;
4926                 break;
4927         }
4928         return;
4929 }
4930
4931
4932 /*
4933  * Enable PCI Wake On Lan capability
4934  */
4935 void
4936 em_enable_wakeup(device_t dev)
4937 {
4938         struct adapter  *adapter = device_get_softc(dev);
4939         struct ifnet    *ifp = adapter->ifp;
4940         u32             pmc, ctrl, ctrl_ext, rctl;
4941         u16             status;
4942
4943         if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0))
4944                 return;
4945
4946         /* Advertise the wakeup capability */
4947         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4948         ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4949         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4950         E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4951
4952         /* ICH workaround code */
4953         if ((adapter->hw.mac.type == e1000_ich8lan) ||
4954             (adapter->hw.mac.type == e1000_pchlan) ||
4955             (adapter->hw.mac.type == e1000_ich9lan) ||
4956             (adapter->hw.mac.type == e1000_ich10lan)) {
4957                 e1000_disable_gig_wol_ich8lan(&adapter->hw);
4958                 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw);
4959         }
4960
4961         /* Keep the laser running on Fiber adapters */
4962         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4963             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4964                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4965                 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4966                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4967         }
4968
4969         /*
4970         ** Determine type of Wakeup: note that wol
4971         ** is set with all bits on by default.
4972         */
4973         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0)
4974                 adapter->wol &= ~E1000_WUFC_MAG;
4975
4976         if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0)
4977                 adapter->wol &= ~E1000_WUFC_MC;
4978         else {
4979                 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4980                 rctl |= E1000_RCTL_MPE;
4981                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4982         }
4983
4984         if (adapter->hw.mac.type == e1000_pchlan) {
4985                 if (em_enable_phy_wakeup(adapter))
4986                         return;
4987         } else {
4988                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4989                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4990         }
4991
4992         if (adapter->hw.phy.type == e1000_phy_igp_3)
4993                 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
4994
4995         /* Request PME */
4996         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4997         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4998         if (ifp->if_capenable & IFCAP_WOL)
4999                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5000         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
5001
5002         return;
5003 }
5004
5005 /*
5006 ** WOL in the newer chipset interfaces (pchlan)
5007 ** require thing to be copied into the phy
5008 */
5009 static int
5010 em_enable_phy_wakeup(struct adapter *adapter)
5011 {
5012         struct e1000_hw *hw = &adapter->hw;
5013         u32 mreg, ret = 0;
5014         u16 preg;
5015
5016         /* copy MAC RARs to PHY RARs */
5017         for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) {
5018                 mreg = E1000_READ_REG(hw, E1000_RAL(i));
5019                 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF));
5020                 e1000_write_phy_reg(hw, BM_RAR_M(i),
5021                     (u16)((mreg >> 16) & 0xFFFF));
5022                 mreg = E1000_READ_REG(hw, E1000_RAH(i));
5023                 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF));
5024                 e1000_write_phy_reg(hw, BM_RAR_CTRL(i),
5025                     (u16)((mreg >> 16) & 0xFFFF));
5026         }
5027
5028         /* copy MAC MTA to PHY MTA */
5029         for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5030                 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
5031                 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
5032                 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
5033                     (u16)((mreg >> 16) & 0xFFFF));
5034         }
5035
5036         /* configure PHY Rx Control register */
5037         e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
5038         mreg = E1000_READ_REG(hw, E1000_RCTL);
5039         if (mreg & E1000_RCTL_UPE)
5040                 preg |= BM_RCTL_UPE;
5041         if (mreg & E1000_RCTL_MPE)
5042                 preg |= BM_RCTL_MPE;
5043         preg &= ~(BM_RCTL_MO_MASK);
5044         if (mreg & E1000_RCTL_MO_3)
5045                 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5046                                 << BM_RCTL_MO_SHIFT);
5047         if (mreg & E1000_RCTL_BAM)
5048                 preg |= BM_RCTL_BAM;
5049         if (mreg & E1000_RCTL_PMCF)
5050                 preg |= BM_RCTL_PMCF;
5051         mreg = E1000_READ_REG(hw, E1000_CTRL);
5052         if (mreg & E1000_CTRL_RFCE)
5053                 preg |= BM_RCTL_RFCE;
5054         e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
5055
5056         /* enable PHY wakeup in MAC register */
5057         E1000_WRITE_REG(hw, E1000_WUC,
5058             E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5059         E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
5060
5061         /* configure and enable PHY wakeup in PHY registers */
5062         e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
5063         e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5064
5065         /* activate PHY wakeup */
5066         ret = hw->phy.ops.acquire(hw);
5067         if (ret) {
5068                 printf("Could not acquire PHY\n");
5069                 return ret;
5070         }
5071         e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
5072                                  (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
5073         ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
5074         if (ret) {
5075                 printf("Could not read PHY page 769\n");
5076                 goto out;
5077         }
5078         preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5079         ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
5080         if (ret)
5081                 printf("Could not set PHY Host Wakeup bit\n");
5082 out:
5083         hw->phy.ops.release(hw);
5084
5085         return ret;
5086 }
5087
5088
5089 /*********************************************************************
5090 * 82544 Coexistence issue workaround.
5091 *    There are 2 issues.
5092 *       1. Transmit Hang issue.
5093 *    To detect this issue, following equation can be used...
5094 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
5095 *         If SUM[3:0] is in between 1 to 4, we will have this issue.
5096 *
5097 *       2. DAC issue.
5098 *    To detect this issue, following equation can be used...
5099 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
5100 *         If SUM[3:0] is in between 9 to c, we will have this issue.
5101 *
5102 *
5103 *    WORKAROUND:
5104 *         Make sure we do not have ending address
5105 *         as 1,2,3,4(Hang) or 9,a,b,c (DAC)
5106 *
5107 *************************************************************************/
5108 static u32
5109 em_fill_descriptors (bus_addr_t address, u32 length,
5110                 PDESC_ARRAY desc_array)
5111 {
5112         u32 safe_terminator;
5113
5114         /* Since issue is sensitive to length and address.*/
5115         /* Let us first check the address...*/
5116         if (length <= 4) {
5117                 desc_array->descriptor[0].address = address;
5118                 desc_array->descriptor[0].length = length;
5119                 desc_array->elements = 1;
5120                 return (desc_array->elements);
5121         }
5122         safe_terminator = (u32)((((u32)address & 0x7) +
5123             (length & 0xF)) & 0xF);
5124         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
5125         if (safe_terminator == 0   ||
5126         (safe_terminator > 4   &&
5127         safe_terminator < 9)   ||
5128         (safe_terminator > 0xC &&
5129         safe_terminator <= 0xF)) {
5130                 desc_array->descriptor[0].address = address;
5131                 desc_array->descriptor[0].length = length;
5132                 desc_array->elements = 1;
5133                 return (desc_array->elements);
5134         }
5135
5136         desc_array->descriptor[0].address = address;
5137         desc_array->descriptor[0].length = length - 4;
5138         desc_array->descriptor[1].address = address + (length - 4);
5139         desc_array->descriptor[1].length = 4;
5140         desc_array->elements = 2;
5141         return (desc_array->elements);
5142 }
5143
5144 /**********************************************************************
5145  *
5146  *  Update the board statistics counters.
5147  *
5148  **********************************************************************/
5149 static void
5150 em_update_stats_counters(struct adapter *adapter)
5151 {
5152         struct ifnet   *ifp;
5153
5154         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5155            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5156                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5157                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5158         }
5159         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5160         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5161         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5162         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5163
5164         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5165         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5166         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5167         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5168         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5169         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5170         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5171         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5172         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5173         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5174         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5175         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5176         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5177         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5178         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5179         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5180         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5181         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5182         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5183         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5184
5185         /* For the 64-bit byte counters the low dword must be read first. */
5186         /* Both registers clear on the read of the high dword */
5187
5188         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
5189         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
5190
5191         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5192         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5193         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5194         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5195         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5196
5197         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5198         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5199
5200         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5201         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5202         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5203         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5204         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5205         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5206         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5207         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5208         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5209         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5210
5211         if (adapter->hw.mac.type >= e1000_82543) {
5212                 adapter->stats.algnerrc +=
5213                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5214                 adapter->stats.rxerrc +=
5215                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5216                 adapter->stats.tncrs +=
5217                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5218                 adapter->stats.cexterr +=
5219                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5220                 adapter->stats.tsctc +=
5221                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5222                 adapter->stats.tsctfc +=
5223                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5224         }
5225         ifp = adapter->ifp;
5226
5227         ifp->if_collisions = adapter->stats.colc;
5228
5229         /* Rx Errors */
5230         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5231             adapter->stats.crcerrs + adapter->stats.algnerrc +
5232             adapter->stats.ruc + adapter->stats.roc +
5233             adapter->stats.mpc + adapter->stats.cexterr;
5234
5235         /* Tx Errors */
5236         ifp->if_oerrors = adapter->stats.ecol +
5237             adapter->stats.latecol + adapter->watchdog_events;
5238 }
5239
5240
5241 /**********************************************************************
5242  *
5243  *  This routine is called only when em_display_debug_stats is enabled.
5244  *  This routine provides a way to take a look at important statistics
5245  *  maintained by the driver and hardware.
5246  *
5247  **********************************************************************/
5248 static void
5249 em_print_debug_info(struct adapter *adapter)
5250 {
5251         device_t dev = adapter->dev;
5252         u8 *hw_addr = adapter->hw.hw_addr;
5253
5254         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
5255         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
5256             E1000_READ_REG(&adapter->hw, E1000_CTRL),
5257             E1000_READ_REG(&adapter->hw, E1000_RCTL));
5258         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
5259             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
5260             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
5261         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
5262             adapter->hw.fc.high_water,
5263             adapter->hw.fc.low_water);
5264         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
5265             E1000_READ_REG(&adapter->hw, E1000_TIDV),
5266             E1000_READ_REG(&adapter->hw, E1000_TADV));
5267         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
5268             E1000_READ_REG(&adapter->hw, E1000_RDTR),
5269             E1000_READ_REG(&adapter->hw, E1000_RADV));
5270         device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
5271             (long long)adapter->tx_fifo_wrk_cnt,
5272             (long long)adapter->tx_fifo_reset_cnt);
5273         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5274             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5275             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5276         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5277             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5278             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5279         device_printf(dev, "Num Tx descriptors avail = %d\n",
5280             adapter->num_tx_desc_avail);
5281         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
5282             adapter->no_tx_desc_avail1);
5283         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
5284             adapter->no_tx_desc_avail2);
5285         device_printf(dev, "Std mbuf failed = %ld\n",
5286             adapter->mbuf_alloc_failed);
5287         device_printf(dev, "Std mbuf cluster failed = %ld\n",
5288             adapter->mbuf_cluster_failed);
5289         device_printf(dev, "Driver dropped packets = %ld\n",
5290             adapter->dropped_pkts);
5291         device_printf(dev, "Driver tx dma failure in encap = %ld\n",
5292                 adapter->no_tx_dma_setup);
5293 }
5294
5295 static void
5296 em_print_hw_stats(struct adapter *adapter)
5297 {
5298         device_t dev = adapter->dev;
5299
5300         device_printf(dev, "Excessive collisions = %lld\n",
5301             (long long)adapter->stats.ecol);
5302 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
5303         device_printf(dev, "Symbol errors = %lld\n",
5304             (long long)adapter->stats.symerrs);
5305 #endif
5306         device_printf(dev, "Sequence errors = %lld\n",
5307             (long long)adapter->stats.sec);
5308         device_printf(dev, "Defer count = %lld\n",
5309             (long long)adapter->stats.dc);
5310         device_printf(dev, "Missed Packets = %lld\n",
5311             (long long)adapter->stats.mpc);
5312         device_printf(dev, "Receive No Buffers = %lld\n",
5313             (long long)adapter->stats.rnbc);
5314         /* RLEC is inaccurate on some hardware, calculate our own. */
5315         device_printf(dev, "Receive Length Errors = %lld\n",
5316             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
5317         device_printf(dev, "Receive errors = %lld\n",
5318             (long long)adapter->stats.rxerrc);
5319         device_printf(dev, "Crc errors = %lld\n",
5320             (long long)adapter->stats.crcerrs);
5321         device_printf(dev, "Alignment errors = %lld\n",
5322             (long long)adapter->stats.algnerrc);
5323         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
5324             (long long)adapter->stats.cexterr);
5325         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
5326         device_printf(dev, "watchdog timeouts = %ld\n",
5327             adapter->watchdog_events);
5328         device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld"
5329             " LINK MSIX IRQ = %ld\n", adapter->rx_irq,
5330             adapter->tx_irq , adapter->link_irq);
5331         device_printf(dev, "XON Rcvd = %lld\n",
5332             (long long)adapter->stats.xonrxc);
5333         device_printf(dev, "XON Xmtd = %lld\n",
5334             (long long)adapter->stats.xontxc);
5335         device_printf(dev, "XOFF Rcvd = %lld\n",
5336             (long long)adapter->stats.xoffrxc);
5337         device_printf(dev, "XOFF Xmtd = %lld\n",
5338             (long long)adapter->stats.xofftxc);
5339         device_printf(dev, "Good Packets Rcvd = %lld\n",
5340             (long long)adapter->stats.gprc);
5341         device_printf(dev, "Good Packets Xmtd = %lld\n",
5342             (long long)adapter->stats.gptc);
5343         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
5344             (long long)adapter->stats.tsctc);
5345         device_printf(dev, "TSO Contexts Failed = %lld\n",
5346             (long long)adapter->stats.tsctfc);
5347 }
5348
5349 /**********************************************************************
5350  *
5351  *  This routine provides a way to dump out the adapter eeprom,
5352  *  often a useful debug/service tool. This only dumps the first
5353  *  32 words, stuff that matters is in that extent.
5354  *
5355  **********************************************************************/
5356 static void
5357 em_print_nvm_info(struct adapter *adapter)
5358 {
5359         u16     eeprom_data;
5360         int     i, j, row = 0;
5361
5362         /* Its a bit crude, but it gets the job done */
5363         printf("\nInterface EEPROM Dump:\n");
5364         printf("Offset\n0x0000  ");
5365         for (i = 0, j = 0; i < 32; i++, j++) {
5366                 if (j == 8) { /* Make the offset block */
5367                         j = 0; ++row;
5368                         printf("\n0x00%x0  ",row);
5369                 }
5370                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5371                 printf("%04x ", eeprom_data);
5372         }
5373         printf("\n");
5374 }
5375
5376 static int
5377 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5378 {
5379         struct adapter *adapter;
5380         int error;
5381         int result;
5382
5383         result = -1;
5384         error = sysctl_handle_int(oidp, &result, 0, req);
5385
5386         if (error || !req->newptr)
5387                 return (error);
5388
5389         if (result == 1) {
5390                 adapter = (struct adapter *)arg1;
5391                 em_print_debug_info(adapter);
5392         }
5393         /*
5394          * This value will cause a hex dump of the
5395          * first 32 16-bit words of the EEPROM to
5396          * the screen.
5397          */
5398         if (result == 2) {
5399                 adapter = (struct adapter *)arg1;
5400                 em_print_nvm_info(adapter);
5401         }
5402
5403         return (error);
5404 }
5405
5406
5407 static int
5408 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
5409 {
5410         struct adapter *adapter;
5411         int error;
5412         int result;
5413
5414         result = -1;
5415         error = sysctl_handle_int(oidp, &result, 0, req);
5416
5417         if (error || !req->newptr)
5418                 return (error);
5419
5420         if (result == 1) {
5421                 adapter = (struct adapter *)arg1;
5422                 em_print_hw_stats(adapter);
5423         }
5424
5425         return (error);
5426 }
5427
5428 static int
5429 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5430 {
5431         struct em_int_delay_info *info;
5432         struct adapter *adapter;
5433         u32 regval;
5434         int error;
5435         int usecs;
5436         int ticks;
5437
5438         info = (struct em_int_delay_info *)arg1;
5439         usecs = info->value;
5440         error = sysctl_handle_int(oidp, &usecs, 0, req);
5441         if (error != 0 || req->newptr == NULL)
5442                 return (error);
5443         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5444                 return (EINVAL);
5445         info->value = usecs;
5446         ticks = EM_USECS_TO_TICKS(usecs);
5447
5448         adapter = info->adapter;
5449
5450         EM_CORE_LOCK(adapter);
5451         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5452         regval = (regval & ~0xffff) | (ticks & 0xffff);
5453         /* Handle a few special cases. */
5454         switch (info->offset) {
5455         case E1000_RDTR:
5456                 break;
5457         case E1000_TIDV:
5458                 if (ticks == 0) {
5459                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5460                         /* Don't write 0 into the TIDV register. */
5461                         regval++;
5462                 } else
5463                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5464                 break;
5465         }
5466         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5467         EM_CORE_UNLOCK(adapter);
5468         return (0);
5469 }
5470
5471 static void
5472 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5473         const char *description, struct em_int_delay_info *info,
5474         int offset, int value)
5475 {
5476         info->adapter = adapter;
5477         info->offset = offset;
5478         info->value = value;
5479         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5480             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5481             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5482             info, 0, em_sysctl_int_delay, "I", description);
5483 }
5484
5485 #ifndef EM_LEGACY_IRQ
5486 static void
5487 em_add_rx_process_limit(struct adapter *adapter, const char *name,
5488         const char *description, int *limit, int value)
5489 {
5490         *limit = value;
5491         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5492             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5493             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5494 }
5495 #endif
5496
5497