sys/dev/e1000/if_lem.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 /*
  36  * Uncomment the following extensions for better performance in a VM,
  37  * especially if you have support in the hypervisor.
  38  * See http://info.iet.unipi.it/~luigi/netmap/
  39  */
  40 // #define BATCH_DISPATCH
  41 // #define NIC_SEND_COMBINING
  42
  43 #include "opt_inet.h"
  44 #include "opt_inet6.h"
  45
  46 #ifdef HAVE_KERNEL_OPTION_HEADERS
  47 #include "opt_device_polling.h"
  48 #endif
  49
  50 #include <sys/param.h>
  51 #include <sys/systm.h>
  52 #include <sys/buf_ring.h>
  53 #include <sys/bus.h>
  54 #include <sys/endian.h>
  55 #include <sys/kernel.h>
  56 #include <sys/kthread.h>
  57 #include <sys/malloc.h>
  58 #include <sys/mbuf.h>
  59 #include <sys/module.h>
  60 #include <sys/rman.h>
  61 #include <sys/socket.h>
  62 #include <sys/sockio.h>
  63 #include <sys/sysctl.h>
  64 #include <sys/taskqueue.h>
  65 #include <sys/eventhandler.h>
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68
  69 #include <net/bpf.h>
  70 #include <net/ethernet.h>
  71 #include <net/if.h>
  72 #include <net/if_var.h>
  73 #include <net/if_arp.h>
  74 #include <net/if_dl.h>
  75 #include <net/if_media.h>
  76
  77 #include <net/if_types.h>
  78 #include <net/if_vlan_var.h>
  79
  80 #include <netinet/in_systm.h>
  81 #include <netinet/in.h>
  82 #include <netinet/if_ether.h>
  83 #include <netinet/ip.h>
  84 #include <netinet/ip6.h>
  85 #include <netinet/tcp.h>
  86 #include <netinet/udp.h>
  87
  88 #include <machine/in_cksum.h>
  89 #include <dev/led/led.h>
  90 #include <dev/pci/pcivar.h>
  91 #include <dev/pci/pcireg.h>
  92
  93 #include "e1000_api.h"
  94 #include "if_lem.h"
  95
  96 /*********************************************************************
  97  *  Legacy Em Driver version:
  98  *********************************************************************/
  99 char lem_driver_version[] = "1.1.0";
 100
 101 /*********************************************************************
 102  *  PCI Device ID Table
 103  *
 104  *  Used by probe to select devices to load on
 105  *  Last field stores an index into e1000_strings
 106  *  Last entry must be all 0s
 107  *
 108  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 109  *********************************************************************/
 110
 111 static em_vendor_info_t lem_vendor_info_array[] =
 112 {
 113         /* Intel(R) PRO/1000 Network Connection */
 114         { 0x8086, E1000_DEV_ID_82540EM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82540EM_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 116         { 0x8086, E1000_DEV_ID_82540EP,         PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82540EP_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 118         { 0x8086, E1000_DEV_ID_82540EP_LP,      PCI_ANY_ID, PCI_ANY_ID, 0},
 119
 120         { 0x8086, E1000_DEV_ID_82541EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82541ER,         PCI_ANY_ID, PCI_ANY_ID, 0},
 122         { 0x8086, E1000_DEV_ID_82541ER_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82541EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 124         { 0x8086, E1000_DEV_ID_82541GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82541GI_LF,      PCI_ANY_ID, PCI_ANY_ID, 0},
 126         { 0x8086, E1000_DEV_ID_82541GI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 127
 128         { 0x8086, E1000_DEV_ID_82542,           PCI_ANY_ID, PCI_ANY_ID, 0},
 129
 130         { 0x8086, E1000_DEV_ID_82543GC_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 131         { 0x8086, E1000_DEV_ID_82543GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 132
 133         { 0x8086, E1000_DEV_ID_82544EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82544EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82544GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 136         { 0x8086, E1000_DEV_ID_82544GC_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 137
 138         { 0x8086, E1000_DEV_ID_82545EM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 139         { 0x8086, E1000_DEV_ID_82545EM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_82545GM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 141         { 0x8086, E1000_DEV_ID_82545GM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 142         { 0x8086, E1000_DEV_ID_82545GM_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 143
 144         { 0x8086, E1000_DEV_ID_82546EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_82546EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_82546GB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_82546GB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_82546GB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_82546GB_PCIE,    PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 152         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
 153                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 154
 155         { 0x8086, E1000_DEV_ID_82547EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_82547EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 157         { 0x8086, E1000_DEV_ID_82547GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 158         /* required last entry */
 159         { 0, 0, 0, 0, 0}
 160 };
 161
 162 /*********************************************************************
 163  *  Table of branding strings for all supported NICs.
 164  *********************************************************************/
 165
 166 static char *lem_strings[] = {
 167         "Intel(R) PRO/1000 Legacy Network Connection"
 168 };
 169
 170 /*********************************************************************
 171  *  Function prototypes
 172  *********************************************************************/
 173 static int      lem_probe(device_t);
 174 static int      lem_attach(device_t);
 175 static int      lem_detach(device_t);
 176 static int      lem_shutdown(device_t);
 177 static int      lem_suspend(device_t);
 178 static int      lem_resume(device_t);
 179 static void     lem_start(if_t);
 180 static void     lem_start_locked(if_t ifp);
 181 static int      lem_ioctl(if_t, u_long, caddr_t);
 182 static uint64_t lem_get_counter(if_t, ift_counter);
 183 static void     lem_init(void *);
 184 static void     lem_init_locked(struct adapter *);
 185 static void     lem_stop(void *);
 186 static void     lem_media_status(if_t, struct ifmediareq *);
 187 static int      lem_media_change(if_t);
 188 static void     lem_identify_hardware(struct adapter *);
 189 static int      lem_allocate_pci_resources(struct adapter *);
 190 static int      lem_allocate_irq(struct adapter *adapter);
 191 static void     lem_free_pci_resources(struct adapter *);
 192 static void     lem_local_timer(void *);
 193 static int      lem_hardware_init(struct adapter *);
 194 static int      lem_setup_interface(device_t, struct adapter *);
 195 static void     lem_setup_transmit_structures(struct adapter *);
 196 static void     lem_initialize_transmit_unit(struct adapter *);
 197 static int      lem_setup_receive_structures(struct adapter *);
 198 static void     lem_initialize_receive_unit(struct adapter *);
 199 static void     lem_enable_intr(struct adapter *);
 200 static void     lem_disable_intr(struct adapter *);
 201 static void     lem_free_transmit_structures(struct adapter *);
 202 static void     lem_free_receive_structures(struct adapter *);
 203 static void     lem_update_stats_counters(struct adapter *);
 204 static void     lem_add_hw_stats(struct adapter *adapter);
 205 static void     lem_txeof(struct adapter *);
 206 static void     lem_tx_purge(struct adapter *);
 207 static int      lem_allocate_receive_structures(struct adapter *);
 208 static int      lem_allocate_transmit_structures(struct adapter *);
 209 static bool     lem_rxeof(struct adapter *, int, int *);
 210 #ifndef __NO_STRICT_ALIGNMENT
 211 static int      lem_fixup_rx(struct adapter *);
 212 #endif
 213 static void     lem_receive_checksum(struct adapter *, struct e1000_rx_desc *,
 214                     struct mbuf *);
 215 static void     lem_transmit_checksum_setup(struct adapter *, struct mbuf *,
 216                     u32 *, u32 *);
 217 static void     lem_set_promisc(struct adapter *);
 218 static void     lem_disable_promisc(struct adapter *);
 219 static void     lem_set_multi(struct adapter *);
 220 static void     lem_update_link_status(struct adapter *);
 221 static int      lem_get_buf(struct adapter *, int);
 222 static void     lem_register_vlan(void *, if_t, u16);
 223 static void     lem_unregister_vlan(void *, if_t, u16);
 224 static void     lem_setup_vlan_hw_support(struct adapter *);
 225 static int      lem_xmit(struct adapter *, struct mbuf **);
 226 static void     lem_smartspeed(struct adapter *);
 227 static int      lem_82547_fifo_workaround(struct adapter *, int);
 228 static void     lem_82547_update_fifo_head(struct adapter *, int);
 229 static int      lem_82547_tx_fifo_reset(struct adapter *);
 230 static void     lem_82547_move_tail(void *);
 231 static int      lem_dma_malloc(struct adapter *, bus_size_t,
 232                     struct em_dma_alloc *, int);
 233 static void     lem_dma_free(struct adapter *, struct em_dma_alloc *);
 234 static int      lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
 235 static void     lem_print_nvm_info(struct adapter *);
 236 static int      lem_is_valid_ether_addr(u8 *);
 237 static u32      lem_fill_descriptors (bus_addr_t address, u32 length,
 238                     PDESC_ARRAY desc_array);
 239 static int      lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 240 static void     lem_add_int_delay_sysctl(struct adapter *, const char *,
 241                     const char *, struct em_int_delay_info *, int, int);
 242 static void     lem_set_flow_cntrl(struct adapter *, const char *,
 243                     const char *, int *, int);
 244 /* Management and WOL Support */
 245 static void     lem_init_manageability(struct adapter *);
 246 static void     lem_release_manageability(struct adapter *);
 247 static void     lem_get_hw_control(struct adapter *);
 248 static void     lem_release_hw_control(struct adapter *);
 249 static void     lem_get_wakeup(device_t);
 250 static void     lem_enable_wakeup(device_t);
 251 static int      lem_enable_phy_wakeup(struct adapter *);
 252 static void     lem_led_func(void *, int);
 253
 254 static void     lem_intr(void *);
 255 static int      lem_irq_fast(void *);
 256 static void     lem_handle_rxtx(void *context, int pending);
 257 static void     lem_handle_link(void *context, int pending);
 258 static void     lem_add_rx_process_limit(struct adapter *, const char *,
 259                     const char *, int *, int);
 260
 261 #ifdef DEVICE_POLLING
 262 static poll_handler_t lem_poll;
 263 #endif /* POLLING */
 264
 265 /*********************************************************************
 266  *  FreeBSD Device Interface Entry Points
 267  *********************************************************************/
 268
 269 static device_method_t lem_methods[] = {
 270         /* Device interface */
 271         DEVMETHOD(device_probe, lem_probe),
 272         DEVMETHOD(device_attach, lem_attach),
 273         DEVMETHOD(device_detach, lem_detach),
 274         DEVMETHOD(device_shutdown, lem_shutdown),
 275         DEVMETHOD(device_suspend, lem_suspend),
 276         DEVMETHOD(device_resume, lem_resume),
 277         DEVMETHOD_END
 278 };
 279
 280 static driver_t lem_driver = {
 281         "em", lem_methods, sizeof(struct adapter),
 282 };
 283
 284 extern devclass_t em_devclass;
 285 DRIVER_MODULE(lem, pci, lem_driver, em_devclass, 0, 0);
 286 MODULE_DEPEND(lem, pci, 1, 1, 1);
 287 MODULE_DEPEND(lem, ether, 1, 1, 1);
 288 #ifdef DEV_NETMAP
 289 MODULE_DEPEND(lem, netmap, 1, 1, 1);
 290 #endif /* DEV_NETMAP */
 291
 292 /*********************************************************************
 293  *  Tunable default values.
 294  *********************************************************************/
 295
 296 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 297 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 298
 299 #define MAX_INTS_PER_SEC        8000
 300 #define DEFAULT_ITR             (1000000000/(MAX_INTS_PER_SEC * 256))
 301
 302 static int lem_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 303 static int lem_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 304 static int lem_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 305 static int lem_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 306 /*
 307  * increase lem_rxd and lem_txd to at least 2048 in netmap mode
 308  * for better performance.
 309  */
 310 static int lem_rxd = EM_DEFAULT_RXD;
 311 static int lem_txd = EM_DEFAULT_TXD;
 312 static int lem_smart_pwr_down = FALSE;
 313
 314 /* Controls whether promiscuous also shows bad packets */
 315 static int lem_debug_sbp = FALSE;
 316
 317 TUNABLE_INT("hw.em.tx_int_delay", &lem_tx_int_delay_dflt);
 318 TUNABLE_INT("hw.em.rx_int_delay", &lem_rx_int_delay_dflt);
 319 TUNABLE_INT("hw.em.tx_abs_int_delay", &lem_tx_abs_int_delay_dflt);
 320 TUNABLE_INT("hw.em.rx_abs_int_delay", &lem_rx_abs_int_delay_dflt);
 321 TUNABLE_INT("hw.em.rxd", &lem_rxd);
 322 TUNABLE_INT("hw.em.txd", &lem_txd);
 323 TUNABLE_INT("hw.em.smart_pwr_down", &lem_smart_pwr_down);
 324 TUNABLE_INT("hw.em.sbp", &lem_debug_sbp);
 325
 326 /* Interrupt style - default to fast */
 327 static int lem_use_legacy_irq = 0;
 328 TUNABLE_INT("hw.em.use_legacy_irq", &lem_use_legacy_irq);
 329
 330 /* How many packets rxeof tries to clean at a time */
 331 static int lem_rx_process_limit = 100;
 332 TUNABLE_INT("hw.em.rx_process_limit", &lem_rx_process_limit);
 333
 334 /* Flow control setting - default to FULL */
 335 static int lem_fc_setting = e1000_fc_full;
 336 TUNABLE_INT("hw.em.fc_setting", &lem_fc_setting);
 337
 338 /* Global used in WOL setup with multiport cards */
 339 static int global_quad_port_a = 0;
 340
 341 #ifdef DEV_NETMAP       /* see ixgbe.c for details */
 342 #include <dev/netmap/if_lem_netmap.h>
 343 #endif /* DEV_NETMAP */
 344
 345 /*********************************************************************
 346  *  Device identification routine
 347  *
 348  *  em_probe determines if the driver should be loaded on
 349  *  adapter based on PCI vendor/device id of the adapter.
 350  *
 351  *  return BUS_PROBE_DEFAULT on success, positive on failure
 352  *********************************************************************/
 353
 354 static int
 355 lem_probe(device_t dev)
 356 {
 357         char            adapter_name[60];
 358         u16             pci_vendor_id = 0;
 359         u16             pci_device_id = 0;
 360         u16             pci_subvendor_id = 0;
 361         u16             pci_subdevice_id = 0;
 362         em_vendor_info_t *ent;
 363
 364         INIT_DEBUGOUT("em_probe: begin");
 365
 366         pci_vendor_id = pci_get_vendor(dev);
 367         if (pci_vendor_id != EM_VENDOR_ID)
 368                 return (ENXIO);
 369
 370         pci_device_id = pci_get_device(dev);
 371         pci_subvendor_id = pci_get_subvendor(dev);
 372         pci_subdevice_id = pci_get_subdevice(dev);
 373
 374         ent = lem_vendor_info_array;
 375         while (ent->vendor_id != 0) {
 376                 if ((pci_vendor_id == ent->vendor_id) &&
 377                     (pci_device_id == ent->device_id) &&
 378
 379                     ((pci_subvendor_id == ent->subvendor_id) ||
 380                     (ent->subvendor_id == PCI_ANY_ID)) &&
 381
 382                     ((pci_subdevice_id == ent->subdevice_id) ||
 383                     (ent->subdevice_id == PCI_ANY_ID))) {
 384                         sprintf(adapter_name, "%s %s",
 385                                 lem_strings[ent->index],
 386                                 lem_driver_version);
 387                         device_set_desc_copy(dev, adapter_name);
 388                         return (BUS_PROBE_DEFAULT);
 389                 }
 390                 ent++;
 391         }
 392
 393         return (ENXIO);
 394 }
 395
 396 /*********************************************************************
 397  *  Device initialization routine
 398  *
 399  *  The attach entry point is called when the driver is being loaded.
 400  *  This routine identifies the type of hardware, allocates all resources
 401  *  and initializes the hardware.
 402  *
 403  *  return 0 on success, positive on failure
 404  *********************************************************************/
 405
 406 static int
 407 lem_attach(device_t dev)
 408 {
 409         struct adapter  *adapter;
 410         int             tsize, rsize;
 411         int             error = 0;
 412
 413         INIT_DEBUGOUT("lem_attach: begin");
 414
 415         adapter = device_get_softc(dev);
 416         adapter->dev = adapter->osdep.dev = dev;
 417         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 418         EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
 419         EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
 420
 421         /* SYSCTL stuff */
 422         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 423             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 424             OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 425             lem_sysctl_nvm_info, "I", "NVM Information");
 426
 427         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 428         callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
 429
 430         /* Determine hardware and mac info */
 431         lem_identify_hardware(adapter);
 432
 433         /* Setup PCI resources */
 434         if (lem_allocate_pci_resources(adapter)) {
 435                 device_printf(dev, "Allocation of PCI resources failed\n");
 436                 error = ENXIO;
 437                 goto err_pci;
 438         }
 439
 440         /* Do Shared Code initialization */
 441         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 442                 device_printf(dev, "Setup of Shared code failed\n");
 443                 error = ENXIO;
 444                 goto err_pci;
 445         }
 446
 447         e1000_get_bus_info(&adapter->hw);
 448
 449         /* Set up some sysctls for the tunable interrupt delays */
 450         lem_add_int_delay_sysctl(adapter, "rx_int_delay",
 451             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 452             E1000_REGISTER(&adapter->hw, E1000_RDTR), lem_rx_int_delay_dflt);
 453         lem_add_int_delay_sysctl(adapter, "tx_int_delay",
 454             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 455             E1000_REGISTER(&adapter->hw, E1000_TIDV), lem_tx_int_delay_dflt);
 456         if (adapter->hw.mac.type >= e1000_82540) {
 457                 lem_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 458                     "receive interrupt delay limit in usecs",
 459                     &adapter->rx_abs_int_delay,
 460                     E1000_REGISTER(&adapter->hw, E1000_RADV),
 461                     lem_rx_abs_int_delay_dflt);
 462                 lem_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 463                     "transmit interrupt delay limit in usecs",
 464                     &adapter->tx_abs_int_delay,
 465                     E1000_REGISTER(&adapter->hw, E1000_TADV),
 466                     lem_tx_abs_int_delay_dflt);
 467                 lem_add_int_delay_sysctl(adapter, "itr",
 468                     "interrupt delay limit in usecs/4",
 469                     &adapter->tx_itr,
 470                     E1000_REGISTER(&adapter->hw, E1000_ITR),
 471                     DEFAULT_ITR);
 472         }
 473
 474         /* Sysctls for limiting the amount of work done in the taskqueue */
 475         lem_add_rx_process_limit(adapter, "rx_processing_limit",
 476             "max number of rx packets to process", &adapter->rx_process_limit,
 477             lem_rx_process_limit);
 478
 479 #ifdef NIC_SEND_COMBINING
 480         /* Sysctls to control mitigation */
 481         lem_add_rx_process_limit(adapter, "sc_enable",
 482             "driver TDT mitigation", &adapter->sc_enable, 0);
 483 #endif /* NIC_SEND_COMBINING */
 484 #ifdef BATCH_DISPATCH
 485         lem_add_rx_process_limit(adapter, "batch_enable",
 486             "driver rx batch", &adapter->batch_enable, 0);
 487 #endif /* BATCH_DISPATCH */
 488
 489         /* Sysctl for setting the interface flow control */
 490         lem_set_flow_cntrl(adapter, "flow_control",
 491             "flow control setting",
 492             &adapter->fc_setting, lem_fc_setting);
 493
 494         /*
 495          * Validate number of transmit and receive descriptors. It
 496          * must not exceed hardware maximum, and must be multiple
 497          * of E1000_DBA_ALIGN.
 498          */
 499         if (((lem_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 500             (adapter->hw.mac.type >= e1000_82544 && lem_txd > EM_MAX_TXD) ||
 501             (adapter->hw.mac.type < e1000_82544 && lem_txd > EM_MAX_TXD_82543) ||
 502             (lem_txd < EM_MIN_TXD)) {
 503                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 504                     EM_DEFAULT_TXD, lem_txd);
 505                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 506         } else
 507                 adapter->num_tx_desc = lem_txd;
 508         if (((lem_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 509             (adapter->hw.mac.type >= e1000_82544 && lem_rxd > EM_MAX_RXD) ||
 510             (adapter->hw.mac.type < e1000_82544 && lem_rxd > EM_MAX_RXD_82543) ||
 511             (lem_rxd < EM_MIN_RXD)) {
 512                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 513                     EM_DEFAULT_RXD, lem_rxd);
 514                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 515         } else
 516                 adapter->num_rx_desc = lem_rxd;
 517
 518         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 519         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 520         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 521         adapter->rx_buffer_len = 2048;
 522
 523         e1000_init_script_state_82541(&adapter->hw, TRUE);
 524         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 525
 526         /* Copper options */
 527         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 528                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 529                 adapter->hw.phy.disable_polarity_correction = FALSE;
 530                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 531         }
 532
 533         /*
 534          * Set the frame limits assuming
 535          * standard ethernet sized frames.
 536          */
 537         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 538         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 539
 540         /*
 541          * This controls when hardware reports transmit completion
 542          * status.
 543          */
 544         adapter->hw.mac.report_tx_early = 1;
 545
 546         /*
 547          * It seems that the descriptor DMA engine on some PCI cards
 548          * fetches memory past the end of the last descriptor in the
 549          * ring.  These reads are problematic when VT-d (DMAR) busdma
 550          * is used.  Allocate the scratch space to avoid getting
 551          * faults from DMAR, by requesting scratch memory for one more
 552          * descriptor.
 553          */
 554         tsize = roundup2((adapter->num_tx_desc + 1) *
 555             sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
 556
 557         /* Allocate Transmit Descriptor ring */
 558         if (lem_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
 559                 device_printf(dev, "Unable to allocate tx_desc memory\n");
 560                 error = ENOMEM;
 561                 goto err_tx_desc;
 562         }
 563         adapter->tx_desc_base =
 564             (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
 565
 566         /*
 567          * See comment above txdma allocation for rationale behind +1.
 568          */
 569         rsize = roundup2((adapter->num_rx_desc + 1) *
 570             sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
 571
 572         /* Allocate Receive Descriptor ring */
 573         if (lem_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 574                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 575                 error = ENOMEM;
 576                 goto err_rx_desc;
 577         }
 578         adapter->rx_desc_base =
 579             (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
 580
 581         /* Allocate multicast array memory. */
 582         adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
 583             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 584         if (adapter->mta == NULL) {
 585                 device_printf(dev, "Can not allocate multicast setup array\n");
 586                 error = ENOMEM;
 587                 goto err_hw_init;
 588         }
 589
 590         /*
 591         ** Start from a known state, this is
 592         ** important in reading the nvm and
 593         ** mac from that.
 594         */
 595         e1000_reset_hw(&adapter->hw);
 596
 597         /* Make sure we have a good EEPROM before we read from it */
 598         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 599                 /*
 600                 ** Some PCI-E parts fail the first check due to
 601                 ** the link being in sleep state, call it again,
 602                 ** if it fails a second time its a real issue.
 603                 */
 604                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 605                         device_printf(dev,
 606                             "The EEPROM Checksum Is Not Valid\n");
 607                         error = EIO;
 608                         goto err_hw_init;
 609                 }
 610         }
 611
 612         /* Copy the permanent MAC address out of the EEPROM */
 613         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 614                 device_printf(dev, "EEPROM read error while reading MAC"
 615                     " address\n");
 616                 error = EIO;
 617                 goto err_hw_init;
 618         }
 619
 620         if (!lem_is_valid_ether_addr(adapter->hw.mac.addr)) {
 621                 device_printf(dev, "Invalid MAC address\n");
 622                 error = EIO;
 623                 goto err_hw_init;
 624         }
 625
 626         /* Initialize the hardware */
 627         if (lem_hardware_init(adapter)) {
 628                 device_printf(dev, "Unable to initialize the hardware\n");
 629                 error = EIO;
 630                 goto err_hw_init;
 631         }
 632
 633         /* Allocate transmit descriptors and buffers */
 634         if (lem_allocate_transmit_structures(adapter)) {
 635                 device_printf(dev, "Could not setup transmit structures\n");
 636                 error = ENOMEM;
 637                 goto err_tx_struct;
 638         }
 639
 640         /* Allocate receive descriptors and buffers */
 641         if (lem_allocate_receive_structures(adapter)) {
 642                 device_printf(dev, "Could not setup receive structures\n");
 643                 error = ENOMEM;
 644                 goto err_rx_struct;
 645         }
 646
 647         /*
 648         **  Do interrupt configuration
 649         */
 650         error = lem_allocate_irq(adapter);
 651         if (error)
 652                 goto err_rx_struct;
 653
 654         /*
 655          * Get Wake-on-Lan and Management info for later use
 656          */
 657         lem_get_wakeup(dev);
 658
 659         /* Setup OS specific network interface */
 660         if (lem_setup_interface(dev, adapter) != 0)
 661                 goto err_rx_struct;
 662
 663         /* Initialize statistics */
 664         lem_update_stats_counters(adapter);
 665
 666         adapter->hw.mac.get_link_status = 1;
 667         lem_update_link_status(adapter);
 668
 669         /* Indicate SOL/IDER usage */
 670         if (e1000_check_reset_block(&adapter->hw))
 671                 device_printf(dev,
 672                     "PHY reset is blocked due to SOL/IDER session.\n");
 673
 674         /* Do we need workaround for 82544 PCI-X adapter? */
 675         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
 676             adapter->hw.mac.type == e1000_82544)
 677                 adapter->pcix_82544 = TRUE;
 678         else
 679                 adapter->pcix_82544 = FALSE;
 680
 681         /* Register for VLAN events */
 682         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 683             lem_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 684         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 685             lem_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 686
 687         lem_add_hw_stats(adapter);
 688
 689         /* Non-AMT based hardware can now take control from firmware */
 690         if (adapter->has_manage && !adapter->has_amt)
 691                 lem_get_hw_control(adapter);
 692
 693         /* Tell the stack that the interface is not active */
 694         if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
 695
 696         adapter->led_dev = led_create(lem_led_func, adapter,
 697             device_get_nameunit(dev));
 698
 699 #ifdef DEV_NETMAP
 700         lem_netmap_attach(adapter);
 701 #endif /* DEV_NETMAP */
 702         INIT_DEBUGOUT("lem_attach: end");
 703
 704         return (0);
 705
 706 err_rx_struct:
 707         lem_free_transmit_structures(adapter);
 708 err_tx_struct:
 709 err_hw_init:
 710         lem_release_hw_control(adapter);
 711         lem_dma_free(adapter, &adapter->rxdma);
 712 err_rx_desc:
 713         lem_dma_free(adapter, &adapter->txdma);
 714 err_tx_desc:
 715 err_pci:
 716         if (adapter->ifp != (void *)NULL)
 717                 if_free(adapter->ifp);
 718         lem_free_pci_resources(adapter);
 719         free(adapter->mta, M_DEVBUF);
 720         EM_TX_LOCK_DESTROY(adapter);
 721         EM_RX_LOCK_DESTROY(adapter);
 722         EM_CORE_LOCK_DESTROY(adapter);
 723
 724         return (error);
 725 }
 726
 727 /*********************************************************************
 728  *  Device removal routine
 729  *
 730  *  The detach entry point is called when the driver is being removed.
 731  *  This routine stops the adapter and deallocates all the resources
 732  *  that were allocated for driver operation.
 733  *
 734  *  return 0 on success, positive on failure
 735  *********************************************************************/
 736
 737 static int
 738 lem_detach(device_t dev)
 739 {
 740         struct adapter  *adapter = device_get_softc(dev);
 741         if_t ifp = adapter->ifp;
 742
 743         INIT_DEBUGOUT("em_detach: begin");
 744
 745         /* Make sure VLANS are not using driver */
 746         if (if_vlantrunkinuse(ifp)) {
 747                 device_printf(dev,"Vlan in use, detach first\n");
 748                 return (EBUSY);
 749         }
 750
 751 #ifdef DEVICE_POLLING
 752         if (if_getcapenable(ifp) & IFCAP_POLLING)
 753                 ether_poll_deregister(ifp);
 754 #endif
 755
 756         if (adapter->led_dev != NULL)
 757                 led_destroy(adapter->led_dev);
 758
 759         EM_CORE_LOCK(adapter);
 760         EM_TX_LOCK(adapter);
 761         adapter->in_detach = 1;
 762         lem_stop(adapter);
 763         e1000_phy_hw_reset(&adapter->hw);
 764
 765         lem_release_manageability(adapter);
 766
 767         EM_TX_UNLOCK(adapter);
 768         EM_CORE_UNLOCK(adapter);
 769
 770         /* Unregister VLAN events */
 771         if (adapter->vlan_attach != NULL)
 772                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 773         if (adapter->vlan_detach != NULL)
 774                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 775
 776         ether_ifdetach(adapter->ifp);
 777         callout_drain(&adapter->timer);
 778         callout_drain(&adapter->tx_fifo_timer);
 779
 780 #ifdef DEV_NETMAP
 781         netmap_detach(ifp);
 782 #endif /* DEV_NETMAP */
 783         lem_free_pci_resources(adapter);
 784         bus_generic_detach(dev);
 785         if_free(ifp);
 786
 787         lem_free_transmit_structures(adapter);
 788         lem_free_receive_structures(adapter);
 789
 790         /* Free Transmit Descriptor ring */
 791         if (adapter->tx_desc_base) {
 792                 lem_dma_free(adapter, &adapter->txdma);
 793                 adapter->tx_desc_base = NULL;
 794         }
 795
 796         /* Free Receive Descriptor ring */
 797         if (adapter->rx_desc_base) {
 798                 lem_dma_free(adapter, &adapter->rxdma);
 799                 adapter->rx_desc_base = NULL;
 800         }
 801
 802         lem_release_hw_control(adapter);
 803         free(adapter->mta, M_DEVBUF);
 804         EM_TX_LOCK_DESTROY(adapter);
 805         EM_RX_LOCK_DESTROY(adapter);
 806         EM_CORE_LOCK_DESTROY(adapter);
 807
 808         return (0);
 809 }
 810
 811 /*********************************************************************
 812  *
 813  *  Shutdown entry point
 814  *
 815  **********************************************************************/
 816
 817 static int
 818 lem_shutdown(device_t dev)
 819 {
 820         return lem_suspend(dev);
 821 }
 822
 823 /*
 824  * Suspend/resume device methods.
 825  */
 826 static int
 827 lem_suspend(device_t dev)
 828 {
 829         struct adapter *adapter = device_get_softc(dev);
 830
 831         EM_CORE_LOCK(adapter);
 832
 833         lem_release_manageability(adapter);
 834         lem_release_hw_control(adapter);
 835         lem_enable_wakeup(dev);
 836
 837         EM_CORE_UNLOCK(adapter);
 838
 839         return bus_generic_suspend(dev);
 840 }
 841
 842 static int
 843 lem_resume(device_t dev)
 844 {
 845         struct adapter *adapter = device_get_softc(dev);
 846         if_t ifp = adapter->ifp;
 847
 848         EM_CORE_LOCK(adapter);
 849         lem_init_locked(adapter);
 850         lem_init_manageability(adapter);
 851         EM_CORE_UNLOCK(adapter);
 852         lem_start(ifp);
 853
 854         return bus_generic_resume(dev);
 855 }
 856
 857
 858 static void
 859 lem_start_locked(if_t ifp)
 860 {
 861         struct adapter  *adapter = if_getsoftc(ifp);
 862         struct mbuf     *m_head;
 863
 864         EM_TX_LOCK_ASSERT(adapter);
 865
 866         if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 867             IFF_DRV_RUNNING)
 868                 return;
 869         if (!adapter->link_active)
 870                 return;
 871
 872         /*
 873          * Force a cleanup if number of TX descriptors
 874          * available hits the threshold
 875          */
 876         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
 877                 lem_txeof(adapter);
 878                 /* Now do we at least have a minimal? */
 879                 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
 880                         adapter->no_tx_desc_avail1++;
 881                         return;
 882                 }
 883         }
 884
 885         while (!if_sendq_empty(ifp)) {
 886                 m_head = if_dequeue(ifp);
 887
 888                 if (m_head == NULL)
 889                         break;
 890                 /*
 891                  *  Encapsulation can modify our pointer, and or make it
 892                  *  NULL on failure.  In that event, we can't requeue.
 893                  */
 894                 if (lem_xmit(adapter, &m_head)) {
 895                         if (m_head == NULL)
 896                                 break;
 897                         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
 898                         if_sendq_prepend(ifp, m_head);
 899                         break;
 900                 }
 901
 902                 /* Send a copy of the frame to the BPF listener */
 903                 if_etherbpfmtap(ifp, m_head);
 904
 905                 /* Set timeout in case hardware has problems transmitting. */
 906                 adapter->watchdog_check = TRUE;
 907                 adapter->watchdog_time = ticks;
 908         }
 909         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
 910                 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
 911
 912         return;
 913 }
 914
 915 static void
 916 lem_start(if_t ifp)
 917 {
 918         struct adapter *adapter = if_getsoftc(ifp);
 919
 920         EM_TX_LOCK(adapter);
 921         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
 922                 lem_start_locked(ifp);
 923         EM_TX_UNLOCK(adapter);
 924 }
 925
 926 /*********************************************************************
 927  *  Ioctl entry point
 928  *
 929  *  em_ioctl is called when the user wants to configure the
 930  *  interface.
 931  *
 932  *  return 0 on success, positive on failure
 933  **********************************************************************/
 934
 935 static int
 936 lem_ioctl(if_t ifp, u_long command, caddr_t data)
 937 {
 938         struct adapter  *adapter = if_getsoftc(ifp);
 939         struct ifreq    *ifr = (struct ifreq *)data;
 940 #if defined(INET) || defined(INET6)
 941         struct ifaddr   *ifa = (struct ifaddr *)data;
 942 #endif
 943         bool            avoid_reset = FALSE;
 944         int             error = 0;
 945
 946         if (adapter->in_detach)
 947                 return (error);
 948
 949         switch (command) {
 950         case SIOCSIFADDR:
 951 #ifdef INET
 952                 if (ifa->ifa_addr->sa_family == AF_INET)
 953                         avoid_reset = TRUE;
 954 #endif
 955 #ifdef INET6
 956                 if (ifa->ifa_addr->sa_family == AF_INET6)
 957                         avoid_reset = TRUE;
 958 #endif
 959                 /*
 960                 ** Calling init results in link renegotiation,
 961                 ** so we avoid doing it when possible.
 962                 */
 963                 if (avoid_reset) {
 964                         if_setflagbits(ifp, IFF_UP, 0);
 965                         if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
 966                                 lem_init(adapter);
 967 #ifdef INET
 968                         if (!(if_getflags(ifp) & IFF_NOARP))
 969                                 arp_ifinit(ifp, ifa);
 970 #endif
 971                 } else
 972                         error = ether_ioctl(ifp, command, data);
 973                 break;
 974         case SIOCSIFMTU:
 975             {
 976                 int max_frame_size;
 977
 978                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 979
 980                 EM_CORE_LOCK(adapter);
 981                 switch (adapter->hw.mac.type) {
 982                 case e1000_82542:
 983                         max_frame_size = ETHER_MAX_LEN;
 984                         break;
 985                 default:
 986                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
 987                 }
 988                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
 989                     ETHER_CRC_LEN) {
 990                         EM_CORE_UNLOCK(adapter);
 991                         error = EINVAL;
 992                         break;
 993                 }
 994
 995                 if_setmtu(ifp, ifr->ifr_mtu);
 996                 adapter->max_frame_size =
 997                     if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN;
 998                 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
 999                         lem_init_locked(adapter);
1000                 EM_CORE_UNLOCK(adapter);
1001                 break;
1002             }
1003         case SIOCSIFFLAGS:
1004                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1005                     SIOCSIFFLAGS (Set Interface Flags)");
1006                 EM_CORE_LOCK(adapter);
1007                 if (if_getflags(ifp) & IFF_UP) {
1008                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
1009                                 if ((if_getflags(ifp) ^ adapter->if_flags) &
1010                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1011                                         lem_disable_promisc(adapter);
1012                                         lem_set_promisc(adapter);
1013                                 }
1014                         } else
1015                                 lem_init_locked(adapter);
1016                 } else
1017                         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1018                                 EM_TX_LOCK(adapter);
1019                                 lem_stop(adapter);
1020                                 EM_TX_UNLOCK(adapter);
1021                         }
1022                 adapter->if_flags = if_getflags(ifp);
1023                 EM_CORE_UNLOCK(adapter);
1024                 break;
1025         case SIOCADDMULTI:
1026         case SIOCDELMULTI:
1027                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1028                 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1029                         EM_CORE_LOCK(adapter);
1030                         lem_disable_intr(adapter);
1031                         lem_set_multi(adapter);
1032                         if (adapter->hw.mac.type == e1000_82542 &&
1033                             adapter->hw.revision_id == E1000_REVISION_2) {
1034                                 lem_initialize_receive_unit(adapter);
1035                         }
1036 #ifdef DEVICE_POLLING
1037                         if (!(if_getcapenable(ifp) & IFCAP_POLLING))
1038 #endif
1039                                 lem_enable_intr(adapter);
1040                         EM_CORE_UNLOCK(adapter);
1041                 }
1042                 break;
1043         case SIOCSIFMEDIA:
1044                 /* Check SOL/IDER usage */
1045                 EM_CORE_LOCK(adapter);
1046                 if (e1000_check_reset_block(&adapter->hw)) {
1047                         EM_CORE_UNLOCK(adapter);
1048                         device_printf(adapter->dev, "Media change is"
1049                             " blocked due to SOL/IDER session.\n");
1050                         break;
1051                 }
1052                 EM_CORE_UNLOCK(adapter);
1053         case SIOCGIFMEDIA:
1054                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1055                     SIOCxIFMEDIA (Get/Set Interface Media)");
1056                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1057                 break;
1058         case SIOCSIFCAP:
1059             {
1060                 int mask, reinit;
1061
1062                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1063                 reinit = 0;
1064                 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
1065 #ifdef DEVICE_POLLING
1066                 if (mask & IFCAP_POLLING) {
1067                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1068                                 error = ether_poll_register(lem_poll, ifp);
1069                                 if (error)
1070                                         return (error);
1071                                 EM_CORE_LOCK(adapter);
1072                                 lem_disable_intr(adapter);
1073                                 if_setcapenablebit(ifp, IFCAP_POLLING, 0);
1074                                 EM_CORE_UNLOCK(adapter);
1075                         } else {
1076                                 error = ether_poll_deregister(ifp);
1077                                 /* Enable interrupt even in error case */
1078                                 EM_CORE_LOCK(adapter);
1079                                 lem_enable_intr(adapter);
1080                                 if_setcapenablebit(ifp, 0, IFCAP_POLLING);
1081                                 EM_CORE_UNLOCK(adapter);
1082                         }
1083                 }
1084 #endif
1085                 if (mask & IFCAP_HWCSUM) {
1086                         if_togglecapenable(ifp, IFCAP_HWCSUM);
1087                         reinit = 1;
1088                 }
1089                 if (mask & IFCAP_VLAN_HWTAGGING) {
1090                         if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
1091                         reinit = 1;
1092                 }
1093                 if ((mask & IFCAP_WOL) &&
1094                     (if_getcapabilities(ifp) & IFCAP_WOL) != 0) {
1095                         if (mask & IFCAP_WOL_MCAST)
1096                                 if_togglecapenable(ifp, IFCAP_WOL_MCAST);
1097                         if (mask & IFCAP_WOL_MAGIC)
1098                                 if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
1099                 }
1100                 if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1101                         lem_init(adapter);
1102                 if_vlancap(ifp);
1103                 break;
1104             }
1105
1106         default:
1107                 error = ether_ioctl(ifp, command, data);
1108                 break;
1109         }
1110
1111         return (error);
1112 }
1113
1114
1115 /*********************************************************************
1116  *  Init entry point
1117  *
1118  *  This routine is used in two ways. It is used by the stack as
1119  *  init entry point in network interface structure. It is also used
1120  *  by the driver as a hw/sw initialization routine to get to a
1121  *  consistent state.
1122  *
1123  *  return 0 on success, positive on failure
1124  **********************************************************************/
1125
1126 static void
1127 lem_init_locked(struct adapter *adapter)
1128 {
1129         if_t ifp = adapter->ifp;
1130         device_t        dev = adapter->dev;
1131         u32             pba;
1132
1133         INIT_DEBUGOUT("lem_init: begin");
1134
1135         EM_CORE_LOCK_ASSERT(adapter);
1136
1137         EM_TX_LOCK(adapter);
1138         lem_stop(adapter);
1139         EM_TX_UNLOCK(adapter);
1140
1141         /*
1142          * Packet Buffer Allocation (PBA)
1143          * Writing PBA sets the receive portion of the buffer
1144          * the remainder is used for the transmit buffer.
1145          *
1146          * Devices before the 82547 had a Packet Buffer of 64K.
1147          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1148          * After the 82547 the buffer was reduced to 40K.
1149          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1150          *   Note: default does not leave enough room for Jumbo Frame >10k.
1151          */
1152         switch (adapter->hw.mac.type) {
1153         case e1000_82547:
1154         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1155                 if (adapter->max_frame_size > 8192)
1156                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1157                 else
1158                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1159                 adapter->tx_fifo_head = 0;
1160                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1161                 adapter->tx_fifo_size =
1162                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1163                 break;
1164         default:
1165                 /* Devices before 82547 had a Packet Buffer of 64K.   */
1166                 if (adapter->max_frame_size > 8192)
1167                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1168                 else
1169                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1170         }
1171
1172         INIT_DEBUGOUT1("lem_init: pba=%dK",pba);
1173         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1174
1175         /* Get the latest mac address, User can use a LAA */
1176         bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr,
1177               ETHER_ADDR_LEN);
1178
1179         /* Put the address into the Receive Address Array */
1180         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1181
1182         /* Initialize the hardware */
1183         if (lem_hardware_init(adapter)) {
1184                 device_printf(dev, "Unable to initialize the hardware\n");
1185                 return;
1186         }
1187         lem_update_link_status(adapter);
1188
1189         /* Setup VLAN support, basic and offload if available */
1190         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1191
1192         /* Set hardware offload abilities */
1193         if_clearhwassist(ifp);
1194         if (adapter->hw.mac.type >= e1000_82543) {
1195                 if (if_getcapenable(ifp) & IFCAP_TXCSUM)
1196                         if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
1197         }
1198
1199         /* Configure for OS presence */
1200         lem_init_manageability(adapter);
1201
1202         /* Prepare transmit descriptors and buffers */
1203         lem_setup_transmit_structures(adapter);
1204         lem_initialize_transmit_unit(adapter);
1205
1206         /* Setup Multicast table */
1207         lem_set_multi(adapter);
1208
1209         /* Prepare receive descriptors and buffers */
1210         if (lem_setup_receive_structures(adapter)) {
1211                 device_printf(dev, "Could not setup receive structures\n");
1212                 EM_TX_LOCK(adapter);
1213                 lem_stop(adapter);
1214                 EM_TX_UNLOCK(adapter);
1215                 return;
1216         }
1217         lem_initialize_receive_unit(adapter);
1218
1219         /* Use real VLAN Filter support? */
1220         if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
1221                 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
1222                         /* Use real VLAN Filter support */
1223                         lem_setup_vlan_hw_support(adapter);
1224                 else {
1225                         u32 ctrl;
1226                         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1227                         ctrl |= E1000_CTRL_VME;
1228                         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1229                 }
1230         }
1231
1232         /* Don't lose promiscuous settings */
1233         lem_set_promisc(adapter);
1234
1235         if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1236
1237         callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
1238         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1239
1240 #ifdef DEVICE_POLLING
1241         /*
1242          * Only enable interrupts if we are not polling, make sure
1243          * they are off otherwise.
1244          */
1245         if (if_getcapenable(ifp) & IFCAP_POLLING)
1246                 lem_disable_intr(adapter);
1247         else
1248 #endif /* DEVICE_POLLING */
1249                 lem_enable_intr(adapter);
1250
1251         /* AMT based hardware can now take control from firmware */
1252         if (adapter->has_manage && adapter->has_amt)
1253                 lem_get_hw_control(adapter);
1254 }
1255
1256 static void
1257 lem_init(void *arg)
1258 {
1259         struct adapter *adapter = arg;
1260
1261         EM_CORE_LOCK(adapter);
1262         lem_init_locked(adapter);
1263         EM_CORE_UNLOCK(adapter);
1264 }
1265
1266
1267 #ifdef DEVICE_POLLING
1268 /*********************************************************************
1269  *
1270  *  Legacy polling routine
1271  *
1272  *********************************************************************/
1273 static int
1274 lem_poll(if_t ifp, enum poll_cmd cmd, int count)
1275 {
1276         struct adapter *adapter = if_getsoftc(ifp);
1277         u32             reg_icr, rx_done = 0;
1278
1279         EM_CORE_LOCK(adapter);
1280         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
1281                 EM_CORE_UNLOCK(adapter);
1282                 return (rx_done);
1283         }
1284
1285         if (cmd == POLL_AND_CHECK_STATUS) {
1286                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1287                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1288                         callout_stop(&adapter->timer);
1289                         adapter->hw.mac.get_link_status = 1;
1290                         lem_update_link_status(adapter);
1291                         callout_reset(&adapter->timer, hz,
1292                             lem_local_timer, adapter);
1293                 }
1294         }
1295         EM_CORE_UNLOCK(adapter);
1296
1297         lem_rxeof(adapter, count, &rx_done);
1298
1299         EM_TX_LOCK(adapter);
1300         lem_txeof(adapter);
1301         if(!if_sendq_empty(ifp))
1302                 lem_start_locked(ifp);
1303         EM_TX_UNLOCK(adapter);
1304         return (rx_done);
1305 }
1306 #endif /* DEVICE_POLLING */
1307
1308 /*********************************************************************
1309  *
1310  *  Legacy Interrupt Service routine
1311  *
1312  *********************************************************************/
1313 static void
1314 lem_intr(void *arg)
1315 {
1316         struct adapter  *adapter = arg;
1317         if_t ifp = adapter->ifp;
1318         u32             reg_icr;
1319
1320
1321         if ((if_getcapenable(ifp) & IFCAP_POLLING) ||
1322             ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
1323                 return;
1324
1325         EM_CORE_LOCK(adapter);
1326         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1327         if (reg_icr & E1000_ICR_RXO)
1328                 adapter->rx_overruns++;
1329
1330         if ((reg_icr == 0xffffffff) || (reg_icr == 0)) {
1331                 EM_CORE_UNLOCK(adapter);
1332                 return;
1333         }
1334
1335         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1336                 callout_stop(&adapter->timer);
1337                 adapter->hw.mac.get_link_status = 1;
1338                 lem_update_link_status(adapter);
1339                 /* Deal with TX cruft when link lost */
1340                 lem_tx_purge(adapter);
1341                 callout_reset(&adapter->timer, hz,
1342                     lem_local_timer, adapter);
1343                 EM_CORE_UNLOCK(adapter);
1344                 return;
1345         }
1346
1347         EM_CORE_UNLOCK(adapter);
1348         lem_rxeof(adapter, -1, NULL);
1349
1350         EM_TX_LOCK(adapter);
1351         lem_txeof(adapter);
1352         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) &&
1353             (!if_sendq_empty(ifp)))
1354                 lem_start_locked(ifp);
1355         EM_TX_UNLOCK(adapter);
1356         return;
1357 }
1358
1359
1360 static void
1361 lem_handle_link(void *context, int pending)
1362 {
1363         struct adapter  *adapter = context;
1364         if_t ifp = adapter->ifp;
1365
1366         if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1367                 return;
1368
1369         EM_CORE_LOCK(adapter);
1370         callout_stop(&adapter->timer);
1371         lem_update_link_status(adapter);
1372         /* Deal with TX cruft when link lost */
1373         lem_tx_purge(adapter);
1374         callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
1375         EM_CORE_UNLOCK(adapter);
1376 }
1377
1378
1379 /* Combined RX/TX handler, used by Legacy and MSI */
1380 static void
1381 lem_handle_rxtx(void *context, int pending)
1382 {
1383         struct adapter  *adapter = context;
1384         if_t ifp = adapter->ifp;
1385
1386
1387         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1388                 bool more = lem_rxeof(adapter, adapter->rx_process_limit, NULL);
1389                 EM_TX_LOCK(adapter);
1390                 lem_txeof(adapter);
1391                 if(!if_sendq_empty(ifp))
1392                         lem_start_locked(ifp);
1393                 EM_TX_UNLOCK(adapter);
1394                 if (more) {
1395                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1396                         return;
1397                 }
1398         }
1399
1400         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
1401                 lem_enable_intr(adapter);
1402 }
1403
1404 /*********************************************************************
1405  *
1406  *  Fast Legacy/MSI Combined Interrupt Service routine
1407  *
1408  *********************************************************************/
1409 static int
1410 lem_irq_fast(void *arg)
1411 {
1412         struct adapter  *adapter = arg;
1413         if_t ifp;
1414         u32             reg_icr;
1415
1416         ifp = adapter->ifp;
1417
1418         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1419
1420         /* Hot eject?  */
1421         if (reg_icr == 0xffffffff)
1422                 return FILTER_STRAY;
1423
1424         /* Definitely not our interrupt.  */
1425         if (reg_icr == 0x0)
1426                 return FILTER_STRAY;
1427
1428         /*
1429          * Mask interrupts until the taskqueue is finished running.  This is
1430          * cheap, just assume that it is needed.  This also works around the
1431          * MSI message reordering errata on certain systems.
1432          */
1433         lem_disable_intr(adapter);
1434         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1435
1436         /* Link status change */
1437         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1438                 adapter->hw.mac.get_link_status = 1;
1439                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1440         }
1441
1442         if (reg_icr & E1000_ICR_RXO)
1443                 adapter->rx_overruns++;
1444         return FILTER_HANDLED;
1445 }
1446
1447
1448 /*********************************************************************
1449  *
1450  *  Media Ioctl callback
1451  *
1452  *  This routine is called whenever the user queries the status of
1453  *  the interface using ifconfig.
1454  *
1455  **********************************************************************/
1456 static void
1457 lem_media_status(if_t ifp, struct ifmediareq *ifmr)
1458 {
1459         struct adapter *adapter = if_getsoftc(ifp);
1460         u_char fiber_type = IFM_1000_SX;
1461
1462         INIT_DEBUGOUT("lem_media_status: begin");
1463
1464         EM_CORE_LOCK(adapter);
1465         lem_update_link_status(adapter);
1466
1467         ifmr->ifm_status = IFM_AVALID;
1468         ifmr->ifm_active = IFM_ETHER;
1469
1470         if (!adapter->link_active) {
1471                 EM_CORE_UNLOCK(adapter);
1472                 return;
1473         }
1474
1475         ifmr->ifm_status |= IFM_ACTIVE;
1476
1477         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1478             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1479                 if (adapter->hw.mac.type == e1000_82545)
1480                         fiber_type = IFM_1000_LX;
1481                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1482         } else {
1483                 switch (adapter->link_speed) {
1484                 case 10:
1485                         ifmr->ifm_active |= IFM_10_T;
1486                         break;
1487                 case 100:
1488                         ifmr->ifm_active |= IFM_100_TX;
1489                         break;
1490                 case 1000:
1491                         ifmr->ifm_active |= IFM_1000_T;
1492                         break;
1493                 }
1494                 if (adapter->link_duplex == FULL_DUPLEX)
1495                         ifmr->ifm_active |= IFM_FDX;
1496                 else
1497                         ifmr->ifm_active |= IFM_HDX;
1498         }
1499         EM_CORE_UNLOCK(adapter);
1500 }
1501
1502 /*********************************************************************
1503  *
1504  *  Media Ioctl callback
1505  *
1506  *  This routine is called when the user changes speed/duplex using
1507  *  media/mediopt option with ifconfig.
1508  *
1509  **********************************************************************/
1510 static int
1511 lem_media_change(if_t ifp)
1512 {
1513         struct adapter *adapter = if_getsoftc(ifp);
1514         struct ifmedia  *ifm = &adapter->media;
1515
1516         INIT_DEBUGOUT("lem_media_change: begin");
1517
1518         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1519                 return (EINVAL);
1520
1521         EM_CORE_LOCK(adapter);
1522         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1523         case IFM_AUTO:
1524                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1525                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1526                 break;
1527         case IFM_1000_LX:
1528         case IFM_1000_SX:
1529         case IFM_1000_T:
1530                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1531                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1532                 break;
1533         case IFM_100_TX:
1534                 adapter->hw.mac.autoneg = FALSE;
1535                 adapter->hw.phy.autoneg_advertised = 0;
1536                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1537                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1538                 else
1539                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1540                 break;
1541         case IFM_10_T:
1542                 adapter->hw.mac.autoneg = FALSE;
1543                 adapter->hw.phy.autoneg_advertised = 0;
1544                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1545                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1546                 else
1547                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1548                 break;
1549         default:
1550                 device_printf(adapter->dev, "Unsupported media type\n");
1551         }
1552
1553         lem_init_locked(adapter);
1554         EM_CORE_UNLOCK(adapter);
1555
1556         return (0);
1557 }
1558
1559 /*********************************************************************
1560  *
1561  *  This routine maps the mbufs to tx descriptors.
1562  *
1563  *  return 0 on success, positive on failure
1564  **********************************************************************/
1565
1566 static int
1567 lem_xmit(struct adapter *adapter, struct mbuf **m_headp)
1568 {
1569         bus_dma_segment_t       segs[EM_MAX_SCATTER];
1570         bus_dmamap_t            map;
1571         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
1572         struct e1000_tx_desc    *ctxd = NULL;
1573         struct mbuf             *m_head;
1574         u32                     txd_upper, txd_lower, txd_used, txd_saved;
1575         int                     error, nsegs, i, j, first, last = 0;
1576
1577         m_head = *m_headp;
1578         txd_upper = txd_lower = txd_used = txd_saved = 0;
1579
1580         /*
1581         ** When doing checksum offload, it is critical to
1582         ** make sure the first mbuf has more than header,
1583         ** because that routine expects data to be present.
1584         */
1585         if ((m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) &&
1586             (m_head->m_len < ETHER_HDR_LEN + sizeof(struct ip))) {
1587                 m_head = m_pullup(m_head, ETHER_HDR_LEN + sizeof(struct ip));
1588                 *m_headp = m_head;
1589                 if (m_head == NULL)
1590                         return (ENOBUFS);
1591         }
1592
1593         /*
1594          * Map the packet for DMA
1595          *
1596          * Capture the first descriptor index,
1597          * this descriptor will have the index
1598          * of the EOP which is the only one that
1599          * now gets a DONE bit writeback.
1600          */
1601         first = adapter->next_avail_tx_desc;
1602         tx_buffer = &adapter->tx_buffer_area[first];
1603         tx_buffer_mapped = tx_buffer;
1604         map = tx_buffer->map;
1605
1606         error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
1607             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1608
1609         /*
1610          * There are two types of errors we can (try) to handle:
1611          * - EFBIG means the mbuf chain was too long and bus_dma ran
1612          *   out of segments.  Defragment the mbuf chain and try again.
1613          * - ENOMEM means bus_dma could not obtain enough bounce buffers
1614          *   at this point in time.  Defer sending and try again later.
1615          * All other errors, in particular EINVAL, are fatal and prevent the
1616          * mbuf chain from ever going through.  Drop it and report error.
1617          */
1618         if (error == EFBIG) {
1619                 struct mbuf *m;
1620
1621                 m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER);
1622                 if (m == NULL) {
1623                         adapter->mbuf_defrag_failed++;
1624                         m_freem(*m_headp);
1625                         *m_headp = NULL;
1626                         return (ENOBUFS);
1627                 }
1628                 *m_headp = m;
1629
1630                 /* Try it again */
1631                 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
1632                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1633
1634                 if (error) {
1635                         adapter->no_tx_dma_setup++;
1636                         m_freem(*m_headp);
1637                         *m_headp = NULL;
1638                         return (error);
1639                 }
1640         } else if (error != 0) {
1641                 adapter->no_tx_dma_setup++;
1642                 return (error);
1643         }
1644
1645         if (adapter->num_tx_desc_avail < (nsegs + 2)) {
1646                 adapter->no_tx_desc_avail2++;
1647                 bus_dmamap_unload(adapter->txtag, map);
1648                 return (ENOBUFS);
1649         }
1650         m_head = *m_headp;
1651
1652         /* Do hardware assists */
1653         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
1654                 lem_transmit_checksum_setup(adapter,  m_head,
1655                     &txd_upper, &txd_lower);
1656
1657         i = adapter->next_avail_tx_desc;
1658         if (adapter->pcix_82544)
1659                 txd_saved = i;
1660
1661         /* Set up our transmit descriptors */
1662         for (j = 0; j < nsegs; j++) {
1663                 bus_size_t seg_len;
1664                 bus_addr_t seg_addr;
1665                 /* If adapter is 82544 and on PCIX bus */
1666                 if(adapter->pcix_82544) {
1667                         DESC_ARRAY      desc_array;
1668                         u32             array_elements, counter;
1669                         /*
1670                          * Check the Address and Length combination and
1671                          * split the data accordingly
1672                          */
1673                         array_elements = lem_fill_descriptors(segs[j].ds_addr,
1674                             segs[j].ds_len, &desc_array);
1675                         for (counter = 0; counter < array_elements; counter++) {
1676                                 if (txd_used == adapter->num_tx_desc_avail) {
1677                                         adapter->next_avail_tx_desc = txd_saved;
1678                                         adapter->no_tx_desc_avail2++;
1679                                         bus_dmamap_unload(adapter->txtag, map);
1680                                         return (ENOBUFS);
1681                                 }
1682                                 tx_buffer = &adapter->tx_buffer_area[i];
1683                                 ctxd = &adapter->tx_desc_base[i];
1684                                 ctxd->buffer_addr = htole64(
1685                                     desc_array.descriptor[counter].address);
1686                                 ctxd->lower.data = htole32(
1687                                     (adapter->txd_cmd | txd_lower | (u16)
1688                                     desc_array.descriptor[counter].length));
1689                                 ctxd->upper.data =
1690                                     htole32((txd_upper));
1691                                 last = i;
1692                                 if (++i == adapter->num_tx_desc)
1693                                          i = 0;
1694                                 tx_buffer->m_head = NULL;
1695                                 tx_buffer->next_eop = -1;
1696                                 txd_used++;
1697                         }
1698                 } else {
1699                         tx_buffer = &adapter->tx_buffer_area[i];
1700                         ctxd = &adapter->tx_desc_base[i];
1701                         seg_addr = segs[j].ds_addr;
1702                         seg_len  = segs[j].ds_len;
1703                         ctxd->buffer_addr = htole64(seg_addr);
1704                         ctxd->lower.data = htole32(
1705                         adapter->txd_cmd | txd_lower | seg_len);
1706                         ctxd->upper.data =
1707                             htole32(txd_upper);
1708                         last = i;
1709                         if (++i == adapter->num_tx_desc)
1710                                 i = 0;
1711                         tx_buffer->m_head = NULL;
1712                         tx_buffer->next_eop = -1;
1713                 }
1714         }
1715
1716         adapter->next_avail_tx_desc = i;
1717
1718         if (adapter->pcix_82544)
1719                 adapter->num_tx_desc_avail -= txd_used;
1720         else
1721                 adapter->num_tx_desc_avail -= nsegs;
1722
1723         if (m_head->m_flags & M_VLANTAG) {
1724                 /* Set the vlan id. */
1725                 ctxd->upper.fields.special =
1726                     htole16(m_head->m_pkthdr.ether_vtag);
1727                 /* Tell hardware to add tag */
1728                 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
1729         }
1730
1731         tx_buffer->m_head = m_head;
1732         tx_buffer_mapped->map = tx_buffer->map;
1733         tx_buffer->map = map;
1734         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1735
1736         /*
1737          * Last Descriptor of Packet
1738          * needs End Of Packet (EOP)
1739          * and Report Status (RS)
1740          */
1741         ctxd->lower.data |=
1742             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1743         /*
1744          * Keep track in the first buffer which
1745          * descriptor will be written back
1746          */
1747         tx_buffer = &adapter->tx_buffer_area[first];
1748         tx_buffer->next_eop = last;
1749         adapter->watchdog_time = ticks;
1750
1751         /*
1752          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
1753          * that this frame is available to transmit.
1754          */
1755         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
1756             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1757
1758 #ifdef NIC_SEND_COMBINING
1759         if (adapter->sc_enable) {
1760                 if (adapter->shadow_tdt & MIT_PENDING_INT) {
1761                         /* signal intr and data pending */
1762                         adapter->shadow_tdt = MIT_PENDING_TDT | (i & 0xffff);
1763                         return (0);
1764                 } else {
1765                         adapter->shadow_tdt = MIT_PENDING_INT;
1766                 }
1767         }
1768 #endif /* NIC_SEND_COMBINING */
1769
1770         if (adapter->hw.mac.type == e1000_82547 &&
1771             adapter->link_duplex == HALF_DUPLEX)
1772                 lem_82547_move_tail(adapter);
1773         else {
1774                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
1775                 if (adapter->hw.mac.type == e1000_82547)
1776                         lem_82547_update_fifo_head(adapter,
1777                             m_head->m_pkthdr.len);
1778         }
1779
1780         return (0);
1781 }
1782
1783 /*********************************************************************
1784  *
1785  * 82547 workaround to avoid controller hang in half-duplex environment.
1786  * The workaround is to avoid queuing a large packet that would span
1787  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1788  * in this case. We do that only when FIFO is quiescent.
1789  *
1790  **********************************************************************/
1791 static void
1792 lem_82547_move_tail(void *arg)
1793 {
1794         struct adapter *adapter = arg;
1795         struct e1000_tx_desc *tx_desc;
1796         u16     hw_tdt, sw_tdt, length = 0;
1797         bool    eop = 0;
1798
1799         EM_TX_LOCK_ASSERT(adapter);
1800
1801         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
1802         sw_tdt = adapter->next_avail_tx_desc;
1803
1804         while (hw_tdt != sw_tdt) {
1805                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1806                 length += tx_desc->lower.flags.length;
1807                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1808                 if (++hw_tdt == adapter->num_tx_desc)
1809                         hw_tdt = 0;
1810
1811                 if (eop) {
1812                         if (lem_82547_fifo_workaround(adapter, length)) {
1813                                 adapter->tx_fifo_wrk_cnt++;
1814                                 callout_reset(&adapter->tx_fifo_timer, 1,
1815                                         lem_82547_move_tail, adapter);
1816                                 break;
1817                         }
1818                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
1819                         lem_82547_update_fifo_head(adapter, length);
1820                         length = 0;
1821                 }
1822         }
1823 }
1824
1825 static int
1826 lem_82547_fifo_workaround(struct adapter *adapter, int len)
1827 {
1828         int fifo_space, fifo_pkt_len;
1829
1830         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1831
1832         if (adapter->link_duplex == HALF_DUPLEX) {
1833                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1834
1835                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1836                         if (lem_82547_tx_fifo_reset(adapter))
1837                                 return (0);
1838                         else
1839                                 return (1);
1840                 }
1841         }
1842
1843         return (0);
1844 }
1845
1846 static void
1847 lem_82547_update_fifo_head(struct adapter *adapter, int len)
1848 {
1849         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1850
1851         /* tx_fifo_head is always 16 byte aligned */
1852         adapter->tx_fifo_head += fifo_pkt_len;
1853         if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
1854                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
1855         }
1856 }
1857
1858
1859 static int
1860 lem_82547_tx_fifo_reset(struct adapter *adapter)
1861 {
1862         u32 tctl;
1863
1864         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
1865             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
1866             (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
1867             E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
1868             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
1869             E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
1870             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
1871                 /* Disable TX unit */
1872                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
1873                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
1874                     tctl & ~E1000_TCTL_EN);
1875
1876                 /* Reset FIFO pointers */
1877                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
1878                     adapter->tx_head_addr);
1879                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
1880                     adapter->tx_head_addr);
1881                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
1882                     adapter->tx_head_addr);
1883                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
1884                     adapter->tx_head_addr);
1885
1886                 /* Re-enable TX unit */
1887                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
1888                 E1000_WRITE_FLUSH(&adapter->hw);
1889
1890                 adapter->tx_fifo_head = 0;
1891                 adapter->tx_fifo_reset_cnt++;
1892
1893                 return (TRUE);
1894         }
1895         else {
1896                 return (FALSE);
1897         }
1898 }
1899
1900 static void
1901 lem_set_promisc(struct adapter *adapter)
1902 {
1903         if_t ifp = adapter->ifp;
1904         u32             reg_rctl;
1905
1906         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1907
1908         if (if_getflags(ifp) & IFF_PROMISC) {
1909                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1910                 /* Turn this on if you want to see bad packets */
1911                 if (lem_debug_sbp)
1912                         reg_rctl |= E1000_RCTL_SBP;
1913                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1914         } else if (if_getflags(ifp) & IFF_ALLMULTI) {
1915                 reg_rctl |= E1000_RCTL_MPE;
1916                 reg_rctl &= ~E1000_RCTL_UPE;
1917                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1918         }
1919 }
1920
1921 static void
1922 lem_disable_promisc(struct adapter *adapter)
1923 {
1924         if_t ifp = adapter->ifp;
1925         u32             reg_rctl;
1926         int             mcnt = 0;
1927
1928         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1929         reg_rctl &=  (~E1000_RCTL_UPE);
1930         if (if_getflags(ifp) & IFF_ALLMULTI)
1931                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1932         else
1933                 mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES);
1934
1935         /* Don't disable if in MAX groups */
1936         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1937                 reg_rctl &=  (~E1000_RCTL_MPE);
1938         reg_rctl &=  (~E1000_RCTL_SBP);
1939         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1940 }
1941
1942
1943 /*********************************************************************
1944  *  Multicast Update
1945  *
1946  *  This routine is called whenever multicast address list is updated.
1947  *
1948  **********************************************************************/
1949
1950 static void
1951 lem_set_multi(struct adapter *adapter)
1952 {
1953         if_t ifp = adapter->ifp;
1954         u32 reg_rctl = 0;
1955         u8  *mta; /* Multicast array memory */
1956         int mcnt = 0;
1957
1958         IOCTL_DEBUGOUT("lem_set_multi: begin");
1959
1960         mta = adapter->mta;
1961         bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
1962
1963         if (adapter->hw.mac.type == e1000_82542 &&
1964             adapter->hw.revision_id == E1000_REVISION_2) {
1965                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1966                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1967                         e1000_pci_clear_mwi(&adapter->hw);
1968                 reg_rctl |= E1000_RCTL_RST;
1969                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1970                 msec_delay(5);
1971         }
1972
1973         if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES);
1974
1975         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1976                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1977                 reg_rctl |= E1000_RCTL_MPE;
1978                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1979         } else
1980                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
1981
1982         if (adapter->hw.mac.type == e1000_82542 &&
1983             adapter->hw.revision_id == E1000_REVISION_2) {
1984                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1985                 reg_rctl &= ~E1000_RCTL_RST;
1986                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1987                 msec_delay(5);
1988                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1989                         e1000_pci_set_mwi(&adapter->hw);
1990         }
1991 }
1992
1993
1994 /*********************************************************************
1995  *  Timer routine
1996  *
1997  *  This routine checks for link status and updates statistics.
1998  *
1999  **********************************************************************/
2000
2001 static void
2002 lem_local_timer(void *arg)
2003 {
2004         struct adapter  *adapter = arg;
2005
2006         EM_CORE_LOCK_ASSERT(adapter);
2007
2008         lem_update_link_status(adapter);
2009         lem_update_stats_counters(adapter);
2010
2011         lem_smartspeed(adapter);
2012
2013         /*
2014          * We check the watchdog: the time since
2015          * the last TX descriptor was cleaned.
2016          * This implies a functional TX engine.
2017          */
2018         if ((adapter->watchdog_check == TRUE) &&
2019             (ticks - adapter->watchdog_time > EM_WATCHDOG))
2020                 goto hung;
2021
2022         callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
2023         return;
2024 hung:
2025         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2026         if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_RUNNING);
2027         adapter->watchdog_events++;
2028         lem_init_locked(adapter);
2029 }
2030
2031 static void
2032 lem_update_link_status(struct adapter *adapter)
2033 {
2034         struct e1000_hw *hw = &adapter->hw;
2035         if_t ifp = adapter->ifp;
2036         device_t dev = adapter->dev;
2037         u32 link_check = 0;
2038
2039         /* Get the cached link value or read phy for real */
2040         switch (hw->phy.media_type) {
2041         case e1000_media_type_copper:
2042                 if (hw->mac.get_link_status) {
2043                         /* Do the work to read phy */
2044                         e1000_check_for_link(hw);
2045                         link_check = !hw->mac.get_link_status;
2046                         if (link_check) /* ESB2 fix */
2047                                 e1000_cfg_on_link_up(hw);
2048                 } else
2049                         link_check = TRUE;
2050                 break;
2051         case e1000_media_type_fiber:
2052                 e1000_check_for_link(hw);
2053                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2054                                  E1000_STATUS_LU);
2055                 break;
2056         case e1000_media_type_internal_serdes:
2057                 e1000_check_for_link(hw);
2058                 link_check = adapter->hw.mac.serdes_has_link;
2059                 break;
2060         default:
2061         case e1000_media_type_unknown:
2062                 break;
2063         }
2064
2065         /* Now check for a transition */
2066         if (link_check && (adapter->link_active == 0)) {
2067                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2068                     &adapter->link_duplex);
2069                 if (bootverbose)
2070                         device_printf(dev, "Link is up %d Mbps %s\n",
2071                             adapter->link_speed,
2072                             ((adapter->link_duplex == FULL_DUPLEX) ?
2073                             "Full Duplex" : "Half Duplex"));
2074                 adapter->link_active = 1;
2075                 adapter->smartspeed = 0;
2076                 if_setbaudrate(ifp, adapter->link_speed * 1000000);
2077                 if_link_state_change(ifp, LINK_STATE_UP);
2078         } else if (!link_check && (adapter->link_active == 1)) {
2079                 if_setbaudrate(ifp, 0);
2080                 adapter->link_speed = 0;
2081                 adapter->link_duplex = 0;
2082                 if (bootverbose)
2083                         device_printf(dev, "Link is Down\n");
2084                 adapter->link_active = 0;
2085                 /* Link down, disable watchdog */
2086                 adapter->watchdog_check = FALSE;
2087                 if_link_state_change(ifp, LINK_STATE_DOWN);
2088         }
2089 }
2090
2091 /*********************************************************************
2092  *
2093  *  This routine disables all traffic on the adapter by issuing a
2094  *  global reset on the MAC and deallocates TX/RX buffers.
2095  *
2096  *  This routine should always be called with BOTH the CORE
2097  *  and TX locks.
2098  **********************************************************************/
2099
2100 static void
2101 lem_stop(void *arg)
2102 {
2103         struct adapter  *adapter = arg;
2104         if_t ifp = adapter->ifp;
2105
2106         EM_CORE_LOCK_ASSERT(adapter);
2107         EM_TX_LOCK_ASSERT(adapter);
2108
2109         INIT_DEBUGOUT("lem_stop: begin");
2110
2111         lem_disable_intr(adapter);
2112         callout_stop(&adapter->timer);
2113         callout_stop(&adapter->tx_fifo_timer);
2114
2115         /* Tell the stack that the interface is no longer active */
2116         if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE));
2117
2118         e1000_reset_hw(&adapter->hw);
2119         if (adapter->hw.mac.type >= e1000_82544)
2120                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2121
2122         e1000_led_off(&adapter->hw);
2123         e1000_cleanup_led(&adapter->hw);
2124 }
2125
2126
2127 /*********************************************************************
2128  *
2129  *  Determine hardware revision.
2130  *
2131  **********************************************************************/
2132 static void
2133 lem_identify_hardware(struct adapter *adapter)
2134 {
2135         device_t dev = adapter->dev;
2136
2137         /* Make sure our PCI config space has the necessary stuff set */
2138         pci_enable_busmaster(dev);
2139         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2140
2141         /* Save off the information about this board */
2142         adapter->hw.vendor_id = pci_get_vendor(dev);
2143         adapter->hw.device_id = pci_get_device(dev);
2144         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2145         adapter->hw.subsystem_vendor_id =
2146             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2147         adapter->hw.subsystem_device_id =
2148             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2149
2150         /* Do Shared Code Init and Setup */
2151         if (e1000_set_mac_type(&adapter->hw)) {
2152                 device_printf(dev, "Setup init failure\n");
2153                 return;
2154         }
2155 }
2156
2157 static int
2158 lem_allocate_pci_resources(struct adapter *adapter)
2159 {
2160         device_t        dev = adapter->dev;
2161         int             val, rid, error = E1000_SUCCESS;
2162
2163         rid = PCIR_BAR(0);
2164         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2165             &rid, RF_ACTIVE);
2166         if (adapter->memory == NULL) {
2167                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2168                 return (ENXIO);
2169         }
2170         adapter->osdep.mem_bus_space_tag =
2171             rman_get_bustag(adapter->memory);
2172         adapter->osdep.mem_bus_space_handle =
2173             rman_get_bushandle(adapter->memory);
2174         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2175
2176         /* Only older adapters use IO mapping */
2177         if (adapter->hw.mac.type > e1000_82543) {
2178                 /* Figure our where our IO BAR is ? */
2179                 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2180                         val = pci_read_config(dev, rid, 4);
2181                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2182                                 adapter->io_rid = rid;
2183                                 break;
2184                         }
2185                         rid += 4;
2186                         /* check for 64bit BAR */
2187                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2188                                 rid += 4;
2189                 }
2190                 if (rid >= PCIR_CIS) {
2191                         device_printf(dev, "Unable to locate IO BAR\n");
2192                         return (ENXIO);
2193                 }
2194                 adapter->ioport = bus_alloc_resource_any(dev,
2195                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2196                 if (adapter->ioport == NULL) {
2197                         device_printf(dev, "Unable to allocate bus resource: "
2198                             "ioport\n");
2199                         return (ENXIO);
2200                 }
2201                 adapter->hw.io_base = 0;
2202                 adapter->osdep.io_bus_space_tag =
2203                     rman_get_bustag(adapter->ioport);
2204                 adapter->osdep.io_bus_space_handle =
2205                     rman_get_bushandle(adapter->ioport);
2206         }
2207
2208         adapter->hw.back = &adapter->osdep;
2209
2210         return (error);
2211 }
2212
2213 /*********************************************************************
2214  *
2215  *  Setup the Legacy or MSI Interrupt handler
2216  *
2217  **********************************************************************/
2218 int
2219 lem_allocate_irq(struct adapter *adapter)
2220 {
2221         device_t dev = adapter->dev;
2222         int error, rid = 0;
2223
2224         /* Manually turn off all interrupts */
2225         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2226
2227         /* We allocate a single interrupt resource */
2228         adapter->res[0] = bus_alloc_resource_any(dev,
2229             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2230         if (adapter->res[0] == NULL) {
2231                 device_printf(dev, "Unable to allocate bus resource: "
2232                     "interrupt\n");
2233                 return (ENXIO);
2234         }
2235
2236         /* Do Legacy setup? */
2237         if (lem_use_legacy_irq) {
2238                 if ((error = bus_setup_intr(dev, adapter->res[0],
2239                     INTR_TYPE_NET | INTR_MPSAFE, NULL, lem_intr, adapter,
2240                     &adapter->tag[0])) != 0) {
2241                         device_printf(dev,
2242                             "Failed to register interrupt handler");
2243                         return (error);
2244                 }
2245                 return (0);
2246         }
2247
2248         /*
2249          * Use a Fast interrupt and the associated
2250          * deferred processing contexts.
2251          */
2252         TASK_INIT(&adapter->rxtx_task, 0, lem_handle_rxtx, adapter);
2253         TASK_INIT(&adapter->link_task, 0, lem_handle_link, adapter);
2254         adapter->tq = taskqueue_create_fast("lem_taskq", M_NOWAIT,
2255             taskqueue_thread_enqueue, &adapter->tq);
2256         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2257             device_get_nameunit(adapter->dev));
2258         if ((error = bus_setup_intr(dev, adapter->res[0],
2259             INTR_TYPE_NET, lem_irq_fast, NULL, adapter,
2260             &adapter->tag[0])) != 0) {
2261                 device_printf(dev, "Failed to register fast interrupt "
2262                             "handler: %d\n", error);
2263                 taskqueue_free(adapter->tq);
2264                 adapter->tq = NULL;
2265                 return (error);
2266         }
2267
2268         return (0);
2269 }
2270
2271
2272 static void
2273 lem_free_pci_resources(struct adapter *adapter)
2274 {
2275         device_t dev = adapter->dev;
2276
2277
2278         if (adapter->tag[0] != NULL) {
2279                 bus_teardown_intr(dev, adapter->res[0],
2280                     adapter->tag[0]);
2281                 adapter->tag[0] = NULL;
2282         }
2283
2284         if (adapter->res[0] != NULL) {
2285                 bus_release_resource(dev, SYS_RES_IRQ,
2286                     0, adapter->res[0]);
2287         }
2288
2289         if (adapter->memory != NULL)
2290                 bus_release_resource(dev, SYS_RES_MEMORY,
2291                     PCIR_BAR(0), adapter->memory);
2292
2293         if (adapter->ioport != NULL)
2294                 bus_release_resource(dev, SYS_RES_IOPORT,
2295                     adapter->io_rid, adapter->ioport);
2296 }
2297
2298
2299 /*********************************************************************
2300  *
2301  *  Initialize the hardware to a configuration
2302  *  as specified by the adapter structure.
2303  *
2304  **********************************************************************/
2305 static int
2306 lem_hardware_init(struct adapter *adapter)
2307 {
2308         device_t dev = adapter->dev;
2309         u16     rx_buffer_size;
2310
2311         INIT_DEBUGOUT("lem_hardware_init: begin");
2312
2313         /* Issue a global reset */
2314         e1000_reset_hw(&adapter->hw);
2315
2316         /* When hardware is reset, fifo_head is also reset */
2317         adapter->tx_fifo_head = 0;
2318
2319         /*
2320          * These parameters control the automatic generation (Tx) and
2321          * response (Rx) to Ethernet PAUSE frames.
2322          * - High water mark should allow for at least two frames to be
2323          *   received after sending an XOFF.
2324          * - Low water mark works best when it is very near the high water mark.
2325          *   This allows the receiver to restart by sending XON when it has
2326          *   drained a bit. Here we use an arbitrary value of 1500 which will
2327          *   restart after one full frame is pulled from the buffer. There
2328          *   could be several smaller frames in the buffer and if so they will
2329          *   not trigger the XON until their total number reduces the buffer
2330          *   by 1500.
2331          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2332          */
2333         rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
2334             0xffff) << 10 );
2335
2336         adapter->hw.fc.high_water = rx_buffer_size -
2337             roundup2(adapter->max_frame_size, 1024);
2338         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
2339
2340         adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
2341         adapter->hw.fc.send_xon = TRUE;
2342
2343         /* Set Flow control, use the tunable location if sane */
2344         if ((lem_fc_setting >= 0) && (lem_fc_setting < 4))
2345                 adapter->hw.fc.requested_mode = lem_fc_setting;
2346         else
2347                 adapter->hw.fc.requested_mode = e1000_fc_none;
2348
2349         if (e1000_init_hw(&adapter->hw) < 0) {
2350                 device_printf(dev, "Hardware Initialization Failed\n");
2351                 return (EIO);
2352         }
2353
2354         e1000_check_for_link(&adapter->hw);
2355
2356         return (0);
2357 }
2358
2359 /*********************************************************************
2360  *
2361  *  Setup networking device structure and register an interface.
2362  *
2363  **********************************************************************/
2364 static int
2365 lem_setup_interface(device_t dev, struct adapter *adapter)
2366 {
2367         if_t ifp;
2368
2369         INIT_DEBUGOUT("lem_setup_interface: begin");
2370
2371         ifp = adapter->ifp = if_gethandle(IFT_ETHER);
2372         if (ifp == (void *)NULL) {
2373                 device_printf(dev, "can not allocate ifnet structure\n");
2374                 return (-1);
2375         }
2376         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2377         if_setinitfn(ifp,  lem_init);
2378         if_setsoftc(ifp, adapter);
2379         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
2380         if_setioctlfn(ifp, lem_ioctl);
2381         if_setstartfn(ifp, lem_start);
2382         if_setgetcounterfn(ifp, lem_get_counter);
2383         if_setsendqlen(ifp, adapter->num_tx_desc - 1);
2384         if_setsendqready(ifp);
2385
2386         ether_ifattach(ifp, adapter->hw.mac.addr);
2387
2388         if_setcapabilities(ifp, 0);
2389
2390         if (adapter->hw.mac.type >= e1000_82543) {
2391                 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0);
2392                 if_setcapenablebit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0);
2393         }
2394
2395         /*
2396          * Tell the upper layer(s) we support long frames.
2397          */
2398         if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
2399         if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0);
2400         if_setcapenablebit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0);
2401
2402         /*
2403         ** Dont turn this on by default, if vlans are
2404         ** created on another pseudo device (eg. lagg)
2405         ** then vlan events are not passed thru, breaking
2406         ** operation, but with HW FILTER off it works. If
2407         ** using vlans directly on the em driver you can
2408         ** enable this and get full hardware tag filtering.
2409         */
2410         if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0);
2411
2412 #ifdef DEVICE_POLLING
2413         if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0);
2414 #endif
2415
2416         /* Enable only WOL MAGIC by default */
2417         if (adapter->wol) {
2418                 if_setcapabilitiesbit(ifp, IFCAP_WOL, 0);
2419                 if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0);
2420         }
2421
2422         /*
2423          * Specify the media types supported by this adapter and register
2424          * callbacks to update media and link information
2425          */
2426         ifmedia_init(&adapter->media, IFM_IMASK,
2427             lem_media_change, lem_media_status);
2428         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2429             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2430                 u_char fiber_type = IFM_1000_SX;        /* default type */
2431
2432                 if (adapter->hw.mac.type == e1000_82545)
2433                         fiber_type = IFM_1000_LX;
2434                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2435                             0, NULL);
2436                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2437         } else {
2438                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2439                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2440                             0, NULL);
2441                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2442                             0, NULL);
2443                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2444                             0, NULL);
2445                 if (adapter->hw.phy.type != e1000_phy_ife) {
2446                         ifmedia_add(&adapter->media,
2447                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2448                         ifmedia_add(&adapter->media,
2449                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2450                 }
2451         }
2452         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2453         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2454         return (0);
2455 }
2456
2457
2458 /*********************************************************************
2459  *
2460  *  Workaround for SmartSpeed on 82541 and 82547 controllers
2461  *
2462  **********************************************************************/
2463 static void
2464 lem_smartspeed(struct adapter *adapter)
2465 {
2466         u16 phy_tmp;
2467
2468         if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
2469             adapter->hw.mac.autoneg == 0 ||
2470             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
2471                 return;
2472
2473         if (adapter->smartspeed == 0) {
2474                 /* If Master/Slave config fault is asserted twice,
2475                  * we assume back-to-back */
2476                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2477                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2478                         return;
2479                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2480                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2481                         e1000_read_phy_reg(&adapter->hw,
2482                             PHY_1000T_CTRL, &phy_tmp);
2483                         if(phy_tmp & CR_1000T_MS_ENABLE) {
2484                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
2485                                 e1000_write_phy_reg(&adapter->hw,
2486                                     PHY_1000T_CTRL, phy_tmp);
2487                                 adapter->smartspeed++;
2488                                 if(adapter->hw.mac.autoneg &&
2489                                    !e1000_copper_link_autoneg(&adapter->hw) &&
2490                                    !e1000_read_phy_reg(&adapter->hw,
2491                                     PHY_CONTROL, &phy_tmp)) {
2492                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
2493                                                     MII_CR_RESTART_AUTO_NEG);
2494                                         e1000_write_phy_reg(&adapter->hw,
2495                                             PHY_CONTROL, phy_tmp);
2496                                 }
2497                         }
2498                 }
2499                 return;
2500         } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2501                 /* If still no link, perhaps using 2/3 pair cable */
2502                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2503                 phy_tmp |= CR_1000T_MS_ENABLE;
2504                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2505                 if(adapter->hw.mac.autoneg &&
2506                    !e1000_copper_link_autoneg(&adapter->hw) &&
2507                    !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
2508                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
2509                                     MII_CR_RESTART_AUTO_NEG);
2510                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
2511                 }
2512         }
2513         /* Restart process after EM_SMARTSPEED_MAX iterations */
2514         if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2515                 adapter->smartspeed = 0;
2516 }
2517
2518
2519 /*
2520  * Manage DMA'able memory.
2521  */
2522 static void
2523 lem_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2524 {
2525         if (error)
2526                 return;
2527         *(bus_addr_t *) arg = segs[0].ds_addr;
2528 }
2529
2530 static int
2531 lem_dma_malloc(struct adapter *adapter, bus_size_t size,
2532         struct em_dma_alloc *dma, int mapflags)
2533 {
2534         int error;
2535
2536         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2537                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
2538                                 BUS_SPACE_MAXADDR,      /* lowaddr */
2539                                 BUS_SPACE_MAXADDR,      /* highaddr */
2540                                 NULL, NULL,             /* filter, filterarg */
2541                                 size,                   /* maxsize */
2542                                 1,                      /* nsegments */
2543                                 size,                   /* maxsegsize */
2544                                 0,                      /* flags */
2545                                 NULL,                   /* lockfunc */
2546                                 NULL,                   /* lockarg */
2547                                 &dma->dma_tag);
2548         if (error) {
2549                 device_printf(adapter->dev,
2550                     "%s: bus_dma_tag_create failed: %d\n",
2551                     __func__, error);
2552                 goto fail_0;
2553         }
2554
2555         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2556             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
2557         if (error) {
2558                 device_printf(adapter->dev,
2559                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2560                     __func__, (uintmax_t)size, error);
2561                 goto fail_2;
2562         }
2563
2564         dma->dma_paddr = 0;
2565         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2566             size, lem_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2567         if (error || dma->dma_paddr == 0) {
2568                 device_printf(adapter->dev,
2569                     "%s: bus_dmamap_load failed: %d\n",
2570                     __func__, error);
2571                 goto fail_3;
2572         }
2573
2574         return (0);
2575
2576 fail_3:
2577         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2578 fail_2:
2579         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2580         bus_dma_tag_destroy(dma->dma_tag);
2581 fail_0:
2582         dma->dma_tag = NULL;
2583
2584         return (error);
2585 }
2586
2587 static void
2588 lem_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2589 {
2590         if (dma->dma_tag == NULL)
2591                 return;
2592         if (dma->dma_paddr != 0) {
2593                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2594                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2595                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2596                 dma->dma_paddr = 0;
2597         }
2598         if (dma->dma_vaddr != NULL) {
2599                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2600                 dma->dma_vaddr = NULL;
2601         }
2602         bus_dma_tag_destroy(dma->dma_tag);
2603         dma->dma_tag = NULL;
2604 }
2605
2606
2607 /*********************************************************************
2608  *
2609  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2610  *  the information needed to transmit a packet on the wire.
2611  *
2612  **********************************************************************/
2613 static int
2614 lem_allocate_transmit_structures(struct adapter *adapter)
2615 {
2616         device_t dev = adapter->dev;
2617         struct em_buffer *tx_buffer;
2618         int error;
2619
2620         /*
2621          * Create DMA tags for tx descriptors
2622          */
2623         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
2624                                 1, 0,                   /* alignment, bounds */
2625                                 BUS_SPACE_MAXADDR,      /* lowaddr */
2626                                 BUS_SPACE_MAXADDR,      /* highaddr */
2627                                 NULL, NULL,             /* filter, filterarg */
2628                                 MCLBYTES * EM_MAX_SCATTER,      /* maxsize */
2629                                 EM_MAX_SCATTER,         /* nsegments */
2630                                 MCLBYTES,               /* maxsegsize */
2631                                 0,                      /* flags */
2632                                 NULL,                   /* lockfunc */
2633                                 NULL,                   /* lockarg */
2634                                 &adapter->txtag)) != 0) {
2635                 device_printf(dev, "Unable to allocate TX DMA tag\n");
2636                 goto fail;
2637         }
2638
2639         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
2640             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
2641         if (adapter->tx_buffer_area == NULL) {
2642                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2643                 error = ENOMEM;
2644                 goto fail;
2645         }
2646
2647         /* Create the descriptor buffer dma maps */
2648         for (int i = 0; i < adapter->num_tx_desc; i++) {
2649                 tx_buffer = &adapter->tx_buffer_area[i];
2650                 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
2651                 if (error != 0) {
2652                         device_printf(dev, "Unable to create TX DMA map\n");
2653                         goto fail;
2654                 }
2655                 tx_buffer->next_eop = -1;
2656         }
2657
2658         return (0);
2659 fail:
2660         lem_free_transmit_structures(adapter);
2661         return (error);
2662 }
2663
2664 /*********************************************************************
2665  *
2666  *  (Re)Initialize transmit structures.
2667  *
2668  **********************************************************************/
2669 static void
2670 lem_setup_transmit_structures(struct adapter *adapter)
2671 {
2672         struct em_buffer *tx_buffer;
2673 #ifdef DEV_NETMAP
2674         /* we are already locked */
2675         struct netmap_adapter *na = netmap_getna(adapter->ifp);
2676         struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0);
2677 #endif /* DEV_NETMAP */
2678
2679         /* Clear the old ring contents */
2680         bzero(adapter->tx_desc_base,
2681             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
2682
2683         /* Free any existing TX buffers */
2684         for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2685                 tx_buffer = &adapter->tx_buffer_area[i];
2686                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2687                     BUS_DMASYNC_POSTWRITE);
2688                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2689                 m_freem(tx_buffer->m_head);
2690                 tx_buffer->m_head = NULL;
2691 #ifdef DEV_NETMAP
2692                 if (slot) {
2693                         /* the i-th NIC entry goes to slot si */
2694                         int si = netmap_idx_n2k(&na->tx_rings[0], i);
2695                         uint64_t paddr;
2696                         void *addr;
2697
2698                         addr = PNMB(na, slot + si, &paddr);
2699                         adapter->tx_desc_base[i].buffer_addr = htole64(paddr);
2700                         /* reload the map for netmap mode */
2701                         netmap_load_map(na, adapter->txtag, tx_buffer->map, addr);
2702                 }
2703 #endif /* DEV_NETMAP */
2704                 tx_buffer->next_eop = -1;
2705         }
2706
2707         /* Reset state */
2708         adapter->last_hw_offload = 0;
2709         adapter->next_avail_tx_desc = 0;
2710         adapter->next_tx_to_clean = 0;
2711         adapter->num_tx_desc_avail = adapter->num_tx_desc;
2712
2713         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2714             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2715
2716         return;
2717 }
2718
2719 /*********************************************************************
2720  *
2721  *  Enable transmit unit.
2722  *
2723  **********************************************************************/
2724 static void
2725 lem_initialize_transmit_unit(struct adapter *adapter)
2726 {
2727         u32     tctl, tipg = 0;
2728         u64     bus_addr;
2729
2730          INIT_DEBUGOUT("lem_initialize_transmit_unit: begin");
2731         /* Setup the Base and Length of the Tx Descriptor Ring */
2732         bus_addr = adapter->txdma.dma_paddr;
2733         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
2734             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
2735         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
2736             (u32)(bus_addr >> 32));
2737         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
2738             (u32)bus_addr);
2739         /* Setup the HW Tx Head and Tail descriptor pointers */
2740         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
2741         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
2742
2743         HW_DEBUGOUT2("Base = %x, Length = %x\n",
2744             E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
2745             E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
2746
2747         /* Set the default values for the Tx Inter Packet Gap timer */
2748         switch (adapter->hw.mac.type) {
2749         case e1000_82542:
2750                 tipg = DEFAULT_82542_TIPG_IPGT;
2751                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2752                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2753                 break;
2754         default:
2755                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2756                     (adapter->hw.phy.media_type ==
2757                     e1000_media_type_internal_serdes))
2758                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2759                 else
2760                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2761                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2762                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2763         }
2764
2765         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
2766         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
2767         if(adapter->hw.mac.type >= e1000_82540)
2768                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
2769                     adapter->tx_abs_int_delay.value);
2770
2771         /* Program the Transmit Control Register */
2772         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2773         tctl &= ~E1000_TCTL_CT;
2774         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
2775                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
2776
2777         /* This write will effectively turn on the transmit unit. */
2778         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2779
2780         /* Setup Transmit Descriptor Base Settings */
2781         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
2782
2783         if (adapter->tx_int_delay.value > 0)
2784                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2785 }
2786
2787 /*********************************************************************
2788  *
2789  *  Free all transmit related data structures.
2790  *
2791  **********************************************************************/
2792 static void
2793 lem_free_transmit_structures(struct adapter *adapter)
2794 {
2795         struct em_buffer *tx_buffer;
2796
2797         INIT_DEBUGOUT("free_transmit_structures: begin");
2798
2799         if (adapter->tx_buffer_area != NULL) {
2800                 for (int i = 0; i < adapter->num_tx_desc; i++) {
2801                         tx_buffer = &adapter->tx_buffer_area[i];
2802                         if (tx_buffer->m_head != NULL) {
2803                                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2804                                     BUS_DMASYNC_POSTWRITE);
2805                                 bus_dmamap_unload(adapter->txtag,
2806                                     tx_buffer->map);
2807                                 m_freem(tx_buffer->m_head);
2808                                 tx_buffer->m_head = NULL;
2809                         } else if (tx_buffer->map != NULL)
2810                                 bus_dmamap_unload(adapter->txtag,
2811                                     tx_buffer->map);
2812                         if (tx_buffer->map != NULL) {
2813                                 bus_dmamap_destroy(adapter->txtag,
2814                                     tx_buffer->map);
2815                                 tx_buffer->map = NULL;
2816                         }
2817                 }
2818         }
2819         if (adapter->tx_buffer_area != NULL) {
2820                 free(adapter->tx_buffer_area, M_DEVBUF);
2821                 adapter->tx_buffer_area = NULL;
2822         }
2823         if (adapter->txtag != NULL) {
2824                 bus_dma_tag_destroy(adapter->txtag);
2825                 adapter->txtag = NULL;
2826         }
2827 }
2828
2829 /*********************************************************************
2830  *
2831  *  The offload context needs to be set when we transfer the first
2832  *  packet of a particular protocol (TCP/UDP). This routine has been
2833  *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
2834  *
2835  *  Added back the old method of keeping the current context type
2836  *  and not setting if unnecessary, as this is reported to be a
2837  *  big performance win.  -jfv
2838  **********************************************************************/
2839 static void
2840 lem_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
2841     u32 *txd_upper, u32 *txd_lower)
2842 {
2843         struct e1000_context_desc *TXD = NULL;
2844         struct em_buffer *tx_buffer;
2845         struct ether_vlan_header *eh;
2846         struct ip *ip = NULL;
2847         struct ip6_hdr *ip6;
2848         int curr_txd, ehdrlen;
2849         u32 cmd, hdr_len, ip_hlen;
2850         u16 etype;
2851         u8 ipproto;
2852
2853
2854         cmd = hdr_len = ipproto = 0;
2855         *txd_upper = *txd_lower = 0;
2856         curr_txd = adapter->next_avail_tx_desc;
2857
2858         /*
2859          * Determine where frame payload starts.
2860          * Jump over vlan headers if already present,
2861          * helpful for QinQ too.
2862          */
2863         eh = mtod(mp, struct ether_vlan_header *);
2864         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2865                 etype = ntohs(eh->evl_proto);
2866                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2867         } else {
2868                 etype = ntohs(eh->evl_encap_proto);
2869                 ehdrlen = ETHER_HDR_LEN;
2870         }
2871
2872         /*
2873          * We only support TCP/UDP for IPv4 and IPv6 for the moment.
2874          * TODO: Support SCTP too when it hits the tree.
2875          */
2876         switch (etype) {
2877         case ETHERTYPE_IP:
2878                 ip = (struct ip *)(mp->m_data + ehdrlen);
2879                 ip_hlen = ip->ip_hl << 2;
2880
2881                 /* Setup of IP header checksum. */
2882                 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
2883                         /*
2884                          * Start offset for header checksum calculation.
2885                          * End offset for header checksum calculation.
2886                          * Offset of place to put the checksum.
2887                          */
2888                         TXD = (struct e1000_context_desc *)
2889                             &adapter->tx_desc_base[curr_txd];
2890                         TXD->lower_setup.ip_fields.ipcss = ehdrlen;
2891                         TXD->lower_setup.ip_fields.ipcse =
2892                             htole16(ehdrlen + ip_hlen);
2893                         TXD->lower_setup.ip_fields.ipcso =
2894                             ehdrlen + offsetof(struct ip, ip_sum);
2895                         cmd |= E1000_TXD_CMD_IP;
2896                         *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2897                 }
2898
2899                 hdr_len = ehdrlen + ip_hlen;
2900                 ipproto = ip->ip_p;
2901
2902                 break;
2903         case ETHERTYPE_IPV6:
2904                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
2905                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
2906
2907                 /* IPv6 doesn't have a header checksum. */
2908
2909                 hdr_len = ehdrlen + ip_hlen;
2910                 ipproto = ip6->ip6_nxt;
2911                 break;
2912
2913         default:
2914                 return;
2915         }
2916
2917         switch (ipproto) {
2918         case IPPROTO_TCP:
2919                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2920                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2921                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2922                         /* no need for context if already set */
2923                         if (adapter->last_hw_offload == CSUM_TCP)
2924                                 return;
2925                         adapter->last_hw_offload = CSUM_TCP;
2926                         /*
2927                          * Start offset for payload checksum calculation.
2928                          * End offset for payload checksum calculation.
2929                          * Offset of place to put the checksum.
2930                          */
2931                         TXD = (struct e1000_context_desc *)
2932                             &adapter->tx_desc_base[curr_txd];
2933                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
2934                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2935                         TXD->upper_setup.tcp_fields.tucso =
2936                             hdr_len + offsetof(struct tcphdr, th_sum);
2937                         cmd |= E1000_TXD_CMD_TCP;
2938                 }
2939                 break;
2940         case IPPROTO_UDP:
2941         {
2942                 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2943                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2944                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2945                         /* no need for context if already set */
2946                         if (adapter->last_hw_offload == CSUM_UDP)
2947                                 return;
2948                         adapter->last_hw_offload = CSUM_UDP;
2949                         /*
2950                          * Start offset for header checksum calculation.
2951                          * End offset for header checksum calculation.
2952                          * Offset of place to put the checksum.
2953                          */
2954                         TXD = (struct e1000_context_desc *)
2955                             &adapter->tx_desc_base[curr_txd];
2956                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
2957                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2958                         TXD->upper_setup.tcp_fields.tucso =
2959                             hdr_len + offsetof(struct udphdr, uh_sum);
2960                 }
2961                 /* Fall Thru */
2962         }
2963         default:
2964                 break;
2965         }
2966
2967         if (TXD == NULL)
2968                 return;
2969         TXD->tcp_seg_setup.data = htole32(0);
2970         TXD->cmd_and_length =
2971             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
2972         tx_buffer = &adapter->tx_buffer_area[curr_txd];
2973         tx_buffer->m_head = NULL;
2974         tx_buffer->next_eop = -1;
2975
2976         if (++curr_txd == adapter->num_tx_desc)
2977                 curr_txd = 0;
2978
2979         adapter->num_tx_desc_avail--;
2980         adapter->next_avail_tx_desc = curr_txd;
2981 }
2982
2983
2984 /**********************************************************************
2985  *
2986  *  Examine each tx_buffer in the used queue. If the hardware is done
2987  *  processing the packet then free associated resources. The
2988  *  tx_buffer is put back on the free queue.
2989  *
2990  **********************************************************************/
2991 static void
2992 lem_txeof(struct adapter *adapter)
2993 {
2994         int first, last, done, num_avail;
2995         struct em_buffer *tx_buffer;
2996         struct e1000_tx_desc   *tx_desc, *eop_desc;
2997         if_t ifp = adapter->ifp;
2998
2999         EM_TX_LOCK_ASSERT(adapter);
3000
3001 #ifdef DEV_NETMAP
3002         if (netmap_tx_irq(ifp, 0))
3003                 return;
3004 #endif /* DEV_NETMAP */
3005         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3006                 return;
3007
3008         num_avail = adapter->num_tx_desc_avail;
3009         first = adapter->next_tx_to_clean;
3010         tx_desc = &adapter->tx_desc_base[first];
3011         tx_buffer = &adapter->tx_buffer_area[first];
3012         last = tx_buffer->next_eop;
3013         eop_desc = &adapter->tx_desc_base[last];
3014
3015         /*
3016          * What this does is get the index of the
3017          * first descriptor AFTER the EOP of the
3018          * first packet, that way we can do the
3019          * simple comparison on the inner while loop.
3020          */
3021         if (++last == adapter->num_tx_desc)
3022                 last = 0;
3023         done = last;
3024
3025         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3026             BUS_DMASYNC_POSTREAD);
3027
3028         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3029                 /* We clean the range of the packet */
3030                 while (first != done) {
3031                         tx_desc->upper.data = 0;
3032                         tx_desc->lower.data = 0;
3033                         tx_desc->buffer_addr = 0;
3034                         ++num_avail;
3035
3036                         if (tx_buffer->m_head) {
3037                                 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
3038                                 bus_dmamap_sync(adapter->txtag,
3039                                     tx_buffer->map,
3040                                     BUS_DMASYNC_POSTWRITE);
3041                                 bus_dmamap_unload(adapter->txtag,
3042                                     tx_buffer->map);
3043
3044                                 m_freem(tx_buffer->m_head);
3045                                 tx_buffer->m_head = NULL;
3046                         }
3047                         tx_buffer->next_eop = -1;
3048                         adapter->watchdog_time = ticks;
3049
3050                         if (++first == adapter->num_tx_desc)
3051                                 first = 0;
3052
3053                         tx_buffer = &adapter->tx_buffer_area[first];
3054                         tx_desc = &adapter->tx_desc_base[first];
3055                 }
3056                 /* See if we can continue to the next packet */
3057                 last = tx_buffer->next_eop;
3058                 if (last != -1) {
3059                         eop_desc = &adapter->tx_desc_base[last];
3060                         /* Get new done point */
3061                         if (++last == adapter->num_tx_desc) last = 0;
3062                         done = last;
3063                 } else
3064                         break;
3065         }
3066         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3067             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3068
3069         adapter->next_tx_to_clean = first;
3070         adapter->num_tx_desc_avail = num_avail;
3071
3072 #ifdef NIC_SEND_COMBINING
3073         if ((adapter->shadow_tdt & MIT_PENDING_TDT) == MIT_PENDING_TDT) {
3074                 /* a tdt write is pending, do it */
3075                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0),
3076                         0xffff & adapter->shadow_tdt);
3077                 adapter->shadow_tdt = MIT_PENDING_INT;
3078         } else {
3079                 adapter->shadow_tdt = 0; // disable
3080         }
3081 #endif /* NIC_SEND_COMBINING */
3082         /*
3083          * If we have enough room, clear IFF_DRV_OACTIVE to
3084          * tell the stack that it is OK to send packets.
3085          * If there are no pending descriptors, clear the watchdog.
3086          */
3087         if (adapter->num_tx_desc_avail > EM_TX_CLEANUP_THRESHOLD) {
3088                 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
3089                 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) {
3090                         adapter->watchdog_check = FALSE;
3091                         return;
3092                 }
3093         }
3094 }
3095
3096 /*********************************************************************
3097  *
3098  *  When Link is lost sometimes there is work still in the TX ring
3099  *  which may result in a watchdog, rather than allow that we do an
3100  *  attempted cleanup and then reinit here. Note that this has been
3101  *  seens mostly with fiber adapters.
3102  *
3103  **********************************************************************/
3104 static void
3105 lem_tx_purge(struct adapter *adapter)
3106 {
3107         if ((!adapter->link_active) && (adapter->watchdog_check)) {
3108                 EM_TX_LOCK(adapter);
3109                 lem_txeof(adapter);
3110                 EM_TX_UNLOCK(adapter);
3111                 if (adapter->watchdog_check) /* Still outstanding? */
3112                         lem_init_locked(adapter);
3113         }
3114 }
3115
3116 /*********************************************************************
3117  *
3118  *  Get a buffer from system mbuf buffer pool.
3119  *
3120  **********************************************************************/
3121 static int
3122 lem_get_buf(struct adapter *adapter, int i)
3123 {
3124         struct mbuf             *m;
3125         bus_dma_segment_t       segs[1];
3126         bus_dmamap_t            map;
3127         struct em_buffer        *rx_buffer;
3128         int                     error, nsegs;
3129
3130         m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
3131         if (m == NULL) {
3132                 adapter->mbuf_cluster_failed++;
3133                 return (ENOBUFS);
3134         }
3135         m->m_len = m->m_pkthdr.len = MCLBYTES;
3136
3137         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3138                 m_adj(m, ETHER_ALIGN);
3139
3140         /*
3141          * Using memory from the mbuf cluster pool, invoke the
3142          * bus_dma machinery to arrange the memory mapping.
3143          */
3144         error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
3145             adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
3146         if (error != 0) {
3147                 m_free(m);
3148                 return (error);
3149         }
3150
3151         /* If nsegs is wrong then the stack is corrupt. */
3152         KASSERT(nsegs == 1, ("Too many segments returned!"));
3153
3154         rx_buffer = &adapter->rx_buffer_area[i];
3155         if (rx_buffer->m_head != NULL)
3156                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3157
3158         map = rx_buffer->map;
3159         rx_buffer->map = adapter->rx_sparemap;
3160         adapter->rx_sparemap = map;
3161         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
3162         rx_buffer->m_head = m;
3163
3164         adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
3165         return (0);
3166 }
3167
3168 /*********************************************************************
3169  *
3170  *  Allocate memory for rx_buffer structures. Since we use one
3171  *  rx_buffer per received packet, the maximum number of rx_buffer's
3172  *  that we'll need is equal to the number of receive descriptors
3173  *  that we've allocated.
3174  *
3175  **********************************************************************/
3176 static int
3177 lem_allocate_receive_structures(struct adapter *adapter)
3178 {
3179         device_t dev = adapter->dev;
3180         struct em_buffer *rx_buffer;
3181         int i, error;
3182
3183         adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
3184             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3185         if (adapter->rx_buffer_area == NULL) {
3186                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3187                 return (ENOMEM);
3188         }
3189
3190         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3191                                 1, 0,                   /* alignment, bounds */
3192                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3193                                 BUS_SPACE_MAXADDR,      /* highaddr */
3194                                 NULL, NULL,             /* filter, filterarg */
3195                                 MCLBYTES,               /* maxsize */
3196                                 1,                      /* nsegments */
3197                                 MCLBYTES,               /* maxsegsize */
3198                                 0,                      /* flags */
3199                                 NULL,                   /* lockfunc */
3200                                 NULL,                   /* lockarg */
3201                                 &adapter->rxtag);
3202         if (error) {
3203                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
3204                     __func__, error);
3205                 goto fail;
3206         }
3207
3208         /* Create the spare map (used by getbuf) */
3209         error = bus_dmamap_create(adapter->rxtag, 0, &adapter->rx_sparemap);
3210         if (error) {
3211                 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3212                     __func__, error);
3213                 goto fail;
3214         }
3215
3216         rx_buffer = adapter->rx_buffer_area;
3217         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3218                 error = bus_dmamap_create(adapter->rxtag, 0, &rx_buffer->map);
3219                 if (error) {
3220                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3221                             __func__, error);
3222                         goto fail;
3223                 }
3224         }
3225
3226         return (0);
3227
3228 fail:
3229         lem_free_receive_structures(adapter);
3230         return (error);
3231 }
3232
3233 /*********************************************************************
3234  *
3235  *  (Re)initialize receive structures.
3236  *
3237  **********************************************************************/
3238 static int
3239 lem_setup_receive_structures(struct adapter *adapter)
3240 {
3241         struct em_buffer *rx_buffer;
3242         int i, error;
3243 #ifdef DEV_NETMAP
3244         /* we are already under lock */
3245         struct netmap_adapter *na = netmap_getna(adapter->ifp);
3246         struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0);
3247 #endif
3248
3249         /* Reset descriptor ring */
3250         bzero(adapter->rx_desc_base,
3251             (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
3252
3253         /* Free current RX buffers. */
3254         rx_buffer = adapter->rx_buffer_area;
3255         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3256                 if (rx_buffer->m_head != NULL) {
3257                         bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
3258                             BUS_DMASYNC_POSTREAD);
3259                         bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3260                         m_freem(rx_buffer->m_head);
3261                         rx_buffer->m_head = NULL;
3262                 }
3263         }
3264
3265         /* Allocate new ones. */
3266         for (i = 0; i < adapter->num_rx_desc; i++) {
3267 #ifdef DEV_NETMAP
3268                 if (slot) {
3269                         /* the i-th NIC entry goes to slot si */
3270                         int si = netmap_idx_n2k(&na->rx_rings[0], i);
3271                         uint64_t paddr;
3272                         void *addr;
3273
3274                         addr = PNMB(na, slot + si, &paddr);
3275                         netmap_load_map(na, adapter->rxtag, rx_buffer->map, addr);
3276                         /* Update descriptor */
3277                         adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
3278                         continue;
3279                 }
3280 #endif /* DEV_NETMAP */
3281                 error = lem_get_buf(adapter, i);
3282                 if (error)
3283                         return (error);
3284         }
3285
3286         /* Setup our descriptor pointers */
3287         adapter->next_rx_desc_to_check = 0;
3288         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3289             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3290
3291         return (0);
3292 }
3293
3294 /*********************************************************************
3295  *
3296  *  Enable receive unit.
3297  *
3298  **********************************************************************/
3299
3300 static void
3301 lem_initialize_receive_unit(struct adapter *adapter)
3302 {
3303         if_t ifp = adapter->ifp;
3304         u64     bus_addr;
3305         u32     rctl, rxcsum;
3306
3307         INIT_DEBUGOUT("lem_initialize_receive_unit: begin");
3308
3309         /*
3310          * Make sure receives are disabled while setting
3311          * up the descriptor ring
3312          */
3313         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
3314         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3315
3316         if (adapter->hw.mac.type >= e1000_82540) {
3317                 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
3318                     adapter->rx_abs_int_delay.value);
3319                 /*
3320                  * Set the interrupt throttling rate. Value is calculated
3321                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
3322                  */
3323                 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
3324         }
3325
3326         /* Setup the Base and Length of the Rx Descriptor Ring */
3327         bus_addr = adapter->rxdma.dma_paddr;
3328         E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
3329             adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3330         E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
3331             (u32)(bus_addr >> 32));
3332         E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
3333             (u32)bus_addr);
3334
3335         /* Setup the Receive Control Register */
3336         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3337         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3338                    E1000_RCTL_RDMTS_HALF |
3339                    (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3340
3341         /* Make sure VLAN Filters are off */
3342         rctl &= ~E1000_RCTL_VFE;
3343
3344         if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
3345                 rctl |= E1000_RCTL_SBP;
3346         else
3347                 rctl &= ~E1000_RCTL_SBP;
3348
3349         switch (adapter->rx_buffer_len) {
3350         default:
3351         case 2048:
3352                 rctl |= E1000_RCTL_SZ_2048;
3353                 break;
3354         case 4096:
3355                 rctl |= E1000_RCTL_SZ_4096 |
3356                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3357                 break;
3358         case 8192:
3359                 rctl |= E1000_RCTL_SZ_8192 |
3360                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3361                 break;
3362         case 16384:
3363                 rctl |= E1000_RCTL_SZ_16384 |
3364                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3365                 break;
3366         }
3367
3368         if (if_getmtu(ifp) > ETHERMTU)
3369                 rctl |= E1000_RCTL_LPE;
3370         else
3371                 rctl &= ~E1000_RCTL_LPE;
3372
3373         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
3374         if ((adapter->hw.mac.type >= e1000_82543) &&
3375             (if_getcapenable(ifp) & IFCAP_RXCSUM)) {
3376                 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
3377                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3378                 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
3379         }
3380
3381         /* Enable Receives */
3382         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
3383
3384         /*
3385          * Setup the HW Rx Head and
3386          * Tail Descriptor Pointers
3387          */
3388         E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
3389         rctl = adapter->num_rx_desc - 1; /* default RDT value */
3390 #ifdef DEV_NETMAP
3391         /* preserve buffers already made available to clients */
3392         if (if_getcapenable(ifp) & IFCAP_NETMAP) {
3393                 struct netmap_adapter *na = netmap_getna(adapter->ifp);
3394                 rctl -= nm_kr_rxspace(&na->rx_rings[0]);
3395         }
3396 #endif /* DEV_NETMAP */
3397         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), rctl);
3398
3399         return;
3400 }
3401
3402 /*********************************************************************
3403  *
3404  *  Free receive related data structures.
3405  *
3406  **********************************************************************/
3407 static void
3408 lem_free_receive_structures(struct adapter *adapter)
3409 {
3410         struct em_buffer *rx_buffer;
3411         int i;
3412
3413         INIT_DEBUGOUT("free_receive_structures: begin");
3414
3415         if (adapter->rx_sparemap) {
3416                 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
3417                 adapter->rx_sparemap = NULL;
3418         }
3419
3420         /* Cleanup any existing buffers */
3421         if (adapter->rx_buffer_area != NULL) {
3422                 rx_buffer = adapter->rx_buffer_area;
3423                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3424                         if (rx_buffer->m_head != NULL) {
3425                                 bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
3426                                     BUS_DMASYNC_POSTREAD);
3427                                 bus_dmamap_unload(adapter->rxtag,
3428                                     rx_buffer->map);
3429                                 m_freem(rx_buffer->m_head);
3430                                 rx_buffer->m_head = NULL;
3431                         } else if (rx_buffer->map != NULL)
3432                                 bus_dmamap_unload(adapter->rxtag,
3433                                     rx_buffer->map);
3434                         if (rx_buffer->map != NULL) {
3435                                 bus_dmamap_destroy(adapter->rxtag,
3436                                     rx_buffer->map);
3437                                 rx_buffer->map = NULL;
3438                         }
3439                 }
3440         }
3441
3442         if (adapter->rx_buffer_area != NULL) {
3443                 free(adapter->rx_buffer_area, M_DEVBUF);
3444                 adapter->rx_buffer_area = NULL;
3445         }
3446
3447         if (adapter->rxtag != NULL) {
3448                 bus_dma_tag_destroy(adapter->rxtag);
3449                 adapter->rxtag = NULL;
3450         }
3451 }
3452
3453 /*********************************************************************
3454  *
3455  *  This routine executes in interrupt context. It replenishes
3456  *  the mbufs in the descriptor and sends data which has been
3457  *  dma'ed into host memory to upper layer.
3458  *
3459  *  We loop at most count times if count is > 0, or until done if
3460  *  count < 0.
3461  *
3462  *  For polling we also now return the number of cleaned packets
3463  *********************************************************************/
3464 static bool
3465 lem_rxeof(struct adapter *adapter, int count, int *done)
3466 {
3467         if_t ifp = adapter->ifp;
3468         struct mbuf     *mp;
3469         u8              status = 0, accept_frame = 0, eop = 0;
3470         u16             len, desc_len, prev_len_adj;
3471         int             i, rx_sent = 0;
3472         struct e1000_rx_desc   *current_desc;
3473
3474 #ifdef BATCH_DISPATCH
3475         struct mbuf *mh = NULL, *mt = NULL;
3476 #endif /* BATCH_DISPATCH */
3477         EM_RX_LOCK(adapter);
3478
3479 #ifdef BATCH_DISPATCH
3480     batch_again:
3481 #endif /* BATCH_DISPATCH */
3482         i = adapter->next_rx_desc_to_check;
3483         current_desc = &adapter->rx_desc_base[i];
3484         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3485             BUS_DMASYNC_POSTREAD);
3486
3487 #ifdef DEV_NETMAP
3488         if (netmap_rx_irq(ifp, 0, &rx_sent)) {
3489                 EM_RX_UNLOCK(adapter);
3490                 return (FALSE);
3491         }
3492 #endif /* DEV_NETMAP */
3493
3494         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
3495                 if (done != NULL)
3496                         *done = rx_sent;
3497                 EM_RX_UNLOCK(adapter);
3498                 return (FALSE);
3499         }
3500
3501         while (count != 0 && if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3502                 struct mbuf *m = NULL;
3503
3504                 status = current_desc->status;
3505                 if ((status & E1000_RXD_STAT_DD) == 0) {
3506                         break;
3507                 }
3508
3509                 mp = adapter->rx_buffer_area[i].m_head;
3510                 /*
3511                  * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3512                  * needs to access the last received byte in the mbuf.
3513                  */
3514                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
3515                     BUS_DMASYNC_POSTREAD);
3516
3517                 accept_frame = 1;
3518                 prev_len_adj = 0;
3519                 desc_len = le16toh(current_desc->length);
3520                 if (status & E1000_RXD_STAT_EOP) {
3521                         count--;
3522                         eop = 1;
3523                         if (desc_len < ETHER_CRC_LEN) {
3524                                 len = 0;
3525                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
3526                         } else
3527                                 len = desc_len - ETHER_CRC_LEN;
3528                 } else {
3529                         eop = 0;
3530                         len = desc_len;
3531                 }
3532
3533                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3534                         u8      last_byte;
3535                         u32     pkt_len = desc_len;
3536
3537                         if (adapter->fmp != NULL)
3538                                 pkt_len += adapter->fmp->m_pkthdr.len;
3539
3540                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3541                         if (TBI_ACCEPT(&adapter->hw, status,
3542                             current_desc->errors, pkt_len, last_byte,
3543                             adapter->min_frame_size, adapter->max_frame_size)) {
3544                                 e1000_tbi_adjust_stats_82543(&adapter->hw,
3545                                     &adapter->stats, pkt_len,
3546                                     adapter->hw.mac.addr,
3547                                     adapter->max_frame_size);
3548                                 if (len > 0)
3549                                         len--;
3550                         } else
3551                                 accept_frame = 0;
3552                 }
3553
3554                 if (accept_frame) {
3555                         if (lem_get_buf(adapter, i) != 0) {
3556                                 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
3557                                 goto discard;
3558                         }
3559
3560                         /* Assign correct length to the current fragment */
3561                         mp->m_len = len;
3562
3563                         if (adapter->fmp == NULL) {
3564                                 mp->m_pkthdr.len = len;
3565                                 adapter->fmp = mp; /* Store the first mbuf */
3566                                 adapter->lmp = mp;
3567                         } else {
3568                                 /* Chain mbuf's together */
3569                                 mp->m_flags &= ~M_PKTHDR;
3570                                 /*
3571                                  * Adjust length of previous mbuf in chain if
3572                                  * we received less than 4 bytes in the last
3573                                  * descriptor.
3574                                  */
3575                                 if (prev_len_adj > 0) {
3576                                         adapter->lmp->m_len -= prev_len_adj;
3577                                         adapter->fmp->m_pkthdr.len -=
3578                                             prev_len_adj;
3579                                 }
3580                                 adapter->lmp->m_next = mp;
3581                                 adapter->lmp = adapter->lmp->m_next;
3582                                 adapter->fmp->m_pkthdr.len += len;
3583                         }
3584
3585                         if (eop) {
3586                                 if_setrcvif(adapter->fmp, ifp);
3587                                 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
3588                                 lem_receive_checksum(adapter, current_desc,
3589                                     adapter->fmp);
3590 #ifndef __NO_STRICT_ALIGNMENT
3591                                 if (adapter->max_frame_size >
3592                                     (MCLBYTES - ETHER_ALIGN) &&
3593                                     lem_fixup_rx(adapter) != 0)
3594                                         goto skip;
3595 #endif
3596                                 if (status & E1000_RXD_STAT_VP) {
3597                                         adapter->fmp->m_pkthdr.ether_vtag =
3598                                             le16toh(current_desc->special);
3599                                         adapter->fmp->m_flags |= M_VLANTAG;
3600                                 }
3601 #ifndef __NO_STRICT_ALIGNMENT
3602 skip:
3603 #endif
3604                                 m = adapter->fmp;
3605                                 adapter->fmp = NULL;
3606                                 adapter->lmp = NULL;
3607                         }
3608                 } else {
3609                         adapter->dropped_pkts++;
3610 discard:
3611                         /* Reuse loaded DMA map and just update mbuf chain */
3612                         mp = adapter->rx_buffer_area[i].m_head;
3613                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
3614                         mp->m_data = mp->m_ext.ext_buf;
3615                         mp->m_next = NULL;
3616                         if (adapter->max_frame_size <=
3617                             (MCLBYTES - ETHER_ALIGN))
3618                                 m_adj(mp, ETHER_ALIGN);
3619                         if (adapter->fmp != NULL) {
3620                                 m_freem(adapter->fmp);
3621                                 adapter->fmp = NULL;
3622                                 adapter->lmp = NULL;
3623                         }
3624                         m = NULL;
3625                 }
3626
3627                 /* Zero out the receive descriptors status. */
3628                 current_desc->status = 0;
3629                 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3630                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3631
3632                 /* Advance our pointers to the next descriptor. */
3633                 if (++i == adapter->num_rx_desc)
3634                         i = 0;
3635                 /* Call into the stack */
3636                 if (m != NULL) {
3637 #ifdef BATCH_DISPATCH
3638                     if (adapter->batch_enable) {
3639                         if (mh == NULL)
3640                                 mh = mt = m;
3641                         else
3642                                 mt->m_nextpkt = m;
3643                         mt = m;
3644                         m->m_nextpkt = NULL;
3645                         rx_sent++;
3646                         current_desc = &adapter->rx_desc_base[i];
3647                         continue;
3648                     }
3649 #endif /* BATCH_DISPATCH */
3650                         adapter->next_rx_desc_to_check = i;
3651                         EM_RX_UNLOCK(adapter);
3652                         if_input(ifp, m);
3653                         EM_RX_LOCK(adapter);
3654                         rx_sent++;
3655                         i = adapter->next_rx_desc_to_check;
3656                 }
3657                 current_desc = &adapter->rx_desc_base[i];
3658         }
3659         adapter->next_rx_desc_to_check = i;
3660 #ifdef BATCH_DISPATCH
3661         if (mh) {
3662                 EM_RX_UNLOCK(adapter);
3663                 while ( (mt = mh) != NULL) {
3664                         mh = mh->m_nextpkt;
3665                         mt->m_nextpkt = NULL;
3666                         if_input(ifp, mt);
3667                 }
3668                 EM_RX_LOCK(adapter);
3669                 i = adapter->next_rx_desc_to_check; /* in case of interrupts */
3670                 if (count > 0)
3671                         goto batch_again;
3672         }
3673 #endif /* BATCH_DISPATCH */
3674
3675         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
3676         if (--i < 0)
3677                 i = adapter->num_rx_desc - 1;
3678         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
3679         if (done != NULL)
3680                 *done = rx_sent;
3681         EM_RX_UNLOCK(adapter);
3682         return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
3683 }
3684
3685 #ifndef __NO_STRICT_ALIGNMENT
3686 /*
3687  * When jumbo frames are enabled we should realign entire payload on
3688  * architecures with strict alignment. This is serious design mistake of 8254x
3689  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
3690  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
3691  * payload. On architecures without strict alignment restrictions 8254x still
3692  * performs unaligned memory access which would reduce the performance too.
3693  * To avoid copying over an entire frame to align, we allocate a new mbuf and
3694  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
3695  * existing mbuf chain.
3696  *
3697  * Be aware, best performance of the 8254x is achieved only when jumbo frame is
3698  * not used at all on architectures with strict alignment.
3699  */
3700 static int
3701 lem_fixup_rx(struct adapter *adapter)
3702 {
3703         struct mbuf *m, *n;
3704         int error;
3705
3706         error = 0;
3707         m = adapter->fmp;
3708         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
3709                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
3710                 m->m_data += ETHER_HDR_LEN;
3711         } else {
3712                 MGETHDR(n, M_NOWAIT, MT_DATA);
3713                 if (n != NULL) {
3714                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
3715                         m->m_data += ETHER_HDR_LEN;
3716                         m->m_len -= ETHER_HDR_LEN;
3717                         n->m_len = ETHER_HDR_LEN;
3718                         M_MOVE_PKTHDR(n, m);
3719                         n->m_next = m;
3720                         adapter->fmp = n;
3721                 } else {
3722                         adapter->dropped_pkts++;
3723                         m_freem(adapter->fmp);
3724                         adapter->fmp = NULL;
3725                         error = ENOMEM;
3726                 }
3727         }
3728
3729         return (error);
3730 }
3731 #endif
3732
3733 /*********************************************************************
3734  *
3735  *  Verify that the hardware indicated that the checksum is valid.
3736  *  Inform the stack about the status of checksum so that stack
3737  *  doesn't spend time verifying the checksum.
3738  *
3739  *********************************************************************/
3740 static void
3741 lem_receive_checksum(struct adapter *adapter,
3742             struct e1000_rx_desc *rx_desc, struct mbuf *mp)
3743 {
3744         /* 82543 or newer only */
3745         if ((adapter->hw.mac.type < e1000_82543) ||
3746             /* Ignore Checksum bit is set */
3747             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
3748                 mp->m_pkthdr.csum_flags = 0;
3749                 return;
3750         }
3751
3752         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
3753                 /* Did it pass? */
3754                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3755                         /* IP Checksum Good */
3756                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3757                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3758
3759                 } else {
3760                         mp->m_pkthdr.csum_flags = 0;
3761                 }
3762         }
3763
3764         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
3765                 /* Did it pass? */
3766                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3767                         mp->m_pkthdr.csum_flags |=
3768                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3769                         mp->m_pkthdr.csum_data = htons(0xffff);
3770                 }
3771         }
3772 }
3773
3774 /*
3775  * This routine is run via an vlan
3776  * config EVENT
3777  */
3778 static void
3779 lem_register_vlan(void *arg, if_t ifp, u16 vtag)
3780 {
3781         struct adapter  *adapter = if_getsoftc(ifp);
3782         u32             index, bit;
3783
3784         if (if_getsoftc(ifp) !=  arg)   /* Not our event */
3785                 return;
3786
3787         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
3788                 return;
3789
3790         EM_CORE_LOCK(adapter);
3791         index = (vtag >> 5) & 0x7F;
3792         bit = vtag & 0x1F;
3793         adapter->shadow_vfta[index] |= (1 << bit);
3794         ++adapter->num_vlans;
3795         /* Re-init to load the changes */
3796         if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3797                 lem_init_locked(adapter);
3798         EM_CORE_UNLOCK(adapter);
3799 }
3800
3801 /*
3802  * This routine is run via an vlan
3803  * unconfig EVENT
3804  */
3805 static void
3806 lem_unregister_vlan(void *arg, if_t ifp, u16 vtag)
3807 {
3808         struct adapter  *adapter = if_getsoftc(ifp);
3809         u32             index, bit;
3810
3811         if (if_getsoftc(ifp) !=  arg)
3812                 return;
3813
3814         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3815                 return;
3816
3817         EM_CORE_LOCK(adapter);
3818         index = (vtag >> 5) & 0x7F;
3819         bit = vtag & 0x1F;
3820         adapter->shadow_vfta[index] &= ~(1 << bit);
3821         --adapter->num_vlans;
3822         /* Re-init to load the changes */
3823         if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3824                 lem_init_locked(adapter);
3825         EM_CORE_UNLOCK(adapter);
3826 }
3827
3828 static void
3829 lem_setup_vlan_hw_support(struct adapter *adapter)
3830 {
3831         struct e1000_hw *hw = &adapter->hw;
3832         u32             reg;
3833
3834         /*
3835         ** We get here thru init_locked, meaning
3836         ** a soft reset, this has already cleared
3837         ** the VFTA and other state, so if there
3838         ** have been no vlan's registered do nothing.
3839         */
3840         if (adapter->num_vlans == 0)
3841                 return;
3842
3843         /*
3844         ** A soft reset zero's out the VFTA, so
3845         ** we need to repopulate it now.
3846         */
3847         for (int i = 0; i < EM_VFTA_SIZE; i++)
3848                 if (adapter->shadow_vfta[i] != 0)
3849                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
3850                             i, adapter->shadow_vfta[i]);
3851
3852         reg = E1000_READ_REG(hw, E1000_CTRL);
3853         reg |= E1000_CTRL_VME;
3854         E1000_WRITE_REG(hw, E1000_CTRL, reg);
3855
3856         /* Enable the Filter Table */
3857         reg = E1000_READ_REG(hw, E1000_RCTL);
3858         reg &= ~E1000_RCTL_CFIEN;
3859         reg |= E1000_RCTL_VFE;
3860         E1000_WRITE_REG(hw, E1000_RCTL, reg);
3861 }
3862
3863 static void
3864 lem_enable_intr(struct adapter *adapter)
3865 {
3866         struct e1000_hw *hw = &adapter->hw;
3867         u32 ims_mask = IMS_ENABLE_MASK;
3868
3869         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
3870 }
3871
3872 static void
3873 lem_disable_intr(struct adapter *adapter)
3874 {
3875         struct e1000_hw *hw = &adapter->hw;
3876
3877         E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff);
3878 }
3879
3880 /*
3881  * Bit of a misnomer, what this really means is
3882  * to enable OS management of the system... aka
3883  * to disable special hardware management features
3884  */
3885 static void
3886 lem_init_manageability(struct adapter *adapter)
3887 {
3888         /* A shared code workaround */
3889         if (adapter->has_manage) {
3890                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3891                 /* disable hardware interception of ARP */
3892                 manc &= ~(E1000_MANC_ARP_EN);
3893                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3894         }
3895 }
3896
3897 /*
3898  * Give control back to hardware management
3899  * controller if there is one.
3900  */
3901 static void
3902 lem_release_manageability(struct adapter *adapter)
3903 {
3904         if (adapter->has_manage) {
3905                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3906
3907                 /* re-enable hardware interception of ARP */
3908                 manc |= E1000_MANC_ARP_EN;
3909                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3910         }
3911 }
3912
3913 /*
3914  * lem_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
3915  * For ASF and Pass Through versions of f/w this means
3916  * that the driver is loaded. For AMT version type f/w
3917  * this means that the network i/f is open.
3918  */
3919 static void
3920 lem_get_hw_control(struct adapter *adapter)
3921 {
3922         u32 ctrl_ext;
3923
3924         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3925         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3926             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
3927         return;
3928 }
3929
3930 /*
3931  * lem_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3932  * For ASF and Pass Through versions of f/w this means that
3933  * the driver is no longer loaded. For AMT versions of the
3934  * f/w this means that the network i/f is closed.
3935  */
3936 static void
3937 lem_release_hw_control(struct adapter *adapter)
3938 {
3939         u32 ctrl_ext;
3940
3941         if (!adapter->has_manage)
3942                 return;
3943
3944         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3945         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3946             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
3947         return;
3948 }
3949
3950 static int
3951 lem_is_valid_ether_addr(u8 *addr)
3952 {
3953         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
3954
3955         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
3956                 return (FALSE);
3957         }
3958
3959         return (TRUE);
3960 }
3961
3962 /*
3963 ** Parse the interface capabilities with regard
3964 ** to both system management and wake-on-lan for
3965 ** later use.
3966 */
3967 static void
3968 lem_get_wakeup(device_t dev)
3969 {
3970         struct adapter  *adapter = device_get_softc(dev);
3971         u16             eeprom_data = 0, device_id, apme_mask;
3972
3973         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
3974         apme_mask = EM_EEPROM_APME;
3975
3976         switch (adapter->hw.mac.type) {
3977         case e1000_82542:
3978         case e1000_82543:
3979                 break;
3980         case e1000_82544:
3981                 e1000_read_nvm(&adapter->hw,
3982                     NVM_INIT_CONTROL2_REG, 1, &eeprom_data);
3983                 apme_mask = EM_82544_APME;
3984                 break;
3985         case e1000_82546:
3986         case e1000_82546_rev_3:
3987                 if (adapter->hw.bus.func == 1) {
3988                         e1000_read_nvm(&adapter->hw,
3989                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
3990                         break;
3991                 } else
3992                         e1000_read_nvm(&adapter->hw,
3993                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
3994                 break;
3995         default:
3996                 e1000_read_nvm(&adapter->hw,
3997                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
3998                 break;
3999         }
4000         if (eeprom_data & apme_mask)
4001                 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4002         /*
4003          * We have the eeprom settings, now apply the special cases
4004          * where the eeprom may be wrong or the board won't support
4005          * wake on lan on a particular port
4006          */
4007         device_id = pci_get_device(dev);
4008         switch (device_id) {
4009         case E1000_DEV_ID_82546GB_PCIE:
4010                 adapter->wol = 0;
4011                 break;
4012         case E1000_DEV_ID_82546EB_FIBER:
4013         case E1000_DEV_ID_82546GB_FIBER:
4014                 /* Wake events only supported on port A for dual fiber
4015                  * regardless of eeprom setting */
4016                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4017                     E1000_STATUS_FUNC_1)
4018                         adapter->wol = 0;
4019                 break;
4020         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
4021                 /* if quad port adapter, disable WoL on all but port A */
4022                 if (global_quad_port_a != 0)
4023                         adapter->wol = 0;
4024                 /* Reset for multiple quad port adapters */
4025                 if (++global_quad_port_a == 4)
4026                         global_quad_port_a = 0;
4027                 break;
4028         }
4029         return;
4030 }
4031
4032
4033 /*
4034  * Enable PCI Wake On Lan capability
4035  */
4036 static void
4037 lem_enable_wakeup(device_t dev)
4038 {
4039         struct adapter  *adapter = device_get_softc(dev);
4040         if_t ifp = adapter->ifp;
4041         u32             pmc, ctrl, ctrl_ext, rctl;
4042         u16             status;
4043
4044         if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
4045                 return;
4046
4047         /* Advertise the wakeup capability */
4048         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4049         ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4050         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4051         E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4052
4053         /* Keep the laser running on Fiber adapters */
4054         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4055             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4056                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4057                 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4058                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4059         }
4060
4061         /*
4062         ** Determine type of Wakeup: note that wol
4063         ** is set with all bits on by default.
4064         */
4065         if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0)
4066                 adapter->wol &= ~E1000_WUFC_MAG;
4067
4068         if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0)
4069                 adapter->wol &= ~E1000_WUFC_MC;
4070         else {
4071                 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4072                 rctl |= E1000_RCTL_MPE;
4073                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4074         }
4075
4076         if (adapter->hw.mac.type == e1000_pchlan) {
4077                 if (lem_enable_phy_wakeup(adapter))
4078                         return;
4079         } else {
4080                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4081                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4082         }
4083
4084
4085         /* Request PME */
4086         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4087         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4088         if (if_getcapenable(ifp) & IFCAP_WOL)
4089                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4090         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
4091
4092         return;
4093 }
4094
4095 /*
4096 ** WOL in the newer chipset interfaces (pchlan)
4097 ** require thing to be copied into the phy
4098 */
4099 static int
4100 lem_enable_phy_wakeup(struct adapter *adapter)
4101 {
4102         struct e1000_hw *hw = &adapter->hw;
4103         u32 mreg, ret = 0;
4104         u16 preg;
4105
4106         /* copy MAC RARs to PHY RARs */
4107         for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) {
4108                 mreg = E1000_READ_REG(hw, E1000_RAL(i));
4109                 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF));
4110                 e1000_write_phy_reg(hw, BM_RAR_M(i),
4111                     (u16)((mreg >> 16) & 0xFFFF));
4112                 mreg = E1000_READ_REG(hw, E1000_RAH(i));
4113                 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF));
4114                 e1000_write_phy_reg(hw, BM_RAR_CTRL(i),
4115                     (u16)((mreg >> 16) & 0xFFFF));
4116         }
4117
4118         /* copy MAC MTA to PHY MTA */
4119         for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
4120                 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
4121                 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
4122                 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
4123                     (u16)((mreg >> 16) & 0xFFFF));
4124         }
4125
4126         /* configure PHY Rx Control register */
4127         e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
4128         mreg = E1000_READ_REG(hw, E1000_RCTL);
4129         if (mreg & E1000_RCTL_UPE)
4130                 preg |= BM_RCTL_UPE;
4131         if (mreg & E1000_RCTL_MPE)
4132                 preg |= BM_RCTL_MPE;
4133         preg &= ~(BM_RCTL_MO_MASK);
4134         if (mreg & E1000_RCTL_MO_3)
4135                 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
4136                                 << BM_RCTL_MO_SHIFT);
4137         if (mreg & E1000_RCTL_BAM)
4138                 preg |= BM_RCTL_BAM;
4139         if (mreg & E1000_RCTL_PMCF)
4140                 preg |= BM_RCTL_PMCF;
4141         mreg = E1000_READ_REG(hw, E1000_CTRL);
4142         if (mreg & E1000_CTRL_RFCE)
4143                 preg |= BM_RCTL_RFCE;
4144         e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
4145
4146         /* enable PHY wakeup in MAC register */
4147         E1000_WRITE_REG(hw, E1000_WUC,
4148             E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
4149         E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
4150
4151         /* configure and enable PHY wakeup in PHY registers */
4152         e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
4153         e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
4154
4155         /* activate PHY wakeup */
4156         ret = hw->phy.ops.acquire(hw);
4157         if (ret) {
4158                 printf("Could not acquire PHY\n");
4159                 return ret;
4160         }
4161         e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4162                                  (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
4163         ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
4164         if (ret) {
4165                 printf("Could not read PHY page 769\n");
4166                 goto out;
4167         }
4168         preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
4169         ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
4170         if (ret)
4171                 printf("Could not set PHY Host Wakeup bit\n");
4172 out:
4173         hw->phy.ops.release(hw);
4174
4175         return ret;
4176 }
4177
4178 static void
4179 lem_led_func(void *arg, int onoff)
4180 {
4181         struct adapter  *adapter = arg;
4182
4183         EM_CORE_LOCK(adapter);
4184         if (onoff) {
4185                 e1000_setup_led(&adapter->hw);
4186                 e1000_led_on(&adapter->hw);
4187         } else {
4188                 e1000_led_off(&adapter->hw);
4189                 e1000_cleanup_led(&adapter->hw);
4190         }
4191         EM_CORE_UNLOCK(adapter);
4192 }
4193
4194 /*********************************************************************
4195 * 82544 Coexistence issue workaround.
4196 *    There are 2 issues.
4197 *       1. Transmit Hang issue.
4198 *    To detect this issue, following equation can be used...
4199 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4200 *         If SUM[3:0] is in between 1 to 4, we will have this issue.
4201 *
4202 *       2. DAC issue.
4203 *    To detect this issue, following equation can be used...
4204 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4205 *         If SUM[3:0] is in between 9 to c, we will have this issue.
4206 *
4207 *
4208 *    WORKAROUND:
4209 *         Make sure we do not have ending address
4210 *         as 1,2,3,4(Hang) or 9,a,b,c (DAC)
4211 *
4212 *************************************************************************/
4213 static u32
4214 lem_fill_descriptors (bus_addr_t address, u32 length,
4215                 PDESC_ARRAY desc_array)
4216 {
4217         u32 safe_terminator;
4218
4219         /* Since issue is sensitive to length and address.*/
4220         /* Let us first check the address...*/
4221         if (length <= 4) {
4222                 desc_array->descriptor[0].address = address;
4223                 desc_array->descriptor[0].length = length;
4224                 desc_array->elements = 1;
4225                 return (desc_array->elements);
4226         }
4227         safe_terminator = (u32)((((u32)address & 0x7) +
4228             (length & 0xF)) & 0xF);
4229         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
4230         if (safe_terminator == 0   ||
4231         (safe_terminator > 4   &&
4232         safe_terminator < 9)   ||
4233         (safe_terminator > 0xC &&
4234         safe_terminator <= 0xF)) {
4235                 desc_array->descriptor[0].address = address;
4236                 desc_array->descriptor[0].length = length;
4237                 desc_array->elements = 1;
4238                 return (desc_array->elements);
4239         }
4240
4241         desc_array->descriptor[0].address = address;
4242         desc_array->descriptor[0].length = length - 4;
4243         desc_array->descriptor[1].address = address + (length - 4);
4244         desc_array->descriptor[1].length = 4;
4245         desc_array->elements = 2;
4246         return (desc_array->elements);
4247 }
4248
4249 /**********************************************************************
4250  *
4251  *  Update the board statistics counters.
4252  *
4253  **********************************************************************/
4254 static void
4255 lem_update_stats_counters(struct adapter *adapter)
4256 {
4257
4258         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
4259            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4260                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4261                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
4262         }
4263         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4264         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4265         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4266         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4267
4268         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4269         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4270         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4271         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4272         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4273         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4274         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4275         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4276         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4277         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4278         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4279         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4280         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4281         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4282         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4283         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4284         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4285         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4286         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4287         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4288
4289         /* For the 64-bit byte counters the low dword must be read first. */
4290         /* Both registers clear on the read of the high dword */
4291
4292         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
4293             ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
4294         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
4295             ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
4296
4297         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4298         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4299         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4300         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4301         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4302
4303         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4304         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4305
4306         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
4307         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
4308         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
4309         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
4310         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
4311         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
4312         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
4313         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
4314         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
4315         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
4316
4317         if (adapter->hw.mac.type >= e1000_82543) {
4318                 adapter->stats.algnerrc +=
4319                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
4320                 adapter->stats.rxerrc +=
4321                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
4322                 adapter->stats.tncrs +=
4323                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
4324                 adapter->stats.cexterr +=
4325                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
4326                 adapter->stats.tsctc +=
4327                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
4328                 adapter->stats.tsctfc +=
4329                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
4330         }
4331 }
4332
4333 static uint64_t
4334 lem_get_counter(if_t ifp, ift_counter cnt)
4335 {
4336         struct adapter *adapter;
4337
4338         adapter = if_getsoftc(ifp);
4339
4340         switch (cnt) {
4341         case IFCOUNTER_COLLISIONS:
4342                 return (adapter->stats.colc);
4343         case IFCOUNTER_IERRORS:
4344                 return (adapter->dropped_pkts + adapter->stats.rxerrc +
4345                     adapter->stats.crcerrs + adapter->stats.algnerrc +
4346                     adapter->stats.ruc + adapter->stats.roc +
4347                     adapter->stats.mpc + adapter->stats.cexterr);
4348         case IFCOUNTER_OERRORS:
4349                 return (adapter->stats.ecol + adapter->stats.latecol +
4350                     adapter->watchdog_events);
4351         default:
4352                 return (if_get_counter_default(ifp, cnt));
4353         }
4354 }
4355
4356 /* Export a single 32-bit register via a read-only sysctl. */
4357 static int
4358 lem_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
4359 {
4360         struct adapter *adapter;
4361         u_int val;
4362
4363         adapter = oidp->oid_arg1;
4364         val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
4365         return (sysctl_handle_int(oidp, &val, 0, req));
4366 }
4367
4368 /*
4369  * Add sysctl variables, one per statistic, to the system.
4370  */
4371 static void
4372 lem_add_hw_stats(struct adapter *adapter)
4373 {
4374         device_t dev = adapter->dev;
4375
4376         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
4377         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
4378         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
4379         struct e1000_hw_stats *stats = &adapter->stats;
4380
4381         struct sysctl_oid *stat_node;
4382         struct sysctl_oid_list *stat_list;
4383
4384         /* Driver Statistics */
4385         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail",
4386                          CTLFLAG_RD, &adapter->mbuf_cluster_failed,
4387                          "Std mbuf cluster failed");
4388         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
4389                          CTLFLAG_RD, &adapter->mbuf_defrag_failed,
4390                          "Defragmenting mbuf chain failed");
4391         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
4392                         CTLFLAG_RD, &adapter->dropped_pkts,
4393                         "Driver dropped packets");
4394         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
4395                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
4396                         "Driver tx dma failure in xmit");
4397         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail1",
4398                         CTLFLAG_RD, &adapter->no_tx_desc_avail1,
4399                         "Not enough tx descriptors failure in xmit");
4400         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail2",
4401                         CTLFLAG_RD, &adapter->no_tx_desc_avail2,
4402                         "Not enough tx descriptors failure in xmit");
4403         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
4404                         CTLFLAG_RD, &adapter->rx_overruns,
4405                         "RX overruns");
4406         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
4407                         CTLFLAG_RD, &adapter->watchdog_events,
4408                         "Watchdog timeouts");
4409
4410         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
4411                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
4412                         lem_sysctl_reg_handler, "IU",
4413                         "Device Control Register");
4414         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
4415                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
4416                         lem_sysctl_reg_handler, "IU",
4417                         "Receiver Control Register");
4418         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
4419                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
4420                         "Flow Control High Watermark");
4421         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
4422                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
4423                         "Flow Control Low Watermark");
4424         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_workaround",
4425                         CTLFLAG_RD, &adapter->tx_fifo_wrk_cnt,
4426                         "TX FIFO workaround events");
4427         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_reset",
4428                         CTLFLAG_RD, &adapter->tx_fifo_reset_cnt,
4429                         "TX FIFO resets");
4430
4431         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_head",
4432                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(0),
4433                         lem_sysctl_reg_handler, "IU",
4434                         "Transmit Descriptor Head");
4435         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_tail",
4436                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(0),
4437                         lem_sysctl_reg_handler, "IU",
4438                         "Transmit Descriptor Tail");
4439         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_head",
4440                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(0),
4441                         lem_sysctl_reg_handler, "IU",
4442                         "Receive Descriptor Head");
4443         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_tail",
4444                         CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(0),
4445                         lem_sysctl_reg_handler, "IU",
4446                         "Receive Descriptor Tail");
4447
4448
4449         /* MAC stats get their own sub node */
4450
4451         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
4452                                     CTLFLAG_RD, NULL, "Statistics");
4453         stat_list = SYSCTL_CHILDREN(stat_node);
4454
4455         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
4456                         CTLFLAG_RD, &stats->ecol,
4457                         "Excessive collisions");
4458         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
4459                         CTLFLAG_RD, &stats->scc,
4460                         "Single collisions");
4461         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
4462                         CTLFLAG_RD, &stats->mcc,
4463                         "Multiple collisions");
4464         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
4465                         CTLFLAG_RD, &stats->latecol,
4466                         "Late collisions");
4467         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
4468                         CTLFLAG_RD, &stats->colc,
4469                         "Collision Count");
4470         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
4471                         CTLFLAG_RD, &adapter->stats.symerrs,
4472                         "Symbol Errors");
4473         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
4474                         CTLFLAG_RD, &adapter->stats.sec,
4475                         "Sequence Errors");
4476         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
4477                         CTLFLAG_RD, &adapter->stats.dc,
4478                         "Defer Count");
4479         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
4480                         CTLFLAG_RD, &adapter->stats.mpc,
4481                         "Missed Packets");
4482         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
4483                         CTLFLAG_RD, &adapter->stats.rnbc,
4484                         "Receive No Buffers");
4485         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
4486                         CTLFLAG_RD, &adapter->stats.ruc,
4487                         "Receive Undersize");
4488         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
4489                         CTLFLAG_RD, &adapter->stats.rfc,
4490                         "Fragmented Packets Received ");
4491         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
4492                         CTLFLAG_RD, &adapter->stats.roc,
4493                         "Oversized Packets Received");
4494         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
4495                         CTLFLAG_RD, &adapter->stats.rjc,
4496                         "Recevied Jabber");
4497         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
4498                         CTLFLAG_RD, &adapter->stats.rxerrc,
4499                         "Receive Errors");
4500         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
4501                         CTLFLAG_RD, &adapter->stats.crcerrs,
4502                         "CRC errors");
4503         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
4504                         CTLFLAG_RD, &adapter->stats.algnerrc,
4505                         "Alignment Errors");
4506         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
4507                         CTLFLAG_RD, &adapter->stats.cexterr,
4508                         "Collision/Carrier extension errors");
4509         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
4510                         CTLFLAG_RD, &adapter->stats.xonrxc,
4511                         "XON Received");
4512         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
4513                         CTLFLAG_RD, &adapter->stats.xontxc,
4514                         "XON Transmitted");
4515         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
4516                         CTLFLAG_RD, &adapter->stats.xoffrxc,
4517                         "XOFF Received");
4518         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
4519                         CTLFLAG_RD, &adapter->stats.xofftxc,
4520                         "XOFF Transmitted");
4521
4522         /* Packet Reception Stats */
4523         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
4524                         CTLFLAG_RD, &adapter->stats.tpr,
4525                         "Total Packets Received ");
4526         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
4527                         CTLFLAG_RD, &adapter->stats.gprc,
4528                         "Good Packets Received");
4529         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
4530                         CTLFLAG_RD, &adapter->stats.bprc,
4531                         "Broadcast Packets Received");
4532         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
4533                         CTLFLAG_RD, &adapter->stats.mprc,
4534                         "Multicast Packets Received");
4535         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
4536                         CTLFLAG_RD, &adapter->stats.prc64,
4537                         "64 byte frames received ");
4538         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
4539                         CTLFLAG_RD, &adapter->stats.prc127,
4540                         "65-127 byte frames received");
4541         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
4542                         CTLFLAG_RD, &adapter->stats.prc255,
4543                         "128-255 byte frames received");
4544         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
4545                         CTLFLAG_RD, &adapter->stats.prc511,
4546                         "256-511 byte frames received");
4547         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
4548                         CTLFLAG_RD, &adapter->stats.prc1023,
4549                         "512-1023 byte frames received");
4550         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
4551                         CTLFLAG_RD, &adapter->stats.prc1522,
4552                         "1023-1522 byte frames received");
4553         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
4554                         CTLFLAG_RD, &adapter->stats.gorc,
4555                         "Good Octets Received");
4556
4557         /* Packet Transmission Stats */
4558         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
4559                         CTLFLAG_RD, &adapter->stats.gotc,
4560                         "Good Octets Transmitted");
4561         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
4562                         CTLFLAG_RD, &adapter->stats.tpt,
4563                         "Total Packets Transmitted");
4564         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
4565                         CTLFLAG_RD, &adapter->stats.gptc,
4566                         "Good Packets Transmitted");
4567         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
4568                         CTLFLAG_RD, &adapter->stats.bptc,
4569                         "Broadcast Packets Transmitted");
4570         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
4571                         CTLFLAG_RD, &adapter->stats.mptc,
4572                         "Multicast Packets Transmitted");
4573         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
4574                         CTLFLAG_RD, &adapter->stats.ptc64,
4575                         "64 byte frames transmitted ");
4576         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
4577                         CTLFLAG_RD, &adapter->stats.ptc127,
4578                         "65-127 byte frames transmitted");
4579         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
4580                         CTLFLAG_RD, &adapter->stats.ptc255,
4581                         "128-255 byte frames transmitted");
4582         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
4583                         CTLFLAG_RD, &adapter->stats.ptc511,
4584                         "256-511 byte frames transmitted");
4585         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
4586                         CTLFLAG_RD, &adapter->stats.ptc1023,
4587                         "512-1023 byte frames transmitted");
4588         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
4589                         CTLFLAG_RD, &adapter->stats.ptc1522,
4590                         "1024-1522 byte frames transmitted");
4591         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
4592                         CTLFLAG_RD, &adapter->stats.tsctc,
4593                         "TSO Contexts Transmitted");
4594         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
4595                         CTLFLAG_RD, &adapter->stats.tsctfc,
4596                         "TSO Contexts Failed");
4597 }
4598
4599 /**********************************************************************
4600  *
4601  *  This routine provides a way to dump out the adapter eeprom,
4602  *  often a useful debug/service tool. This only dumps the first
4603  *  32 words, stuff that matters is in that extent.
4604  *
4605  **********************************************************************/
4606
4607 static int
4608 lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
4609 {
4610         struct adapter *adapter;
4611         int error;
4612         int result;
4613
4614         result = -1;
4615         error = sysctl_handle_int(oidp, &result, 0, req);
4616
4617         if (error || !req->newptr)
4618                 return (error);
4619
4620         /*
4621          * This value will cause a hex dump of the
4622          * first 32 16-bit words of the EEPROM to
4623          * the screen.
4624          */
4625         if (result == 1) {
4626                 adapter = (struct adapter *)arg1;
4627                 lem_print_nvm_info(adapter);
4628         }
4629
4630         return (error);
4631 }
4632
4633 static void
4634 lem_print_nvm_info(struct adapter *adapter)
4635 {
4636         u16     eeprom_data;
4637         int     i, j, row = 0;
4638
4639         /* Its a bit crude, but it gets the job done */
4640         printf("\nInterface EEPROM Dump:\n");
4641         printf("Offset\n0x0000  ");
4642         for (i = 0, j = 0; i < 32; i++, j++) {
4643                 if (j == 8) { /* Make the offset block */
4644                         j = 0; ++row;
4645                         printf("\n0x00%x0  ",row);
4646                 }
4647                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
4648                 printf("%04x ", eeprom_data);
4649         }
4650         printf("\n");
4651 }
4652
4653 static int
4654 lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
4655 {
4656         struct em_int_delay_info *info;
4657         struct adapter *adapter;
4658         u32 regval;
4659         int error;
4660         int usecs;
4661         int ticks;
4662
4663         info = (struct em_int_delay_info *)arg1;
4664         usecs = info->value;
4665         error = sysctl_handle_int(oidp, &usecs, 0, req);
4666         if (error != 0 || req->newptr == NULL)
4667                 return (error);
4668         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
4669                 return (EINVAL);
4670         info->value = usecs;
4671         ticks = EM_USECS_TO_TICKS(usecs);
4672         if (info->offset == E1000_ITR)  /* units are 256ns here */
4673                 ticks *= 4;
4674
4675         adapter = info->adapter;
4676
4677         EM_CORE_LOCK(adapter);
4678         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
4679         regval = (regval & ~0xffff) | (ticks & 0xffff);
4680         /* Handle a few special cases. */
4681         switch (info->offset) {
4682         case E1000_RDTR:
4683                 break;
4684         case E1000_TIDV:
4685                 if (ticks == 0) {
4686                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
4687                         /* Don't write 0 into the TIDV register. */
4688                         regval++;
4689                 } else
4690                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
4691                 break;
4692         }
4693         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
4694         EM_CORE_UNLOCK(adapter);
4695         return (0);
4696 }
4697
4698 static void
4699 lem_add_int_delay_sysctl(struct adapter *adapter, const char *name,
4700         const char *description, struct em_int_delay_info *info,
4701         int offset, int value)
4702 {
4703         info->adapter = adapter;
4704         info->offset = offset;
4705         info->value = value;
4706         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
4707             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4708             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
4709             info, 0, lem_sysctl_int_delay, "I", description);
4710 }
4711
4712 static void
4713 lem_set_flow_cntrl(struct adapter *adapter, const char *name,
4714         const char *description, int *limit, int value)
4715 {
4716         *limit = value;
4717         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4718             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4719             OID_AUTO, name, CTLFLAG_RW, limit, value, description);
4720 }
4721
4722 static void
4723 lem_add_rx_process_limit(struct adapter *adapter, const char *name,
4724         const char *description, int *limit, int value)
4725 {
4726         *limit = value;
4727         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4728             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4729             OID_AUTO, name, CTLFLAG_RW, limit, value, description);
4730 }