1 /******************************************************************************
3 Copyright (c) 2001-2011, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
35 #ifdef HAVE_KERNEL_OPTION_HEADERS
36 #include "opt_device_polling.h"
38 #include "opt_inet6.h"
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #if __FreeBSD_version >= 800000
44 #include <sys/buf_ring.h>
47 #include <sys/endian.h>
48 #include <sys/kernel.h>
49 #include <sys/kthread.h>
50 #include <sys/malloc.h>
52 #include <sys/module.h>
54 #include <sys/socket.h>
55 #include <sys/sockio.h>
56 #include <sys/sysctl.h>
57 #include <sys/taskqueue.h>
58 #include <sys/eventhandler.h>
59 #include <machine/bus.h>
60 #include <machine/resource.h>
63 #include <net/ethernet.h>
65 #include <net/if_arp.h>
66 #include <net/if_dl.h>
67 #include <net/if_media.h>
69 #include <net/if_types.h>
70 #include <net/if_vlan_var.h>
72 #include <netinet/in_systm.h>
73 #include <netinet/in.h>
74 #include <netinet/if_ether.h>
75 #include <netinet/ip.h>
76 #include <netinet/ip6.h>
77 #include <netinet/tcp.h>
78 #include <netinet/udp.h>
80 #include <machine/in_cksum.h>
81 #include <dev/led/led.h>
82 #include <dev/pci/pcivar.h>
83 #include <dev/pci/pcireg.h>
85 #include "e1000_api.h"
86 #include "e1000_82571.h"
89 /*********************************************************************
90 * Set this to one to display debug statistics
91 *********************************************************************/
92 int em_display_debug_stats = 0;
94 /*********************************************************************
96 *********************************************************************/
97 char em_driver_version[] = "7.3.2";
99 /*********************************************************************
100 * PCI Device ID Table
102 * Used by probe to select devices to load on
103 * Last field stores an index into e1000_strings
104 * Last entry must be all 0s
106 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
107 *********************************************************************/
109 static em_vendor_info_t em_vendor_info_array[] =
111 /* Intel(R) PRO/1000 Network Connection */
112 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
113 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
114 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
115 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
116 PCI_ANY_ID, PCI_ANY_ID, 0},
117 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
118 PCI_ANY_ID, PCI_ANY_ID, 0},
119 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
120 PCI_ANY_ID, PCI_ANY_ID, 0},
121 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
122 PCI_ANY_ID, PCI_ANY_ID, 0},
123 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
124 PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
126 PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
128 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
129 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
130 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0},
132 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0},
133 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0},
134 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0},
135 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0},
136 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
137 PCI_ANY_ID, PCI_ANY_ID, 0},
138 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
139 PCI_ANY_ID, PCI_ANY_ID, 0},
140 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
141 PCI_ANY_ID, PCI_ANY_ID, 0},
142 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
143 PCI_ANY_ID, PCI_ANY_ID, 0},
144 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
145 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
147 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
148 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
151 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0},
152 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
153 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
154 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
155 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
156 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0},
157 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
158 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
159 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
160 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0},
161 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0},
162 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0},
163 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
164 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
165 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
166 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
167 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
168 { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
169 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
170 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0},
171 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0},
172 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0},
173 { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
174 { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0},
175 /* required last entry */
179 /*********************************************************************
180 * Table of branding strings for all supported NICs.
181 *********************************************************************/
183 static char *em_strings[] = {
184 "Intel(R) PRO/1000 Network Connection"
187 /*********************************************************************
188 * Function prototypes
189 *********************************************************************/
190 static int em_probe(device_t);
191 static int em_attach(device_t);
192 static int em_detach(device_t);
193 static int em_shutdown(device_t);
194 static int em_suspend(device_t);
195 static int em_resume(device_t);
196 static void em_start(struct ifnet *);
197 static void em_start_locked(struct ifnet *, struct tx_ring *);
199 static int em_mq_start(struct ifnet *, struct mbuf *);
200 static int em_mq_start_locked(struct ifnet *,
201 struct tx_ring *, struct mbuf *);
202 static void em_qflush(struct ifnet *);
204 static int em_ioctl(struct ifnet *, u_long, caddr_t);
205 static void em_init(void *);
206 static void em_init_locked(struct adapter *);
207 static void em_stop(void *);
208 static void em_media_status(struct ifnet *, struct ifmediareq *);
209 static int em_media_change(struct ifnet *);
210 static void em_identify_hardware(struct adapter *);
211 static int em_allocate_pci_resources(struct adapter *);
212 static int em_allocate_legacy(struct adapter *);
213 static int em_allocate_msix(struct adapter *);
214 static int em_allocate_queues(struct adapter *);
215 static int em_setup_msix(struct adapter *);
216 static void em_free_pci_resources(struct adapter *);
217 static void em_local_timer(void *);
218 static void em_reset(struct adapter *);
219 static int em_setup_interface(device_t, struct adapter *);
221 static void em_setup_transmit_structures(struct adapter *);
222 static void em_initialize_transmit_unit(struct adapter *);
223 static int em_allocate_transmit_buffers(struct tx_ring *);
224 static void em_free_transmit_structures(struct adapter *);
225 static void em_free_transmit_buffers(struct tx_ring *);
227 static int em_setup_receive_structures(struct adapter *);
228 static int em_allocate_receive_buffers(struct rx_ring *);
229 static void em_initialize_receive_unit(struct adapter *);
230 static void em_free_receive_structures(struct adapter *);
231 static void em_free_receive_buffers(struct rx_ring *);
233 static void em_enable_intr(struct adapter *);
234 static void em_disable_intr(struct adapter *);
235 static void em_update_stats_counters(struct adapter *);
236 static void em_add_hw_stats(struct adapter *adapter);
237 static bool em_txeof(struct tx_ring *);
238 static bool em_rxeof(struct rx_ring *, int, int *);
239 #ifndef __NO_STRICT_ALIGNMENT
240 static int em_fixup_rx(struct rx_ring *);
242 static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *);
243 static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int,
244 struct ip *, u32 *, u32 *);
245 static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *,
246 struct tcphdr *, u32 *, u32 *);
247 static void em_set_promisc(struct adapter *);
248 static void em_disable_promisc(struct adapter *);
249 static void em_set_multi(struct adapter *);
250 static void em_update_link_status(struct adapter *);
251 static void em_refresh_mbufs(struct rx_ring *, int);
252 static void em_register_vlan(void *, struct ifnet *, u16);
253 static void em_unregister_vlan(void *, struct ifnet *, u16);
254 static void em_setup_vlan_hw_support(struct adapter *);
255 static int em_xmit(struct tx_ring *, struct mbuf **);
256 static int em_dma_malloc(struct adapter *, bus_size_t,
257 struct em_dma_alloc *, int);
258 static void em_dma_free(struct adapter *, struct em_dma_alloc *);
259 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
260 static void em_print_nvm_info(struct adapter *);
261 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
262 static void em_print_debug_info(struct adapter *);
263 static int em_is_valid_ether_addr(u8 *);
264 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
265 static void em_add_int_delay_sysctl(struct adapter *, const char *,
266 const char *, struct em_int_delay_info *, int, int);
267 /* Management and WOL Support */
268 static void em_init_manageability(struct adapter *);
269 static void em_release_manageability(struct adapter *);
270 static void em_get_hw_control(struct adapter *);
271 static void em_release_hw_control(struct adapter *);
272 static void em_get_wakeup(device_t);
273 static void em_enable_wakeup(device_t);
274 static int em_enable_phy_wakeup(struct adapter *);
275 static void em_led_func(void *, int);
276 static void em_disable_aspm(struct adapter *);
278 static int em_irq_fast(void *);
281 static void em_msix_tx(void *);
282 static void em_msix_rx(void *);
283 static void em_msix_link(void *);
284 static void em_handle_tx(void *context, int pending);
285 static void em_handle_rx(void *context, int pending);
286 static void em_handle_link(void *context, int pending);
288 static void em_set_sysctl_value(struct adapter *, const char *,
289 const char *, int *, int);
290 static int em_set_flowcntl(SYSCTL_HANDLER_ARGS);
292 static __inline void em_rx_discard(struct rx_ring *, int);
294 #ifdef DEVICE_POLLING
295 static poll_handler_t em_poll;
298 /*********************************************************************
299 * FreeBSD Device Interface Entry Points
300 *********************************************************************/
302 static device_method_t em_methods[] = {
303 /* Device interface */
304 DEVMETHOD(device_probe, em_probe),
305 DEVMETHOD(device_attach, em_attach),
306 DEVMETHOD(device_detach, em_detach),
307 DEVMETHOD(device_shutdown, em_shutdown),
308 DEVMETHOD(device_suspend, em_suspend),
309 DEVMETHOD(device_resume, em_resume),
313 static driver_t em_driver = {
314 "em", em_methods, sizeof(struct adapter),
317 devclass_t em_devclass;
318 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
319 MODULE_DEPEND(em, pci, 1, 1, 1);
320 MODULE_DEPEND(em, ether, 1, 1, 1);
322 /*********************************************************************
323 * Tunable default values.
324 *********************************************************************/
326 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000)
327 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024)
330 /* Allow common code without TSO */
335 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
337 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
338 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
339 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
340 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
341 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &em_tx_int_delay_dflt,
342 0, "Default transmit interrupt delay in usecs");
343 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt,
344 0, "Default receive interrupt delay in usecs");
346 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
347 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
348 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
349 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
350 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN,
351 &em_tx_abs_int_delay_dflt, 0,
352 "Default transmit interrupt delay limit in usecs");
353 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN,
354 &em_rx_abs_int_delay_dflt, 0,
355 "Default receive interrupt delay limit in usecs");
357 static int em_rxd = EM_DEFAULT_RXD;
358 static int em_txd = EM_DEFAULT_TXD;
359 TUNABLE_INT("hw.em.rxd", &em_rxd);
360 TUNABLE_INT("hw.em.txd", &em_txd);
361 SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0,
362 "Number of receive descriptors per queue");
363 SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0,
364 "Number of transmit descriptors per queue");
366 static int em_smart_pwr_down = FALSE;
367 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
368 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down,
369 0, "Set to true to leave smart power down enabled on newer adapters");
371 /* Controls whether promiscuous also shows bad packets */
372 static int em_debug_sbp = FALSE;
373 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
374 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0,
375 "Show bad packets in promiscuous mode");
377 static int em_enable_msix = TRUE;
378 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix);
379 SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0,
380 "Enable MSI-X interrupts");
382 /* How many packets rxeof tries to clean at a time */
383 static int em_rx_process_limit = 100;
384 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
385 SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
386 &em_rx_process_limit, 0,
387 "Maximum number of received packets to process "
388 "at a time, -1 means unlimited");
390 /* Energy efficient ethernet - default to OFF */
391 static int eee_setting = 0;
392 TUNABLE_INT("hw.em.eee_setting", &eee_setting);
393 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0,
394 "Enable Energy Efficient Ethernet");
396 /* Global used in WOL setup with multiport cards */
397 static int global_quad_port_a = 0;
399 #ifdef DEV_NETMAP /* see ixgbe.c for details */
400 #include <dev/netmap/if_em_netmap.h>
401 #endif /* DEV_NETMAP */
403 /*********************************************************************
404 * Device identification routine
406 * em_probe determines if the driver should be loaded on
407 * adapter based on PCI vendor/device id of the adapter.
409 * return BUS_PROBE_DEFAULT on success, positive on failure
410 *********************************************************************/
413 em_probe(device_t dev)
415 char adapter_name[60];
416 u16 pci_vendor_id = 0;
417 u16 pci_device_id = 0;
418 u16 pci_subvendor_id = 0;
419 u16 pci_subdevice_id = 0;
420 em_vendor_info_t *ent;
422 INIT_DEBUGOUT("em_probe: begin");
424 pci_vendor_id = pci_get_vendor(dev);
425 if (pci_vendor_id != EM_VENDOR_ID)
428 pci_device_id = pci_get_device(dev);
429 pci_subvendor_id = pci_get_subvendor(dev);
430 pci_subdevice_id = pci_get_subdevice(dev);
432 ent = em_vendor_info_array;
433 while (ent->vendor_id != 0) {
434 if ((pci_vendor_id == ent->vendor_id) &&
435 (pci_device_id == ent->device_id) &&
437 ((pci_subvendor_id == ent->subvendor_id) ||
438 (ent->subvendor_id == PCI_ANY_ID)) &&
440 ((pci_subdevice_id == ent->subdevice_id) ||
441 (ent->subdevice_id == PCI_ANY_ID))) {
442 sprintf(adapter_name, "%s %s",
443 em_strings[ent->index],
445 device_set_desc_copy(dev, adapter_name);
446 return (BUS_PROBE_DEFAULT);
454 /*********************************************************************
455 * Device initialization routine
457 * The attach entry point is called when the driver is being loaded.
458 * This routine identifies the type of hardware, allocates all resources
459 * and initializes the hardware.
461 * return 0 on success, positive on failure
462 *********************************************************************/
465 em_attach(device_t dev)
467 struct adapter *adapter;
471 INIT_DEBUGOUT("em_attach: begin");
473 if (resource_disabled("em", device_get_unit(dev))) {
474 device_printf(dev, "Disabled by device hint\n");
478 adapter = device_get_softc(dev);
479 adapter->dev = adapter->osdep.dev = dev;
481 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
484 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
485 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
486 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
487 em_sysctl_nvm_info, "I", "NVM Information");
489 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
490 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
491 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
492 em_sysctl_debug_info, "I", "Debug Information");
494 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
495 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
496 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
497 em_set_flowcntl, "I", "Flow Control");
499 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
501 /* Determine hardware and mac info */
502 em_identify_hardware(adapter);
504 /* Setup PCI resources */
505 if (em_allocate_pci_resources(adapter)) {
506 device_printf(dev, "Allocation of PCI resources failed\n");
512 ** For ICH8 and family we need to
513 ** map the flash memory, and this
514 ** must happen after the MAC is
517 if ((hw->mac.type == e1000_ich8lan) ||
518 (hw->mac.type == e1000_ich9lan) ||
519 (hw->mac.type == e1000_ich10lan) ||
520 (hw->mac.type == e1000_pchlan) ||
521 (hw->mac.type == e1000_pch2lan)) {
522 int rid = EM_BAR_TYPE_FLASH;
523 adapter->flash = bus_alloc_resource_any(dev,
524 SYS_RES_MEMORY, &rid, RF_ACTIVE);
525 if (adapter->flash == NULL) {
526 device_printf(dev, "Mapping of Flash failed\n");
530 /* This is used in the shared code */
531 hw->flash_address = (u8 *)adapter->flash;
532 adapter->osdep.flash_bus_space_tag =
533 rman_get_bustag(adapter->flash);
534 adapter->osdep.flash_bus_space_handle =
535 rman_get_bushandle(adapter->flash);
538 /* Do Shared Code initialization */
539 if (e1000_setup_init_funcs(hw, TRUE)) {
540 device_printf(dev, "Setup of Shared code failed\n");
545 e1000_get_bus_info(hw);
547 /* Set up some sysctls for the tunable interrupt delays */
548 em_add_int_delay_sysctl(adapter, "rx_int_delay",
549 "receive interrupt delay in usecs", &adapter->rx_int_delay,
550 E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt);
551 em_add_int_delay_sysctl(adapter, "tx_int_delay",
552 "transmit interrupt delay in usecs", &adapter->tx_int_delay,
553 E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt);
554 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
555 "receive interrupt delay limit in usecs",
556 &adapter->rx_abs_int_delay,
557 E1000_REGISTER(hw, E1000_RADV),
558 em_rx_abs_int_delay_dflt);
559 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
560 "transmit interrupt delay limit in usecs",
561 &adapter->tx_abs_int_delay,
562 E1000_REGISTER(hw, E1000_TADV),
563 em_tx_abs_int_delay_dflt);
565 /* Sysctl for limiting the amount of work done in the taskqueue */
566 em_set_sysctl_value(adapter, "rx_processing_limit",
567 "max number of rx packets to process", &adapter->rx_process_limit,
568 em_rx_process_limit);
571 * Validate number of transmit and receive descriptors. It
572 * must not exceed hardware maximum, and must be multiple
573 * of E1000_DBA_ALIGN.
575 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
576 (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) {
577 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
578 EM_DEFAULT_TXD, em_txd);
579 adapter->num_tx_desc = EM_DEFAULT_TXD;
581 adapter->num_tx_desc = em_txd;
583 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
584 (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) {
585 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
586 EM_DEFAULT_RXD, em_rxd);
587 adapter->num_rx_desc = EM_DEFAULT_RXD;
589 adapter->num_rx_desc = em_rxd;
591 hw->mac.autoneg = DO_AUTO_NEG;
592 hw->phy.autoneg_wait_to_complete = FALSE;
593 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
596 if (hw->phy.media_type == e1000_media_type_copper) {
597 hw->phy.mdix = AUTO_ALL_MODES;
598 hw->phy.disable_polarity_correction = FALSE;
599 hw->phy.ms_type = EM_MASTER_SLAVE;
603 * Set the frame limits assuming
604 * standard ethernet sized frames.
606 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
607 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
610 * This controls when hardware reports transmit completion
613 hw->mac.report_tx_early = 1;
616 ** Get queue/ring memory
618 if (em_allocate_queues(adapter)) {
623 /* Allocate multicast array memory. */
624 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
625 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
626 if (adapter->mta == NULL) {
627 device_printf(dev, "Can not allocate multicast setup array\n");
632 /* Check SOL/IDER usage */
633 if (e1000_check_reset_block(hw))
634 device_printf(dev, "PHY reset is blocked"
635 " due to SOL/IDER session.\n");
637 /* Sysctl for setting Energy Efficient Ethernet */
638 em_set_sysctl_value(adapter, "eee_control",
639 "enable Energy Efficient Ethernet",
640 &hw->dev_spec.ich8lan.eee_disable, eee_setting);
643 ** Start from a known state, this is
644 ** important in reading the nvm and
650 /* Make sure we have a good EEPROM before we read from it */
651 if (e1000_validate_nvm_checksum(hw) < 0) {
653 ** Some PCI-E parts fail the first check due to
654 ** the link being in sleep state, call it again,
655 ** if it fails a second time its a real issue.
657 if (e1000_validate_nvm_checksum(hw) < 0) {
659 "The EEPROM Checksum Is Not Valid\n");
665 /* Copy the permanent MAC address out of the EEPROM */
666 if (e1000_read_mac_addr(hw) < 0) {
667 device_printf(dev, "EEPROM read error while reading MAC"
673 if (!em_is_valid_ether_addr(hw->mac.addr)) {
674 device_printf(dev, "Invalid MAC address\n");
680 ** Do interrupt configuration
682 if (adapter->msix > 1) /* Do MSIX */
683 error = em_allocate_msix(adapter);
684 else /* MSI or Legacy */
685 error = em_allocate_legacy(adapter);
690 * Get Wake-on-Lan and Management info for later use
694 /* Setup OS specific network interface */
695 if (em_setup_interface(dev, adapter) != 0)
700 /* Initialize statistics */
701 em_update_stats_counters(adapter);
703 hw->mac.get_link_status = 1;
704 em_update_link_status(adapter);
706 /* Register for VLAN events */
707 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
708 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
709 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
710 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
712 em_add_hw_stats(adapter);
714 /* Non-AMT based hardware can now take control from firmware */
715 if (adapter->has_manage && !adapter->has_amt)
716 em_get_hw_control(adapter);
718 /* Tell the stack that the interface is not active */
719 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
720 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
722 adapter->led_dev = led_create(em_led_func, adapter,
723 device_get_nameunit(dev));
725 em_netmap_attach(adapter);
726 #endif /* DEV_NETMAP */
728 INIT_DEBUGOUT("em_attach: end");
733 em_free_transmit_structures(adapter);
734 em_free_receive_structures(adapter);
735 em_release_hw_control(adapter);
736 if (adapter->ifp != NULL)
737 if_free(adapter->ifp);
739 em_free_pci_resources(adapter);
740 free(adapter->mta, M_DEVBUF);
741 EM_CORE_LOCK_DESTROY(adapter);
746 /*********************************************************************
747 * Device removal routine
749 * The detach entry point is called when the driver is being removed.
750 * This routine stops the adapter and deallocates all the resources
751 * that were allocated for driver operation.
753 * return 0 on success, positive on failure
754 *********************************************************************/
757 em_detach(device_t dev)
759 struct adapter *adapter = device_get_softc(dev);
760 struct ifnet *ifp = adapter->ifp;
762 INIT_DEBUGOUT("em_detach: begin");
764 /* Make sure VLANS are not using driver */
765 if (adapter->ifp->if_vlantrunk != NULL) {
766 device_printf(dev,"Vlan in use, detach first\n");
770 #ifdef DEVICE_POLLING
771 if (ifp->if_capenable & IFCAP_POLLING)
772 ether_poll_deregister(ifp);
775 if (adapter->led_dev != NULL)
776 led_destroy(adapter->led_dev);
778 EM_CORE_LOCK(adapter);
779 adapter->in_detach = 1;
781 EM_CORE_UNLOCK(adapter);
782 EM_CORE_LOCK_DESTROY(adapter);
784 e1000_phy_hw_reset(&adapter->hw);
786 em_release_manageability(adapter);
787 em_release_hw_control(adapter);
789 /* Unregister VLAN events */
790 if (adapter->vlan_attach != NULL)
791 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
792 if (adapter->vlan_detach != NULL)
793 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
795 ether_ifdetach(adapter->ifp);
796 callout_drain(&adapter->timer);
800 #endif /* DEV_NETMAP */
802 em_free_pci_resources(adapter);
803 bus_generic_detach(dev);
806 em_free_transmit_structures(adapter);
807 em_free_receive_structures(adapter);
809 em_release_hw_control(adapter);
810 free(adapter->mta, M_DEVBUF);
815 /*********************************************************************
817 * Shutdown entry point
819 **********************************************************************/
822 em_shutdown(device_t dev)
824 return em_suspend(dev);
828 * Suspend/resume device methods.
831 em_suspend(device_t dev)
833 struct adapter *adapter = device_get_softc(dev);
835 EM_CORE_LOCK(adapter);
837 em_release_manageability(adapter);
838 em_release_hw_control(adapter);
839 em_enable_wakeup(dev);
841 EM_CORE_UNLOCK(adapter);
843 return bus_generic_suspend(dev);
847 em_resume(device_t dev)
849 struct adapter *adapter = device_get_softc(dev);
850 struct ifnet *ifp = adapter->ifp;
852 EM_CORE_LOCK(adapter);
853 if (adapter->hw.mac.type == e1000_pch2lan)
854 e1000_resume_workarounds_pchlan(&adapter->hw);
855 em_init_locked(adapter);
856 em_init_manageability(adapter);
857 EM_CORE_UNLOCK(adapter);
860 return bus_generic_resume(dev);
865 /*********************************************************************
866 * Multiqueue Transmit routines
868 * em_mq_start is called by the stack to initiate a transmit.
869 * however, if busy the driver can queue the request rather
870 * than do an immediate send. It is this that is an advantage
871 * in this driver, rather than also having multiple tx queues.
872 **********************************************************************/
874 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
876 struct adapter *adapter = txr->adapter;
878 int err = 0, enq = 0;
880 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
881 IFF_DRV_RUNNING || adapter->link_active == 0) {
883 err = drbr_enqueue(ifp, txr->br, m);
889 next = drbr_dequeue(ifp, txr->br);
890 } else if (drbr_needs_enqueue(ifp, txr->br)) {
891 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
893 next = drbr_dequeue(ifp, txr->br);
897 /* Process the queue */
898 while (next != NULL) {
899 if ((err = em_xmit(txr, &next)) != 0) {
901 err = drbr_enqueue(ifp, txr->br, next);
905 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
906 ETHER_BPF_MTAP(ifp, next);
907 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
909 next = drbr_dequeue(ifp, txr->br);
913 /* Set the watchdog */
914 txr->queue_status = EM_QUEUE_WORKING;
915 txr->watchdog_time = ticks;
918 if (txr->tx_avail < EM_MAX_SCATTER)
920 if (txr->tx_avail < EM_MAX_SCATTER)
921 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
926 ** Multiqueue capable stack interface
929 em_mq_start(struct ifnet *ifp, struct mbuf *m)
931 struct adapter *adapter = ifp->if_softc;
932 struct tx_ring *txr = adapter->tx_rings;
935 if (EM_TX_TRYLOCK(txr)) {
936 error = em_mq_start_locked(ifp, txr, m);
939 error = drbr_enqueue(ifp, txr->br, m);
945 ** Flush all ring buffers
948 em_qflush(struct ifnet *ifp)
950 struct adapter *adapter = ifp->if_softc;
951 struct tx_ring *txr = adapter->tx_rings;
954 for (int i = 0; i < adapter->num_queues; i++, txr++) {
956 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
962 #endif /* EM_MULTIQUEUE */
965 em_start_locked(struct ifnet *ifp, struct tx_ring *txr)
967 struct adapter *adapter = ifp->if_softc;
970 EM_TX_LOCK_ASSERT(txr);
972 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
976 if (!adapter->link_active)
979 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
980 /* Call cleanup if number of TX descriptors low */
981 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
983 if (txr->tx_avail < EM_MAX_SCATTER) {
984 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
987 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
991 * Encapsulation can modify our pointer, and or make it
992 * NULL on failure. In that event, we can't requeue.
994 if (em_xmit(txr, &m_head)) {
997 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1001 /* Send a copy of the frame to the BPF listener */
1002 ETHER_BPF_MTAP(ifp, m_head);
1004 /* Set timeout in case hardware has problems transmitting. */
1005 txr->watchdog_time = ticks;
1006 txr->queue_status = EM_QUEUE_WORKING;
1013 em_start(struct ifnet *ifp)
1015 struct adapter *adapter = ifp->if_softc;
1016 struct tx_ring *txr = adapter->tx_rings;
1018 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1020 em_start_locked(ifp, txr);
1024 ** If we went inactive schedule
1025 ** a task to clean up.
1027 if (ifp->if_drv_flags & IFF_DRV_OACTIVE)
1028 taskqueue_enqueue(txr->tq, &txr->tx_task);
1032 /*********************************************************************
1035 * em_ioctl is called when the user wants to configure the
1038 * return 0 on success, positive on failure
1039 **********************************************************************/
1042 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1044 struct adapter *adapter = ifp->if_softc;
1045 struct ifreq *ifr = (struct ifreq *)data;
1046 #if defined(INET) || defined(INET6)
1047 struct ifaddr *ifa = (struct ifaddr *)data;
1049 bool avoid_reset = FALSE;
1052 if (adapter->in_detach)
1058 if (ifa->ifa_addr->sa_family == AF_INET)
1062 if (ifa->ifa_addr->sa_family == AF_INET6)
1066 ** Calling init results in link renegotiation,
1067 ** so we avoid doing it when possible.
1070 ifp->if_flags |= IFF_UP;
1071 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1074 if (!(ifp->if_flags & IFF_NOARP))
1075 arp_ifinit(ifp, ifa);
1078 error = ether_ioctl(ifp, command, data);
1084 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1086 EM_CORE_LOCK(adapter);
1087 switch (adapter->hw.mac.type) {
1091 case e1000_ich10lan:
1095 case e1000_80003es2lan: /* 9K Jumbo Frame size */
1096 max_frame_size = 9234;
1099 max_frame_size = 4096;
1101 /* Adapters that do not support jumbo frames */
1103 max_frame_size = ETHER_MAX_LEN;
1106 max_frame_size = MAX_JUMBO_FRAME_SIZE;
1108 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1110 EM_CORE_UNLOCK(adapter);
1115 ifp->if_mtu = ifr->ifr_mtu;
1116 adapter->max_frame_size =
1117 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1118 em_init_locked(adapter);
1119 EM_CORE_UNLOCK(adapter);
1123 IOCTL_DEBUGOUT("ioctl rcv'd:\
1124 SIOCSIFFLAGS (Set Interface Flags)");
1125 EM_CORE_LOCK(adapter);
1126 if (ifp->if_flags & IFF_UP) {
1127 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1128 if ((ifp->if_flags ^ adapter->if_flags) &
1129 (IFF_PROMISC | IFF_ALLMULTI)) {
1130 em_disable_promisc(adapter);
1131 em_set_promisc(adapter);
1134 em_init_locked(adapter);
1136 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1138 adapter->if_flags = ifp->if_flags;
1139 EM_CORE_UNLOCK(adapter);
1143 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1144 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1145 EM_CORE_LOCK(adapter);
1146 em_disable_intr(adapter);
1147 em_set_multi(adapter);
1148 #ifdef DEVICE_POLLING
1149 if (!(ifp->if_capenable & IFCAP_POLLING))
1151 em_enable_intr(adapter);
1152 EM_CORE_UNLOCK(adapter);
1156 /* Check SOL/IDER usage */
1157 EM_CORE_LOCK(adapter);
1158 if (e1000_check_reset_block(&adapter->hw)) {
1159 EM_CORE_UNLOCK(adapter);
1160 device_printf(adapter->dev, "Media change is"
1161 " blocked due to SOL/IDER session.\n");
1164 EM_CORE_UNLOCK(adapter);
1167 IOCTL_DEBUGOUT("ioctl rcv'd: \
1168 SIOCxIFMEDIA (Get/Set Interface Media)");
1169 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1175 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1177 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1178 #ifdef DEVICE_POLLING
1179 if (mask & IFCAP_POLLING) {
1180 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1181 error = ether_poll_register(em_poll, ifp);
1184 EM_CORE_LOCK(adapter);
1185 em_disable_intr(adapter);
1186 ifp->if_capenable |= IFCAP_POLLING;
1187 EM_CORE_UNLOCK(adapter);
1189 error = ether_poll_deregister(ifp);
1190 /* Enable interrupt even in error case */
1191 EM_CORE_LOCK(adapter);
1192 em_enable_intr(adapter);
1193 ifp->if_capenable &= ~IFCAP_POLLING;
1194 EM_CORE_UNLOCK(adapter);
1198 if (mask & IFCAP_HWCSUM) {
1199 ifp->if_capenable ^= IFCAP_HWCSUM;
1202 if (mask & IFCAP_TSO4) {
1203 ifp->if_capenable ^= IFCAP_TSO4;
1206 if (mask & IFCAP_VLAN_HWTAGGING) {
1207 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1210 if (mask & IFCAP_VLAN_HWFILTER) {
1211 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1214 if (mask & IFCAP_VLAN_HWTSO) {
1215 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1218 if ((mask & IFCAP_WOL) &&
1219 (ifp->if_capabilities & IFCAP_WOL) != 0) {
1220 if (mask & IFCAP_WOL_MCAST)
1221 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1222 if (mask & IFCAP_WOL_MAGIC)
1223 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1225 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1227 VLAN_CAPABILITIES(ifp);
1232 error = ether_ioctl(ifp, command, data);
1240 /*********************************************************************
1243 * This routine is used in two ways. It is used by the stack as
1244 * init entry point in network interface structure. It is also used
1245 * by the driver as a hw/sw initialization routine to get to a
1248 * return 0 on success, positive on failure
1249 **********************************************************************/
1252 em_init_locked(struct adapter *adapter)
1254 struct ifnet *ifp = adapter->ifp;
1255 device_t dev = adapter->dev;
1257 INIT_DEBUGOUT("em_init: begin");
1259 EM_CORE_LOCK_ASSERT(adapter);
1261 em_disable_intr(adapter);
1262 callout_stop(&adapter->timer);
1264 /* Get the latest mac address, User can use a LAA */
1265 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1268 /* Put the address into the Receive Address Array */
1269 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1272 * With the 82571 adapter, RAR[0] may be overwritten
1273 * when the other port is reset, we make a duplicate
1274 * in RAR[14] for that eventuality, this assures
1275 * the interface continues to function.
1277 if (adapter->hw.mac.type == e1000_82571) {
1278 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1279 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1280 E1000_RAR_ENTRIES - 1);
1283 /* Initialize the hardware */
1285 em_update_link_status(adapter);
1287 /* Setup VLAN support, basic and offload if available */
1288 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1290 /* Set hardware offload abilities */
1291 ifp->if_hwassist = 0;
1292 if (ifp->if_capenable & IFCAP_TXCSUM)
1293 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1294 if (ifp->if_capenable & IFCAP_TSO4)
1295 ifp->if_hwassist |= CSUM_TSO;
1297 /* Configure for OS presence */
1298 em_init_manageability(adapter);
1300 /* Prepare transmit descriptors and buffers */
1301 em_setup_transmit_structures(adapter);
1302 em_initialize_transmit_unit(adapter);
1304 /* Setup Multicast table */
1305 em_set_multi(adapter);
1308 ** Figure out the desired mbuf
1309 ** pool for doing jumbos
1311 if (adapter->max_frame_size <= 2048)
1312 adapter->rx_mbuf_sz = MCLBYTES;
1313 else if (adapter->max_frame_size <= 4096)
1314 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1316 adapter->rx_mbuf_sz = MJUM9BYTES;
1318 /* Prepare receive descriptors and buffers */
1319 if (em_setup_receive_structures(adapter)) {
1320 device_printf(dev, "Could not setup receive structures\n");
1324 em_initialize_receive_unit(adapter);
1326 /* Use real VLAN Filter support? */
1327 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1328 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1329 /* Use real VLAN Filter support */
1330 em_setup_vlan_hw_support(adapter);
1333 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1334 ctrl |= E1000_CTRL_VME;
1335 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1339 /* Don't lose promiscuous settings */
1340 em_set_promisc(adapter);
1342 /* Set the interface as ACTIVE */
1343 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1344 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1346 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1347 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1349 /* MSI/X configuration for 82574 */
1350 if (adapter->hw.mac.type == e1000_82574) {
1352 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1353 tmp |= E1000_CTRL_EXT_PBA_CLR;
1354 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1355 /* Set the IVAR - interrupt vector routing. */
1356 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
1359 #ifdef DEVICE_POLLING
1361 * Only enable interrupts if we are not polling, make sure
1362 * they are off otherwise.
1364 if (ifp->if_capenable & IFCAP_POLLING)
1365 em_disable_intr(adapter);
1367 #endif /* DEVICE_POLLING */
1368 em_enable_intr(adapter);
1370 /* AMT based hardware can now take control from firmware */
1371 if (adapter->has_manage && adapter->has_amt)
1372 em_get_hw_control(adapter);
1378 struct adapter *adapter = arg;
1380 EM_CORE_LOCK(adapter);
1381 em_init_locked(adapter);
1382 EM_CORE_UNLOCK(adapter);
1386 #ifdef DEVICE_POLLING
1387 /*********************************************************************
1389 * Legacy polling routine: note this only works with single queue
1391 *********************************************************************/
1393 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1395 struct adapter *adapter = ifp->if_softc;
1396 struct tx_ring *txr = adapter->tx_rings;
1397 struct rx_ring *rxr = adapter->rx_rings;
1401 EM_CORE_LOCK(adapter);
1402 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1403 EM_CORE_UNLOCK(adapter);
1407 if (cmd == POLL_AND_CHECK_STATUS) {
1408 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1409 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1410 callout_stop(&adapter->timer);
1411 adapter->hw.mac.get_link_status = 1;
1412 em_update_link_status(adapter);
1413 callout_reset(&adapter->timer, hz,
1414 em_local_timer, adapter);
1417 EM_CORE_UNLOCK(adapter);
1419 em_rxeof(rxr, count, &rx_done);
1423 #ifdef EM_MULTIQUEUE
1424 if (!drbr_empty(ifp, txr->br))
1425 em_mq_start_locked(ifp, txr, NULL);
1427 em_start_locked(ifp, txr);
1433 #endif /* DEVICE_POLLING */
1436 /*********************************************************************
1438 * Fast Legacy/MSI Combined Interrupt Service routine
1440 *********************************************************************/
1442 em_irq_fast(void *arg)
1444 struct adapter *adapter = arg;
1450 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1453 if (reg_icr == 0xffffffff)
1454 return FILTER_STRAY;
1456 /* Definitely not our interrupt. */
1458 return FILTER_STRAY;
1461 * Starting with the 82571 chip, bit 31 should be used to
1462 * determine whether the interrupt belongs to us.
1464 if (adapter->hw.mac.type >= e1000_82571 &&
1465 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1466 return FILTER_STRAY;
1468 em_disable_intr(adapter);
1469 taskqueue_enqueue(adapter->tq, &adapter->que_task);
1471 /* Link status change */
1472 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1473 adapter->hw.mac.get_link_status = 1;
1474 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1477 if (reg_icr & E1000_ICR_RXO)
1478 adapter->rx_overruns++;
1479 return FILTER_HANDLED;
1482 /* Combined RX/TX handler, used by Legacy and MSI */
1484 em_handle_que(void *context, int pending)
1486 struct adapter *adapter = context;
1487 struct ifnet *ifp = adapter->ifp;
1488 struct tx_ring *txr = adapter->tx_rings;
1489 struct rx_ring *rxr = adapter->rx_rings;
1492 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1493 bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1496 #ifdef EM_MULTIQUEUE
1497 if (!drbr_empty(ifp, txr->br))
1498 em_mq_start_locked(ifp, txr, NULL);
1500 em_start_locked(ifp, txr);
1503 if (more || (ifp->if_drv_flags & IFF_DRV_OACTIVE)) {
1504 taskqueue_enqueue(adapter->tq, &adapter->que_task);
1509 em_enable_intr(adapter);
1514 /*********************************************************************
1516 * MSIX Interrupt Service Routines
1518 **********************************************************************/
1520 em_msix_tx(void *arg)
1522 struct tx_ring *txr = arg;
1523 struct adapter *adapter = txr->adapter;
1528 more = em_txeof(txr);
1531 taskqueue_enqueue(txr->tq, &txr->tx_task);
1533 /* Reenable this interrupt */
1534 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1538 /*********************************************************************
1540 * MSIX RX Interrupt Service routine
1542 **********************************************************************/
1545 em_msix_rx(void *arg)
1547 struct rx_ring *rxr = arg;
1548 struct adapter *adapter = rxr->adapter;
1552 more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1554 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1556 /* Reenable this interrupt */
1557 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1561 /*********************************************************************
1563 * MSIX Link Fast Interrupt Service routine
1565 **********************************************************************/
1567 em_msix_link(void *arg)
1569 struct adapter *adapter = arg;
1572 ++adapter->link_irq;
1573 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1575 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1576 adapter->hw.mac.get_link_status = 1;
1577 em_handle_link(adapter, 0);
1579 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1580 EM_MSIX_LINK | E1000_IMS_LSC);
1585 em_handle_rx(void *context, int pending)
1587 struct rx_ring *rxr = context;
1588 struct adapter *adapter = rxr->adapter;
1591 more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1593 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1595 /* Reenable this interrupt */
1596 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1600 em_handle_tx(void *context, int pending)
1602 struct tx_ring *txr = context;
1603 struct adapter *adapter = txr->adapter;
1604 struct ifnet *ifp = adapter->ifp;
1608 #ifdef EM_MULTIQUEUE
1609 if (!drbr_empty(ifp, txr->br))
1610 em_mq_start_locked(ifp, txr, NULL);
1612 em_start_locked(ifp, txr);
1614 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1619 em_handle_link(void *context, int pending)
1621 struct adapter *adapter = context;
1622 struct ifnet *ifp = adapter->ifp;
1624 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1627 EM_CORE_LOCK(adapter);
1628 callout_stop(&adapter->timer);
1629 em_update_link_status(adapter);
1630 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1631 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1632 EM_MSIX_LINK | E1000_IMS_LSC);
1633 EM_CORE_UNLOCK(adapter);
1637 /*********************************************************************
1639 * Media Ioctl callback
1641 * This routine is called whenever the user queries the status of
1642 * the interface using ifconfig.
1644 **********************************************************************/
1646 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1648 struct adapter *adapter = ifp->if_softc;
1649 u_char fiber_type = IFM_1000_SX;
1651 INIT_DEBUGOUT("em_media_status: begin");
1653 EM_CORE_LOCK(adapter);
1654 em_update_link_status(adapter);
1656 ifmr->ifm_status = IFM_AVALID;
1657 ifmr->ifm_active = IFM_ETHER;
1659 if (!adapter->link_active) {
1660 EM_CORE_UNLOCK(adapter);
1664 ifmr->ifm_status |= IFM_ACTIVE;
1666 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1667 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1668 ifmr->ifm_active |= fiber_type | IFM_FDX;
1670 switch (adapter->link_speed) {
1672 ifmr->ifm_active |= IFM_10_T;
1675 ifmr->ifm_active |= IFM_100_TX;
1678 ifmr->ifm_active |= IFM_1000_T;
1681 if (adapter->link_duplex == FULL_DUPLEX)
1682 ifmr->ifm_active |= IFM_FDX;
1684 ifmr->ifm_active |= IFM_HDX;
1686 EM_CORE_UNLOCK(adapter);
1689 /*********************************************************************
1691 * Media Ioctl callback
1693 * This routine is called when the user changes speed/duplex using
1694 * media/mediopt option with ifconfig.
1696 **********************************************************************/
1698 em_media_change(struct ifnet *ifp)
1700 struct adapter *adapter = ifp->if_softc;
1701 struct ifmedia *ifm = &adapter->media;
1703 INIT_DEBUGOUT("em_media_change: begin");
1705 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1708 EM_CORE_LOCK(adapter);
1709 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1711 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1712 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1717 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1718 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1721 adapter->hw.mac.autoneg = FALSE;
1722 adapter->hw.phy.autoneg_advertised = 0;
1723 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1724 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1726 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1729 adapter->hw.mac.autoneg = FALSE;
1730 adapter->hw.phy.autoneg_advertised = 0;
1731 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1732 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1734 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1737 device_printf(adapter->dev, "Unsupported media type\n");
1740 em_init_locked(adapter);
1741 EM_CORE_UNLOCK(adapter);
1746 /*********************************************************************
1748 * This routine maps the mbufs to tx descriptors.
1750 * return 0 on success, positive on failure
1751 **********************************************************************/
1754 em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1756 struct adapter *adapter = txr->adapter;
1757 bus_dma_segment_t segs[EM_MAX_SCATTER];
1759 struct em_buffer *tx_buffer, *tx_buffer_mapped;
1760 struct e1000_tx_desc *ctxd = NULL;
1761 struct mbuf *m_head;
1762 struct ether_header *eh;
1763 struct ip *ip = NULL;
1764 struct tcphdr *tp = NULL;
1765 u32 txd_upper, txd_lower, txd_used, txd_saved;
1767 int nsegs, i, j, first, last = 0;
1768 int error, do_tso, tso_desc = 0, remap = 1;
1772 txd_upper = txd_lower = txd_used = txd_saved = 0;
1773 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1777 * Intel recommends entire IP/TCP header length reside in a single
1778 * buffer. If multiple descriptors are used to describe the IP and
1779 * TCP header, each descriptor should describe one or more
1780 * complete headers; descriptors referencing only parts of headers
1781 * are not supported. If all layer headers are not coalesced into
1782 * a single buffer, each buffer should not cross a 4KB boundary,
1783 * or be larger than the maximum read request size.
1784 * Controller also requires modifing IP/TCP header to make TSO work
1785 * so we firstly get a writable mbuf chain then coalesce ethernet/
1786 * IP/TCP header into a single buffer to meet the requirement of
1787 * controller. This also simplifies IP/TCP/UDP checksum offloading
1788 * which also has similiar restrictions.
1790 if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
1791 if (do_tso || (m_head->m_next != NULL &&
1792 m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) {
1793 if (M_WRITABLE(*m_headp) == 0) {
1794 m_head = m_dup(*m_headp, M_DONTWAIT);
1796 if (m_head == NULL) {
1805 * Assume IPv4, we don't have TSO/checksum offload support
1808 ip_off = sizeof(struct ether_header);
1809 m_head = m_pullup(m_head, ip_off);
1810 if (m_head == NULL) {
1814 eh = mtod(m_head, struct ether_header *);
1815 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1816 ip_off = sizeof(struct ether_vlan_header);
1817 m_head = m_pullup(m_head, ip_off);
1818 if (m_head == NULL) {
1823 m_head = m_pullup(m_head, ip_off + sizeof(struct ip));
1824 if (m_head == NULL) {
1828 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1829 poff = ip_off + (ip->ip_hl << 2);
1831 m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1832 if (m_head == NULL) {
1836 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1839 * pull 4 more bytes of data into it.
1841 m_head = m_pullup(m_head, poff + (tp->th_off << 2) + 4);
1842 if (m_head == NULL) {
1846 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1850 * The pseudo TCP checksum does not include TCP payload
1851 * length so driver should recompute the checksum here
1852 * what hardware expect to see. This is adherence of
1853 * Microsoft's Large Send specification.
1855 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1856 tp->th_sum = in_pseudo(ip->ip_src.s_addr,
1857 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
1858 } else if (m_head->m_pkthdr.csum_flags & CSUM_TCP) {
1859 m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1860 if (m_head == NULL) {
1864 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1865 m_head = m_pullup(m_head, poff + (tp->th_off << 2));
1866 if (m_head == NULL) {
1870 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1871 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1872 } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) {
1873 m_head = m_pullup(m_head, poff + sizeof(struct udphdr));
1874 if (m_head == NULL) {
1878 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1884 * Map the packet for DMA
1886 * Capture the first descriptor index,
1887 * this descriptor will have the index
1888 * of the EOP which is the only one that
1889 * now gets a DONE bit writeback.
1891 first = txr->next_avail_desc;
1892 tx_buffer = &txr->tx_buffers[first];
1893 tx_buffer_mapped = tx_buffer;
1894 map = tx_buffer->map;
1896 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1897 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1900 * There are two types of errors we can (try) to handle:
1901 * - EFBIG means the mbuf chain was too long and bus_dma ran
1902 * out of segments. Defragment the mbuf chain and try again.
1903 * - ENOMEM means bus_dma could not obtain enough bounce buffers
1904 * at this point in time. Defer sending and try again later.
1905 * All other errors, in particular EINVAL, are fatal and prevent the
1906 * mbuf chain from ever going through. Drop it and report error.
1908 if (error == EFBIG && remap) {
1911 m = m_defrag(*m_headp, M_DONTWAIT);
1913 adapter->mbuf_alloc_failed++;
1920 /* Try it again, but only once */
1923 } else if (error == ENOMEM) {
1924 adapter->no_tx_dma_setup++;
1926 } else if (error != 0) {
1927 adapter->no_tx_dma_setup++;
1934 * TSO Hardware workaround, if this packet is not
1935 * TSO, and is only a single descriptor long, and
1936 * it follows a TSO burst, then we need to add a
1937 * sentinel descriptor to prevent premature writeback.
1939 if ((do_tso == 0) && (txr->tx_tso == TRUE)) {
1942 txr->tx_tso = FALSE;
1945 if (nsegs > (txr->tx_avail - 2)) {
1946 txr->no_desc_avail++;
1947 bus_dmamap_unload(txr->txtag, map);
1952 /* Do hardware assists */
1953 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1954 em_tso_setup(txr, m_head, ip_off, ip, tp,
1955 &txd_upper, &txd_lower);
1956 /* we need to make a final sentinel transmit desc */
1958 } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
1959 em_transmit_checksum_setup(txr, m_head,
1960 ip_off, ip, &txd_upper, &txd_lower);
1962 if (m_head->m_flags & M_VLANTAG) {
1963 /* Set the vlan id. */
1965 (htole16(m_head->m_pkthdr.ether_vtag) << 16);
1966 /* Tell hardware to add tag */
1967 txd_lower |= htole32(E1000_TXD_CMD_VLE);
1970 i = txr->next_avail_desc;
1972 /* Set up our transmit descriptors */
1973 for (j = 0; j < nsegs; j++) {
1975 bus_addr_t seg_addr;
1977 tx_buffer = &txr->tx_buffers[i];
1978 ctxd = &txr->tx_base[i];
1979 seg_addr = segs[j].ds_addr;
1980 seg_len = segs[j].ds_len;
1983 ** If this is the last descriptor, we want to
1984 ** split it so we have a small final sentinel
1986 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
1988 ctxd->buffer_addr = htole64(seg_addr);
1989 ctxd->lower.data = htole32(
1990 adapter->txd_cmd | txd_lower | seg_len);
1993 if (++i == adapter->num_tx_desc)
1995 /* Now make the sentinel */
1996 ++txd_used; /* using an extra txd */
1997 ctxd = &txr->tx_base[i];
1998 tx_buffer = &txr->tx_buffers[i];
2000 htole64(seg_addr + seg_len);
2001 ctxd->lower.data = htole32(
2002 adapter->txd_cmd | txd_lower | 4);
2006 if (++i == adapter->num_tx_desc)
2009 ctxd->buffer_addr = htole64(seg_addr);
2010 ctxd->lower.data = htole32(
2011 adapter->txd_cmd | txd_lower | seg_len);
2015 if (++i == adapter->num_tx_desc)
2018 tx_buffer->m_head = NULL;
2019 tx_buffer->next_eop = -1;
2022 txr->next_avail_desc = i;
2023 txr->tx_avail -= nsegs;
2024 if (tso_desc) /* TSO used an extra for sentinel */
2025 txr->tx_avail -= txd_used;
2027 tx_buffer->m_head = m_head;
2029 ** Here we swap the map so the last descriptor,
2030 ** which gets the completion interrupt has the
2031 ** real map, and the first descriptor gets the
2032 ** unused map from this descriptor.
2034 tx_buffer_mapped->map = tx_buffer->map;
2035 tx_buffer->map = map;
2036 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
2039 * Last Descriptor of Packet
2040 * needs End Of Packet (EOP)
2041 * and Report Status (RS)
2044 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2046 * Keep track in the first buffer which
2047 * descriptor will be written back
2049 tx_buffer = &txr->tx_buffers[first];
2050 tx_buffer->next_eop = last;
2051 /* Update the watchdog time early and often */
2052 txr->watchdog_time = ticks;
2055 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2056 * that this frame is available to transmit.
2058 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2059 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2060 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
2066 em_set_promisc(struct adapter *adapter)
2068 struct ifnet *ifp = adapter->ifp;
2071 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2073 if (ifp->if_flags & IFF_PROMISC) {
2074 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2075 /* Turn this on if you want to see bad packets */
2077 reg_rctl |= E1000_RCTL_SBP;
2078 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2079 } else if (ifp->if_flags & IFF_ALLMULTI) {
2080 reg_rctl |= E1000_RCTL_MPE;
2081 reg_rctl &= ~E1000_RCTL_UPE;
2082 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2087 em_disable_promisc(struct adapter *adapter)
2091 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2093 reg_rctl &= (~E1000_RCTL_UPE);
2094 reg_rctl &= (~E1000_RCTL_MPE);
2095 reg_rctl &= (~E1000_RCTL_SBP);
2096 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2100 /*********************************************************************
2103 * This routine is called whenever multicast address list is updated.
2105 **********************************************************************/
2108 em_set_multi(struct adapter *adapter)
2110 struct ifnet *ifp = adapter->ifp;
2111 struct ifmultiaddr *ifma;
2113 u8 *mta; /* Multicast array memory */
2116 IOCTL_DEBUGOUT("em_set_multi: begin");
2119 bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
2121 if (adapter->hw.mac.type == e1000_82542 &&
2122 adapter->hw.revision_id == E1000_REVISION_2) {
2123 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2124 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2125 e1000_pci_clear_mwi(&adapter->hw);
2126 reg_rctl |= E1000_RCTL_RST;
2127 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2131 #if __FreeBSD_version < 800000
2134 if_maddr_rlock(ifp);
2136 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2137 if (ifma->ifma_addr->sa_family != AF_LINK)
2140 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2143 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2144 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2147 #if __FreeBSD_version < 800000
2148 IF_ADDR_UNLOCK(ifp);
2150 if_maddr_runlock(ifp);
2152 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2153 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2154 reg_rctl |= E1000_RCTL_MPE;
2155 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2157 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2159 if (adapter->hw.mac.type == e1000_82542 &&
2160 adapter->hw.revision_id == E1000_REVISION_2) {
2161 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2162 reg_rctl &= ~E1000_RCTL_RST;
2163 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2165 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2166 e1000_pci_set_mwi(&adapter->hw);
2171 /*********************************************************************
2174 * This routine checks for link status and updates statistics.
2176 **********************************************************************/
2179 em_local_timer(void *arg)
2181 struct adapter *adapter = arg;
2182 struct ifnet *ifp = adapter->ifp;
2183 struct tx_ring *txr = adapter->tx_rings;
2184 struct rx_ring *rxr = adapter->rx_rings;
2187 EM_CORE_LOCK_ASSERT(adapter);
2189 em_update_link_status(adapter);
2190 em_update_stats_counters(adapter);
2192 /* Reset LAA into RAR[0] on 82571 */
2193 if ((adapter->hw.mac.type == e1000_82571) &&
2194 e1000_get_laa_state_82571(&adapter->hw))
2195 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2197 /* Mask to use in the irq trigger */
2198 if (adapter->msix_mem)
2199 trigger = rxr->ims; /* RX for 82574 */
2201 trigger = E1000_ICS_RXDMT0;
2204 ** Check on the state of the TX queue(s), this
2205 ** can be done without the lock because its RO
2206 ** and the HUNG state will be static if set.
2208 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2209 if ((txr->queue_status == EM_QUEUE_HUNG) &&
2210 (adapter->pause_frames == 0))
2212 /* Schedule a TX tasklet if needed */
2213 if (txr->tx_avail <= EM_MAX_SCATTER)
2214 taskqueue_enqueue(txr->tq, &txr->tx_task);
2217 adapter->pause_frames = 0;
2218 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2219 #ifndef DEVICE_POLLING
2220 /* Trigger an RX interrupt to guarantee mbuf refresh */
2221 E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger);
2225 /* Looks like we're hung */
2226 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2227 device_printf(adapter->dev,
2228 "Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2229 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2230 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2231 device_printf(adapter->dev,"TX(%d) desc avail = %d,"
2232 "Next TX to Clean = %d\n",
2233 txr->me, txr->tx_avail, txr->next_to_clean);
2234 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2235 adapter->watchdog_events++;
2236 adapter->pause_frames = 0;
2237 em_init_locked(adapter);
2242 em_update_link_status(struct adapter *adapter)
2244 struct e1000_hw *hw = &adapter->hw;
2245 struct ifnet *ifp = adapter->ifp;
2246 device_t dev = adapter->dev;
2247 struct tx_ring *txr = adapter->tx_rings;
2250 /* Get the cached link value or read phy for real */
2251 switch (hw->phy.media_type) {
2252 case e1000_media_type_copper:
2253 if (hw->mac.get_link_status) {
2254 /* Do the work to read phy */
2255 e1000_check_for_link(hw);
2256 link_check = !hw->mac.get_link_status;
2257 if (link_check) /* ESB2 fix */
2258 e1000_cfg_on_link_up(hw);
2262 case e1000_media_type_fiber:
2263 e1000_check_for_link(hw);
2264 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2267 case e1000_media_type_internal_serdes:
2268 e1000_check_for_link(hw);
2269 link_check = adapter->hw.mac.serdes_has_link;
2272 case e1000_media_type_unknown:
2276 /* Now check for a transition */
2277 if (link_check && (adapter->link_active == 0)) {
2278 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2279 &adapter->link_duplex);
2280 /* Check if we must disable SPEED_MODE bit on PCI-E */
2281 if ((adapter->link_speed != SPEED_1000) &&
2282 ((hw->mac.type == e1000_82571) ||
2283 (hw->mac.type == e1000_82572))) {
2285 tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2286 tarc0 &= ~SPEED_MODE_BIT;
2287 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2290 device_printf(dev, "Link is up %d Mbps %s\n",
2291 adapter->link_speed,
2292 ((adapter->link_duplex == FULL_DUPLEX) ?
2293 "Full Duplex" : "Half Duplex"));
2294 adapter->link_active = 1;
2295 adapter->smartspeed = 0;
2296 ifp->if_baudrate = adapter->link_speed * 1000000;
2297 if_link_state_change(ifp, LINK_STATE_UP);
2298 } else if (!link_check && (adapter->link_active == 1)) {
2299 ifp->if_baudrate = adapter->link_speed = 0;
2300 adapter->link_duplex = 0;
2302 device_printf(dev, "Link is Down\n");
2303 adapter->link_active = 0;
2304 /* Link down, disable watchdog */
2305 for (int i = 0; i < adapter->num_queues; i++, txr++)
2306 txr->queue_status = EM_QUEUE_IDLE;
2307 if_link_state_change(ifp, LINK_STATE_DOWN);
2311 /*********************************************************************
2313 * This routine disables all traffic on the adapter by issuing a
2314 * global reset on the MAC and deallocates TX/RX buffers.
2316 * This routine should always be called with BOTH the CORE
2318 **********************************************************************/
2323 struct adapter *adapter = arg;
2324 struct ifnet *ifp = adapter->ifp;
2325 struct tx_ring *txr = adapter->tx_rings;
2327 EM_CORE_LOCK_ASSERT(adapter);
2329 INIT_DEBUGOUT("em_stop: begin");
2331 em_disable_intr(adapter);
2332 callout_stop(&adapter->timer);
2334 /* Tell the stack that the interface is no longer active */
2335 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2336 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2338 /* Unarm watchdog timer. */
2339 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2341 txr->queue_status = EM_QUEUE_IDLE;
2345 e1000_reset_hw(&adapter->hw);
2346 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2348 e1000_led_off(&adapter->hw);
2349 e1000_cleanup_led(&adapter->hw);
2353 /*********************************************************************
2355 * Determine hardware revision.
2357 **********************************************************************/
2359 em_identify_hardware(struct adapter *adapter)
2361 device_t dev = adapter->dev;
2363 /* Make sure our PCI config space has the necessary stuff set */
2364 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2365 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2366 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2367 device_printf(dev, "Memory Access and/or Bus Master bits "
2369 adapter->hw.bus.pci_cmd_word |=
2370 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2371 pci_write_config(dev, PCIR_COMMAND,
2372 adapter->hw.bus.pci_cmd_word, 2);
2375 /* Save off the information about this board */
2376 adapter->hw.vendor_id = pci_get_vendor(dev);
2377 adapter->hw.device_id = pci_get_device(dev);
2378 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2379 adapter->hw.subsystem_vendor_id =
2380 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2381 adapter->hw.subsystem_device_id =
2382 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2384 /* Do Shared Code Init and Setup */
2385 if (e1000_set_mac_type(&adapter->hw)) {
2386 device_printf(dev, "Setup init failure\n");
2392 em_allocate_pci_resources(struct adapter *adapter)
2394 device_t dev = adapter->dev;
2398 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2400 if (adapter->memory == NULL) {
2401 device_printf(dev, "Unable to allocate bus resource: memory\n");
2404 adapter->osdep.mem_bus_space_tag =
2405 rman_get_bustag(adapter->memory);
2406 adapter->osdep.mem_bus_space_handle =
2407 rman_get_bushandle(adapter->memory);
2408 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2410 /* Default to a single queue */
2411 adapter->num_queues = 1;
2414 * Setup MSI/X or MSI if PCI Express
2416 adapter->msix = em_setup_msix(adapter);
2418 adapter->hw.back = &adapter->osdep;
2423 /*********************************************************************
2425 * Setup the Legacy or MSI Interrupt handler
2427 **********************************************************************/
2429 em_allocate_legacy(struct adapter *adapter)
2431 device_t dev = adapter->dev;
2432 struct tx_ring *txr = adapter->tx_rings;
2435 /* Manually turn off all interrupts */
2436 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2438 if (adapter->msix == 1) /* using MSI */
2440 /* We allocate a single interrupt resource */
2441 adapter->res = bus_alloc_resource_any(dev,
2442 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2443 if (adapter->res == NULL) {
2444 device_printf(dev, "Unable to allocate bus resource: "
2450 * Allocate a fast interrupt and the associated
2451 * deferred processing contexts.
2453 TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter);
2454 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2455 taskqueue_thread_enqueue, &adapter->tq);
2456 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que",
2457 device_get_nameunit(adapter->dev));
2458 /* Use a TX only tasklet for local timer */
2459 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2460 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2461 taskqueue_thread_enqueue, &txr->tq);
2462 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2463 device_get_nameunit(adapter->dev));
2464 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2465 if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET,
2466 em_irq_fast, NULL, adapter, &adapter->tag)) != 0) {
2467 device_printf(dev, "Failed to register fast interrupt "
2468 "handler: %d\n", error);
2469 taskqueue_free(adapter->tq);
2477 /*********************************************************************
2479 * Setup the MSIX Interrupt handlers
2480 * This is not really Multiqueue, rather
2481 * its just seperate interrupt vectors
2482 * for TX, RX, and Link.
2484 **********************************************************************/
2486 em_allocate_msix(struct adapter *adapter)
2488 device_t dev = adapter->dev;
2489 struct tx_ring *txr = adapter->tx_rings;
2490 struct rx_ring *rxr = adapter->rx_rings;
2491 int error, rid, vector = 0;
2494 /* Make sure all interrupts are disabled */
2495 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2497 /* First set up ring resources */
2498 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
2503 rxr->res = bus_alloc_resource_any(dev,
2504 SYS_RES_IRQ, &rid, RF_ACTIVE);
2505 if (rxr->res == NULL) {
2507 "Unable to allocate bus resource: "
2508 "RX MSIX Interrupt %d\n", i);
2511 if ((error = bus_setup_intr(dev, rxr->res,
2512 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx,
2513 rxr, &rxr->tag)) != 0) {
2514 device_printf(dev, "Failed to register RX handler");
2517 #if __FreeBSD_version >= 800504
2518 bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i);
2520 rxr->msix = vector++; /* NOTE increment vector for TX */
2521 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr);
2522 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT,
2523 taskqueue_thread_enqueue, &rxr->tq);
2524 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq",
2525 device_get_nameunit(adapter->dev));
2527 ** Set the bit to enable interrupt
2528 ** in E1000_IMS -- bits 20 and 21
2529 ** are for RX0 and RX1, note this has
2530 ** NOTHING to do with the MSIX vector
2532 rxr->ims = 1 << (20 + i);
2533 adapter->ivars |= (8 | rxr->msix) << (i * 4);
2537 txr->res = bus_alloc_resource_any(dev,
2538 SYS_RES_IRQ, &rid, RF_ACTIVE);
2539 if (txr->res == NULL) {
2541 "Unable to allocate bus resource: "
2542 "TX MSIX Interrupt %d\n", i);
2545 if ((error = bus_setup_intr(dev, txr->res,
2546 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx,
2547 txr, &txr->tag)) != 0) {
2548 device_printf(dev, "Failed to register TX handler");
2551 #if __FreeBSD_version >= 800504
2552 bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i);
2554 txr->msix = vector++; /* Increment vector for next pass */
2555 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2556 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2557 taskqueue_thread_enqueue, &txr->tq);
2558 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2559 device_get_nameunit(adapter->dev));
2561 ** Set the bit to enable interrupt
2562 ** in E1000_IMS -- bits 22 and 23
2563 ** are for TX0 and TX1, note this has
2564 ** NOTHING to do with the MSIX vector
2566 txr->ims = 1 << (22 + i);
2567 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4));
2570 /* Link interrupt */
2572 adapter->res = bus_alloc_resource_any(dev,
2573 SYS_RES_IRQ, &rid, RF_ACTIVE);
2574 if (!adapter->res) {
2575 device_printf(dev,"Unable to allocate "
2576 "bus resource: Link interrupt [%d]\n", rid);
2579 /* Set the link handler function */
2580 error = bus_setup_intr(dev, adapter->res,
2581 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2582 em_msix_link, adapter, &adapter->tag);
2584 adapter->res = NULL;
2585 device_printf(dev, "Failed to register LINK handler");
2588 #if __FreeBSD_version >= 800504
2589 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2591 adapter->linkvec = vector;
2592 adapter->ivars |= (8 | vector) << 16;
2593 adapter->ivars |= 0x80000000;
2600 em_free_pci_resources(struct adapter *adapter)
2602 device_t dev = adapter->dev;
2603 struct tx_ring *txr;
2604 struct rx_ring *rxr;
2609 ** Release all the queue interrupt resources:
2611 for (int i = 0; i < adapter->num_queues; i++) {
2612 txr = &adapter->tx_rings[i];
2613 rxr = &adapter->rx_rings[i];
2614 /* an early abort? */
2615 if ((txr == NULL) || (rxr == NULL))
2618 if (txr->tag != NULL) {
2619 bus_teardown_intr(dev, txr->res, txr->tag);
2622 if (txr->res != NULL)
2623 bus_release_resource(dev, SYS_RES_IRQ,
2626 if (rxr->tag != NULL) {
2627 bus_teardown_intr(dev, rxr->res, rxr->tag);
2630 if (rxr->res != NULL)
2631 bus_release_resource(dev, SYS_RES_IRQ,
2635 if (adapter->linkvec) /* we are doing MSIX */
2636 rid = adapter->linkvec + 1;
2638 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2640 if (adapter->tag != NULL) {
2641 bus_teardown_intr(dev, adapter->res, adapter->tag);
2642 adapter->tag = NULL;
2645 if (adapter->res != NULL)
2646 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2650 pci_release_msi(dev);
2652 if (adapter->msix_mem != NULL)
2653 bus_release_resource(dev, SYS_RES_MEMORY,
2654 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2656 if (adapter->memory != NULL)
2657 bus_release_resource(dev, SYS_RES_MEMORY,
2658 PCIR_BAR(0), adapter->memory);
2660 if (adapter->flash != NULL)
2661 bus_release_resource(dev, SYS_RES_MEMORY,
2662 EM_FLASH, adapter->flash);
2666 * Setup MSI or MSI/X
2669 em_setup_msix(struct adapter *adapter)
2671 device_t dev = adapter->dev;
2675 ** Setup MSI/X for Hartwell: tests have shown
2676 ** use of two queues to be unstable, and to
2677 ** provide no great gain anyway, so we simply
2678 ** seperate the interrupts and use a single queue.
2680 if ((adapter->hw.mac.type == e1000_82574) &&
2681 (em_enable_msix == TRUE)) {
2682 /* Map the MSIX BAR */
2683 int rid = PCIR_BAR(EM_MSIX_BAR);
2684 adapter->msix_mem = bus_alloc_resource_any(dev,
2685 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2686 if (!adapter->msix_mem) {
2687 /* May not be enabled */
2688 device_printf(adapter->dev,
2689 "Unable to map MSIX table \n");
2692 val = pci_msix_count(dev);
2693 /* We only need 3 vectors */
2696 if ((val != 3) && (val != 5)) {
2697 bus_release_resource(dev, SYS_RES_MEMORY,
2698 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2699 adapter->msix_mem = NULL;
2700 device_printf(adapter->dev,
2701 "MSIX: incorrect vectors, using MSI\n");
2705 if (pci_alloc_msix(dev, &val) == 0) {
2706 device_printf(adapter->dev,
2707 "Using MSIX interrupts "
2708 "with %d vectors\n", val);
2714 val = pci_msi_count(dev);
2715 if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2717 device_printf(adapter->dev,"Using an MSI interrupt\n");
2720 /* Should only happen due to manual configuration */
2721 device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n");
2726 /*********************************************************************
2728 * Initialize the hardware to a configuration
2729 * as specified by the adapter structure.
2731 **********************************************************************/
2733 em_reset(struct adapter *adapter)
2735 device_t dev = adapter->dev;
2736 struct ifnet *ifp = adapter->ifp;
2737 struct e1000_hw *hw = &adapter->hw;
2741 INIT_DEBUGOUT("em_reset: begin");
2743 /* Set up smart power down as default off on newer adapters. */
2744 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
2745 hw->mac.type == e1000_82572)) {
2748 /* Speed up time to link by disabling smart power down. */
2749 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2750 phy_tmp &= ~IGP02E1000_PM_SPD;
2751 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2755 * Packet Buffer Allocation (PBA)
2756 * Writing PBA sets the receive portion of the buffer
2757 * the remainder is used for the transmit buffer.
2759 switch (hw->mac.type) {
2760 /* Total Packet Buffer on these is 48K */
2763 case e1000_80003es2lan:
2764 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
2766 case e1000_82573: /* 82573: Total Packet Buffer is 32K */
2767 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
2771 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
2777 case e1000_ich10lan:
2778 /* Boost Receive side for jumbo frames */
2779 if (adapter->max_frame_size > 4096)
2780 pba = E1000_PBA_14K;
2782 pba = E1000_PBA_10K;
2786 pba = E1000_PBA_26K;
2789 if (adapter->max_frame_size > 8192)
2790 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
2792 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
2794 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
2797 * These parameters control the automatic generation (Tx) and
2798 * response (Rx) to Ethernet PAUSE frames.
2799 * - High water mark should allow for at least two frames to be
2800 * received after sending an XOFF.
2801 * - Low water mark works best when it is very near the high water mark.
2802 * This allows the receiver to restart by sending XON when it has
2803 * drained a bit. Here we use an arbitary value of 1500 which will
2804 * restart after one full frame is pulled from the buffer. There
2805 * could be several smaller frames in the buffer and if so they will
2806 * not trigger the XON until their total number reduces the buffer
2808 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2810 rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 );
2811 hw->fc.high_water = rx_buffer_size -
2812 roundup2(adapter->max_frame_size, 1024);
2813 hw->fc.low_water = hw->fc.high_water - 1500;
2815 if (adapter->fc) /* locally set flow control value? */
2816 hw->fc.requested_mode = adapter->fc;
2818 hw->fc.requested_mode = e1000_fc_full;
2820 if (hw->mac.type == e1000_80003es2lan)
2821 hw->fc.pause_time = 0xFFFF;
2823 hw->fc.pause_time = EM_FC_PAUSE_TIME;
2825 hw->fc.send_xon = TRUE;
2827 /* Device specific overrides/settings */
2828 switch (hw->mac.type) {
2830 /* Workaround: no TX flow ctrl for PCH */
2831 hw->fc.requested_mode = e1000_fc_rx_pause;
2832 hw->fc.pause_time = 0xFFFF; /* override */
2833 if (ifp->if_mtu > ETHERMTU) {
2834 hw->fc.high_water = 0x3500;
2835 hw->fc.low_water = 0x1500;
2837 hw->fc.high_water = 0x5000;
2838 hw->fc.low_water = 0x3000;
2840 hw->fc.refresh_time = 0x1000;
2843 hw->fc.high_water = 0x5C20;
2844 hw->fc.low_water = 0x5048;
2845 hw->fc.pause_time = 0x0650;
2846 hw->fc.refresh_time = 0x0400;
2847 /* Jumbos need adjusted PBA */
2848 if (ifp->if_mtu > ETHERMTU)
2849 E1000_WRITE_REG(hw, E1000_PBA, 12);
2851 E1000_WRITE_REG(hw, E1000_PBA, 26);
2854 case e1000_ich10lan:
2855 if (ifp->if_mtu > ETHERMTU) {
2856 hw->fc.high_water = 0x2800;
2857 hw->fc.low_water = hw->fc.high_water - 8;
2860 /* else fall thru */
2862 if (hw->mac.type == e1000_80003es2lan)
2863 hw->fc.pause_time = 0xFFFF;
2867 /* Issue a global reset */
2869 E1000_WRITE_REG(hw, E1000_WUC, 0);
2870 em_disable_aspm(adapter);
2872 if (e1000_init_hw(hw) < 0) {
2873 device_printf(dev, "Hardware Initialization Failed\n");
2877 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
2878 e1000_get_phy_info(hw);
2879 e1000_check_for_link(hw);
2883 /*********************************************************************
2885 * Setup networking device structure and register an interface.
2887 **********************************************************************/
2889 em_setup_interface(device_t dev, struct adapter *adapter)
2893 INIT_DEBUGOUT("em_setup_interface: begin");
2895 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2897 device_printf(dev, "can not allocate ifnet structure\n");
2900 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2901 ifp->if_init = em_init;
2902 ifp->if_softc = adapter;
2903 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2904 ifp->if_ioctl = em_ioctl;
2905 ifp->if_start = em_start;
2906 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2907 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2908 IFQ_SET_READY(&ifp->if_snd);
2910 ether_ifattach(ifp, adapter->hw.mac.addr);
2912 ifp->if_capabilities = ifp->if_capenable = 0;
2914 #ifdef EM_MULTIQUEUE
2915 /* Multiqueue stack interface */
2916 ifp->if_transmit = em_mq_start;
2917 ifp->if_qflush = em_qflush;
2920 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2921 ifp->if_capabilities |= IFCAP_TSO4;
2923 * Tell the upper layer(s) we
2924 * support full VLAN capability
2926 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2927 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2930 ifp->if_capenable = ifp->if_capabilities;
2933 ** Don't turn this on by default, if vlans are
2934 ** created on another pseudo device (eg. lagg)
2935 ** then vlan events are not passed thru, breaking
2936 ** operation, but with HW FILTER off it works. If
2937 ** using vlans directly on the em driver you can
2938 ** enable this and get full hardware tag filtering.
2940 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2942 #ifdef DEVICE_POLLING
2943 ifp->if_capabilities |= IFCAP_POLLING;
2946 /* Enable only WOL MAGIC by default */
2948 ifp->if_capabilities |= IFCAP_WOL;
2949 ifp->if_capenable |= IFCAP_WOL_MAGIC;
2953 * Specify the media types supported by this adapter and register
2954 * callbacks to update media and link information
2956 ifmedia_init(&adapter->media, IFM_IMASK,
2957 em_media_change, em_media_status);
2958 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2959 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2960 u_char fiber_type = IFM_1000_SX; /* default type */
2962 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2964 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2966 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2967 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2969 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2971 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2973 if (adapter->hw.phy.type != e1000_phy_ife) {
2974 ifmedia_add(&adapter->media,
2975 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2976 ifmedia_add(&adapter->media,
2977 IFM_ETHER | IFM_1000_T, 0, NULL);
2980 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2981 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2987 * Manage DMA'able memory.
2990 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2994 *(bus_addr_t *) arg = segs[0].ds_addr;
2998 em_dma_malloc(struct adapter *adapter, bus_size_t size,
2999 struct em_dma_alloc *dma, int mapflags)
3003 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3004 EM_DBA_ALIGN, 0, /* alignment, bounds */
3005 BUS_SPACE_MAXADDR, /* lowaddr */
3006 BUS_SPACE_MAXADDR, /* highaddr */
3007 NULL, NULL, /* filter, filterarg */
3010 size, /* maxsegsize */
3012 NULL, /* lockfunc */
3016 device_printf(adapter->dev,
3017 "%s: bus_dma_tag_create failed: %d\n",
3022 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3023 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3025 device_printf(adapter->dev,
3026 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3027 __func__, (uintmax_t)size, error);
3032 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3033 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3034 if (error || dma->dma_paddr == 0) {
3035 device_printf(adapter->dev,
3036 "%s: bus_dmamap_load failed: %d\n",
3044 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3046 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3047 bus_dma_tag_destroy(dma->dma_tag);
3049 dma->dma_map = NULL;
3050 dma->dma_tag = NULL;
3056 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3058 if (dma->dma_tag == NULL)
3060 if (dma->dma_map != NULL) {
3061 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3062 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3063 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3064 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3065 dma->dma_map = NULL;
3067 bus_dma_tag_destroy(dma->dma_tag);
3068 dma->dma_tag = NULL;
3072 /*********************************************************************
3074 * Allocate memory for the transmit and receive rings, and then
3075 * the descriptors associated with each, called only once at attach.
3077 **********************************************************************/
3079 em_allocate_queues(struct adapter *adapter)
3081 device_t dev = adapter->dev;
3082 struct tx_ring *txr = NULL;
3083 struct rx_ring *rxr = NULL;
3084 int rsize, tsize, error = E1000_SUCCESS;
3085 int txconf = 0, rxconf = 0;
3088 /* Allocate the TX ring struct memory */
3089 if (!(adapter->tx_rings =
3090 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3091 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3092 device_printf(dev, "Unable to allocate TX ring memory\n");
3097 /* Now allocate the RX */
3098 if (!(adapter->rx_rings =
3099 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3100 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3101 device_printf(dev, "Unable to allocate RX ring memory\n");
3106 tsize = roundup2(adapter->num_tx_desc *
3107 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
3109 * Now set up the TX queues, txconf is needed to handle the
3110 * possibility that things fail midcourse and we need to
3111 * undo memory gracefully
3113 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3114 /* Set up some basics */
3115 txr = &adapter->tx_rings[i];
3116 txr->adapter = adapter;
3119 /* Initialize the TX lock */
3120 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3121 device_get_nameunit(dev), txr->me);
3122 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3124 if (em_dma_malloc(adapter, tsize,
3125 &txr->txdma, BUS_DMA_NOWAIT)) {
3127 "Unable to allocate TX Descriptor memory\n");
3131 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
3132 bzero((void *)txr->tx_base, tsize);
3134 if (em_allocate_transmit_buffers(txr)) {
3136 "Critical Failure setting up transmit buffers\n");
3140 #if __FreeBSD_version >= 800000
3141 /* Allocate a buf ring */
3142 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3143 M_WAITOK, &txr->tx_mtx);
3148 * Next the RX queues...
3150 rsize = roundup2(adapter->num_rx_desc *
3151 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
3152 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3153 rxr = &adapter->rx_rings[i];
3154 rxr->adapter = adapter;
3157 /* Initialize the RX lock */
3158 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3159 device_get_nameunit(dev), txr->me);
3160 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3162 if (em_dma_malloc(adapter, rsize,
3163 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3165 "Unable to allocate RxDescriptor memory\n");
3169 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr;
3170 bzero((void *)rxr->rx_base, rsize);
3172 /* Allocate receive buffers for the ring*/
3173 if (em_allocate_receive_buffers(rxr)) {
3175 "Critical Failure setting up receive buffers\n");
3184 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3185 em_dma_free(adapter, &rxr->rxdma);
3187 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3188 em_dma_free(adapter, &txr->txdma);
3189 free(adapter->rx_rings, M_DEVBUF);
3191 #if __FreeBSD_version >= 800000
3192 buf_ring_free(txr->br, M_DEVBUF);
3194 free(adapter->tx_rings, M_DEVBUF);
3200 /*********************************************************************
3202 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3203 * the information needed to transmit a packet on the wire. This is
3204 * called only once at attach, setup is done every reset.
3206 **********************************************************************/
3208 em_allocate_transmit_buffers(struct tx_ring *txr)
3210 struct adapter *adapter = txr->adapter;
3211 device_t dev = adapter->dev;
3212 struct em_buffer *txbuf;
3216 * Setup DMA descriptor areas.
3218 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3219 1, 0, /* alignment, bounds */
3220 BUS_SPACE_MAXADDR, /* lowaddr */
3221 BUS_SPACE_MAXADDR, /* highaddr */
3222 NULL, NULL, /* filter, filterarg */
3223 EM_TSO_SIZE, /* maxsize */
3224 EM_MAX_SCATTER, /* nsegments */
3225 PAGE_SIZE, /* maxsegsize */
3227 NULL, /* lockfunc */
3228 NULL, /* lockfuncarg */
3230 device_printf(dev,"Unable to allocate TX DMA tag\n");
3234 if (!(txr->tx_buffers =
3235 (struct em_buffer *) malloc(sizeof(struct em_buffer) *
3236 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3237 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3242 /* Create the descriptor buffer dma maps */
3243 txbuf = txr->tx_buffers;
3244 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3245 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3247 device_printf(dev, "Unable to create TX DMA map\n");
3254 /* We free all, it handles case where we are in the middle */
3255 em_free_transmit_structures(adapter);
3259 /*********************************************************************
3261 * Initialize a transmit ring.
3263 **********************************************************************/
3265 em_setup_transmit_ring(struct tx_ring *txr)
3267 struct adapter *adapter = txr->adapter;
3268 struct em_buffer *txbuf;
3271 struct netmap_adapter *na = NA(adapter->ifp);
3272 struct netmap_slot *slot;
3273 #endif /* DEV_NETMAP */
3275 /* Clear the old descriptor contents */
3278 slot = netmap_reset(na, NR_TX, txr->me, 0);
3279 #endif /* DEV_NETMAP */
3281 bzero((void *)txr->tx_base,
3282 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3284 txr->next_avail_desc = 0;
3285 txr->next_to_clean = 0;
3287 /* Free any existing tx buffers. */
3288 txbuf = txr->tx_buffers;
3289 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3290 if (txbuf->m_head != NULL) {
3291 bus_dmamap_sync(txr->txtag, txbuf->map,
3292 BUS_DMASYNC_POSTWRITE);
3293 bus_dmamap_unload(txr->txtag, txbuf->map);
3294 m_freem(txbuf->m_head);
3295 txbuf->m_head = NULL;
3299 int si = netmap_tidx_n2k(na, txr->me, i);
3303 addr = PNMB(slot + si, &paddr);
3304 txr->tx_base[i].buffer_addr = htole64(paddr);
3305 /* reload the map for netmap mode */
3306 netmap_load_map(txr->txtag, txbuf->map, addr);
3308 #endif /* DEV_NETMAP */
3310 /* clear the watch index */
3311 txbuf->next_eop = -1;
3314 /* Set number of descriptors available */
3315 txr->tx_avail = adapter->num_tx_desc;
3316 txr->queue_status = EM_QUEUE_IDLE;
3318 /* Clear checksum offload context. */
3319 txr->last_hw_offload = 0;
3320 txr->last_hw_ipcss = 0;
3321 txr->last_hw_ipcso = 0;
3322 txr->last_hw_tucss = 0;
3323 txr->last_hw_tucso = 0;
3325 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3326 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3330 /*********************************************************************
3332 * Initialize all transmit rings.
3334 **********************************************************************/
3336 em_setup_transmit_structures(struct adapter *adapter)
3338 struct tx_ring *txr = adapter->tx_rings;
3340 for (int i = 0; i < adapter->num_queues; i++, txr++)
3341 em_setup_transmit_ring(txr);
3346 /*********************************************************************
3348 * Enable transmit unit.
3350 **********************************************************************/
3352 em_initialize_transmit_unit(struct adapter *adapter)
3354 struct tx_ring *txr = adapter->tx_rings;
3355 struct e1000_hw *hw = &adapter->hw;
3356 u32 tctl, tarc, tipg = 0;
3358 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3360 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3361 u64 bus_addr = txr->txdma.dma_paddr;
3362 /* Base and Len of TX Ring */
3363 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3364 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3365 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3366 (u32)(bus_addr >> 32));
3367 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3369 /* Init the HEAD/TAIL indices */
3370 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3371 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3373 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3374 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
3375 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
3377 txr->queue_status = EM_QUEUE_IDLE;
3380 /* Set the default values for the Tx Inter Packet Gap timer */
3381 switch (adapter->hw.mac.type) {
3382 case e1000_80003es2lan:
3383 tipg = DEFAULT_82543_TIPG_IPGR1;
3384 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3385 E1000_TIPG_IPGR2_SHIFT;
3388 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3389 (adapter->hw.phy.media_type ==
3390 e1000_media_type_internal_serdes))
3391 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3393 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3394 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3395 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3398 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3399 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3401 if(adapter->hw.mac.type >= e1000_82540)
3402 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3403 adapter->tx_abs_int_delay.value);
3405 if ((adapter->hw.mac.type == e1000_82571) ||
3406 (adapter->hw.mac.type == e1000_82572)) {
3407 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3408 tarc |= SPEED_MODE_BIT;
3409 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3410 } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3411 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3413 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3414 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3416 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3419 adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3420 if (adapter->tx_int_delay.value > 0)
3421 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3423 /* Program the Transmit Control Register */
3424 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3425 tctl &= ~E1000_TCTL_CT;
3426 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3427 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3429 if (adapter->hw.mac.type >= e1000_82571)
3430 tctl |= E1000_TCTL_MULR;
3432 /* This write will effectively turn on the transmit unit. */
3433 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3438 /*********************************************************************
3440 * Free all transmit rings.
3442 **********************************************************************/
3444 em_free_transmit_structures(struct adapter *adapter)
3446 struct tx_ring *txr = adapter->tx_rings;
3448 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3450 em_free_transmit_buffers(txr);
3451 em_dma_free(adapter, &txr->txdma);
3453 EM_TX_LOCK_DESTROY(txr);
3456 free(adapter->tx_rings, M_DEVBUF);
3459 /*********************************************************************
3461 * Free transmit ring related data structures.
3463 **********************************************************************/
3465 em_free_transmit_buffers(struct tx_ring *txr)
3467 struct adapter *adapter = txr->adapter;
3468 struct em_buffer *txbuf;
3470 INIT_DEBUGOUT("free_transmit_ring: begin");
3472 if (txr->tx_buffers == NULL)
3475 for (int i = 0; i < adapter->num_tx_desc; i++) {
3476 txbuf = &txr->tx_buffers[i];
3477 if (txbuf->m_head != NULL) {
3478 bus_dmamap_sync(txr->txtag, txbuf->map,
3479 BUS_DMASYNC_POSTWRITE);
3480 bus_dmamap_unload(txr->txtag,
3482 m_freem(txbuf->m_head);
3483 txbuf->m_head = NULL;
3484 if (txbuf->map != NULL) {
3485 bus_dmamap_destroy(txr->txtag,
3489 } else if (txbuf->map != NULL) {
3490 bus_dmamap_unload(txr->txtag,
3492 bus_dmamap_destroy(txr->txtag,
3497 #if __FreeBSD_version >= 800000
3498 if (txr->br != NULL)
3499 buf_ring_free(txr->br, M_DEVBUF);
3501 if (txr->tx_buffers != NULL) {
3502 free(txr->tx_buffers, M_DEVBUF);
3503 txr->tx_buffers = NULL;
3505 if (txr->txtag != NULL) {
3506 bus_dma_tag_destroy(txr->txtag);
3513 /*********************************************************************
3514 * The offload context is protocol specific (TCP/UDP) and thus
3515 * only needs to be set when the protocol changes. The occasion
3516 * of a context change can be a performance detriment, and
3517 * might be better just disabled. The reason arises in the way
3518 * in which the controller supports pipelined requests from the
3519 * Tx data DMA. Up to four requests can be pipelined, and they may
3520 * belong to the same packet or to multiple packets. However all
3521 * requests for one packet are issued before a request is issued
3522 * for a subsequent packet and if a request for the next packet
3523 * requires a context change, that request will be stalled
3524 * until the previous request completes. This means setting up
3525 * a new context effectively disables pipelined Tx data DMA which
3526 * in turn greatly slow down performance to send small sized
3528 **********************************************************************/
3530 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3531 struct ip *ip, u32 *txd_upper, u32 *txd_lower)
3533 struct adapter *adapter = txr->adapter;
3534 struct e1000_context_desc *TXD = NULL;
3535 struct em_buffer *tx_buffer;
3539 u8 ipcso, ipcss, tucso, tucss;
3541 ipcss = ipcso = tucss = tucso = 0;
3542 hdr_len = ip_off + (ip->ip_hl << 2);
3543 cur = txr->next_avail_desc;
3545 /* Setup of IP header checksum. */
3546 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3547 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3550 ipcso = ip_off + offsetof(struct ip, ip_sum);
3552 * Start offset for header checksum calculation.
3553 * End offset for header checksum calculation.
3554 * Offset of place to put the checksum.
3556 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3557 TXD->lower_setup.ip_fields.ipcss = ipcss;
3558 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
3559 TXD->lower_setup.ip_fields.ipcso = ipcso;
3560 cmd |= E1000_TXD_CMD_IP;
3563 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3564 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3565 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3566 offload |= CSUM_TCP;
3568 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
3570 * Setting up new checksum offload context for every frames
3571 * takes a lot of processing time for hardware. This also
3572 * reduces performance a lot for small sized frames so avoid
3573 * it if driver can use previously configured checksum
3576 if (txr->last_hw_offload == offload) {
3577 if (offload & CSUM_IP) {
3578 if (txr->last_hw_ipcss == ipcss &&
3579 txr->last_hw_ipcso == ipcso &&
3580 txr->last_hw_tucss == tucss &&
3581 txr->last_hw_tucso == tucso)
3584 if (txr->last_hw_tucss == tucss &&
3585 txr->last_hw_tucso == tucso)
3589 txr->last_hw_offload = offload;
3590 txr->last_hw_tucss = tucss;
3591 txr->last_hw_tucso = tucso;
3593 * Start offset for payload checksum calculation.
3594 * End offset for payload checksum calculation.
3595 * Offset of place to put the checksum.
3597 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3598 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3599 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3600 TXD->upper_setup.tcp_fields.tucso = tucso;
3601 cmd |= E1000_TXD_CMD_TCP;
3602 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3603 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3604 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3606 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
3608 * Setting up new checksum offload context for every frames
3609 * takes a lot of processing time for hardware. This also
3610 * reduces performance a lot for small sized frames so avoid
3611 * it if driver can use previously configured checksum
3614 if (txr->last_hw_offload == offload) {
3615 if (offload & CSUM_IP) {
3616 if (txr->last_hw_ipcss == ipcss &&
3617 txr->last_hw_ipcso == ipcso &&
3618 txr->last_hw_tucss == tucss &&
3619 txr->last_hw_tucso == tucso)
3622 if (txr->last_hw_tucss == tucss &&
3623 txr->last_hw_tucso == tucso)
3627 txr->last_hw_offload = offload;
3628 txr->last_hw_tucss = tucss;
3629 txr->last_hw_tucso = tucso;
3631 * Start offset for header checksum calculation.
3632 * End offset for header checksum calculation.
3633 * Offset of place to put the checksum.
3635 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3636 TXD->upper_setup.tcp_fields.tucss = tucss;
3637 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3638 TXD->upper_setup.tcp_fields.tucso = tucso;
3641 if (offload & CSUM_IP) {
3642 txr->last_hw_ipcss = ipcss;
3643 txr->last_hw_ipcso = ipcso;
3646 TXD->tcp_seg_setup.data = htole32(0);
3647 TXD->cmd_and_length =
3648 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3649 tx_buffer = &txr->tx_buffers[cur];
3650 tx_buffer->m_head = NULL;
3651 tx_buffer->next_eop = -1;
3653 if (++cur == adapter->num_tx_desc)
3657 txr->next_avail_desc = cur;
3661 /**********************************************************************
3663 * Setup work for hardware segmentation offload (TSO)
3665 **********************************************************************/
3667 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3668 struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower)
3670 struct adapter *adapter = txr->adapter;
3671 struct e1000_context_desc *TXD;
3672 struct em_buffer *tx_buffer;
3676 * In theory we can use the same TSO context if and only if
3677 * frame is the same type(IP/TCP) and the same MSS. However
3678 * checking whether a frame has the same IP/TCP structure is
3679 * hard thing so just ignore that and always restablish a
3682 hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2);
3683 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
3684 E1000_TXD_DTYP_D | /* Data descr type */
3685 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
3687 /* IP and/or TCP header checksum calculation and insertion. */
3688 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
3690 cur = txr->next_avail_desc;
3691 tx_buffer = &txr->tx_buffers[cur];
3692 TXD = (struct e1000_context_desc *) &txr->tx_base[cur];
3695 * Start offset for header checksum calculation.
3696 * End offset for header checksum calculation.
3697 * Offset of place put the checksum.
3699 TXD->lower_setup.ip_fields.ipcss = ip_off;
3700 TXD->lower_setup.ip_fields.ipcse =
3701 htole16(ip_off + (ip->ip_hl << 2) - 1);
3702 TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum);
3704 * Start offset for payload checksum calculation.
3705 * End offset for payload checksum calculation.
3706 * Offset of place to put the checksum.
3708 TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2);
3709 TXD->upper_setup.tcp_fields.tucse = 0;
3710 TXD->upper_setup.tcp_fields.tucso =
3711 ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum);
3713 * Payload size per packet w/o any headers.
3714 * Length of all headers up to payload.
3716 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3717 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3719 TXD->cmd_and_length = htole32(adapter->txd_cmd |
3720 E1000_TXD_CMD_DEXT | /* Extended descr */
3721 E1000_TXD_CMD_TSE | /* TSE context */
3722 E1000_TXD_CMD_IP | /* Do IP csum */
3723 E1000_TXD_CMD_TCP | /* Do TCP checksum */
3724 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3726 tx_buffer->m_head = NULL;
3727 tx_buffer->next_eop = -1;
3729 if (++cur == adapter->num_tx_desc)
3733 txr->next_avail_desc = cur;
3738 /**********************************************************************
3740 * Examine each tx_buffer in the used queue. If the hardware is done
3741 * processing the packet then free associated resources. The
3742 * tx_buffer is put back on the free queue.
3744 **********************************************************************/
3746 em_txeof(struct tx_ring *txr)
3748 struct adapter *adapter = txr->adapter;
3749 int first, last, done, processed;
3750 struct em_buffer *tx_buffer;
3751 struct e1000_tx_desc *tx_desc, *eop_desc;
3752 struct ifnet *ifp = adapter->ifp;
3754 EM_TX_LOCK_ASSERT(txr);
3756 if (ifp->if_capenable & IFCAP_NETMAP) {
3757 struct netmap_adapter *na = NA(ifp);
3759 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3761 EM_CORE_LOCK(adapter);
3762 selwakeuppri(&na->tx_rings[na->num_queues + 1].si, PI_NET);
3763 EM_CORE_UNLOCK(adapter);
3767 #endif /* DEV_NETMAP */
3769 /* No work, make sure watchdog is off */
3770 if (txr->tx_avail == adapter->num_tx_desc) {
3771 txr->queue_status = EM_QUEUE_IDLE;
3776 first = txr->next_to_clean;
3777 tx_desc = &txr->tx_base[first];
3778 tx_buffer = &txr->tx_buffers[first];
3779 last = tx_buffer->next_eop;
3780 eop_desc = &txr->tx_base[last];
3783 * What this does is get the index of the
3784 * first descriptor AFTER the EOP of the
3785 * first packet, that way we can do the
3786 * simple comparison on the inner while loop.
3788 if (++last == adapter->num_tx_desc)
3792 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3793 BUS_DMASYNC_POSTREAD);
3795 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3796 /* We clean the range of the packet */
3797 while (first != done) {
3798 tx_desc->upper.data = 0;
3799 tx_desc->lower.data = 0;
3800 tx_desc->buffer_addr = 0;
3804 if (tx_buffer->m_head) {
3805 bus_dmamap_sync(txr->txtag,
3807 BUS_DMASYNC_POSTWRITE);
3808 bus_dmamap_unload(txr->txtag,
3810 m_freem(tx_buffer->m_head);
3811 tx_buffer->m_head = NULL;
3813 tx_buffer->next_eop = -1;
3814 txr->watchdog_time = ticks;
3816 if (++first == adapter->num_tx_desc)
3819 tx_buffer = &txr->tx_buffers[first];
3820 tx_desc = &txr->tx_base[first];
3823 /* See if we can continue to the next packet */
3824 last = tx_buffer->next_eop;
3826 eop_desc = &txr->tx_base[last];
3827 /* Get new done point */
3828 if (++last == adapter->num_tx_desc) last = 0;
3833 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3834 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3836 txr->next_to_clean = first;
3839 ** Watchdog calculation, we know there's
3840 ** work outstanding or the first return
3841 ** would have been taken, so none processed
3842 ** for too long indicates a hang. local timer
3843 ** will examine this and do a reset if needed.
3845 if ((!processed) && ((ticks - txr->watchdog_time) > EM_WATCHDOG))
3846 txr->queue_status = EM_QUEUE_HUNG;
3849 * If we have a minimum free, clear IFF_DRV_OACTIVE
3850 * to tell the stack that it is OK to send packets.
3851 * Notice that all writes of OACTIVE happen under the
3852 * TX lock which, with a single queue, guarantees
3855 if (txr->tx_avail >= EM_MAX_SCATTER)
3856 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3858 /* Disable watchdog if all clean */
3859 if (txr->tx_avail == adapter->num_tx_desc) {
3860 txr->queue_status = EM_QUEUE_IDLE;
3868 /*********************************************************************
3870 * Refresh RX descriptor mbufs from system mbuf buffer pool.
3872 **********************************************************************/
3874 em_refresh_mbufs(struct rx_ring *rxr, int limit)
3876 struct adapter *adapter = rxr->adapter;
3878 bus_dma_segment_t segs[1];
3879 struct em_buffer *rxbuf;
3880 int i, j, error, nsegs;
3881 bool cleaned = FALSE;
3883 i = j = rxr->next_to_refresh;
3885 ** Get one descriptor beyond
3886 ** our work mark to control
3889 if (++j == adapter->num_rx_desc)
3892 while (j != limit) {
3893 rxbuf = &rxr->rx_buffers[i];
3894 if (rxbuf->m_head == NULL) {
3895 m = m_getjcl(M_DONTWAIT, MT_DATA,
3896 M_PKTHDR, adapter->rx_mbuf_sz);
3898 ** If we have a temporary resource shortage
3899 ** that causes a failure, just abort refresh
3900 ** for now, we will return to this point when
3901 ** reinvoked from em_rxeof.
3908 m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz;
3909 m->m_flags |= M_PKTHDR;
3910 m->m_data = m->m_ext.ext_buf;
3912 /* Use bus_dma machinery to setup the memory mapping */
3913 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map,
3914 m, segs, &nsegs, BUS_DMA_NOWAIT);
3916 printf("Refresh mbufs: hdr dmamap load"
3917 " failure - %d\n", error);
3919 rxbuf->m_head = NULL;
3923 bus_dmamap_sync(rxr->rxtag,
3924 rxbuf->map, BUS_DMASYNC_PREREAD);
3925 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr);
3928 i = j; /* Next is precalulated for us */
3929 rxr->next_to_refresh = i;
3930 /* Calculate next controlling index */
3931 if (++j == adapter->num_rx_desc)
3936 ** Update the tail pointer only if,
3937 ** and as far as we have refreshed.
3940 E1000_WRITE_REG(&adapter->hw,
3941 E1000_RDT(rxr->me), rxr->next_to_refresh);
3947 /*********************************************************************
3949 * Allocate memory for rx_buffer structures. Since we use one
3950 * rx_buffer per received packet, the maximum number of rx_buffer's
3951 * that we'll need is equal to the number of receive descriptors
3952 * that we've allocated.
3954 **********************************************************************/
3956 em_allocate_receive_buffers(struct rx_ring *rxr)
3958 struct adapter *adapter = rxr->adapter;
3959 device_t dev = adapter->dev;
3960 struct em_buffer *rxbuf;
3963 rxr->rx_buffers = malloc(sizeof(struct em_buffer) *
3964 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3965 if (rxr->rx_buffers == NULL) {
3966 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3970 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3971 1, 0, /* alignment, bounds */
3972 BUS_SPACE_MAXADDR, /* lowaddr */
3973 BUS_SPACE_MAXADDR, /* highaddr */
3974 NULL, NULL, /* filter, filterarg */
3975 MJUM9BYTES, /* maxsize */
3977 MJUM9BYTES, /* maxsegsize */
3979 NULL, /* lockfunc */
3983 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
3988 rxbuf = rxr->rx_buffers;
3989 for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3990 rxbuf = &rxr->rx_buffers[i];
3991 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
3994 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4003 em_free_receive_structures(adapter);
4008 /*********************************************************************
4010 * Initialize a receive ring and its buffers.
4012 **********************************************************************/
4014 em_setup_receive_ring(struct rx_ring *rxr)
4016 struct adapter *adapter = rxr->adapter;
4017 struct em_buffer *rxbuf;
4018 bus_dma_segment_t seg[1];
4019 int rsize, nsegs, error;
4021 struct netmap_adapter *na = NA(adapter->ifp);
4022 struct netmap_slot *slot;
4026 /* Clear the ring contents */
4028 rsize = roundup2(adapter->num_rx_desc *
4029 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
4030 bzero((void *)rxr->rx_base, rsize);
4032 slot = netmap_reset(na, NR_RX, 0, 0);
4036 ** Free current RX buffer structs and their mbufs
4038 for (int i = 0; i < adapter->num_rx_desc; i++) {
4039 rxbuf = &rxr->rx_buffers[i];
4040 if (rxbuf->m_head != NULL) {
4041 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4042 BUS_DMASYNC_POSTREAD);
4043 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4044 m_freem(rxbuf->m_head);
4045 rxbuf->m_head = NULL; /* mark as freed */
4049 /* Now replenish the mbufs */
4050 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4051 rxbuf = &rxr->rx_buffers[j];
4054 int si = netmap_ridx_n2k(na, rxr->me, j);
4058 addr = PNMB(slot + si, &paddr);
4059 netmap_load_map(rxr->rxtag, rxbuf->map, addr);
4060 /* Update descriptor */
4061 rxr->rx_base[j].buffer_addr = htole64(paddr);
4064 #endif /* DEV_NETMAP */
4065 rxbuf->m_head = m_getjcl(M_DONTWAIT, MT_DATA,
4066 M_PKTHDR, adapter->rx_mbuf_sz);
4067 if (rxbuf->m_head == NULL) {
4071 rxbuf->m_head->m_len = adapter->rx_mbuf_sz;
4072 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */
4073 rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz;
4075 /* Get the memory mapping */
4076 error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
4077 rxbuf->map, rxbuf->m_head, seg,
4078 &nsegs, BUS_DMA_NOWAIT);
4080 m_freem(rxbuf->m_head);
4081 rxbuf->m_head = NULL;
4084 bus_dmamap_sync(rxr->rxtag,
4085 rxbuf->map, BUS_DMASYNC_PREREAD);
4087 /* Update descriptor */
4088 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr);
4090 rxr->next_to_check = 0;
4091 rxr->next_to_refresh = 0;
4092 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4093 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4100 /*********************************************************************
4102 * Initialize all receive rings.
4104 **********************************************************************/
4106 em_setup_receive_structures(struct adapter *adapter)
4108 struct rx_ring *rxr = adapter->rx_rings;
4111 for (q = 0; q < adapter->num_queues; q++, rxr++)
4112 if (em_setup_receive_ring(rxr))
4118 * Free RX buffers allocated so far, we will only handle
4119 * the rings that completed, the failing case will have
4120 * cleaned up for itself. 'q' failed, so its the terminus.
4122 for (int i = 0; i < q; ++i) {
4123 rxr = &adapter->rx_rings[i];
4124 for (int n = 0; n < adapter->num_rx_desc; n++) {
4125 struct em_buffer *rxbuf;
4126 rxbuf = &rxr->rx_buffers[n];
4127 if (rxbuf->m_head != NULL) {
4128 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4129 BUS_DMASYNC_POSTREAD);
4130 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4131 m_freem(rxbuf->m_head);
4132 rxbuf->m_head = NULL;
4135 rxr->next_to_check = 0;
4136 rxr->next_to_refresh = 0;
4142 /*********************************************************************
4144 * Free all receive rings.
4146 **********************************************************************/
4148 em_free_receive_structures(struct adapter *adapter)
4150 struct rx_ring *rxr = adapter->rx_rings;
4152 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4153 em_free_receive_buffers(rxr);
4154 /* Free the ring memory as well */
4155 em_dma_free(adapter, &rxr->rxdma);
4156 EM_RX_LOCK_DESTROY(rxr);
4159 free(adapter->rx_rings, M_DEVBUF);
4163 /*********************************************************************
4165 * Free receive ring data structures
4167 **********************************************************************/
4169 em_free_receive_buffers(struct rx_ring *rxr)
4171 struct adapter *adapter = rxr->adapter;
4172 struct em_buffer *rxbuf = NULL;
4174 INIT_DEBUGOUT("free_receive_buffers: begin");
4176 if (rxr->rx_buffers != NULL) {
4177 for (int i = 0; i < adapter->num_rx_desc; i++) {
4178 rxbuf = &rxr->rx_buffers[i];
4179 if (rxbuf->map != NULL) {
4180 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4181 BUS_DMASYNC_POSTREAD);
4182 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4183 bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
4185 if (rxbuf->m_head != NULL) {
4186 m_freem(rxbuf->m_head);
4187 rxbuf->m_head = NULL;
4190 free(rxr->rx_buffers, M_DEVBUF);
4191 rxr->rx_buffers = NULL;
4192 rxr->next_to_check = 0;
4193 rxr->next_to_refresh = 0;
4196 if (rxr->rxtag != NULL) {
4197 bus_dma_tag_destroy(rxr->rxtag);
4205 /*********************************************************************
4207 * Enable receive unit.
4209 **********************************************************************/
4210 #define MAX_INTS_PER_SEC 8000
4211 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256)
4214 em_initialize_receive_unit(struct adapter *adapter)
4216 struct rx_ring *rxr = adapter->rx_rings;
4217 struct ifnet *ifp = adapter->ifp;
4218 struct e1000_hw *hw = &adapter->hw;
4222 INIT_DEBUGOUT("em_initialize_receive_units: begin");
4225 * Make sure receives are disabled while setting
4226 * up the descriptor ring
4228 rctl = E1000_READ_REG(hw, E1000_RCTL);
4229 /* Do not disable if ever enabled on this hardware */
4230 if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583))
4231 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4233 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4234 adapter->rx_abs_int_delay.value);
4236 * Set the interrupt throttling rate. Value is calculated
4237 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4239 E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
4242 ** When using MSIX interrupts we need to throttle
4243 ** using the EITR register (82574 only)
4245 if (hw->mac.type == e1000_82574) {
4246 for (int i = 0; i < 4; i++)
4247 E1000_WRITE_REG(hw, E1000_EITR_82574(i),
4249 /* Disable accelerated acknowledge */
4250 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4253 if (ifp->if_capenable & IFCAP_RXCSUM) {
4254 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4255 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4256 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4260 ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4261 ** long latencies are observed, like Lenovo X60. This
4262 ** change eliminates the problem, but since having positive
4263 ** values in RDTR is a known source of problems on other
4264 ** platforms another solution is being sought.
4266 if (hw->mac.type == e1000_82573)
4267 E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
4269 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4270 /* Setup the Base and Length of the Rx Descriptor Ring */
4271 bus_addr = rxr->rxdma.dma_paddr;
4272 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4273 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4274 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
4275 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
4276 /* Setup the Head and Tail Descriptor Pointers */
4277 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4280 * an init() while a netmap client is active must
4281 * preserve the rx buffers passed to userspace.
4282 * In this driver it means we adjust RDT to
4283 * something different from na->num_rx_desc - 1.
4285 if (ifp->if_capenable & IFCAP_NETMAP) {
4286 struct netmap_adapter *na = NA(adapter->ifp);
4287 struct netmap_kring *kring = &na->rx_rings[i];
4288 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
4290 E1000_WRITE_REG(hw, E1000_RDT(i), t);
4292 #endif /* DEV_NETMAP */
4293 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1);
4296 /* Set PTHRESH for improved jumbo performance */
4297 if (((adapter->hw.mac.type == e1000_ich9lan) ||
4298 (adapter->hw.mac.type == e1000_pch2lan) ||
4299 (adapter->hw.mac.type == e1000_ich10lan)) &&
4300 (ifp->if_mtu > ETHERMTU)) {
4301 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
4302 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3);
4305 if (adapter->hw.mac.type == e1000_pch2lan) {
4306 if (ifp->if_mtu > ETHERMTU)
4307 e1000_lv_jumbo_workaround_ich8lan(hw, TRUE);
4309 e1000_lv_jumbo_workaround_ich8lan(hw, FALSE);
4312 /* Setup the Receive Control Register */
4313 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4314 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
4315 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
4316 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4319 rctl |= E1000_RCTL_SECRC;
4321 /* Make sure VLAN Filters are off */
4322 rctl &= ~E1000_RCTL_VFE;
4323 rctl &= ~E1000_RCTL_SBP;
4325 if (adapter->rx_mbuf_sz == MCLBYTES)
4326 rctl |= E1000_RCTL_SZ_2048;
4327 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
4328 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4329 else if (adapter->rx_mbuf_sz > MJUMPAGESIZE)
4330 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4332 if (ifp->if_mtu > ETHERMTU)
4333 rctl |= E1000_RCTL_LPE;
4335 rctl &= ~E1000_RCTL_LPE;
4337 /* Write out the settings */
4338 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4344 /*********************************************************************
4346 * This routine executes in interrupt context. It replenishes
4347 * the mbufs in the descriptor and sends data which has been
4348 * dma'ed into host memory to upper layer.
4350 * We loop at most count times if count is > 0, or until done if
4353 * For polling we also now return the number of cleaned packets
4354 *********************************************************************/
4356 em_rxeof(struct rx_ring *rxr, int count, int *done)
4358 struct adapter *adapter = rxr->adapter;
4359 struct ifnet *ifp = adapter->ifp;
4360 struct mbuf *mp, *sendmp;
4363 int i, processed, rxdone = 0;
4365 struct e1000_rx_desc *cur;
4370 if (ifp->if_capenable & IFCAP_NETMAP) {
4371 struct netmap_adapter *na = NA(ifp);
4373 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4375 EM_CORE_LOCK(adapter);
4376 selwakeuppri(&na->rx_rings[na->num_queues + 1].si, PI_NET);
4377 EM_CORE_UNLOCK(adapter);
4380 #endif /* DEV_NETMAP */
4382 for (i = rxr->next_to_check, processed = 0; count != 0;) {
4384 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4387 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4388 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4390 cur = &rxr->rx_base[i];
4391 status = cur->status;
4394 if ((status & E1000_RXD_STAT_DD) == 0)
4397 len = le16toh(cur->length);
4398 eop = (status & E1000_RXD_STAT_EOP) != 0;
4400 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) ||
4401 (rxr->discard == TRUE)) {
4403 ++rxr->rx_discarded;
4404 if (!eop) /* Catch subsequent segs */
4405 rxr->discard = TRUE;
4407 rxr->discard = FALSE;
4408 em_rx_discard(rxr, i);
4412 /* Assign correct length to the current fragment */
4413 mp = rxr->rx_buffers[i].m_head;
4416 /* Trigger for refresh */
4417 rxr->rx_buffers[i].m_head = NULL;
4419 /* First segment? */
4420 if (rxr->fmp == NULL) {
4421 mp->m_pkthdr.len = len;
4422 rxr->fmp = rxr->lmp = mp;
4424 /* Chain mbuf's together */
4425 mp->m_flags &= ~M_PKTHDR;
4426 rxr->lmp->m_next = mp;
4428 rxr->fmp->m_pkthdr.len += len;
4434 sendmp->m_pkthdr.rcvif = ifp;
4436 em_receive_checksum(cur, sendmp);
4437 #ifndef __NO_STRICT_ALIGNMENT
4438 if (adapter->max_frame_size >
4439 (MCLBYTES - ETHER_ALIGN) &&
4440 em_fixup_rx(rxr) != 0)
4443 if (status & E1000_RXD_STAT_VP) {
4444 sendmp->m_pkthdr.ether_vtag =
4445 le16toh(cur->special);
4446 sendmp->m_flags |= M_VLANTAG;
4448 #ifndef __NO_STRICT_ALIGNMENT
4451 rxr->fmp = rxr->lmp = NULL;
4454 /* Zero out the receive descriptors status. */
4456 ++rxdone; /* cumulative for POLL */
4459 /* Advance our pointers to the next descriptor. */
4460 if (++i == adapter->num_rx_desc)
4463 /* Send to the stack */
4464 if (sendmp != NULL) {
4465 rxr->next_to_check = i;
4467 (*ifp->if_input)(ifp, sendmp);
4469 i = rxr->next_to_check;
4472 /* Only refresh mbufs every 8 descriptors */
4473 if (processed == 8) {
4474 em_refresh_mbufs(rxr, i);
4479 /* Catch any remaining refresh work */
4480 if (e1000_rx_unrefreshed(rxr))
4481 em_refresh_mbufs(rxr, i);
4483 rxr->next_to_check = i;
4488 return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4491 static __inline void
4492 em_rx_discard(struct rx_ring *rxr, int i)
4494 struct em_buffer *rbuf;
4496 rbuf = &rxr->rx_buffers[i];
4497 /* Free any previous pieces */
4498 if (rxr->fmp != NULL) {
4499 rxr->fmp->m_flags |= M_PKTHDR;
4505 ** Free buffer and allow em_refresh_mbufs()
4506 ** to clean up and recharge buffer.
4509 m_free(rbuf->m_head);
4510 rbuf->m_head = NULL;
4515 #ifndef __NO_STRICT_ALIGNMENT
4517 * When jumbo frames are enabled we should realign entire payload on
4518 * architecures with strict alignment. This is serious design mistake of 8254x
4519 * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4520 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4521 * payload. On architecures without strict alignment restrictions 8254x still
4522 * performs unaligned memory access which would reduce the performance too.
4523 * To avoid copying over an entire frame to align, we allocate a new mbuf and
4524 * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4525 * existing mbuf chain.
4527 * Be aware, best performance of the 8254x is achived only when jumbo frame is
4528 * not used at all on architectures with strict alignment.
4531 em_fixup_rx(struct rx_ring *rxr)
4533 struct adapter *adapter = rxr->adapter;
4539 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4540 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4541 m->m_data += ETHER_HDR_LEN;
4543 MGETHDR(n, M_DONTWAIT, MT_DATA);
4545 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4546 m->m_data += ETHER_HDR_LEN;
4547 m->m_len -= ETHER_HDR_LEN;
4548 n->m_len = ETHER_HDR_LEN;
4549 M_MOVE_PKTHDR(n, m);
4553 adapter->dropped_pkts++;
4564 /*********************************************************************
4566 * Verify that the hardware indicated that the checksum is valid.
4567 * Inform the stack about the status of checksum so that stack
4568 * doesn't spend time verifying the checksum.
4570 *********************************************************************/
4572 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4574 /* Ignore Checksum bit is set */
4575 if (rx_desc->status & E1000_RXD_STAT_IXSM) {
4576 mp->m_pkthdr.csum_flags = 0;
4580 if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4582 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4583 /* IP Checksum Good */
4584 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4585 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4588 mp->m_pkthdr.csum_flags = 0;
4592 if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4594 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4595 mp->m_pkthdr.csum_flags |=
4596 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4597 mp->m_pkthdr.csum_data = htons(0xffff);
4603 * This routine is run via an vlan
4607 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4609 struct adapter *adapter = ifp->if_softc;
4612 if (ifp->if_softc != arg) /* Not our event */
4615 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */
4618 EM_CORE_LOCK(adapter);
4619 index = (vtag >> 5) & 0x7F;
4621 adapter->shadow_vfta[index] |= (1 << bit);
4622 ++adapter->num_vlans;
4623 /* Re-init to load the changes */
4624 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4625 em_init_locked(adapter);
4626 EM_CORE_UNLOCK(adapter);
4630 * This routine is run via an vlan
4634 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4636 struct adapter *adapter = ifp->if_softc;
4639 if (ifp->if_softc != arg)
4642 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4645 EM_CORE_LOCK(adapter);
4646 index = (vtag >> 5) & 0x7F;
4648 adapter->shadow_vfta[index] &= ~(1 << bit);
4649 --adapter->num_vlans;
4650 /* Re-init to load the changes */
4651 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4652 em_init_locked(adapter);
4653 EM_CORE_UNLOCK(adapter);
4657 em_setup_vlan_hw_support(struct adapter *adapter)
4659 struct e1000_hw *hw = &adapter->hw;
4663 ** We get here thru init_locked, meaning
4664 ** a soft reset, this has already cleared
4665 ** the VFTA and other state, so if there
4666 ** have been no vlan's registered do nothing.
4668 if (adapter->num_vlans == 0)
4672 ** A soft reset zero's out the VFTA, so
4673 ** we need to repopulate it now.
4675 for (int i = 0; i < EM_VFTA_SIZE; i++)
4676 if (adapter->shadow_vfta[i] != 0)
4677 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4678 i, adapter->shadow_vfta[i]);
4680 reg = E1000_READ_REG(hw, E1000_CTRL);
4681 reg |= E1000_CTRL_VME;
4682 E1000_WRITE_REG(hw, E1000_CTRL, reg);
4684 /* Enable the Filter Table */
4685 reg = E1000_READ_REG(hw, E1000_RCTL);
4686 reg &= ~E1000_RCTL_CFIEN;
4687 reg |= E1000_RCTL_VFE;
4688 E1000_WRITE_REG(hw, E1000_RCTL, reg);
4692 em_enable_intr(struct adapter *adapter)
4694 struct e1000_hw *hw = &adapter->hw;
4695 u32 ims_mask = IMS_ENABLE_MASK;
4697 if (hw->mac.type == e1000_82574) {
4698 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4699 ims_mask |= EM_MSIX_MASK;
4701 E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4705 em_disable_intr(struct adapter *adapter)
4707 struct e1000_hw *hw = &adapter->hw;
4709 if (hw->mac.type == e1000_82574)
4710 E1000_WRITE_REG(hw, EM_EIAC, 0);
4711 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4715 * Bit of a misnomer, what this really means is
4716 * to enable OS management of the system... aka
4717 * to disable special hardware management features
4720 em_init_manageability(struct adapter *adapter)
4722 /* A shared code workaround */
4723 #define E1000_82542_MANC2H E1000_MANC2H
4724 if (adapter->has_manage) {
4725 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4726 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4728 /* disable hardware interception of ARP */
4729 manc &= ~(E1000_MANC_ARP_EN);
4731 /* enable receiving management packets to the host */
4732 manc |= E1000_MANC_EN_MNG2HOST;
4733 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4734 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4735 manc2h |= E1000_MNG2HOST_PORT_623;
4736 manc2h |= E1000_MNG2HOST_PORT_664;
4737 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4738 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4743 * Give control back to hardware management
4744 * controller if there is one.
4747 em_release_manageability(struct adapter *adapter)
4749 if (adapter->has_manage) {
4750 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4752 /* re-enable hardware interception of ARP */
4753 manc |= E1000_MANC_ARP_EN;
4754 manc &= ~E1000_MANC_EN_MNG2HOST;
4756 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4761 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4762 * For ASF and Pass Through versions of f/w this means
4763 * that the driver is loaded. For AMT version type f/w
4764 * this means that the network i/f is open.
4767 em_get_hw_control(struct adapter *adapter)
4771 if (adapter->hw.mac.type == e1000_82573) {
4772 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4773 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4774 swsm | E1000_SWSM_DRV_LOAD);
4778 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4779 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4780 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4785 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4786 * For ASF and Pass Through versions of f/w this means that
4787 * the driver is no longer loaded. For AMT versions of the
4788 * f/w this means that the network i/f is closed.
4791 em_release_hw_control(struct adapter *adapter)
4795 if (!adapter->has_manage)
4798 if (adapter->hw.mac.type == e1000_82573) {
4799 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4800 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4801 swsm & ~E1000_SWSM_DRV_LOAD);
4805 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4806 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4807 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4812 em_is_valid_ether_addr(u8 *addr)
4814 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4816 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4824 ** Parse the interface capabilities with regard
4825 ** to both system management and wake-on-lan for
4829 em_get_wakeup(device_t dev)
4831 struct adapter *adapter = device_get_softc(dev);
4832 u16 eeprom_data = 0, device_id, apme_mask;
4834 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4835 apme_mask = EM_EEPROM_APME;
4837 switch (adapter->hw.mac.type) {
4840 adapter->has_amt = TRUE;
4844 case e1000_80003es2lan:
4845 if (adapter->hw.bus.func == 1) {
4846 e1000_read_nvm(&adapter->hw,
4847 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4850 e1000_read_nvm(&adapter->hw,
4851 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4855 case e1000_ich10lan:
4858 apme_mask = E1000_WUC_APME;
4859 adapter->has_amt = TRUE;
4860 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
4863 e1000_read_nvm(&adapter->hw,
4864 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4867 if (eeprom_data & apme_mask)
4868 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4870 * We have the eeprom settings, now apply the special cases
4871 * where the eeprom may be wrong or the board won't support
4872 * wake on lan on a particular port
4874 device_id = pci_get_device(dev);
4875 switch (device_id) {
4876 case E1000_DEV_ID_82571EB_FIBER:
4877 /* Wake events only supported on port A for dual fiber
4878 * regardless of eeprom setting */
4879 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4880 E1000_STATUS_FUNC_1)
4883 case E1000_DEV_ID_82571EB_QUAD_COPPER:
4884 case E1000_DEV_ID_82571EB_QUAD_FIBER:
4885 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
4886 /* if quad port adapter, disable WoL on all but port A */
4887 if (global_quad_port_a != 0)
4889 /* Reset for multiple quad port adapters */
4890 if (++global_quad_port_a == 4)
4891 global_quad_port_a = 0;
4899 * Enable PCI Wake On Lan capability
4902 em_enable_wakeup(device_t dev)
4904 struct adapter *adapter = device_get_softc(dev);
4905 struct ifnet *ifp = adapter->ifp;
4906 u32 pmc, ctrl, ctrl_ext, rctl;
4909 if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
4912 /* Advertise the wakeup capability */
4913 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4914 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4915 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4916 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4918 if ((adapter->hw.mac.type == e1000_ich8lan) ||
4919 (adapter->hw.mac.type == e1000_pchlan) ||
4920 (adapter->hw.mac.type == e1000_ich9lan) ||
4921 (adapter->hw.mac.type == e1000_ich10lan))
4922 e1000_suspend_workarounds_ich8lan(&adapter->hw);
4924 /* Keep the laser running on Fiber adapters */
4925 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4926 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4927 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4928 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4929 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4933 ** Determine type of Wakeup: note that wol
4934 ** is set with all bits on by default.
4936 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0)
4937 adapter->wol &= ~E1000_WUFC_MAG;
4939 if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0)
4940 adapter->wol &= ~E1000_WUFC_MC;
4942 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4943 rctl |= E1000_RCTL_MPE;
4944 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4947 if ((adapter->hw.mac.type == e1000_pchlan) ||
4948 (adapter->hw.mac.type == e1000_pch2lan)) {
4949 if (em_enable_phy_wakeup(adapter))
4952 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4953 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4956 if (adapter->hw.phy.type == e1000_phy_igp_3)
4957 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
4960 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4961 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4962 if (ifp->if_capenable & IFCAP_WOL)
4963 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4964 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
4970 ** WOL in the newer chipset interfaces (pchlan)
4971 ** require thing to be copied into the phy
4974 em_enable_phy_wakeup(struct adapter *adapter)
4976 struct e1000_hw *hw = &adapter->hw;
4980 /* copy MAC RARs to PHY RARs */
4981 e1000_copy_rx_addrs_to_phy_ich8lan(hw);
4983 /* copy MAC MTA to PHY MTA */
4984 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
4985 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
4986 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
4987 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
4988 (u16)((mreg >> 16) & 0xFFFF));
4991 /* configure PHY Rx Control register */
4992 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
4993 mreg = E1000_READ_REG(hw, E1000_RCTL);
4994 if (mreg & E1000_RCTL_UPE)
4995 preg |= BM_RCTL_UPE;
4996 if (mreg & E1000_RCTL_MPE)
4997 preg |= BM_RCTL_MPE;
4998 preg &= ~(BM_RCTL_MO_MASK);
4999 if (mreg & E1000_RCTL_MO_3)
5000 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5001 << BM_RCTL_MO_SHIFT);
5002 if (mreg & E1000_RCTL_BAM)
5003 preg |= BM_RCTL_BAM;
5004 if (mreg & E1000_RCTL_PMCF)
5005 preg |= BM_RCTL_PMCF;
5006 mreg = E1000_READ_REG(hw, E1000_CTRL);
5007 if (mreg & E1000_CTRL_RFCE)
5008 preg |= BM_RCTL_RFCE;
5009 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
5011 /* enable PHY wakeup in MAC register */
5012 E1000_WRITE_REG(hw, E1000_WUC,
5013 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5014 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
5016 /* configure and enable PHY wakeup in PHY registers */
5017 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
5018 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5020 /* activate PHY wakeup */
5021 ret = hw->phy.ops.acquire(hw);
5023 printf("Could not acquire PHY\n");
5026 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
5027 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
5028 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
5030 printf("Could not read PHY page 769\n");
5033 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5034 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
5036 printf("Could not set PHY Host Wakeup bit\n");
5038 hw->phy.ops.release(hw);
5044 em_led_func(void *arg, int onoff)
5046 struct adapter *adapter = arg;
5048 EM_CORE_LOCK(adapter);
5050 e1000_setup_led(&adapter->hw);
5051 e1000_led_on(&adapter->hw);
5053 e1000_led_off(&adapter->hw);
5054 e1000_cleanup_led(&adapter->hw);
5056 EM_CORE_UNLOCK(adapter);
5060 ** Disable the L0S and L1 LINK states
5063 em_disable_aspm(struct adapter *adapter)
5066 u16 link_cap,link_ctrl;
5067 device_t dev = adapter->dev;
5069 switch (adapter->hw.mac.type) {
5077 if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0)
5079 reg = base + PCIR_EXPRESS_LINK_CAP;
5080 link_cap = pci_read_config(dev, reg, 2);
5081 if ((link_cap & PCIM_LINK_CAP_ASPM) == 0)
5083 reg = base + PCIR_EXPRESS_LINK_CTL;
5084 link_ctrl = pci_read_config(dev, reg, 2);
5085 link_ctrl &= 0xFFFC; /* turn off bit 1 and 2 */
5086 pci_write_config(dev, reg, link_ctrl, 2);
5090 /**********************************************************************
5092 * Update the board statistics counters.
5094 **********************************************************************/
5096 em_update_stats_counters(struct adapter *adapter)
5100 if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5101 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5102 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5103 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5105 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5106 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5107 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5108 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5110 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5111 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5112 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5113 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5114 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5115 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5116 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5118 ** For watchdog management we need to know if we have been
5119 ** paused during the last interval, so capture that here.
5121 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5122 adapter->stats.xoffrxc += adapter->pause_frames;
5123 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5124 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5125 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5126 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5127 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5128 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5129 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5130 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5131 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5132 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5133 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5134 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5136 /* For the 64-bit byte counters the low dword must be read first. */
5137 /* Both registers clear on the read of the high dword */
5139 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
5140 ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
5141 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
5142 ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
5144 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5145 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5146 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5147 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5148 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5150 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5151 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5153 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5154 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5155 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5156 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5157 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5158 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5159 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5160 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5161 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5162 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5164 /* Interrupt Counts */
5166 adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC);
5167 adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC);
5168 adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC);
5169 adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC);
5170 adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC);
5171 adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC);
5172 adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC);
5173 adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC);
5174 adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC);
5176 if (adapter->hw.mac.type >= e1000_82543) {
5177 adapter->stats.algnerrc +=
5178 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5179 adapter->stats.rxerrc +=
5180 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5181 adapter->stats.tncrs +=
5182 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5183 adapter->stats.cexterr +=
5184 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5185 adapter->stats.tsctc +=
5186 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5187 adapter->stats.tsctfc +=
5188 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5192 ifp->if_collisions = adapter->stats.colc;
5195 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5196 adapter->stats.crcerrs + adapter->stats.algnerrc +
5197 adapter->stats.ruc + adapter->stats.roc +
5198 adapter->stats.mpc + adapter->stats.cexterr;
5201 ifp->if_oerrors = adapter->stats.ecol +
5202 adapter->stats.latecol + adapter->watchdog_events;
5205 /* Export a single 32-bit register via a read-only sysctl. */
5207 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5209 struct adapter *adapter;
5212 adapter = oidp->oid_arg1;
5213 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5214 return (sysctl_handle_int(oidp, &val, 0, req));
5218 * Add sysctl variables, one per statistic, to the system.
5221 em_add_hw_stats(struct adapter *adapter)
5223 device_t dev = adapter->dev;
5225 struct tx_ring *txr = adapter->tx_rings;
5226 struct rx_ring *rxr = adapter->rx_rings;
5228 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5229 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5230 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5231 struct e1000_hw_stats *stats = &adapter->stats;
5233 struct sysctl_oid *stat_node, *queue_node, *int_node;
5234 struct sysctl_oid_list *stat_list, *queue_list, *int_list;
5236 #define QUEUE_NAME_LEN 32
5237 char namebuf[QUEUE_NAME_LEN];
5239 /* Driver Statistics */
5240 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5241 CTLFLAG_RD, &adapter->link_irq,
5242 "Link MSIX IRQ Handled");
5243 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail",
5244 CTLFLAG_RD, &adapter->mbuf_alloc_failed,
5246 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail",
5247 CTLFLAG_RD, &adapter->mbuf_cluster_failed,
5248 "Std mbuf cluster failed");
5249 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5250 CTLFLAG_RD, &adapter->dropped_pkts,
5251 "Driver dropped packets");
5252 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5253 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5254 "Driver tx dma failure in xmit");
5255 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5256 CTLFLAG_RD, &adapter->rx_overruns,
5258 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5259 CTLFLAG_RD, &adapter->watchdog_events,
5260 "Watchdog timeouts");
5262 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
5263 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
5264 em_sysctl_reg_handler, "IU",
5265 "Device Control Register");
5266 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
5267 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
5268 em_sysctl_reg_handler, "IU",
5269 "Receiver Control Register");
5270 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5271 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5272 "Flow Control High Watermark");
5273 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5274 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5275 "Flow Control Low Watermark");
5277 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5278 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5279 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5280 CTLFLAG_RD, NULL, "Queue Name");
5281 queue_list = SYSCTL_CHILDREN(queue_node);
5283 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5284 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5286 em_sysctl_reg_handler, "IU",
5287 "Transmit Descriptor Head");
5288 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5289 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5291 em_sysctl_reg_handler, "IU",
5292 "Transmit Descriptor Tail");
5293 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
5294 CTLFLAG_RD, &txr->tx_irq,
5295 "Queue MSI-X Transmit Interrupts");
5296 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail",
5297 CTLFLAG_RD, &txr->no_desc_avail,
5298 "Queue No Descriptor Available");
5300 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5301 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5303 em_sysctl_reg_handler, "IU",
5304 "Receive Descriptor Head");
5305 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5306 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5308 em_sysctl_reg_handler, "IU",
5309 "Receive Descriptor Tail");
5310 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
5311 CTLFLAG_RD, &rxr->rx_irq,
5312 "Queue MSI-X Receive Interrupts");
5315 /* MAC stats get their own sub node */
5317 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5318 CTLFLAG_RD, NULL, "Statistics");
5319 stat_list = SYSCTL_CHILDREN(stat_node);
5321 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5322 CTLFLAG_RD, &stats->ecol,
5323 "Excessive collisions");
5324 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
5325 CTLFLAG_RD, &stats->scc,
5326 "Single collisions");
5327 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5328 CTLFLAG_RD, &stats->mcc,
5329 "Multiple collisions");
5330 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
5331 CTLFLAG_RD, &stats->latecol,
5333 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
5334 CTLFLAG_RD, &stats->colc,
5336 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5337 CTLFLAG_RD, &adapter->stats.symerrs,
5339 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5340 CTLFLAG_RD, &adapter->stats.sec,
5342 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
5343 CTLFLAG_RD, &adapter->stats.dc,
5345 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5346 CTLFLAG_RD, &adapter->stats.mpc,
5348 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5349 CTLFLAG_RD, &adapter->stats.rnbc,
5350 "Receive No Buffers");
5351 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5352 CTLFLAG_RD, &adapter->stats.ruc,
5353 "Receive Undersize");
5354 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5355 CTLFLAG_RD, &adapter->stats.rfc,
5356 "Fragmented Packets Received ");
5357 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5358 CTLFLAG_RD, &adapter->stats.roc,
5359 "Oversized Packets Received");
5360 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5361 CTLFLAG_RD, &adapter->stats.rjc,
5363 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5364 CTLFLAG_RD, &adapter->stats.rxerrc,
5366 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5367 CTLFLAG_RD, &adapter->stats.crcerrs,
5369 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5370 CTLFLAG_RD, &adapter->stats.algnerrc,
5371 "Alignment Errors");
5372 /* On 82575 these are collision counts */
5373 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5374 CTLFLAG_RD, &adapter->stats.cexterr,
5375 "Collision/Carrier extension errors");
5376 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5377 CTLFLAG_RD, &adapter->stats.xonrxc,
5379 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5380 CTLFLAG_RD, &adapter->stats.xontxc,
5382 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5383 CTLFLAG_RD, &adapter->stats.xoffrxc,
5385 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5386 CTLFLAG_RD, &adapter->stats.xofftxc,
5387 "XOFF Transmitted");
5389 /* Packet Reception Stats */
5390 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5391 CTLFLAG_RD, &adapter->stats.tpr,
5392 "Total Packets Received ");
5393 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5394 CTLFLAG_RD, &adapter->stats.gprc,
5395 "Good Packets Received");
5396 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5397 CTLFLAG_RD, &adapter->stats.bprc,
5398 "Broadcast Packets Received");
5399 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5400 CTLFLAG_RD, &adapter->stats.mprc,
5401 "Multicast Packets Received");
5402 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5403 CTLFLAG_RD, &adapter->stats.prc64,
5404 "64 byte frames received ");
5405 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5406 CTLFLAG_RD, &adapter->stats.prc127,
5407 "65-127 byte frames received");
5408 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5409 CTLFLAG_RD, &adapter->stats.prc255,
5410 "128-255 byte frames received");
5411 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5412 CTLFLAG_RD, &adapter->stats.prc511,
5413 "256-511 byte frames received");
5414 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5415 CTLFLAG_RD, &adapter->stats.prc1023,
5416 "512-1023 byte frames received");
5417 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5418 CTLFLAG_RD, &adapter->stats.prc1522,
5419 "1023-1522 byte frames received");
5420 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5421 CTLFLAG_RD, &adapter->stats.gorc,
5422 "Good Octets Received");
5424 /* Packet Transmission Stats */
5425 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5426 CTLFLAG_RD, &adapter->stats.gotc,
5427 "Good Octets Transmitted");
5428 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5429 CTLFLAG_RD, &adapter->stats.tpt,
5430 "Total Packets Transmitted");
5431 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5432 CTLFLAG_RD, &adapter->stats.gptc,
5433 "Good Packets Transmitted");
5434 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5435 CTLFLAG_RD, &adapter->stats.bptc,
5436 "Broadcast Packets Transmitted");
5437 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5438 CTLFLAG_RD, &adapter->stats.mptc,
5439 "Multicast Packets Transmitted");
5440 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5441 CTLFLAG_RD, &adapter->stats.ptc64,
5442 "64 byte frames transmitted ");
5443 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5444 CTLFLAG_RD, &adapter->stats.ptc127,
5445 "65-127 byte frames transmitted");
5446 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5447 CTLFLAG_RD, &adapter->stats.ptc255,
5448 "128-255 byte frames transmitted");
5449 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5450 CTLFLAG_RD, &adapter->stats.ptc511,
5451 "256-511 byte frames transmitted");
5452 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5453 CTLFLAG_RD, &adapter->stats.ptc1023,
5454 "512-1023 byte frames transmitted");
5455 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5456 CTLFLAG_RD, &adapter->stats.ptc1522,
5457 "1024-1522 byte frames transmitted");
5458 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5459 CTLFLAG_RD, &adapter->stats.tsctc,
5460 "TSO Contexts Transmitted");
5461 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5462 CTLFLAG_RD, &adapter->stats.tsctfc,
5463 "TSO Contexts Failed");
5466 /* Interrupt Stats */
5468 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5469 CTLFLAG_RD, NULL, "Interrupt Statistics");
5470 int_list = SYSCTL_CHILDREN(int_node);
5472 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts",
5473 CTLFLAG_RD, &adapter->stats.iac,
5474 "Interrupt Assertion Count");
5476 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5477 CTLFLAG_RD, &adapter->stats.icrxptc,
5478 "Interrupt Cause Rx Pkt Timer Expire Count");
5480 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5481 CTLFLAG_RD, &adapter->stats.icrxatc,
5482 "Interrupt Cause Rx Abs Timer Expire Count");
5484 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5485 CTLFLAG_RD, &adapter->stats.ictxptc,
5486 "Interrupt Cause Tx Pkt Timer Expire Count");
5488 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5489 CTLFLAG_RD, &adapter->stats.ictxatc,
5490 "Interrupt Cause Tx Abs Timer Expire Count");
5492 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5493 CTLFLAG_RD, &adapter->stats.ictxqec,
5494 "Interrupt Cause Tx Queue Empty Count");
5496 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5497 CTLFLAG_RD, &adapter->stats.ictxqmtc,
5498 "Interrupt Cause Tx Queue Min Thresh Count");
5500 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5501 CTLFLAG_RD, &adapter->stats.icrxdmtc,
5502 "Interrupt Cause Rx Desc Min Thresh Count");
5504 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5505 CTLFLAG_RD, &adapter->stats.icrxoc,
5506 "Interrupt Cause Receiver Overrun Count");
5509 /**********************************************************************
5511 * This routine provides a way to dump out the adapter eeprom,
5512 * often a useful debug/service tool. This only dumps the first
5513 * 32 words, stuff that matters is in that extent.
5515 **********************************************************************/
5517 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5519 struct adapter *adapter = (struct adapter *)arg1;
5524 error = sysctl_handle_int(oidp, &result, 0, req);
5526 if (error || !req->newptr)
5530 * This value will cause a hex dump of the
5531 * first 32 16-bit words of the EEPROM to
5535 em_print_nvm_info(adapter);
5541 em_print_nvm_info(struct adapter *adapter)
5546 /* Its a bit crude, but it gets the job done */
5547 printf("\nInterface EEPROM Dump:\n");
5548 printf("Offset\n0x0000 ");
5549 for (i = 0, j = 0; i < 32; i++, j++) {
5550 if (j == 8) { /* Make the offset block */
5552 printf("\n0x00%x0 ",row);
5554 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5555 printf("%04x ", eeprom_data);
5561 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5563 struct em_int_delay_info *info;
5564 struct adapter *adapter;
5566 int error, usecs, ticks;
5568 info = (struct em_int_delay_info *)arg1;
5569 usecs = info->value;
5570 error = sysctl_handle_int(oidp, &usecs, 0, req);
5571 if (error != 0 || req->newptr == NULL)
5573 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5575 info->value = usecs;
5576 ticks = EM_USECS_TO_TICKS(usecs);
5578 adapter = info->adapter;
5580 EM_CORE_LOCK(adapter);
5581 regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5582 regval = (regval & ~0xffff) | (ticks & 0xffff);
5583 /* Handle a few special cases. */
5584 switch (info->offset) {
5589 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5590 /* Don't write 0 into the TIDV register. */
5593 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5596 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5597 EM_CORE_UNLOCK(adapter);
5602 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5603 const char *description, struct em_int_delay_info *info,
5604 int offset, int value)
5606 info->adapter = adapter;
5607 info->offset = offset;
5608 info->value = value;
5609 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5610 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5611 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5612 info, 0, em_sysctl_int_delay, "I", description);
5616 em_set_sysctl_value(struct adapter *adapter, const char *name,
5617 const char *description, int *limit, int value)
5620 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5621 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5622 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5627 ** Set flow control using sysctl:
5628 ** Flow control values:
5635 em_set_flowcntl(SYSCTL_HANDLER_ARGS)
5638 static int input = 3; /* default is full */
5639 struct adapter *adapter = (struct adapter *) arg1;
5641 error = sysctl_handle_int(oidp, &input, 0, req);
5643 if ((error) || (req->newptr == NULL))
5646 if (input == adapter->fc) /* no change? */
5650 case e1000_fc_rx_pause:
5651 case e1000_fc_tx_pause:
5654 adapter->hw.fc.requested_mode = input;
5655 adapter->fc = input;
5662 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
5663 e1000_force_mac_fc(&adapter->hw);
5669 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5671 struct adapter *adapter;
5676 error = sysctl_handle_int(oidp, &result, 0, req);
5678 if (error || !req->newptr)
5682 adapter = (struct adapter *)arg1;
5683 em_print_debug_info(adapter);
5690 ** This routine is meant to be fluid, add whatever is
5691 ** needed for debugging a problem. -jfv
5694 em_print_debug_info(struct adapter *adapter)
5696 device_t dev = adapter->dev;
5697 struct tx_ring *txr = adapter->tx_rings;
5698 struct rx_ring *rxr = adapter->rx_rings;
5700 if (adapter->ifp->if_drv_flags & IFF_DRV_RUNNING)
5701 printf("Interface is RUNNING ");
5703 printf("Interface is NOT RUNNING\n");
5705 if (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE)
5706 printf("and INACTIVE\n");
5708 printf("and ACTIVE\n");
5710 device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5711 E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5712 E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5713 device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5714 E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5715 E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5716 device_printf(dev, "Tx Queue Status = %d\n", txr->queue_status);
5717 device_printf(dev, "TX descriptors avail = %d\n",
5719 device_printf(dev, "Tx Descriptors avail failure = %ld\n",
5720 txr->no_desc_avail);
5721 device_printf(dev, "RX discarded packets = %ld\n",
5723 device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check);
5724 device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh);