1 /******************************************************************************
3 Copyright (c) 2001-2011, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
35 #ifdef HAVE_KERNEL_OPTION_HEADERS
36 #include "opt_device_polling.h"
38 #include "opt_inet6.h"
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #if __FreeBSD_version >= 800000
44 #include <sys/buf_ring.h>
47 #include <sys/endian.h>
48 #include <sys/kernel.h>
49 #include <sys/kthread.h>
50 #include <sys/malloc.h>
52 #include <sys/module.h>
54 #include <sys/socket.h>
55 #include <sys/sockio.h>
56 #include <sys/sysctl.h>
57 #include <sys/taskqueue.h>
58 #include <sys/eventhandler.h>
59 #include <machine/bus.h>
60 #include <machine/resource.h>
63 #include <net/ethernet.h>
65 #include <net/if_arp.h>
66 #include <net/if_dl.h>
67 #include <net/if_media.h>
69 #include <net/if_types.h>
70 #include <net/if_vlan_var.h>
72 #include <netinet/in_systm.h>
73 #include <netinet/in.h>
74 #include <netinet/if_ether.h>
75 #include <netinet/ip.h>
76 #include <netinet/ip6.h>
77 #include <netinet/tcp.h>
78 #include <netinet/udp.h>
80 #include <machine/in_cksum.h>
81 #include <dev/led/led.h>
82 #include <dev/pci/pcivar.h>
83 #include <dev/pci/pcireg.h>
85 #include "e1000_api.h"
86 #include "e1000_82571.h"
89 /*********************************************************************
90 * Set this to one to display debug statistics
91 *********************************************************************/
92 int em_display_debug_stats = 0;
94 /*********************************************************************
96 *********************************************************************/
97 char em_driver_version[] = "7.3.2";
99 /*********************************************************************
100 * PCI Device ID Table
102 * Used by probe to select devices to load on
103 * Last field stores an index into e1000_strings
104 * Last entry must be all 0s
106 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
107 *********************************************************************/
109 static em_vendor_info_t em_vendor_info_array[] =
111 /* Intel(R) PRO/1000 Network Connection */
112 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
113 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
114 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
115 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
116 PCI_ANY_ID, PCI_ANY_ID, 0},
117 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
118 PCI_ANY_ID, PCI_ANY_ID, 0},
119 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
120 PCI_ANY_ID, PCI_ANY_ID, 0},
121 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
122 PCI_ANY_ID, PCI_ANY_ID, 0},
123 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
124 PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
126 PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
128 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
129 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
130 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0},
132 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0},
133 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0},
134 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0},
135 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0},
136 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
137 PCI_ANY_ID, PCI_ANY_ID, 0},
138 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
139 PCI_ANY_ID, PCI_ANY_ID, 0},
140 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
141 PCI_ANY_ID, PCI_ANY_ID, 0},
142 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
143 PCI_ANY_ID, PCI_ANY_ID, 0},
144 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
145 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
147 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
148 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
151 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0},
152 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
153 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
154 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
155 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
156 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0},
157 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
158 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
159 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
160 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0},
161 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0},
162 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0},
163 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
164 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
165 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
166 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
167 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
168 { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
169 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
170 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0},
171 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0},
172 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0},
173 { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
174 { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0},
175 /* required last entry */
179 /*********************************************************************
180 * Table of branding strings for all supported NICs.
181 *********************************************************************/
183 static char *em_strings[] = {
184 "Intel(R) PRO/1000 Network Connection"
187 /*********************************************************************
188 * Function prototypes
189 *********************************************************************/
190 static int em_probe(device_t);
191 static int em_attach(device_t);
192 static int em_detach(device_t);
193 static int em_shutdown(device_t);
194 static int em_suspend(device_t);
195 static int em_resume(device_t);
197 static int em_mq_start(struct ifnet *, struct mbuf *);
198 static int em_mq_start_locked(struct ifnet *,
199 struct tx_ring *, struct mbuf *);
200 static void em_qflush(struct ifnet *);
202 static void em_start(struct ifnet *);
203 static void em_start_locked(struct ifnet *, struct tx_ring *);
205 static int em_ioctl(struct ifnet *, u_long, caddr_t);
206 static void em_init(void *);
207 static void em_init_locked(struct adapter *);
208 static void em_stop(void *);
209 static void em_media_status(struct ifnet *, struct ifmediareq *);
210 static int em_media_change(struct ifnet *);
211 static void em_identify_hardware(struct adapter *);
212 static int em_allocate_pci_resources(struct adapter *);
213 static int em_allocate_legacy(struct adapter *);
214 static int em_allocate_msix(struct adapter *);
215 static int em_allocate_queues(struct adapter *);
216 static int em_setup_msix(struct adapter *);
217 static void em_free_pci_resources(struct adapter *);
218 static void em_local_timer(void *);
219 static void em_reset(struct adapter *);
220 static int em_setup_interface(device_t, struct adapter *);
222 static void em_setup_transmit_structures(struct adapter *);
223 static void em_initialize_transmit_unit(struct adapter *);
224 static int em_allocate_transmit_buffers(struct tx_ring *);
225 static void em_free_transmit_structures(struct adapter *);
226 static void em_free_transmit_buffers(struct tx_ring *);
228 static int em_setup_receive_structures(struct adapter *);
229 static int em_allocate_receive_buffers(struct rx_ring *);
230 static void em_initialize_receive_unit(struct adapter *);
231 static void em_free_receive_structures(struct adapter *);
232 static void em_free_receive_buffers(struct rx_ring *);
234 static void em_enable_intr(struct adapter *);
235 static void em_disable_intr(struct adapter *);
236 static void em_update_stats_counters(struct adapter *);
237 static void em_add_hw_stats(struct adapter *adapter);
238 static void em_txeof(struct tx_ring *);
239 static bool em_rxeof(struct rx_ring *, int, int *);
240 #ifndef __NO_STRICT_ALIGNMENT
241 static int em_fixup_rx(struct rx_ring *);
243 static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *);
244 static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int,
245 struct ip *, u32 *, u32 *);
246 static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *,
247 struct tcphdr *, u32 *, u32 *);
248 static void em_set_promisc(struct adapter *);
249 static void em_disable_promisc(struct adapter *);
250 static void em_set_multi(struct adapter *);
251 static void em_update_link_status(struct adapter *);
252 static void em_refresh_mbufs(struct rx_ring *, int);
253 static void em_register_vlan(void *, struct ifnet *, u16);
254 static void em_unregister_vlan(void *, struct ifnet *, u16);
255 static void em_setup_vlan_hw_support(struct adapter *);
256 static int em_xmit(struct tx_ring *, struct mbuf **);
257 static int em_dma_malloc(struct adapter *, bus_size_t,
258 struct em_dma_alloc *, int);
259 static void em_dma_free(struct adapter *, struct em_dma_alloc *);
260 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
261 static void em_print_nvm_info(struct adapter *);
262 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
263 static void em_print_debug_info(struct adapter *);
264 static int em_is_valid_ether_addr(u8 *);
265 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
266 static void em_add_int_delay_sysctl(struct adapter *, const char *,
267 const char *, struct em_int_delay_info *, int, int);
268 /* Management and WOL Support */
269 static void em_init_manageability(struct adapter *);
270 static void em_release_manageability(struct adapter *);
271 static void em_get_hw_control(struct adapter *);
272 static void em_release_hw_control(struct adapter *);
273 static void em_get_wakeup(device_t);
274 static void em_enable_wakeup(device_t);
275 static int em_enable_phy_wakeup(struct adapter *);
276 static void em_led_func(void *, int);
277 static void em_disable_aspm(struct adapter *);
279 static int em_irq_fast(void *);
282 static void em_msix_tx(void *);
283 static void em_msix_rx(void *);
284 static void em_msix_link(void *);
285 static void em_handle_tx(void *context, int pending);
286 static void em_handle_rx(void *context, int pending);
287 static void em_handle_link(void *context, int pending);
289 static void em_set_sysctl_value(struct adapter *, const char *,
290 const char *, int *, int);
291 static int em_set_flowcntl(SYSCTL_HANDLER_ARGS);
292 static int em_sysctl_eee(SYSCTL_HANDLER_ARGS);
294 static __inline void em_rx_discard(struct rx_ring *, int);
296 #ifdef DEVICE_POLLING
297 static poll_handler_t em_poll;
300 /*********************************************************************
301 * FreeBSD Device Interface Entry Points
302 *********************************************************************/
304 static device_method_t em_methods[] = {
305 /* Device interface */
306 DEVMETHOD(device_probe, em_probe),
307 DEVMETHOD(device_attach, em_attach),
308 DEVMETHOD(device_detach, em_detach),
309 DEVMETHOD(device_shutdown, em_shutdown),
310 DEVMETHOD(device_suspend, em_suspend),
311 DEVMETHOD(device_resume, em_resume),
315 static driver_t em_driver = {
316 "em", em_methods, sizeof(struct adapter),
319 devclass_t em_devclass;
320 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
321 MODULE_DEPEND(em, pci, 1, 1, 1);
322 MODULE_DEPEND(em, ether, 1, 1, 1);
324 /*********************************************************************
325 * Tunable default values.
326 *********************************************************************/
328 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000)
329 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024)
332 /* Allow common code without TSO */
337 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
339 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
340 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
341 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
342 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
343 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &em_tx_int_delay_dflt,
344 0, "Default transmit interrupt delay in usecs");
345 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt,
346 0, "Default receive interrupt delay in usecs");
348 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
349 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
350 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
351 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
352 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN,
353 &em_tx_abs_int_delay_dflt, 0,
354 "Default transmit interrupt delay limit in usecs");
355 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN,
356 &em_rx_abs_int_delay_dflt, 0,
357 "Default receive interrupt delay limit in usecs");
359 static int em_rxd = EM_DEFAULT_RXD;
360 static int em_txd = EM_DEFAULT_TXD;
361 TUNABLE_INT("hw.em.rxd", &em_rxd);
362 TUNABLE_INT("hw.em.txd", &em_txd);
363 SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0,
364 "Number of receive descriptors per queue");
365 SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0,
366 "Number of transmit descriptors per queue");
368 static int em_smart_pwr_down = FALSE;
369 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
370 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down,
371 0, "Set to true to leave smart power down enabled on newer adapters");
373 /* Controls whether promiscuous also shows bad packets */
374 static int em_debug_sbp = FALSE;
375 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
376 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0,
377 "Show bad packets in promiscuous mode");
379 static int em_enable_msix = TRUE;
380 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix);
381 SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0,
382 "Enable MSI-X interrupts");
384 /* How many packets rxeof tries to clean at a time */
385 static int em_rx_process_limit = 100;
386 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
387 SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
388 &em_rx_process_limit, 0,
389 "Maximum number of received packets to process "
390 "at a time, -1 means unlimited");
392 /* Energy efficient ethernet - default to OFF */
393 static int eee_setting = 1;
394 TUNABLE_INT("hw.em.eee_setting", &eee_setting);
395 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0,
396 "Enable Energy Efficient Ethernet");
398 /* Global used in WOL setup with multiport cards */
399 static int global_quad_port_a = 0;
401 #ifdef DEV_NETMAP /* see ixgbe.c for details */
402 #include <dev/netmap/if_em_netmap.h>
403 #endif /* DEV_NETMAP */
405 /*********************************************************************
406 * Device identification routine
408 * em_probe determines if the driver should be loaded on
409 * adapter based on PCI vendor/device id of the adapter.
411 * return BUS_PROBE_DEFAULT on success, positive on failure
412 *********************************************************************/
415 em_probe(device_t dev)
417 char adapter_name[60];
418 u16 pci_vendor_id = 0;
419 u16 pci_device_id = 0;
420 u16 pci_subvendor_id = 0;
421 u16 pci_subdevice_id = 0;
422 em_vendor_info_t *ent;
424 INIT_DEBUGOUT("em_probe: begin");
426 pci_vendor_id = pci_get_vendor(dev);
427 if (pci_vendor_id != EM_VENDOR_ID)
430 pci_device_id = pci_get_device(dev);
431 pci_subvendor_id = pci_get_subvendor(dev);
432 pci_subdevice_id = pci_get_subdevice(dev);
434 ent = em_vendor_info_array;
435 while (ent->vendor_id != 0) {
436 if ((pci_vendor_id == ent->vendor_id) &&
437 (pci_device_id == ent->device_id) &&
439 ((pci_subvendor_id == ent->subvendor_id) ||
440 (ent->subvendor_id == PCI_ANY_ID)) &&
442 ((pci_subdevice_id == ent->subdevice_id) ||
443 (ent->subdevice_id == PCI_ANY_ID))) {
444 sprintf(adapter_name, "%s %s",
445 em_strings[ent->index],
447 device_set_desc_copy(dev, adapter_name);
448 return (BUS_PROBE_DEFAULT);
456 /*********************************************************************
457 * Device initialization routine
459 * The attach entry point is called when the driver is being loaded.
460 * This routine identifies the type of hardware, allocates all resources
461 * and initializes the hardware.
463 * return 0 on success, positive on failure
464 *********************************************************************/
467 em_attach(device_t dev)
469 struct adapter *adapter;
473 INIT_DEBUGOUT("em_attach: begin");
475 adapter = device_get_softc(dev);
476 adapter->dev = adapter->osdep.dev = dev;
478 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
481 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
482 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
483 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
484 em_sysctl_nvm_info, "I", "NVM Information");
486 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
487 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
488 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
489 em_sysctl_debug_info, "I", "Debug Information");
491 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
492 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
493 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
494 em_set_flowcntl, "I", "Flow Control");
496 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
498 /* Determine hardware and mac info */
499 em_identify_hardware(adapter);
501 /* Setup PCI resources */
502 if (em_allocate_pci_resources(adapter)) {
503 device_printf(dev, "Allocation of PCI resources failed\n");
509 ** For ICH8 and family we need to
510 ** map the flash memory, and this
511 ** must happen after the MAC is
514 if ((hw->mac.type == e1000_ich8lan) ||
515 (hw->mac.type == e1000_ich9lan) ||
516 (hw->mac.type == e1000_ich10lan) ||
517 (hw->mac.type == e1000_pchlan) ||
518 (hw->mac.type == e1000_pch2lan)) {
519 int rid = EM_BAR_TYPE_FLASH;
520 adapter->flash = bus_alloc_resource_any(dev,
521 SYS_RES_MEMORY, &rid, RF_ACTIVE);
522 if (adapter->flash == NULL) {
523 device_printf(dev, "Mapping of Flash failed\n");
527 /* This is used in the shared code */
528 hw->flash_address = (u8 *)adapter->flash;
529 adapter->osdep.flash_bus_space_tag =
530 rman_get_bustag(adapter->flash);
531 adapter->osdep.flash_bus_space_handle =
532 rman_get_bushandle(adapter->flash);
535 /* Do Shared Code initialization */
536 if (e1000_setup_init_funcs(hw, TRUE)) {
537 device_printf(dev, "Setup of Shared code failed\n");
542 e1000_get_bus_info(hw);
544 /* Set up some sysctls for the tunable interrupt delays */
545 em_add_int_delay_sysctl(adapter, "rx_int_delay",
546 "receive interrupt delay in usecs", &adapter->rx_int_delay,
547 E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt);
548 em_add_int_delay_sysctl(adapter, "tx_int_delay",
549 "transmit interrupt delay in usecs", &adapter->tx_int_delay,
550 E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt);
551 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
552 "receive interrupt delay limit in usecs",
553 &adapter->rx_abs_int_delay,
554 E1000_REGISTER(hw, E1000_RADV),
555 em_rx_abs_int_delay_dflt);
556 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
557 "transmit interrupt delay limit in usecs",
558 &adapter->tx_abs_int_delay,
559 E1000_REGISTER(hw, E1000_TADV),
560 em_tx_abs_int_delay_dflt);
562 /* Sysctl for limiting the amount of work done in the taskqueue */
563 em_set_sysctl_value(adapter, "rx_processing_limit",
564 "max number of rx packets to process", &adapter->rx_process_limit,
565 em_rx_process_limit);
568 * Validate number of transmit and receive descriptors. It
569 * must not exceed hardware maximum, and must be multiple
570 * of E1000_DBA_ALIGN.
572 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
573 (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) {
574 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
575 EM_DEFAULT_TXD, em_txd);
576 adapter->num_tx_desc = EM_DEFAULT_TXD;
578 adapter->num_tx_desc = em_txd;
580 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
581 (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) {
582 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
583 EM_DEFAULT_RXD, em_rxd);
584 adapter->num_rx_desc = EM_DEFAULT_RXD;
586 adapter->num_rx_desc = em_rxd;
588 hw->mac.autoneg = DO_AUTO_NEG;
589 hw->phy.autoneg_wait_to_complete = FALSE;
590 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
593 if (hw->phy.media_type == e1000_media_type_copper) {
594 hw->phy.mdix = AUTO_ALL_MODES;
595 hw->phy.disable_polarity_correction = FALSE;
596 hw->phy.ms_type = EM_MASTER_SLAVE;
600 * Set the frame limits assuming
601 * standard ethernet sized frames.
603 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
604 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
607 * This controls when hardware reports transmit completion
610 hw->mac.report_tx_early = 1;
613 ** Get queue/ring memory
615 if (em_allocate_queues(adapter)) {
620 /* Allocate multicast array memory. */
621 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
622 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
623 if (adapter->mta == NULL) {
624 device_printf(dev, "Can not allocate multicast setup array\n");
629 /* Check SOL/IDER usage */
630 if (e1000_check_reset_block(hw))
631 device_printf(dev, "PHY reset is blocked"
632 " due to SOL/IDER session.\n");
634 /* Sysctl for setting Energy Efficient Ethernet */
635 hw->dev_spec.ich8lan.eee_disable = eee_setting;
636 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
637 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
638 OID_AUTO, "eee_control", CTLTYPE_INT|CTLFLAG_RW,
639 adapter, 0, em_sysctl_eee, "I",
640 "Disable Energy Efficient Ethernet");
643 ** Start from a known state, this is
644 ** important in reading the nvm and
650 /* Make sure we have a good EEPROM before we read from it */
651 if (e1000_validate_nvm_checksum(hw) < 0) {
653 ** Some PCI-E parts fail the first check due to
654 ** the link being in sleep state, call it again,
655 ** if it fails a second time its a real issue.
657 if (e1000_validate_nvm_checksum(hw) < 0) {
659 "The EEPROM Checksum Is Not Valid\n");
665 /* Copy the permanent MAC address out of the EEPROM */
666 if (e1000_read_mac_addr(hw) < 0) {
667 device_printf(dev, "EEPROM read error while reading MAC"
673 if (!em_is_valid_ether_addr(hw->mac.addr)) {
674 device_printf(dev, "Invalid MAC address\n");
680 ** Do interrupt configuration
682 if (adapter->msix > 1) /* Do MSIX */
683 error = em_allocate_msix(adapter);
684 else /* MSI or Legacy */
685 error = em_allocate_legacy(adapter);
690 * Get Wake-on-Lan and Management info for later use
694 /* Setup OS specific network interface */
695 if (em_setup_interface(dev, adapter) != 0)
700 /* Initialize statistics */
701 em_update_stats_counters(adapter);
703 hw->mac.get_link_status = 1;
704 em_update_link_status(adapter);
706 /* Register for VLAN events */
707 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
708 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
709 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
710 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
712 em_add_hw_stats(adapter);
714 /* Non-AMT based hardware can now take control from firmware */
715 if (adapter->has_manage && !adapter->has_amt)
716 em_get_hw_control(adapter);
718 /* Tell the stack that the interface is not active */
719 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
720 adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE;
722 adapter->led_dev = led_create(em_led_func, adapter,
723 device_get_nameunit(dev));
725 em_netmap_attach(adapter);
726 #endif /* DEV_NETMAP */
728 INIT_DEBUGOUT("em_attach: end");
733 em_free_transmit_structures(adapter);
734 em_free_receive_structures(adapter);
735 em_release_hw_control(adapter);
736 if (adapter->ifp != NULL)
737 if_free(adapter->ifp);
739 em_free_pci_resources(adapter);
740 free(adapter->mta, M_DEVBUF);
741 EM_CORE_LOCK_DESTROY(adapter);
746 /*********************************************************************
747 * Device removal routine
749 * The detach entry point is called when the driver is being removed.
750 * This routine stops the adapter and deallocates all the resources
751 * that were allocated for driver operation.
753 * return 0 on success, positive on failure
754 *********************************************************************/
757 em_detach(device_t dev)
759 struct adapter *adapter = device_get_softc(dev);
760 struct ifnet *ifp = adapter->ifp;
762 INIT_DEBUGOUT("em_detach: begin");
764 /* Make sure VLANS are not using driver */
765 if (adapter->ifp->if_vlantrunk != NULL) {
766 device_printf(dev,"Vlan in use, detach first\n");
770 #ifdef DEVICE_POLLING
771 if (ifp->if_capenable & IFCAP_POLLING)
772 ether_poll_deregister(ifp);
775 if (adapter->led_dev != NULL)
776 led_destroy(adapter->led_dev);
778 EM_CORE_LOCK(adapter);
779 adapter->in_detach = 1;
781 EM_CORE_UNLOCK(adapter);
782 EM_CORE_LOCK_DESTROY(adapter);
784 e1000_phy_hw_reset(&adapter->hw);
786 em_release_manageability(adapter);
787 em_release_hw_control(adapter);
789 /* Unregister VLAN events */
790 if (adapter->vlan_attach != NULL)
791 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
792 if (adapter->vlan_detach != NULL)
793 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
795 ether_ifdetach(adapter->ifp);
796 callout_drain(&adapter->timer);
800 #endif /* DEV_NETMAP */
802 em_free_pci_resources(adapter);
803 bus_generic_detach(dev);
806 em_free_transmit_structures(adapter);
807 em_free_receive_structures(adapter);
809 em_release_hw_control(adapter);
810 free(adapter->mta, M_DEVBUF);
815 /*********************************************************************
817 * Shutdown entry point
819 **********************************************************************/
822 em_shutdown(device_t dev)
824 return em_suspend(dev);
828 * Suspend/resume device methods.
831 em_suspend(device_t dev)
833 struct adapter *adapter = device_get_softc(dev);
835 EM_CORE_LOCK(adapter);
837 em_release_manageability(adapter);
838 em_release_hw_control(adapter);
839 em_enable_wakeup(dev);
841 EM_CORE_UNLOCK(adapter);
843 return bus_generic_suspend(dev);
847 em_resume(device_t dev)
849 struct adapter *adapter = device_get_softc(dev);
850 struct tx_ring *txr = adapter->tx_rings;
851 struct ifnet *ifp = adapter->ifp;
853 EM_CORE_LOCK(adapter);
854 if (adapter->hw.mac.type == e1000_pch2lan)
855 e1000_resume_workarounds_pchlan(&adapter->hw);
856 em_init_locked(adapter);
857 em_init_manageability(adapter);
859 if ((ifp->if_flags & IFF_UP) &&
860 (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
861 for (int i = 0; i < adapter->num_queues; i++, txr++) {
864 if (!drbr_empty(ifp, txr->br))
865 em_mq_start_locked(ifp, txr, NULL);
867 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
868 em_start_locked(ifp, txr);
873 EM_CORE_UNLOCK(adapter);
875 return bus_generic_resume(dev);
880 /*********************************************************************
881 * Multiqueue Transmit routines
883 * em_mq_start is called by the stack to initiate a transmit.
884 * however, if busy the driver can queue the request rather
885 * than do an immediate send. It is this that is an advantage
886 * in this driver, rather than also having multiple tx queues.
887 **********************************************************************/
889 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
891 struct adapter *adapter = txr->adapter;
893 int err = 0, enq = 0;
895 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
896 IFF_DRV_RUNNING || adapter->link_active == 0) {
898 err = drbr_enqueue(ifp, txr->br, m);
904 next = drbr_dequeue(ifp, txr->br);
905 } else if (drbr_needs_enqueue(ifp, txr->br)) {
906 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
908 next = drbr_dequeue(ifp, txr->br);
912 /* Process the queue */
913 while (next != NULL) {
914 if ((err = em_xmit(txr, &next)) != 0) {
916 err = drbr_enqueue(ifp, txr->br, next);
920 ifp->if_obytes += next->m_pkthdr.len;
921 if (next->m_flags & M_MCAST)
923 ETHER_BPF_MTAP(ifp, next);
924 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
926 next = drbr_dequeue(ifp, txr->br);
930 /* Set the watchdog */
931 txr->queue_status = EM_QUEUE_WORKING;
932 txr->watchdog_time = ticks;
935 if (txr->tx_avail < EM_MAX_SCATTER)
937 if (txr->tx_avail < EM_MAX_SCATTER)
938 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
943 ** Multiqueue capable stack interface
946 em_mq_start(struct ifnet *ifp, struct mbuf *m)
948 struct adapter *adapter = ifp->if_softc;
949 struct tx_ring *txr = adapter->tx_rings;
952 if (EM_TX_TRYLOCK(txr)) {
953 error = em_mq_start_locked(ifp, txr, m);
956 error = drbr_enqueue(ifp, txr->br, m);
962 ** Flush all ring buffers
965 em_qflush(struct ifnet *ifp)
967 struct adapter *adapter = ifp->if_softc;
968 struct tx_ring *txr = adapter->tx_rings;
971 for (int i = 0; i < adapter->num_queues; i++, txr++) {
973 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
979 #else /* !EM_MULTIQUEUE */
982 em_start_locked(struct ifnet *ifp, struct tx_ring *txr)
984 struct adapter *adapter = ifp->if_softc;
987 EM_TX_LOCK_ASSERT(txr);
989 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
993 if (!adapter->link_active)
996 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
997 /* Call cleanup if number of TX descriptors low */
998 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
1000 if (txr->tx_avail < EM_MAX_SCATTER) {
1001 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1004 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
1008 * Encapsulation can modify our pointer, and or make it
1009 * NULL on failure. In that event, we can't requeue.
1011 if (em_xmit(txr, &m_head)) {
1014 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1018 /* Send a copy of the frame to the BPF listener */
1019 ETHER_BPF_MTAP(ifp, m_head);
1021 /* Set timeout in case hardware has problems transmitting. */
1022 txr->watchdog_time = ticks;
1023 txr->queue_status = EM_QUEUE_WORKING;
1030 em_start(struct ifnet *ifp)
1032 struct adapter *adapter = ifp->if_softc;
1033 struct tx_ring *txr = adapter->tx_rings;
1035 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1037 em_start_locked(ifp, txr);
1042 #endif /* EM_MULTIQUEUE */
1044 /*********************************************************************
1047 * em_ioctl is called when the user wants to configure the
1050 * return 0 on success, positive on failure
1051 **********************************************************************/
1054 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1056 struct adapter *adapter = ifp->if_softc;
1057 struct ifreq *ifr = (struct ifreq *)data;
1058 #if defined(INET) || defined(INET6)
1059 struct ifaddr *ifa = (struct ifaddr *)data;
1061 bool avoid_reset = FALSE;
1064 if (adapter->in_detach)
1070 if (ifa->ifa_addr->sa_family == AF_INET)
1074 if (ifa->ifa_addr->sa_family == AF_INET6)
1078 ** Calling init results in link renegotiation,
1079 ** so we avoid doing it when possible.
1082 ifp->if_flags |= IFF_UP;
1083 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1086 if (!(ifp->if_flags & IFF_NOARP))
1087 arp_ifinit(ifp, ifa);
1090 error = ether_ioctl(ifp, command, data);
1096 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1098 EM_CORE_LOCK(adapter);
1099 switch (adapter->hw.mac.type) {
1103 case e1000_ich10lan:
1107 case e1000_80003es2lan: /* 9K Jumbo Frame size */
1108 max_frame_size = 9234;
1111 max_frame_size = 4096;
1113 /* Adapters that do not support jumbo frames */
1115 max_frame_size = ETHER_MAX_LEN;
1118 max_frame_size = MAX_JUMBO_FRAME_SIZE;
1120 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1122 EM_CORE_UNLOCK(adapter);
1127 ifp->if_mtu = ifr->ifr_mtu;
1128 adapter->max_frame_size =
1129 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1130 em_init_locked(adapter);
1131 EM_CORE_UNLOCK(adapter);
1135 IOCTL_DEBUGOUT("ioctl rcv'd:\
1136 SIOCSIFFLAGS (Set Interface Flags)");
1137 EM_CORE_LOCK(adapter);
1138 if (ifp->if_flags & IFF_UP) {
1139 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1140 if ((ifp->if_flags ^ adapter->if_flags) &
1141 (IFF_PROMISC | IFF_ALLMULTI)) {
1142 em_disable_promisc(adapter);
1143 em_set_promisc(adapter);
1146 em_init_locked(adapter);
1148 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1150 adapter->if_flags = ifp->if_flags;
1151 EM_CORE_UNLOCK(adapter);
1155 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1156 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1157 EM_CORE_LOCK(adapter);
1158 em_disable_intr(adapter);
1159 em_set_multi(adapter);
1160 #ifdef DEVICE_POLLING
1161 if (!(ifp->if_capenable & IFCAP_POLLING))
1163 em_enable_intr(adapter);
1164 EM_CORE_UNLOCK(adapter);
1168 /* Check SOL/IDER usage */
1169 EM_CORE_LOCK(adapter);
1170 if (e1000_check_reset_block(&adapter->hw)) {
1171 EM_CORE_UNLOCK(adapter);
1172 device_printf(adapter->dev, "Media change is"
1173 " blocked due to SOL/IDER session.\n");
1176 EM_CORE_UNLOCK(adapter);
1179 IOCTL_DEBUGOUT("ioctl rcv'd: \
1180 SIOCxIFMEDIA (Get/Set Interface Media)");
1181 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1187 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1189 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1190 #ifdef DEVICE_POLLING
1191 if (mask & IFCAP_POLLING) {
1192 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1193 error = ether_poll_register(em_poll, ifp);
1196 EM_CORE_LOCK(adapter);
1197 em_disable_intr(adapter);
1198 ifp->if_capenable |= IFCAP_POLLING;
1199 EM_CORE_UNLOCK(adapter);
1201 error = ether_poll_deregister(ifp);
1202 /* Enable interrupt even in error case */
1203 EM_CORE_LOCK(adapter);
1204 em_enable_intr(adapter);
1205 ifp->if_capenable &= ~IFCAP_POLLING;
1206 EM_CORE_UNLOCK(adapter);
1210 if (mask & IFCAP_HWCSUM) {
1211 ifp->if_capenable ^= IFCAP_HWCSUM;
1214 if (mask & IFCAP_TSO4) {
1215 ifp->if_capenable ^= IFCAP_TSO4;
1218 if (mask & IFCAP_VLAN_HWTAGGING) {
1219 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1222 if (mask & IFCAP_VLAN_HWFILTER) {
1223 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1226 if (mask & IFCAP_VLAN_HWTSO) {
1227 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1230 if ((mask & IFCAP_WOL) &&
1231 (ifp->if_capabilities & IFCAP_WOL) != 0) {
1232 if (mask & IFCAP_WOL_MCAST)
1233 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1234 if (mask & IFCAP_WOL_MAGIC)
1235 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1237 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1239 VLAN_CAPABILITIES(ifp);
1244 error = ether_ioctl(ifp, command, data);
1252 /*********************************************************************
1255 * This routine is used in two ways. It is used by the stack as
1256 * init entry point in network interface structure. It is also used
1257 * by the driver as a hw/sw initialization routine to get to a
1260 * return 0 on success, positive on failure
1261 **********************************************************************/
1264 em_init_locked(struct adapter *adapter)
1266 struct ifnet *ifp = adapter->ifp;
1267 device_t dev = adapter->dev;
1269 INIT_DEBUGOUT("em_init: begin");
1271 EM_CORE_LOCK_ASSERT(adapter);
1273 em_disable_intr(adapter);
1274 callout_stop(&adapter->timer);
1276 /* Get the latest mac address, User can use a LAA */
1277 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1280 /* Put the address into the Receive Address Array */
1281 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1284 * With the 82571 adapter, RAR[0] may be overwritten
1285 * when the other port is reset, we make a duplicate
1286 * in RAR[14] for that eventuality, this assures
1287 * the interface continues to function.
1289 if (adapter->hw.mac.type == e1000_82571) {
1290 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1291 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1292 E1000_RAR_ENTRIES - 1);
1295 /* Initialize the hardware */
1297 em_update_link_status(adapter);
1299 /* Setup VLAN support, basic and offload if available */
1300 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1302 /* Set hardware offload abilities */
1303 ifp->if_hwassist = 0;
1304 if (ifp->if_capenable & IFCAP_TXCSUM)
1305 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1306 if (ifp->if_capenable & IFCAP_TSO4)
1307 ifp->if_hwassist |= CSUM_TSO;
1309 /* Configure for OS presence */
1310 em_init_manageability(adapter);
1312 /* Prepare transmit descriptors and buffers */
1313 em_setup_transmit_structures(adapter);
1314 em_initialize_transmit_unit(adapter);
1316 /* Setup Multicast table */
1317 em_set_multi(adapter);
1320 ** Figure out the desired mbuf
1321 ** pool for doing jumbos
1323 if (adapter->max_frame_size <= 2048)
1324 adapter->rx_mbuf_sz = MCLBYTES;
1325 else if (adapter->max_frame_size <= 4096)
1326 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1328 adapter->rx_mbuf_sz = MJUM9BYTES;
1330 /* Prepare receive descriptors and buffers */
1331 if (em_setup_receive_structures(adapter)) {
1332 device_printf(dev, "Could not setup receive structures\n");
1336 em_initialize_receive_unit(adapter);
1338 /* Use real VLAN Filter support? */
1339 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1340 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1341 /* Use real VLAN Filter support */
1342 em_setup_vlan_hw_support(adapter);
1345 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1346 ctrl |= E1000_CTRL_VME;
1347 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1351 /* Don't lose promiscuous settings */
1352 em_set_promisc(adapter);
1354 /* Set the interface as ACTIVE */
1355 ifp->if_drv_flags |= IFF_DRV_RUNNING;
1356 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1358 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1359 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1361 /* MSI/X configuration for 82574 */
1362 if (adapter->hw.mac.type == e1000_82574) {
1364 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1365 tmp |= E1000_CTRL_EXT_PBA_CLR;
1366 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1367 /* Set the IVAR - interrupt vector routing. */
1368 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
1371 #ifdef DEVICE_POLLING
1373 * Only enable interrupts if we are not polling, make sure
1374 * they are off otherwise.
1376 if (ifp->if_capenable & IFCAP_POLLING)
1377 em_disable_intr(adapter);
1379 #endif /* DEVICE_POLLING */
1380 em_enable_intr(adapter);
1382 /* AMT based hardware can now take control from firmware */
1383 if (adapter->has_manage && adapter->has_amt)
1384 em_get_hw_control(adapter);
1390 struct adapter *adapter = arg;
1392 EM_CORE_LOCK(adapter);
1393 em_init_locked(adapter);
1394 EM_CORE_UNLOCK(adapter);
1398 #ifdef DEVICE_POLLING
1399 /*********************************************************************
1401 * Legacy polling routine: note this only works with single queue
1403 *********************************************************************/
1405 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1407 struct adapter *adapter = ifp->if_softc;
1408 struct tx_ring *txr = adapter->tx_rings;
1409 struct rx_ring *rxr = adapter->rx_rings;
1413 EM_CORE_LOCK(adapter);
1414 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1415 EM_CORE_UNLOCK(adapter);
1419 if (cmd == POLL_AND_CHECK_STATUS) {
1420 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1421 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1422 callout_stop(&adapter->timer);
1423 adapter->hw.mac.get_link_status = 1;
1424 em_update_link_status(adapter);
1425 callout_reset(&adapter->timer, hz,
1426 em_local_timer, adapter);
1429 EM_CORE_UNLOCK(adapter);
1431 em_rxeof(rxr, count, &rx_done);
1435 #ifdef EM_MULTIQUEUE
1436 if (!drbr_empty(ifp, txr->br))
1437 em_mq_start_locked(ifp, txr, NULL);
1439 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1440 em_start_locked(ifp, txr);
1446 #endif /* DEVICE_POLLING */
1449 /*********************************************************************
1451 * Fast Legacy/MSI Combined Interrupt Service routine
1453 *********************************************************************/
1455 em_irq_fast(void *arg)
1457 struct adapter *adapter = arg;
1463 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1466 if (reg_icr == 0xffffffff)
1467 return FILTER_STRAY;
1469 /* Definitely not our interrupt. */
1471 return FILTER_STRAY;
1474 * Starting with the 82571 chip, bit 31 should be used to
1475 * determine whether the interrupt belongs to us.
1477 if (adapter->hw.mac.type >= e1000_82571 &&
1478 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1479 return FILTER_STRAY;
1481 em_disable_intr(adapter);
1482 taskqueue_enqueue(adapter->tq, &adapter->que_task);
1484 /* Link status change */
1485 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1486 adapter->hw.mac.get_link_status = 1;
1487 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1490 if (reg_icr & E1000_ICR_RXO)
1491 adapter->rx_overruns++;
1492 return FILTER_HANDLED;
1495 /* Combined RX/TX handler, used by Legacy and MSI */
1497 em_handle_que(void *context, int pending)
1499 struct adapter *adapter = context;
1500 struct ifnet *ifp = adapter->ifp;
1501 struct tx_ring *txr = adapter->tx_rings;
1502 struct rx_ring *rxr = adapter->rx_rings;
1505 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1506 bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1509 #ifdef EM_MULTIQUEUE
1510 if (!drbr_empty(ifp, txr->br))
1511 em_mq_start_locked(ifp, txr, NULL);
1513 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1514 em_start_locked(ifp, txr);
1518 taskqueue_enqueue(adapter->tq, &adapter->que_task);
1523 em_enable_intr(adapter);
1528 /*********************************************************************
1530 * MSIX Interrupt Service Routines
1532 **********************************************************************/
1534 em_msix_tx(void *arg)
1536 struct tx_ring *txr = arg;
1537 struct adapter *adapter = txr->adapter;
1538 struct ifnet *ifp = adapter->ifp;
1543 #ifdef EM_MULTIQUEUE
1544 if (!drbr_empty(ifp, txr->br))
1545 em_mq_start_locked(ifp, txr, NULL);
1547 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1548 em_start_locked(ifp, txr);
1550 /* Reenable this interrupt */
1551 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1556 /*********************************************************************
1558 * MSIX RX Interrupt Service routine
1560 **********************************************************************/
1563 em_msix_rx(void *arg)
1565 struct rx_ring *rxr = arg;
1566 struct adapter *adapter = rxr->adapter;
1570 if (!(adapter->ifp->if_drv_flags & IFF_DRV_RUNNING))
1572 more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1574 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1576 /* Reenable this interrupt */
1577 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1581 /*********************************************************************
1583 * MSIX Link Fast Interrupt Service routine
1585 **********************************************************************/
1587 em_msix_link(void *arg)
1589 struct adapter *adapter = arg;
1592 ++adapter->link_irq;
1593 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1595 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1596 adapter->hw.mac.get_link_status = 1;
1597 em_handle_link(adapter, 0);
1599 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1600 EM_MSIX_LINK | E1000_IMS_LSC);
1605 em_handle_rx(void *context, int pending)
1607 struct rx_ring *rxr = context;
1608 struct adapter *adapter = rxr->adapter;
1611 more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1613 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1615 /* Reenable this interrupt */
1616 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1620 em_handle_tx(void *context, int pending)
1622 struct tx_ring *txr = context;
1623 struct adapter *adapter = txr->adapter;
1624 struct ifnet *ifp = adapter->ifp;
1628 #ifdef EM_MULTIQUEUE
1629 if (!drbr_empty(ifp, txr->br))
1630 em_mq_start_locked(ifp, txr, NULL);
1632 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1633 em_start_locked(ifp, txr);
1635 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1640 em_handle_link(void *context, int pending)
1642 struct adapter *adapter = context;
1643 struct tx_ring *txr = adapter->tx_rings;
1644 struct ifnet *ifp = adapter->ifp;
1646 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1649 EM_CORE_LOCK(adapter);
1650 callout_stop(&adapter->timer);
1651 em_update_link_status(adapter);
1652 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1653 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1654 EM_MSIX_LINK | E1000_IMS_LSC);
1655 if (adapter->link_active) {
1656 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1658 #ifdef EM_MULTIQUEUE
1659 if (!drbr_empty(ifp, txr->br))
1660 em_mq_start_locked(ifp, txr, NULL);
1662 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1663 em_start_locked(ifp, txr);
1668 EM_CORE_UNLOCK(adapter);
1672 /*********************************************************************
1674 * Media Ioctl callback
1676 * This routine is called whenever the user queries the status of
1677 * the interface using ifconfig.
1679 **********************************************************************/
1681 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1683 struct adapter *adapter = ifp->if_softc;
1684 u_char fiber_type = IFM_1000_SX;
1686 INIT_DEBUGOUT("em_media_status: begin");
1688 EM_CORE_LOCK(adapter);
1689 em_update_link_status(adapter);
1691 ifmr->ifm_status = IFM_AVALID;
1692 ifmr->ifm_active = IFM_ETHER;
1694 if (!adapter->link_active) {
1695 EM_CORE_UNLOCK(adapter);
1699 ifmr->ifm_status |= IFM_ACTIVE;
1701 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1702 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1703 ifmr->ifm_active |= fiber_type | IFM_FDX;
1705 switch (adapter->link_speed) {
1707 ifmr->ifm_active |= IFM_10_T;
1710 ifmr->ifm_active |= IFM_100_TX;
1713 ifmr->ifm_active |= IFM_1000_T;
1716 if (adapter->link_duplex == FULL_DUPLEX)
1717 ifmr->ifm_active |= IFM_FDX;
1719 ifmr->ifm_active |= IFM_HDX;
1721 EM_CORE_UNLOCK(adapter);
1724 /*********************************************************************
1726 * Media Ioctl callback
1728 * This routine is called when the user changes speed/duplex using
1729 * media/mediopt option with ifconfig.
1731 **********************************************************************/
1733 em_media_change(struct ifnet *ifp)
1735 struct adapter *adapter = ifp->if_softc;
1736 struct ifmedia *ifm = &adapter->media;
1738 INIT_DEBUGOUT("em_media_change: begin");
1740 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1743 EM_CORE_LOCK(adapter);
1744 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1746 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1747 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1752 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1753 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1756 adapter->hw.mac.autoneg = FALSE;
1757 adapter->hw.phy.autoneg_advertised = 0;
1758 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1759 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1761 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1764 adapter->hw.mac.autoneg = FALSE;
1765 adapter->hw.phy.autoneg_advertised = 0;
1766 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1767 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1769 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1772 device_printf(adapter->dev, "Unsupported media type\n");
1775 em_init_locked(adapter);
1776 EM_CORE_UNLOCK(adapter);
1781 /*********************************************************************
1783 * This routine maps the mbufs to tx descriptors.
1785 * return 0 on success, positive on failure
1786 **********************************************************************/
1789 em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1791 struct adapter *adapter = txr->adapter;
1792 bus_dma_segment_t segs[EM_MAX_SCATTER];
1794 struct em_buffer *tx_buffer, *tx_buffer_mapped;
1795 struct e1000_tx_desc *ctxd = NULL;
1796 struct mbuf *m_head;
1797 struct ether_header *eh;
1798 struct ip *ip = NULL;
1799 struct tcphdr *tp = NULL;
1800 u32 txd_upper, txd_lower, txd_used, txd_saved;
1802 int nsegs, i, j, first, last = 0;
1803 int error, do_tso, tso_desc = 0, remap = 1;
1807 txd_upper = txd_lower = txd_used = txd_saved = 0;
1808 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1812 * Intel recommends entire IP/TCP header length reside in a single
1813 * buffer. If multiple descriptors are used to describe the IP and
1814 * TCP header, each descriptor should describe one or more
1815 * complete headers; descriptors referencing only parts of headers
1816 * are not supported. If all layer headers are not coalesced into
1817 * a single buffer, each buffer should not cross a 4KB boundary,
1818 * or be larger than the maximum read request size.
1819 * Controller also requires modifing IP/TCP header to make TSO work
1820 * so we firstly get a writable mbuf chain then coalesce ethernet/
1821 * IP/TCP header into a single buffer to meet the requirement of
1822 * controller. This also simplifies IP/TCP/UDP checksum offloading
1823 * which also has similiar restrictions.
1825 if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
1826 if (do_tso || (m_head->m_next != NULL &&
1827 m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) {
1828 if (M_WRITABLE(*m_headp) == 0) {
1829 m_head = m_dup(*m_headp, M_DONTWAIT);
1831 if (m_head == NULL) {
1840 * Assume IPv4, we don't have TSO/checksum offload support
1843 ip_off = sizeof(struct ether_header);
1844 m_head = m_pullup(m_head, ip_off);
1845 if (m_head == NULL) {
1849 eh = mtod(m_head, struct ether_header *);
1850 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1851 ip_off = sizeof(struct ether_vlan_header);
1852 m_head = m_pullup(m_head, ip_off);
1853 if (m_head == NULL) {
1858 m_head = m_pullup(m_head, ip_off + sizeof(struct ip));
1859 if (m_head == NULL) {
1863 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1864 poff = ip_off + (ip->ip_hl << 2);
1866 m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1867 if (m_head == NULL) {
1871 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1874 * pull 4 more bytes of data into it.
1876 m_head = m_pullup(m_head, poff + (tp->th_off << 2) + 4);
1877 if (m_head == NULL) {
1881 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1885 * The pseudo TCP checksum does not include TCP payload
1886 * length so driver should recompute the checksum here
1887 * what hardware expect to see. This is adherence of
1888 * Microsoft's Large Send specification.
1890 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1891 tp->th_sum = in_pseudo(ip->ip_src.s_addr,
1892 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
1893 } else if (m_head->m_pkthdr.csum_flags & CSUM_TCP) {
1894 m_head = m_pullup(m_head, poff + sizeof(struct tcphdr));
1895 if (m_head == NULL) {
1899 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1900 m_head = m_pullup(m_head, poff + (tp->th_off << 2));
1901 if (m_head == NULL) {
1905 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1906 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1907 } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) {
1908 m_head = m_pullup(m_head, poff + sizeof(struct udphdr));
1909 if (m_head == NULL) {
1913 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1919 * Map the packet for DMA
1921 * Capture the first descriptor index,
1922 * this descriptor will have the index
1923 * of the EOP which is the only one that
1924 * now gets a DONE bit writeback.
1926 first = txr->next_avail_desc;
1927 tx_buffer = &txr->tx_buffers[first];
1928 tx_buffer_mapped = tx_buffer;
1929 map = tx_buffer->map;
1931 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1932 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1935 * There are two types of errors we can (try) to handle:
1936 * - EFBIG means the mbuf chain was too long and bus_dma ran
1937 * out of segments. Defragment the mbuf chain and try again.
1938 * - ENOMEM means bus_dma could not obtain enough bounce buffers
1939 * at this point in time. Defer sending and try again later.
1940 * All other errors, in particular EINVAL, are fatal and prevent the
1941 * mbuf chain from ever going through. Drop it and report error.
1943 if (error == EFBIG && remap) {
1946 m = m_defrag(*m_headp, M_DONTWAIT);
1948 adapter->mbuf_alloc_failed++;
1955 /* Try it again, but only once */
1958 } else if (error == ENOMEM) {
1959 adapter->no_tx_dma_setup++;
1961 } else if (error != 0) {
1962 adapter->no_tx_dma_setup++;
1969 * TSO Hardware workaround, if this packet is not
1970 * TSO, and is only a single descriptor long, and
1971 * it follows a TSO burst, then we need to add a
1972 * sentinel descriptor to prevent premature writeback.
1974 if ((do_tso == 0) && (txr->tx_tso == TRUE)) {
1977 txr->tx_tso = FALSE;
1980 if (nsegs > (txr->tx_avail - 2)) {
1981 txr->no_desc_avail++;
1982 bus_dmamap_unload(txr->txtag, map);
1987 /* Do hardware assists */
1988 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1989 em_tso_setup(txr, m_head, ip_off, ip, tp,
1990 &txd_upper, &txd_lower);
1991 /* we need to make a final sentinel transmit desc */
1993 } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
1994 em_transmit_checksum_setup(txr, m_head,
1995 ip_off, ip, &txd_upper, &txd_lower);
1997 if (m_head->m_flags & M_VLANTAG) {
1998 /* Set the vlan id. */
2000 (htole16(m_head->m_pkthdr.ether_vtag) << 16);
2001 /* Tell hardware to add tag */
2002 txd_lower |= htole32(E1000_TXD_CMD_VLE);
2005 i = txr->next_avail_desc;
2007 /* Set up our transmit descriptors */
2008 for (j = 0; j < nsegs; j++) {
2010 bus_addr_t seg_addr;
2012 tx_buffer = &txr->tx_buffers[i];
2013 ctxd = &txr->tx_base[i];
2014 seg_addr = segs[j].ds_addr;
2015 seg_len = segs[j].ds_len;
2018 ** If this is the last descriptor, we want to
2019 ** split it so we have a small final sentinel
2021 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2023 ctxd->buffer_addr = htole64(seg_addr);
2024 ctxd->lower.data = htole32(
2025 adapter->txd_cmd | txd_lower | seg_len);
2028 if (++i == adapter->num_tx_desc)
2030 /* Now make the sentinel */
2031 ++txd_used; /* using an extra txd */
2032 ctxd = &txr->tx_base[i];
2033 tx_buffer = &txr->tx_buffers[i];
2035 htole64(seg_addr + seg_len);
2036 ctxd->lower.data = htole32(
2037 adapter->txd_cmd | txd_lower | 4);
2041 if (++i == adapter->num_tx_desc)
2044 ctxd->buffer_addr = htole64(seg_addr);
2045 ctxd->lower.data = htole32(
2046 adapter->txd_cmd | txd_lower | seg_len);
2050 if (++i == adapter->num_tx_desc)
2053 tx_buffer->m_head = NULL;
2054 tx_buffer->next_eop = -1;
2057 txr->next_avail_desc = i;
2058 txr->tx_avail -= nsegs;
2059 if (tso_desc) /* TSO used an extra for sentinel */
2060 txr->tx_avail -= txd_used;
2062 tx_buffer->m_head = m_head;
2064 ** Here we swap the map so the last descriptor,
2065 ** which gets the completion interrupt has the
2066 ** real map, and the first descriptor gets the
2067 ** unused map from this descriptor.
2069 tx_buffer_mapped->map = tx_buffer->map;
2070 tx_buffer->map = map;
2071 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
2074 * Last Descriptor of Packet
2075 * needs End Of Packet (EOP)
2076 * and Report Status (RS)
2079 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2081 * Keep track in the first buffer which
2082 * descriptor will be written back
2084 tx_buffer = &txr->tx_buffers[first];
2085 tx_buffer->next_eop = last;
2086 /* Update the watchdog time early and often */
2087 txr->watchdog_time = ticks;
2090 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2091 * that this frame is available to transmit.
2093 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2094 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2095 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
2101 em_set_promisc(struct adapter *adapter)
2103 struct ifnet *ifp = adapter->ifp;
2106 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2108 if (ifp->if_flags & IFF_PROMISC) {
2109 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2110 /* Turn this on if you want to see bad packets */
2112 reg_rctl |= E1000_RCTL_SBP;
2113 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2114 } else if (ifp->if_flags & IFF_ALLMULTI) {
2115 reg_rctl |= E1000_RCTL_MPE;
2116 reg_rctl &= ~E1000_RCTL_UPE;
2117 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2122 em_disable_promisc(struct adapter *adapter)
2126 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2128 reg_rctl &= (~E1000_RCTL_UPE);
2129 reg_rctl &= (~E1000_RCTL_MPE);
2130 reg_rctl &= (~E1000_RCTL_SBP);
2131 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2135 /*********************************************************************
2138 * This routine is called whenever multicast address list is updated.
2140 **********************************************************************/
2143 em_set_multi(struct adapter *adapter)
2145 struct ifnet *ifp = adapter->ifp;
2146 struct ifmultiaddr *ifma;
2148 u8 *mta; /* Multicast array memory */
2151 IOCTL_DEBUGOUT("em_set_multi: begin");
2154 bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
2156 if (adapter->hw.mac.type == e1000_82542 &&
2157 adapter->hw.revision_id == E1000_REVISION_2) {
2158 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2159 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2160 e1000_pci_clear_mwi(&adapter->hw);
2161 reg_rctl |= E1000_RCTL_RST;
2162 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2166 #if __FreeBSD_version < 800000
2169 if_maddr_rlock(ifp);
2171 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2172 if (ifma->ifma_addr->sa_family != AF_LINK)
2175 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2178 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2179 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2182 #if __FreeBSD_version < 800000
2183 IF_ADDR_UNLOCK(ifp);
2185 if_maddr_runlock(ifp);
2187 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2188 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2189 reg_rctl |= E1000_RCTL_MPE;
2190 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2192 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2194 if (adapter->hw.mac.type == e1000_82542 &&
2195 adapter->hw.revision_id == E1000_REVISION_2) {
2196 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2197 reg_rctl &= ~E1000_RCTL_RST;
2198 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2200 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2201 e1000_pci_set_mwi(&adapter->hw);
2206 /*********************************************************************
2209 * This routine checks for link status and updates statistics.
2211 **********************************************************************/
2214 em_local_timer(void *arg)
2216 struct adapter *adapter = arg;
2217 struct ifnet *ifp = adapter->ifp;
2218 struct tx_ring *txr = adapter->tx_rings;
2219 struct rx_ring *rxr = adapter->rx_rings;
2222 EM_CORE_LOCK_ASSERT(adapter);
2224 em_update_link_status(adapter);
2225 em_update_stats_counters(adapter);
2227 /* Reset LAA into RAR[0] on 82571 */
2228 if ((adapter->hw.mac.type == e1000_82571) &&
2229 e1000_get_laa_state_82571(&adapter->hw))
2230 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2232 /* Mask to use in the irq trigger */
2233 if (adapter->msix_mem)
2234 trigger = rxr->ims; /* RX for 82574 */
2236 trigger = E1000_ICS_RXDMT0;
2239 ** Check on the state of the TX queue(s), this
2240 ** can be done without the lock because its RO
2241 ** and the HUNG state will be static if set.
2243 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2244 if ((txr->queue_status == EM_QUEUE_HUNG) &&
2245 (adapter->pause_frames == 0))
2247 /* Schedule a TX tasklet if needed */
2248 if (txr->tx_avail <= EM_MAX_SCATTER)
2249 taskqueue_enqueue(txr->tq, &txr->tx_task);
2252 adapter->pause_frames = 0;
2253 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2254 #ifndef DEVICE_POLLING
2255 /* Trigger an RX interrupt to guarantee mbuf refresh */
2256 E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger);
2260 /* Looks like we're hung */
2261 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2262 device_printf(adapter->dev,
2263 "Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2264 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
2265 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
2266 device_printf(adapter->dev,"TX(%d) desc avail = %d,"
2267 "Next TX to Clean = %d\n",
2268 txr->me, txr->tx_avail, txr->next_to_clean);
2269 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2270 adapter->watchdog_events++;
2271 adapter->pause_frames = 0;
2272 em_init_locked(adapter);
2277 em_update_link_status(struct adapter *adapter)
2279 struct e1000_hw *hw = &adapter->hw;
2280 struct ifnet *ifp = adapter->ifp;
2281 device_t dev = adapter->dev;
2282 struct tx_ring *txr = adapter->tx_rings;
2285 /* Get the cached link value or read phy for real */
2286 switch (hw->phy.media_type) {
2287 case e1000_media_type_copper:
2288 if (hw->mac.get_link_status) {
2289 /* Do the work to read phy */
2290 e1000_check_for_link(hw);
2291 link_check = !hw->mac.get_link_status;
2292 if (link_check) /* ESB2 fix */
2293 e1000_cfg_on_link_up(hw);
2297 case e1000_media_type_fiber:
2298 e1000_check_for_link(hw);
2299 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2302 case e1000_media_type_internal_serdes:
2303 e1000_check_for_link(hw);
2304 link_check = adapter->hw.mac.serdes_has_link;
2307 case e1000_media_type_unknown:
2311 /* Now check for a transition */
2312 if (link_check && (adapter->link_active == 0)) {
2313 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2314 &adapter->link_duplex);
2315 /* Check if we must disable SPEED_MODE bit on PCI-E */
2316 if ((adapter->link_speed != SPEED_1000) &&
2317 ((hw->mac.type == e1000_82571) ||
2318 (hw->mac.type == e1000_82572))) {
2320 tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2321 tarc0 &= ~SPEED_MODE_BIT;
2322 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2325 device_printf(dev, "Link is up %d Mbps %s\n",
2326 adapter->link_speed,
2327 ((adapter->link_duplex == FULL_DUPLEX) ?
2328 "Full Duplex" : "Half Duplex"));
2329 adapter->link_active = 1;
2330 adapter->smartspeed = 0;
2331 ifp->if_baudrate = adapter->link_speed * 1000000;
2332 if_link_state_change(ifp, LINK_STATE_UP);
2333 } else if (!link_check && (adapter->link_active == 1)) {
2334 ifp->if_baudrate = adapter->link_speed = 0;
2335 adapter->link_duplex = 0;
2337 device_printf(dev, "Link is Down\n");
2338 adapter->link_active = 0;
2339 /* Link down, disable watchdog */
2340 for (int i = 0; i < adapter->num_queues; i++, txr++)
2341 txr->queue_status = EM_QUEUE_IDLE;
2342 if_link_state_change(ifp, LINK_STATE_DOWN);
2346 /*********************************************************************
2348 * This routine disables all traffic on the adapter by issuing a
2349 * global reset on the MAC and deallocates TX/RX buffers.
2351 * This routine should always be called with BOTH the CORE
2353 **********************************************************************/
2358 struct adapter *adapter = arg;
2359 struct ifnet *ifp = adapter->ifp;
2360 struct tx_ring *txr = adapter->tx_rings;
2362 EM_CORE_LOCK_ASSERT(adapter);
2364 INIT_DEBUGOUT("em_stop: begin");
2366 em_disable_intr(adapter);
2367 callout_stop(&adapter->timer);
2369 /* Tell the stack that the interface is no longer active */
2370 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2371 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2373 /* Unarm watchdog timer. */
2374 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2376 txr->queue_status = EM_QUEUE_IDLE;
2380 e1000_reset_hw(&adapter->hw);
2381 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2383 e1000_led_off(&adapter->hw);
2384 e1000_cleanup_led(&adapter->hw);
2388 /*********************************************************************
2390 * Determine hardware revision.
2392 **********************************************************************/
2394 em_identify_hardware(struct adapter *adapter)
2396 device_t dev = adapter->dev;
2398 /* Make sure our PCI config space has the necessary stuff set */
2399 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2400 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2401 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2402 device_printf(dev, "Memory Access and/or Bus Master bits "
2404 adapter->hw.bus.pci_cmd_word |=
2405 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2406 pci_write_config(dev, PCIR_COMMAND,
2407 adapter->hw.bus.pci_cmd_word, 2);
2410 /* Save off the information about this board */
2411 adapter->hw.vendor_id = pci_get_vendor(dev);
2412 adapter->hw.device_id = pci_get_device(dev);
2413 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2414 adapter->hw.subsystem_vendor_id =
2415 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2416 adapter->hw.subsystem_device_id =
2417 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2419 /* Do Shared Code Init and Setup */
2420 if (e1000_set_mac_type(&adapter->hw)) {
2421 device_printf(dev, "Setup init failure\n");
2427 em_allocate_pci_resources(struct adapter *adapter)
2429 device_t dev = adapter->dev;
2433 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2435 if (adapter->memory == NULL) {
2436 device_printf(dev, "Unable to allocate bus resource: memory\n");
2439 adapter->osdep.mem_bus_space_tag =
2440 rman_get_bustag(adapter->memory);
2441 adapter->osdep.mem_bus_space_handle =
2442 rman_get_bushandle(adapter->memory);
2443 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2445 /* Default to a single queue */
2446 adapter->num_queues = 1;
2449 * Setup MSI/X or MSI if PCI Express
2451 adapter->msix = em_setup_msix(adapter);
2453 adapter->hw.back = &adapter->osdep;
2458 /*********************************************************************
2460 * Setup the Legacy or MSI Interrupt handler
2462 **********************************************************************/
2464 em_allocate_legacy(struct adapter *adapter)
2466 device_t dev = adapter->dev;
2467 struct tx_ring *txr = adapter->tx_rings;
2470 /* Manually turn off all interrupts */
2471 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2473 if (adapter->msix == 1) /* using MSI */
2475 /* We allocate a single interrupt resource */
2476 adapter->res = bus_alloc_resource_any(dev,
2477 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2478 if (adapter->res == NULL) {
2479 device_printf(dev, "Unable to allocate bus resource: "
2485 * Allocate a fast interrupt and the associated
2486 * deferred processing contexts.
2488 TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter);
2489 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2490 taskqueue_thread_enqueue, &adapter->tq);
2491 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que",
2492 device_get_nameunit(adapter->dev));
2493 /* Use a TX only tasklet for local timer */
2494 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2495 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2496 taskqueue_thread_enqueue, &txr->tq);
2497 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2498 device_get_nameunit(adapter->dev));
2499 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2500 if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET,
2501 em_irq_fast, NULL, adapter, &adapter->tag)) != 0) {
2502 device_printf(dev, "Failed to register fast interrupt "
2503 "handler: %d\n", error);
2504 taskqueue_free(adapter->tq);
2512 /*********************************************************************
2514 * Setup the MSIX Interrupt handlers
2515 * This is not really Multiqueue, rather
2516 * its just seperate interrupt vectors
2517 * for TX, RX, and Link.
2519 **********************************************************************/
2521 em_allocate_msix(struct adapter *adapter)
2523 device_t dev = adapter->dev;
2524 struct tx_ring *txr = adapter->tx_rings;
2525 struct rx_ring *rxr = adapter->rx_rings;
2526 int error, rid, vector = 0;
2529 /* Make sure all interrupts are disabled */
2530 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2532 /* First set up ring resources */
2533 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
2538 rxr->res = bus_alloc_resource_any(dev,
2539 SYS_RES_IRQ, &rid, RF_ACTIVE);
2540 if (rxr->res == NULL) {
2542 "Unable to allocate bus resource: "
2543 "RX MSIX Interrupt %d\n", i);
2546 if ((error = bus_setup_intr(dev, rxr->res,
2547 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx,
2548 rxr, &rxr->tag)) != 0) {
2549 device_printf(dev, "Failed to register RX handler");
2552 #if __FreeBSD_version >= 800504
2553 bus_describe_intr(dev, rxr->res, rxr->tag, "rx %d", i);
2555 rxr->msix = vector++; /* NOTE increment vector for TX */
2556 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr);
2557 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT,
2558 taskqueue_thread_enqueue, &rxr->tq);
2559 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq",
2560 device_get_nameunit(adapter->dev));
2562 ** Set the bit to enable interrupt
2563 ** in E1000_IMS -- bits 20 and 21
2564 ** are for RX0 and RX1, note this has
2565 ** NOTHING to do with the MSIX vector
2567 rxr->ims = 1 << (20 + i);
2568 adapter->ivars |= (8 | rxr->msix) << (i * 4);
2572 txr->res = bus_alloc_resource_any(dev,
2573 SYS_RES_IRQ, &rid, RF_ACTIVE);
2574 if (txr->res == NULL) {
2576 "Unable to allocate bus resource: "
2577 "TX MSIX Interrupt %d\n", i);
2580 if ((error = bus_setup_intr(dev, txr->res,
2581 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx,
2582 txr, &txr->tag)) != 0) {
2583 device_printf(dev, "Failed to register TX handler");
2586 #if __FreeBSD_version >= 800504
2587 bus_describe_intr(dev, txr->res, txr->tag, "tx %d", i);
2589 txr->msix = vector++; /* Increment vector for next pass */
2590 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2591 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2592 taskqueue_thread_enqueue, &txr->tq);
2593 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2594 device_get_nameunit(adapter->dev));
2596 ** Set the bit to enable interrupt
2597 ** in E1000_IMS -- bits 22 and 23
2598 ** are for TX0 and TX1, note this has
2599 ** NOTHING to do with the MSIX vector
2601 txr->ims = 1 << (22 + i);
2602 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4));
2605 /* Link interrupt */
2607 adapter->res = bus_alloc_resource_any(dev,
2608 SYS_RES_IRQ, &rid, RF_ACTIVE);
2609 if (!adapter->res) {
2610 device_printf(dev,"Unable to allocate "
2611 "bus resource: Link interrupt [%d]\n", rid);
2614 /* Set the link handler function */
2615 error = bus_setup_intr(dev, adapter->res,
2616 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2617 em_msix_link, adapter, &adapter->tag);
2619 adapter->res = NULL;
2620 device_printf(dev, "Failed to register LINK handler");
2623 #if __FreeBSD_version >= 800504
2624 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2626 adapter->linkvec = vector;
2627 adapter->ivars |= (8 | vector) << 16;
2628 adapter->ivars |= 0x80000000;
2635 em_free_pci_resources(struct adapter *adapter)
2637 device_t dev = adapter->dev;
2638 struct tx_ring *txr;
2639 struct rx_ring *rxr;
2644 ** Release all the queue interrupt resources:
2646 for (int i = 0; i < adapter->num_queues; i++) {
2647 txr = &adapter->tx_rings[i];
2648 rxr = &adapter->rx_rings[i];
2649 /* an early abort? */
2650 if ((txr == NULL) || (rxr == NULL))
2653 if (txr->tag != NULL) {
2654 bus_teardown_intr(dev, txr->res, txr->tag);
2657 if (txr->res != NULL)
2658 bus_release_resource(dev, SYS_RES_IRQ,
2661 if (rxr->tag != NULL) {
2662 bus_teardown_intr(dev, rxr->res, rxr->tag);
2665 if (rxr->res != NULL)
2666 bus_release_resource(dev, SYS_RES_IRQ,
2670 if (adapter->linkvec) /* we are doing MSIX */
2671 rid = adapter->linkvec + 1;
2673 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2675 if (adapter->tag != NULL) {
2676 bus_teardown_intr(dev, adapter->res, adapter->tag);
2677 adapter->tag = NULL;
2680 if (adapter->res != NULL)
2681 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2685 pci_release_msi(dev);
2687 if (adapter->msix_mem != NULL)
2688 bus_release_resource(dev, SYS_RES_MEMORY,
2689 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2691 if (adapter->memory != NULL)
2692 bus_release_resource(dev, SYS_RES_MEMORY,
2693 PCIR_BAR(0), adapter->memory);
2695 if (adapter->flash != NULL)
2696 bus_release_resource(dev, SYS_RES_MEMORY,
2697 EM_FLASH, adapter->flash);
2701 * Setup MSI or MSI/X
2704 em_setup_msix(struct adapter *adapter)
2706 device_t dev = adapter->dev;
2710 ** Setup MSI/X for Hartwell: tests have shown
2711 ** use of two queues to be unstable, and to
2712 ** provide no great gain anyway, so we simply
2713 ** seperate the interrupts and use a single queue.
2715 if ((adapter->hw.mac.type == e1000_82574) &&
2716 (em_enable_msix == TRUE)) {
2717 /* Map the MSIX BAR */
2718 int rid = PCIR_BAR(EM_MSIX_BAR);
2719 adapter->msix_mem = bus_alloc_resource_any(dev,
2720 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2721 if (!adapter->msix_mem) {
2722 /* May not be enabled */
2723 device_printf(adapter->dev,
2724 "Unable to map MSIX table \n");
2727 val = pci_msix_count(dev);
2728 /* We only need 3 vectors */
2731 if ((val != 3) && (val != 5)) {
2732 bus_release_resource(dev, SYS_RES_MEMORY,
2733 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2734 adapter->msix_mem = NULL;
2735 device_printf(adapter->dev,
2736 "MSIX: incorrect vectors, using MSI\n");
2740 if (pci_alloc_msix(dev, &val) == 0) {
2741 device_printf(adapter->dev,
2742 "Using MSIX interrupts "
2743 "with %d vectors\n", val);
2749 val = pci_msi_count(dev);
2750 if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2752 device_printf(adapter->dev,"Using an MSI interrupt\n");
2755 /* Should only happen due to manual configuration */
2756 device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n");
2761 /*********************************************************************
2763 * Initialize the hardware to a configuration
2764 * as specified by the adapter structure.
2766 **********************************************************************/
2768 em_reset(struct adapter *adapter)
2770 device_t dev = adapter->dev;
2771 struct ifnet *ifp = adapter->ifp;
2772 struct e1000_hw *hw = &adapter->hw;
2776 INIT_DEBUGOUT("em_reset: begin");
2778 /* Set up smart power down as default off on newer adapters. */
2779 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
2780 hw->mac.type == e1000_82572)) {
2783 /* Speed up time to link by disabling smart power down. */
2784 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2785 phy_tmp &= ~IGP02E1000_PM_SPD;
2786 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2790 * Packet Buffer Allocation (PBA)
2791 * Writing PBA sets the receive portion of the buffer
2792 * the remainder is used for the transmit buffer.
2794 switch (hw->mac.type) {
2795 /* Total Packet Buffer on these is 48K */
2798 case e1000_80003es2lan:
2799 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
2801 case e1000_82573: /* 82573: Total Packet Buffer is 32K */
2802 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
2806 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
2812 case e1000_ich10lan:
2813 /* Boost Receive side for jumbo frames */
2814 if (adapter->max_frame_size > 4096)
2815 pba = E1000_PBA_14K;
2817 pba = E1000_PBA_10K;
2821 pba = E1000_PBA_26K;
2824 if (adapter->max_frame_size > 8192)
2825 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
2827 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
2829 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
2832 * These parameters control the automatic generation (Tx) and
2833 * response (Rx) to Ethernet PAUSE frames.
2834 * - High water mark should allow for at least two frames to be
2835 * received after sending an XOFF.
2836 * - Low water mark works best when it is very near the high water mark.
2837 * This allows the receiver to restart by sending XON when it has
2838 * drained a bit. Here we use an arbitary value of 1500 which will
2839 * restart after one full frame is pulled from the buffer. There
2840 * could be several smaller frames in the buffer and if so they will
2841 * not trigger the XON until their total number reduces the buffer
2843 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2845 rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 );
2846 hw->fc.high_water = rx_buffer_size -
2847 roundup2(adapter->max_frame_size, 1024);
2848 hw->fc.low_water = hw->fc.high_water - 1500;
2850 if (adapter->fc) /* locally set flow control value? */
2851 hw->fc.requested_mode = adapter->fc;
2853 hw->fc.requested_mode = e1000_fc_full;
2855 if (hw->mac.type == e1000_80003es2lan)
2856 hw->fc.pause_time = 0xFFFF;
2858 hw->fc.pause_time = EM_FC_PAUSE_TIME;
2860 hw->fc.send_xon = TRUE;
2862 /* Device specific overrides/settings */
2863 switch (hw->mac.type) {
2865 /* Workaround: no TX flow ctrl for PCH */
2866 hw->fc.requested_mode = e1000_fc_rx_pause;
2867 hw->fc.pause_time = 0xFFFF; /* override */
2868 if (ifp->if_mtu > ETHERMTU) {
2869 hw->fc.high_water = 0x3500;
2870 hw->fc.low_water = 0x1500;
2872 hw->fc.high_water = 0x5000;
2873 hw->fc.low_water = 0x3000;
2875 hw->fc.refresh_time = 0x1000;
2878 hw->fc.high_water = 0x5C20;
2879 hw->fc.low_water = 0x5048;
2880 hw->fc.pause_time = 0x0650;
2881 hw->fc.refresh_time = 0x0400;
2882 /* Jumbos need adjusted PBA */
2883 if (ifp->if_mtu > ETHERMTU)
2884 E1000_WRITE_REG(hw, E1000_PBA, 12);
2886 E1000_WRITE_REG(hw, E1000_PBA, 26);
2889 case e1000_ich10lan:
2890 if (ifp->if_mtu > ETHERMTU) {
2891 hw->fc.high_water = 0x2800;
2892 hw->fc.low_water = hw->fc.high_water - 8;
2895 /* else fall thru */
2897 if (hw->mac.type == e1000_80003es2lan)
2898 hw->fc.pause_time = 0xFFFF;
2902 /* Issue a global reset */
2904 E1000_WRITE_REG(hw, E1000_WUC, 0);
2905 em_disable_aspm(adapter);
2907 if (e1000_init_hw(hw) < 0) {
2908 device_printf(dev, "Hardware Initialization Failed\n");
2912 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
2913 e1000_get_phy_info(hw);
2914 e1000_check_for_link(hw);
2918 /*********************************************************************
2920 * Setup networking device structure and register an interface.
2922 **********************************************************************/
2924 em_setup_interface(device_t dev, struct adapter *adapter)
2928 INIT_DEBUGOUT("em_setup_interface: begin");
2930 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2932 device_printf(dev, "can not allocate ifnet structure\n");
2935 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2936 ifp->if_init = em_init;
2937 ifp->if_softc = adapter;
2938 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2939 ifp->if_ioctl = em_ioctl;
2940 #ifdef EM_MULTIQUEUE
2941 /* Multiqueue stack interface */
2942 ifp->if_transmit = em_mq_start;
2943 ifp->if_qflush = em_qflush;
2945 ifp->if_start = em_start;
2946 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2947 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2948 IFQ_SET_READY(&ifp->if_snd);
2951 ether_ifattach(ifp, adapter->hw.mac.addr);
2953 ifp->if_capabilities = ifp->if_capenable = 0;
2956 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2957 ifp->if_capabilities |= IFCAP_TSO4;
2959 * Tell the upper layer(s) we
2960 * support full VLAN capability
2962 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2963 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2966 ifp->if_capenable = ifp->if_capabilities;
2969 ** Don't turn this on by default, if vlans are
2970 ** created on another pseudo device (eg. lagg)
2971 ** then vlan events are not passed thru, breaking
2972 ** operation, but with HW FILTER off it works. If
2973 ** using vlans directly on the em driver you can
2974 ** enable this and get full hardware tag filtering.
2976 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2978 #ifdef DEVICE_POLLING
2979 ifp->if_capabilities |= IFCAP_POLLING;
2982 /* Enable only WOL MAGIC by default */
2984 ifp->if_capabilities |= IFCAP_WOL;
2985 ifp->if_capenable |= IFCAP_WOL_MAGIC;
2989 * Specify the media types supported by this adapter and register
2990 * callbacks to update media and link information
2992 ifmedia_init(&adapter->media, IFM_IMASK,
2993 em_media_change, em_media_status);
2994 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2995 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2996 u_char fiber_type = IFM_1000_SX; /* default type */
2998 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3000 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3002 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3003 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3005 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3007 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3009 if (adapter->hw.phy.type != e1000_phy_ife) {
3010 ifmedia_add(&adapter->media,
3011 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3012 ifmedia_add(&adapter->media,
3013 IFM_ETHER | IFM_1000_T, 0, NULL);
3016 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3017 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3023 * Manage DMA'able memory.
3026 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3030 *(bus_addr_t *) arg = segs[0].ds_addr;
3034 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3035 struct em_dma_alloc *dma, int mapflags)
3039 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3040 EM_DBA_ALIGN, 0, /* alignment, bounds */
3041 BUS_SPACE_MAXADDR, /* lowaddr */
3042 BUS_SPACE_MAXADDR, /* highaddr */
3043 NULL, NULL, /* filter, filterarg */
3046 size, /* maxsegsize */
3048 NULL, /* lockfunc */
3052 device_printf(adapter->dev,
3053 "%s: bus_dma_tag_create failed: %d\n",
3058 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3059 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3061 device_printf(adapter->dev,
3062 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3063 __func__, (uintmax_t)size, error);
3068 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3069 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3070 if (error || dma->dma_paddr == 0) {
3071 device_printf(adapter->dev,
3072 "%s: bus_dmamap_load failed: %d\n",
3080 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3082 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3083 bus_dma_tag_destroy(dma->dma_tag);
3085 dma->dma_map = NULL;
3086 dma->dma_tag = NULL;
3092 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3094 if (dma->dma_tag == NULL)
3096 if (dma->dma_map != NULL) {
3097 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3098 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3099 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3100 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3101 dma->dma_map = NULL;
3103 bus_dma_tag_destroy(dma->dma_tag);
3104 dma->dma_tag = NULL;
3108 /*********************************************************************
3110 * Allocate memory for the transmit and receive rings, and then
3111 * the descriptors associated with each, called only once at attach.
3113 **********************************************************************/
3115 em_allocate_queues(struct adapter *adapter)
3117 device_t dev = adapter->dev;
3118 struct tx_ring *txr = NULL;
3119 struct rx_ring *rxr = NULL;
3120 int rsize, tsize, error = E1000_SUCCESS;
3121 int txconf = 0, rxconf = 0;
3124 /* Allocate the TX ring struct memory */
3125 if (!(adapter->tx_rings =
3126 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3127 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3128 device_printf(dev, "Unable to allocate TX ring memory\n");
3133 /* Now allocate the RX */
3134 if (!(adapter->rx_rings =
3135 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3136 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3137 device_printf(dev, "Unable to allocate RX ring memory\n");
3142 tsize = roundup2(adapter->num_tx_desc *
3143 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
3145 * Now set up the TX queues, txconf is needed to handle the
3146 * possibility that things fail midcourse and we need to
3147 * undo memory gracefully
3149 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3150 /* Set up some basics */
3151 txr = &adapter->tx_rings[i];
3152 txr->adapter = adapter;
3155 /* Initialize the TX lock */
3156 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3157 device_get_nameunit(dev), txr->me);
3158 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3160 if (em_dma_malloc(adapter, tsize,
3161 &txr->txdma, BUS_DMA_NOWAIT)) {
3163 "Unable to allocate TX Descriptor memory\n");
3167 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
3168 bzero((void *)txr->tx_base, tsize);
3170 if (em_allocate_transmit_buffers(txr)) {
3172 "Critical Failure setting up transmit buffers\n");
3176 #if __FreeBSD_version >= 800000
3177 /* Allocate a buf ring */
3178 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3179 M_WAITOK, &txr->tx_mtx);
3184 * Next the RX queues...
3186 rsize = roundup2(adapter->num_rx_desc *
3187 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
3188 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3189 rxr = &adapter->rx_rings[i];
3190 rxr->adapter = adapter;
3193 /* Initialize the RX lock */
3194 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3195 device_get_nameunit(dev), txr->me);
3196 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3198 if (em_dma_malloc(adapter, rsize,
3199 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3201 "Unable to allocate RxDescriptor memory\n");
3205 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr;
3206 bzero((void *)rxr->rx_base, rsize);
3208 /* Allocate receive buffers for the ring*/
3209 if (em_allocate_receive_buffers(rxr)) {
3211 "Critical Failure setting up receive buffers\n");
3220 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3221 em_dma_free(adapter, &rxr->rxdma);
3223 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3224 em_dma_free(adapter, &txr->txdma);
3225 free(adapter->rx_rings, M_DEVBUF);
3227 #if __FreeBSD_version >= 800000
3228 buf_ring_free(txr->br, M_DEVBUF);
3230 free(adapter->tx_rings, M_DEVBUF);
3236 /*********************************************************************
3238 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3239 * the information needed to transmit a packet on the wire. This is
3240 * called only once at attach, setup is done every reset.
3242 **********************************************************************/
3244 em_allocate_transmit_buffers(struct tx_ring *txr)
3246 struct adapter *adapter = txr->adapter;
3247 device_t dev = adapter->dev;
3248 struct em_buffer *txbuf;
3252 * Setup DMA descriptor areas.
3254 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3255 1, 0, /* alignment, bounds */
3256 BUS_SPACE_MAXADDR, /* lowaddr */
3257 BUS_SPACE_MAXADDR, /* highaddr */
3258 NULL, NULL, /* filter, filterarg */
3259 EM_TSO_SIZE, /* maxsize */
3260 EM_MAX_SCATTER, /* nsegments */
3261 PAGE_SIZE, /* maxsegsize */
3263 NULL, /* lockfunc */
3264 NULL, /* lockfuncarg */
3266 device_printf(dev,"Unable to allocate TX DMA tag\n");
3270 if (!(txr->tx_buffers =
3271 (struct em_buffer *) malloc(sizeof(struct em_buffer) *
3272 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3273 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3278 /* Create the descriptor buffer dma maps */
3279 txbuf = txr->tx_buffers;
3280 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3281 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3283 device_printf(dev, "Unable to create TX DMA map\n");
3290 /* We free all, it handles case where we are in the middle */
3291 em_free_transmit_structures(adapter);
3295 /*********************************************************************
3297 * Initialize a transmit ring.
3299 **********************************************************************/
3301 em_setup_transmit_ring(struct tx_ring *txr)
3303 struct adapter *adapter = txr->adapter;
3304 struct em_buffer *txbuf;
3307 struct netmap_adapter *na = NA(adapter->ifp);
3308 struct netmap_slot *slot;
3309 #endif /* DEV_NETMAP */
3311 /* Clear the old descriptor contents */
3314 slot = netmap_reset(na, NR_TX, txr->me, 0);
3315 #endif /* DEV_NETMAP */
3317 bzero((void *)txr->tx_base,
3318 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3320 txr->next_avail_desc = 0;
3321 txr->next_to_clean = 0;
3323 /* Free any existing tx buffers. */
3324 txbuf = txr->tx_buffers;
3325 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3326 if (txbuf->m_head != NULL) {
3327 bus_dmamap_sync(txr->txtag, txbuf->map,
3328 BUS_DMASYNC_POSTWRITE);
3329 bus_dmamap_unload(txr->txtag, txbuf->map);
3330 m_freem(txbuf->m_head);
3331 txbuf->m_head = NULL;
3335 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3339 addr = PNMB(slot + si, &paddr);
3340 txr->tx_base[i].buffer_addr = htole64(paddr);
3341 /* reload the map for netmap mode */
3342 netmap_load_map(txr->txtag, txbuf->map, addr);
3344 #endif /* DEV_NETMAP */
3346 /* clear the watch index */
3347 txbuf->next_eop = -1;
3350 /* Set number of descriptors available */
3351 txr->tx_avail = adapter->num_tx_desc;
3352 txr->queue_status = EM_QUEUE_IDLE;
3354 /* Clear checksum offload context. */
3355 txr->last_hw_offload = 0;
3356 txr->last_hw_ipcss = 0;
3357 txr->last_hw_ipcso = 0;
3358 txr->last_hw_tucss = 0;
3359 txr->last_hw_tucso = 0;
3361 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3362 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3366 /*********************************************************************
3368 * Initialize all transmit rings.
3370 **********************************************************************/
3372 em_setup_transmit_structures(struct adapter *adapter)
3374 struct tx_ring *txr = adapter->tx_rings;
3376 for (int i = 0; i < adapter->num_queues; i++, txr++)
3377 em_setup_transmit_ring(txr);
3382 /*********************************************************************
3384 * Enable transmit unit.
3386 **********************************************************************/
3388 em_initialize_transmit_unit(struct adapter *adapter)
3390 struct tx_ring *txr = adapter->tx_rings;
3391 struct e1000_hw *hw = &adapter->hw;
3392 u32 tctl, tarc, tipg = 0;
3394 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3396 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3397 u64 bus_addr = txr->txdma.dma_paddr;
3398 /* Base and Len of TX Ring */
3399 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3400 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3401 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3402 (u32)(bus_addr >> 32));
3403 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3405 /* Init the HEAD/TAIL indices */
3406 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3407 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3409 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3410 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
3411 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
3413 txr->queue_status = EM_QUEUE_IDLE;
3416 /* Set the default values for the Tx Inter Packet Gap timer */
3417 switch (adapter->hw.mac.type) {
3418 case e1000_80003es2lan:
3419 tipg = DEFAULT_82543_TIPG_IPGR1;
3420 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3421 E1000_TIPG_IPGR2_SHIFT;
3424 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3425 (adapter->hw.phy.media_type ==
3426 e1000_media_type_internal_serdes))
3427 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3429 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3430 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3431 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3434 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3435 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3437 if(adapter->hw.mac.type >= e1000_82540)
3438 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3439 adapter->tx_abs_int_delay.value);
3441 if ((adapter->hw.mac.type == e1000_82571) ||
3442 (adapter->hw.mac.type == e1000_82572)) {
3443 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3444 tarc |= SPEED_MODE_BIT;
3445 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3446 } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3447 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3449 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3450 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3452 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3455 adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3456 if (adapter->tx_int_delay.value > 0)
3457 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3459 /* Program the Transmit Control Register */
3460 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3461 tctl &= ~E1000_TCTL_CT;
3462 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3463 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3465 if (adapter->hw.mac.type >= e1000_82571)
3466 tctl |= E1000_TCTL_MULR;
3468 /* This write will effectively turn on the transmit unit. */
3469 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3474 /*********************************************************************
3476 * Free all transmit rings.
3478 **********************************************************************/
3480 em_free_transmit_structures(struct adapter *adapter)
3482 struct tx_ring *txr = adapter->tx_rings;
3484 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3486 em_free_transmit_buffers(txr);
3487 em_dma_free(adapter, &txr->txdma);
3489 EM_TX_LOCK_DESTROY(txr);
3492 free(adapter->tx_rings, M_DEVBUF);
3495 /*********************************************************************
3497 * Free transmit ring related data structures.
3499 **********************************************************************/
3501 em_free_transmit_buffers(struct tx_ring *txr)
3503 struct adapter *adapter = txr->adapter;
3504 struct em_buffer *txbuf;
3506 INIT_DEBUGOUT("free_transmit_ring: begin");
3508 if (txr->tx_buffers == NULL)
3511 for (int i = 0; i < adapter->num_tx_desc; i++) {
3512 txbuf = &txr->tx_buffers[i];
3513 if (txbuf->m_head != NULL) {
3514 bus_dmamap_sync(txr->txtag, txbuf->map,
3515 BUS_DMASYNC_POSTWRITE);
3516 bus_dmamap_unload(txr->txtag,
3518 m_freem(txbuf->m_head);
3519 txbuf->m_head = NULL;
3520 if (txbuf->map != NULL) {
3521 bus_dmamap_destroy(txr->txtag,
3525 } else if (txbuf->map != NULL) {
3526 bus_dmamap_unload(txr->txtag,
3528 bus_dmamap_destroy(txr->txtag,
3533 #if __FreeBSD_version >= 800000
3534 if (txr->br != NULL)
3535 buf_ring_free(txr->br, M_DEVBUF);
3537 if (txr->tx_buffers != NULL) {
3538 free(txr->tx_buffers, M_DEVBUF);
3539 txr->tx_buffers = NULL;
3541 if (txr->txtag != NULL) {
3542 bus_dma_tag_destroy(txr->txtag);
3549 /*********************************************************************
3550 * The offload context is protocol specific (TCP/UDP) and thus
3551 * only needs to be set when the protocol changes. The occasion
3552 * of a context change can be a performance detriment, and
3553 * might be better just disabled. The reason arises in the way
3554 * in which the controller supports pipelined requests from the
3555 * Tx data DMA. Up to four requests can be pipelined, and they may
3556 * belong to the same packet or to multiple packets. However all
3557 * requests for one packet are issued before a request is issued
3558 * for a subsequent packet and if a request for the next packet
3559 * requires a context change, that request will be stalled
3560 * until the previous request completes. This means setting up
3561 * a new context effectively disables pipelined Tx data DMA which
3562 * in turn greatly slow down performance to send small sized
3564 **********************************************************************/
3566 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3567 struct ip *ip, u32 *txd_upper, u32 *txd_lower)
3569 struct adapter *adapter = txr->adapter;
3570 struct e1000_context_desc *TXD = NULL;
3571 struct em_buffer *tx_buffer;
3575 u8 ipcso, ipcss, tucso, tucss;
3577 ipcss = ipcso = tucss = tucso = 0;
3578 hdr_len = ip_off + (ip->ip_hl << 2);
3579 cur = txr->next_avail_desc;
3581 /* Setup of IP header checksum. */
3582 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3583 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3586 ipcso = ip_off + offsetof(struct ip, ip_sum);
3588 * Start offset for header checksum calculation.
3589 * End offset for header checksum calculation.
3590 * Offset of place to put the checksum.
3592 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3593 TXD->lower_setup.ip_fields.ipcss = ipcss;
3594 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
3595 TXD->lower_setup.ip_fields.ipcso = ipcso;
3596 cmd |= E1000_TXD_CMD_IP;
3599 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3600 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3601 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3602 offload |= CSUM_TCP;
3604 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
3606 * Setting up new checksum offload context for every frames
3607 * takes a lot of processing time for hardware. This also
3608 * reduces performance a lot for small sized frames so avoid
3609 * it if driver can use previously configured checksum
3612 if (txr->last_hw_offload == offload) {
3613 if (offload & CSUM_IP) {
3614 if (txr->last_hw_ipcss == ipcss &&
3615 txr->last_hw_ipcso == ipcso &&
3616 txr->last_hw_tucss == tucss &&
3617 txr->last_hw_tucso == tucso)
3620 if (txr->last_hw_tucss == tucss &&
3621 txr->last_hw_tucso == tucso)
3625 txr->last_hw_offload = offload;
3626 txr->last_hw_tucss = tucss;
3627 txr->last_hw_tucso = tucso;
3629 * Start offset for payload checksum calculation.
3630 * End offset for payload checksum calculation.
3631 * Offset of place to put the checksum.
3633 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3634 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3635 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3636 TXD->upper_setup.tcp_fields.tucso = tucso;
3637 cmd |= E1000_TXD_CMD_TCP;
3638 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3639 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3640 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3642 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
3644 * Setting up new checksum offload context for every frames
3645 * takes a lot of processing time for hardware. This also
3646 * reduces performance a lot for small sized frames so avoid
3647 * it if driver can use previously configured checksum
3650 if (txr->last_hw_offload == offload) {
3651 if (offload & CSUM_IP) {
3652 if (txr->last_hw_ipcss == ipcss &&
3653 txr->last_hw_ipcso == ipcso &&
3654 txr->last_hw_tucss == tucss &&
3655 txr->last_hw_tucso == tucso)
3658 if (txr->last_hw_tucss == tucss &&
3659 txr->last_hw_tucso == tucso)
3663 txr->last_hw_offload = offload;
3664 txr->last_hw_tucss = tucss;
3665 txr->last_hw_tucso = tucso;
3667 * Start offset for header checksum calculation.
3668 * End offset for header checksum calculation.
3669 * Offset of place to put the checksum.
3671 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3672 TXD->upper_setup.tcp_fields.tucss = tucss;
3673 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3674 TXD->upper_setup.tcp_fields.tucso = tucso;
3677 if (offload & CSUM_IP) {
3678 txr->last_hw_ipcss = ipcss;
3679 txr->last_hw_ipcso = ipcso;
3682 TXD->tcp_seg_setup.data = htole32(0);
3683 TXD->cmd_and_length =
3684 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3685 tx_buffer = &txr->tx_buffers[cur];
3686 tx_buffer->m_head = NULL;
3687 tx_buffer->next_eop = -1;
3689 if (++cur == adapter->num_tx_desc)
3693 txr->next_avail_desc = cur;
3697 /**********************************************************************
3699 * Setup work for hardware segmentation offload (TSO)
3701 **********************************************************************/
3703 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3704 struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower)
3706 struct adapter *adapter = txr->adapter;
3707 struct e1000_context_desc *TXD;
3708 struct em_buffer *tx_buffer;
3712 * In theory we can use the same TSO context if and only if
3713 * frame is the same type(IP/TCP) and the same MSS. However
3714 * checking whether a frame has the same IP/TCP structure is
3715 * hard thing so just ignore that and always restablish a
3718 hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2);
3719 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
3720 E1000_TXD_DTYP_D | /* Data descr type */
3721 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
3723 /* IP and/or TCP header checksum calculation and insertion. */
3724 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
3726 cur = txr->next_avail_desc;
3727 tx_buffer = &txr->tx_buffers[cur];
3728 TXD = (struct e1000_context_desc *) &txr->tx_base[cur];
3731 * Start offset for header checksum calculation.
3732 * End offset for header checksum calculation.
3733 * Offset of place put the checksum.
3735 TXD->lower_setup.ip_fields.ipcss = ip_off;
3736 TXD->lower_setup.ip_fields.ipcse =
3737 htole16(ip_off + (ip->ip_hl << 2) - 1);
3738 TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum);
3740 * Start offset for payload checksum calculation.
3741 * End offset for payload checksum calculation.
3742 * Offset of place to put the checksum.
3744 TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2);
3745 TXD->upper_setup.tcp_fields.tucse = 0;
3746 TXD->upper_setup.tcp_fields.tucso =
3747 ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum);
3749 * Payload size per packet w/o any headers.
3750 * Length of all headers up to payload.
3752 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3753 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3755 TXD->cmd_and_length = htole32(adapter->txd_cmd |
3756 E1000_TXD_CMD_DEXT | /* Extended descr */
3757 E1000_TXD_CMD_TSE | /* TSE context */
3758 E1000_TXD_CMD_IP | /* Do IP csum */
3759 E1000_TXD_CMD_TCP | /* Do TCP checksum */
3760 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3762 tx_buffer->m_head = NULL;
3763 tx_buffer->next_eop = -1;
3765 if (++cur == adapter->num_tx_desc)
3769 txr->next_avail_desc = cur;
3774 /**********************************************************************
3776 * Examine each tx_buffer in the used queue. If the hardware is done
3777 * processing the packet then free associated resources. The
3778 * tx_buffer is put back on the free queue.
3780 **********************************************************************/
3782 em_txeof(struct tx_ring *txr)
3784 struct adapter *adapter = txr->adapter;
3785 int first, last, done, processed;
3786 struct em_buffer *tx_buffer;
3787 struct e1000_tx_desc *tx_desc, *eop_desc;
3788 struct ifnet *ifp = adapter->ifp;
3790 EM_TX_LOCK_ASSERT(txr);
3792 if (ifp->if_capenable & IFCAP_NETMAP) {
3793 struct netmap_adapter *na = NA(ifp);
3795 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3797 EM_CORE_LOCK(adapter);
3798 selwakeuppri(&na->tx_si, PI_NET);
3799 EM_CORE_UNLOCK(adapter);
3803 #endif /* DEV_NETMAP */
3805 /* No work, make sure watchdog is off */
3806 if (txr->tx_avail == adapter->num_tx_desc) {
3807 txr->queue_status = EM_QUEUE_IDLE;
3812 first = txr->next_to_clean;
3813 tx_desc = &txr->tx_base[first];
3814 tx_buffer = &txr->tx_buffers[first];
3815 last = tx_buffer->next_eop;
3816 eop_desc = &txr->tx_base[last];
3819 * What this does is get the index of the
3820 * first descriptor AFTER the EOP of the
3821 * first packet, that way we can do the
3822 * simple comparison on the inner while loop.
3824 if (++last == adapter->num_tx_desc)
3828 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3829 BUS_DMASYNC_POSTREAD);
3831 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3832 /* We clean the range of the packet */
3833 while (first != done) {
3834 tx_desc->upper.data = 0;
3835 tx_desc->lower.data = 0;
3836 tx_desc->buffer_addr = 0;
3840 if (tx_buffer->m_head) {
3841 bus_dmamap_sync(txr->txtag,
3843 BUS_DMASYNC_POSTWRITE);
3844 bus_dmamap_unload(txr->txtag,
3846 m_freem(tx_buffer->m_head);
3847 tx_buffer->m_head = NULL;
3849 tx_buffer->next_eop = -1;
3850 txr->watchdog_time = ticks;
3852 if (++first == adapter->num_tx_desc)
3855 tx_buffer = &txr->tx_buffers[first];
3856 tx_desc = &txr->tx_base[first];
3859 /* See if we can continue to the next packet */
3860 last = tx_buffer->next_eop;
3862 eop_desc = &txr->tx_base[last];
3863 /* Get new done point */
3864 if (++last == adapter->num_tx_desc) last = 0;
3869 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3870 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3872 txr->next_to_clean = first;
3875 ** Watchdog calculation, we know there's
3876 ** work outstanding or the first return
3877 ** would have been taken, so none processed
3878 ** for too long indicates a hang. local timer
3879 ** will examine this and do a reset if needed.
3881 if ((!processed) && ((ticks - txr->watchdog_time) > EM_WATCHDOG))
3882 txr->queue_status = EM_QUEUE_HUNG;
3885 * If we have a minimum free, clear IFF_DRV_OACTIVE
3886 * to tell the stack that it is OK to send packets.
3887 * Notice that all writes of OACTIVE happen under the
3888 * TX lock which, with a single queue, guarantees
3891 if (txr->tx_avail >= EM_MAX_SCATTER)
3892 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3894 /* Disable watchdog if all clean */
3895 if (txr->tx_avail == adapter->num_tx_desc) {
3896 txr->queue_status = EM_QUEUE_IDLE;
3901 /*********************************************************************
3903 * Refresh RX descriptor mbufs from system mbuf buffer pool.
3905 **********************************************************************/
3907 em_refresh_mbufs(struct rx_ring *rxr, int limit)
3909 struct adapter *adapter = rxr->adapter;
3911 bus_dma_segment_t segs[1];
3912 struct em_buffer *rxbuf;
3913 int i, j, error, nsegs;
3914 bool cleaned = FALSE;
3916 i = j = rxr->next_to_refresh;
3918 ** Get one descriptor beyond
3919 ** our work mark to control
3922 if (++j == adapter->num_rx_desc)
3925 while (j != limit) {
3926 rxbuf = &rxr->rx_buffers[i];
3927 if (rxbuf->m_head == NULL) {
3928 m = m_getjcl(M_DONTWAIT, MT_DATA,
3929 M_PKTHDR, adapter->rx_mbuf_sz);
3931 ** If we have a temporary resource shortage
3932 ** that causes a failure, just abort refresh
3933 ** for now, we will return to this point when
3934 ** reinvoked from em_rxeof.
3941 m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz;
3942 m->m_flags |= M_PKTHDR;
3943 m->m_data = m->m_ext.ext_buf;
3945 /* Use bus_dma machinery to setup the memory mapping */
3946 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map,
3947 m, segs, &nsegs, BUS_DMA_NOWAIT);
3949 printf("Refresh mbufs: hdr dmamap load"
3950 " failure - %d\n", error);
3952 rxbuf->m_head = NULL;
3956 bus_dmamap_sync(rxr->rxtag,
3957 rxbuf->map, BUS_DMASYNC_PREREAD);
3958 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr);
3961 i = j; /* Next is precalulated for us */
3962 rxr->next_to_refresh = i;
3963 /* Calculate next controlling index */
3964 if (++j == adapter->num_rx_desc)
3969 ** Update the tail pointer only if,
3970 ** and as far as we have refreshed.
3973 E1000_WRITE_REG(&adapter->hw,
3974 E1000_RDT(rxr->me), rxr->next_to_refresh);
3980 /*********************************************************************
3982 * Allocate memory for rx_buffer structures. Since we use one
3983 * rx_buffer per received packet, the maximum number of rx_buffer's
3984 * that we'll need is equal to the number of receive descriptors
3985 * that we've allocated.
3987 **********************************************************************/
3989 em_allocate_receive_buffers(struct rx_ring *rxr)
3991 struct adapter *adapter = rxr->adapter;
3992 device_t dev = adapter->dev;
3993 struct em_buffer *rxbuf;
3996 rxr->rx_buffers = malloc(sizeof(struct em_buffer) *
3997 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3998 if (rxr->rx_buffers == NULL) {
3999 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4003 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
4004 1, 0, /* alignment, bounds */
4005 BUS_SPACE_MAXADDR, /* lowaddr */
4006 BUS_SPACE_MAXADDR, /* highaddr */
4007 NULL, NULL, /* filter, filterarg */
4008 MJUM9BYTES, /* maxsize */
4010 MJUM9BYTES, /* maxsegsize */
4012 NULL, /* lockfunc */
4016 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
4021 rxbuf = rxr->rx_buffers;
4022 for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
4023 rxbuf = &rxr->rx_buffers[i];
4024 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
4027 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4036 em_free_receive_structures(adapter);
4041 /*********************************************************************
4043 * Initialize a receive ring and its buffers.
4045 **********************************************************************/
4047 em_setup_receive_ring(struct rx_ring *rxr)
4049 struct adapter *adapter = rxr->adapter;
4050 struct em_buffer *rxbuf;
4051 bus_dma_segment_t seg[1];
4052 int rsize, nsegs, error = 0;
4054 struct netmap_adapter *na = NA(adapter->ifp);
4055 struct netmap_slot *slot;
4059 /* Clear the ring contents */
4061 rsize = roundup2(adapter->num_rx_desc *
4062 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
4063 bzero((void *)rxr->rx_base, rsize);
4065 slot = netmap_reset(na, NR_RX, 0, 0);
4069 ** Free current RX buffer structs and their mbufs
4071 for (int i = 0; i < adapter->num_rx_desc; i++) {
4072 rxbuf = &rxr->rx_buffers[i];
4073 if (rxbuf->m_head != NULL) {
4074 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4075 BUS_DMASYNC_POSTREAD);
4076 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4077 m_freem(rxbuf->m_head);
4078 rxbuf->m_head = NULL; /* mark as freed */
4082 /* Now replenish the mbufs */
4083 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4084 rxbuf = &rxr->rx_buffers[j];
4087 int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4091 addr = PNMB(slot + si, &paddr);
4092 netmap_load_map(rxr->rxtag, rxbuf->map, addr);
4093 /* Update descriptor */
4094 rxr->rx_base[j].buffer_addr = htole64(paddr);
4097 #endif /* DEV_NETMAP */
4098 rxbuf->m_head = m_getjcl(M_DONTWAIT, MT_DATA,
4099 M_PKTHDR, adapter->rx_mbuf_sz);
4100 if (rxbuf->m_head == NULL) {
4104 rxbuf->m_head->m_len = adapter->rx_mbuf_sz;
4105 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */
4106 rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz;
4108 /* Get the memory mapping */
4109 error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
4110 rxbuf->map, rxbuf->m_head, seg,
4111 &nsegs, BUS_DMA_NOWAIT);
4113 m_freem(rxbuf->m_head);
4114 rxbuf->m_head = NULL;
4117 bus_dmamap_sync(rxr->rxtag,
4118 rxbuf->map, BUS_DMASYNC_PREREAD);
4120 /* Update descriptor */
4121 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr);
4123 rxr->next_to_check = 0;
4124 rxr->next_to_refresh = 0;
4125 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4126 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4133 /*********************************************************************
4135 * Initialize all receive rings.
4137 **********************************************************************/
4139 em_setup_receive_structures(struct adapter *adapter)
4141 struct rx_ring *rxr = adapter->rx_rings;
4144 for (q = 0; q < adapter->num_queues; q++, rxr++)
4145 if (em_setup_receive_ring(rxr))
4151 * Free RX buffers allocated so far, we will only handle
4152 * the rings that completed, the failing case will have
4153 * cleaned up for itself. 'q' failed, so its the terminus.
4155 for (int i = 0; i < q; ++i) {
4156 rxr = &adapter->rx_rings[i];
4157 for (int n = 0; n < adapter->num_rx_desc; n++) {
4158 struct em_buffer *rxbuf;
4159 rxbuf = &rxr->rx_buffers[n];
4160 if (rxbuf->m_head != NULL) {
4161 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4162 BUS_DMASYNC_POSTREAD);
4163 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4164 m_freem(rxbuf->m_head);
4165 rxbuf->m_head = NULL;
4168 rxr->next_to_check = 0;
4169 rxr->next_to_refresh = 0;
4175 /*********************************************************************
4177 * Free all receive rings.
4179 **********************************************************************/
4181 em_free_receive_structures(struct adapter *adapter)
4183 struct rx_ring *rxr = adapter->rx_rings;
4185 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4186 em_free_receive_buffers(rxr);
4187 /* Free the ring memory as well */
4188 em_dma_free(adapter, &rxr->rxdma);
4189 EM_RX_LOCK_DESTROY(rxr);
4192 free(adapter->rx_rings, M_DEVBUF);
4196 /*********************************************************************
4198 * Free receive ring data structures
4200 **********************************************************************/
4202 em_free_receive_buffers(struct rx_ring *rxr)
4204 struct adapter *adapter = rxr->adapter;
4205 struct em_buffer *rxbuf = NULL;
4207 INIT_DEBUGOUT("free_receive_buffers: begin");
4209 if (rxr->rx_buffers != NULL) {
4210 for (int i = 0; i < adapter->num_rx_desc; i++) {
4211 rxbuf = &rxr->rx_buffers[i];
4212 if (rxbuf->map != NULL) {
4213 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4214 BUS_DMASYNC_POSTREAD);
4215 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4216 bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
4218 if (rxbuf->m_head != NULL) {
4219 m_freem(rxbuf->m_head);
4220 rxbuf->m_head = NULL;
4223 free(rxr->rx_buffers, M_DEVBUF);
4224 rxr->rx_buffers = NULL;
4225 rxr->next_to_check = 0;
4226 rxr->next_to_refresh = 0;
4229 if (rxr->rxtag != NULL) {
4230 bus_dma_tag_destroy(rxr->rxtag);
4238 /*********************************************************************
4240 * Enable receive unit.
4242 **********************************************************************/
4243 #define MAX_INTS_PER_SEC 8000
4244 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256)
4247 em_initialize_receive_unit(struct adapter *adapter)
4249 struct rx_ring *rxr = adapter->rx_rings;
4250 struct ifnet *ifp = adapter->ifp;
4251 struct e1000_hw *hw = &adapter->hw;
4255 INIT_DEBUGOUT("em_initialize_receive_units: begin");
4258 * Make sure receives are disabled while setting
4259 * up the descriptor ring
4261 rctl = E1000_READ_REG(hw, E1000_RCTL);
4262 /* Do not disable if ever enabled on this hardware */
4263 if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583))
4264 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4266 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4267 adapter->rx_abs_int_delay.value);
4269 * Set the interrupt throttling rate. Value is calculated
4270 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4272 E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
4275 ** When using MSIX interrupts we need to throttle
4276 ** using the EITR register (82574 only)
4278 if (hw->mac.type == e1000_82574) {
4279 for (int i = 0; i < 4; i++)
4280 E1000_WRITE_REG(hw, E1000_EITR_82574(i),
4282 /* Disable accelerated acknowledge */
4283 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4286 if (ifp->if_capenable & IFCAP_RXCSUM) {
4287 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4288 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4289 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4293 ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4294 ** long latencies are observed, like Lenovo X60. This
4295 ** change eliminates the problem, but since having positive
4296 ** values in RDTR is a known source of problems on other
4297 ** platforms another solution is being sought.
4299 if (hw->mac.type == e1000_82573)
4300 E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
4302 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4303 /* Setup the Base and Length of the Rx Descriptor Ring */
4304 bus_addr = rxr->rxdma.dma_paddr;
4305 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4306 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4307 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
4308 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
4309 /* Setup the Head and Tail Descriptor Pointers */
4310 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4313 * an init() while a netmap client is active must
4314 * preserve the rx buffers passed to userspace.
4315 * In this driver it means we adjust RDT to
4316 * something different from na->num_rx_desc - 1.
4318 if (ifp->if_capenable & IFCAP_NETMAP) {
4319 struct netmap_adapter *na = NA(adapter->ifp);
4320 struct netmap_kring *kring = &na->rx_rings[i];
4321 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
4323 E1000_WRITE_REG(hw, E1000_RDT(i), t);
4325 #endif /* DEV_NETMAP */
4326 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1);
4329 /* Set PTHRESH for improved jumbo performance */
4330 if (((adapter->hw.mac.type == e1000_ich9lan) ||
4331 (adapter->hw.mac.type == e1000_pch2lan) ||
4332 (adapter->hw.mac.type == e1000_ich10lan)) &&
4333 (ifp->if_mtu > ETHERMTU)) {
4334 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
4335 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3);
4338 if (adapter->hw.mac.type == e1000_pch2lan) {
4339 if (ifp->if_mtu > ETHERMTU)
4340 e1000_lv_jumbo_workaround_ich8lan(hw, TRUE);
4342 e1000_lv_jumbo_workaround_ich8lan(hw, FALSE);
4345 /* Setup the Receive Control Register */
4346 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4347 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
4348 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
4349 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4352 rctl |= E1000_RCTL_SECRC;
4354 /* Make sure VLAN Filters are off */
4355 rctl &= ~E1000_RCTL_VFE;
4356 rctl &= ~E1000_RCTL_SBP;
4358 if (adapter->rx_mbuf_sz == MCLBYTES)
4359 rctl |= E1000_RCTL_SZ_2048;
4360 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
4361 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4362 else if (adapter->rx_mbuf_sz > MJUMPAGESIZE)
4363 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4365 if (ifp->if_mtu > ETHERMTU)
4366 rctl |= E1000_RCTL_LPE;
4368 rctl &= ~E1000_RCTL_LPE;
4370 /* Write out the settings */
4371 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4377 /*********************************************************************
4379 * This routine executes in interrupt context. It replenishes
4380 * the mbufs in the descriptor and sends data which has been
4381 * dma'ed into host memory to upper layer.
4383 * We loop at most count times if count is > 0, or until done if
4386 * For polling we also now return the number of cleaned packets
4387 *********************************************************************/
4389 em_rxeof(struct rx_ring *rxr, int count, int *done)
4391 struct adapter *adapter = rxr->adapter;
4392 struct ifnet *ifp = adapter->ifp;
4393 struct mbuf *mp, *sendmp;
4396 int i, processed, rxdone = 0;
4398 struct e1000_rx_desc *cur;
4403 if (ifp->if_capenable & IFCAP_NETMAP) {
4404 struct netmap_adapter *na = NA(ifp);
4406 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4407 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4409 EM_CORE_LOCK(adapter);
4410 selwakeuppri(&na->rx_si, PI_NET);
4411 EM_CORE_UNLOCK(adapter);
4414 #endif /* DEV_NETMAP */
4416 for (i = rxr->next_to_check, processed = 0; count != 0;) {
4418 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4421 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4422 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4424 cur = &rxr->rx_base[i];
4425 status = cur->status;
4428 if ((status & E1000_RXD_STAT_DD) == 0)
4431 len = le16toh(cur->length);
4432 eop = (status & E1000_RXD_STAT_EOP) != 0;
4434 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) ||
4435 (rxr->discard == TRUE)) {
4436 adapter->dropped_pkts++;
4437 ++rxr->rx_discarded;
4438 if (!eop) /* Catch subsequent segs */
4439 rxr->discard = TRUE;
4441 rxr->discard = FALSE;
4442 em_rx_discard(rxr, i);
4446 /* Assign correct length to the current fragment */
4447 mp = rxr->rx_buffers[i].m_head;
4450 /* Trigger for refresh */
4451 rxr->rx_buffers[i].m_head = NULL;
4453 /* First segment? */
4454 if (rxr->fmp == NULL) {
4455 mp->m_pkthdr.len = len;
4456 rxr->fmp = rxr->lmp = mp;
4458 /* Chain mbuf's together */
4459 mp->m_flags &= ~M_PKTHDR;
4460 rxr->lmp->m_next = mp;
4462 rxr->fmp->m_pkthdr.len += len;
4468 sendmp->m_pkthdr.rcvif = ifp;
4470 em_receive_checksum(cur, sendmp);
4471 #ifndef __NO_STRICT_ALIGNMENT
4472 if (adapter->max_frame_size >
4473 (MCLBYTES - ETHER_ALIGN) &&
4474 em_fixup_rx(rxr) != 0)
4477 if (status & E1000_RXD_STAT_VP) {
4478 sendmp->m_pkthdr.ether_vtag =
4479 le16toh(cur->special);
4480 sendmp->m_flags |= M_VLANTAG;
4482 #ifndef __NO_STRICT_ALIGNMENT
4485 rxr->fmp = rxr->lmp = NULL;
4488 /* Zero out the receive descriptors status. */
4490 ++rxdone; /* cumulative for POLL */
4493 /* Advance our pointers to the next descriptor. */
4494 if (++i == adapter->num_rx_desc)
4497 /* Send to the stack */
4498 if (sendmp != NULL) {
4499 rxr->next_to_check = i;
4501 (*ifp->if_input)(ifp, sendmp);
4503 i = rxr->next_to_check;
4506 /* Only refresh mbufs every 8 descriptors */
4507 if (processed == 8) {
4508 em_refresh_mbufs(rxr, i);
4513 /* Catch any remaining refresh work */
4514 if (e1000_rx_unrefreshed(rxr))
4515 em_refresh_mbufs(rxr, i);
4517 rxr->next_to_check = i;
4522 return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4525 static __inline void
4526 em_rx_discard(struct rx_ring *rxr, int i)
4528 struct em_buffer *rbuf;
4530 rbuf = &rxr->rx_buffers[i];
4531 /* Free any previous pieces */
4532 if (rxr->fmp != NULL) {
4533 rxr->fmp->m_flags |= M_PKTHDR;
4539 ** Free buffer and allow em_refresh_mbufs()
4540 ** to clean up and recharge buffer.
4543 m_free(rbuf->m_head);
4544 rbuf->m_head = NULL;
4549 #ifndef __NO_STRICT_ALIGNMENT
4551 * When jumbo frames are enabled we should realign entire payload on
4552 * architecures with strict alignment. This is serious design mistake of 8254x
4553 * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4554 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4555 * payload. On architecures without strict alignment restrictions 8254x still
4556 * performs unaligned memory access which would reduce the performance too.
4557 * To avoid copying over an entire frame to align, we allocate a new mbuf and
4558 * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4559 * existing mbuf chain.
4561 * Be aware, best performance of the 8254x is achived only when jumbo frame is
4562 * not used at all on architectures with strict alignment.
4565 em_fixup_rx(struct rx_ring *rxr)
4567 struct adapter *adapter = rxr->adapter;
4573 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4574 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4575 m->m_data += ETHER_HDR_LEN;
4577 MGETHDR(n, M_DONTWAIT, MT_DATA);
4579 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4580 m->m_data += ETHER_HDR_LEN;
4581 m->m_len -= ETHER_HDR_LEN;
4582 n->m_len = ETHER_HDR_LEN;
4583 M_MOVE_PKTHDR(n, m);
4587 adapter->dropped_pkts++;
4598 /*********************************************************************
4600 * Verify that the hardware indicated that the checksum is valid.
4601 * Inform the stack about the status of checksum so that stack
4602 * doesn't spend time verifying the checksum.
4604 *********************************************************************/
4606 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4608 /* Ignore Checksum bit is set */
4609 if (rx_desc->status & E1000_RXD_STAT_IXSM) {
4610 mp->m_pkthdr.csum_flags = 0;
4614 if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4616 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4617 /* IP Checksum Good */
4618 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4619 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4622 mp->m_pkthdr.csum_flags = 0;
4626 if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4628 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4629 mp->m_pkthdr.csum_flags |=
4630 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4631 mp->m_pkthdr.csum_data = htons(0xffff);
4637 * This routine is run via an vlan
4641 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4643 struct adapter *adapter = ifp->if_softc;
4646 if (ifp->if_softc != arg) /* Not our event */
4649 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */
4652 EM_CORE_LOCK(adapter);
4653 index = (vtag >> 5) & 0x7F;
4655 adapter->shadow_vfta[index] |= (1 << bit);
4656 ++adapter->num_vlans;
4657 /* Re-init to load the changes */
4658 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4659 em_init_locked(adapter);
4660 EM_CORE_UNLOCK(adapter);
4664 * This routine is run via an vlan
4668 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4670 struct adapter *adapter = ifp->if_softc;
4673 if (ifp->if_softc != arg)
4676 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4679 EM_CORE_LOCK(adapter);
4680 index = (vtag >> 5) & 0x7F;
4682 adapter->shadow_vfta[index] &= ~(1 << bit);
4683 --adapter->num_vlans;
4684 /* Re-init to load the changes */
4685 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
4686 em_init_locked(adapter);
4687 EM_CORE_UNLOCK(adapter);
4691 em_setup_vlan_hw_support(struct adapter *adapter)
4693 struct e1000_hw *hw = &adapter->hw;
4697 ** We get here thru init_locked, meaning
4698 ** a soft reset, this has already cleared
4699 ** the VFTA and other state, so if there
4700 ** have been no vlan's registered do nothing.
4702 if (adapter->num_vlans == 0)
4706 ** A soft reset zero's out the VFTA, so
4707 ** we need to repopulate it now.
4709 for (int i = 0; i < EM_VFTA_SIZE; i++)
4710 if (adapter->shadow_vfta[i] != 0)
4711 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4712 i, adapter->shadow_vfta[i]);
4714 reg = E1000_READ_REG(hw, E1000_CTRL);
4715 reg |= E1000_CTRL_VME;
4716 E1000_WRITE_REG(hw, E1000_CTRL, reg);
4718 /* Enable the Filter Table */
4719 reg = E1000_READ_REG(hw, E1000_RCTL);
4720 reg &= ~E1000_RCTL_CFIEN;
4721 reg |= E1000_RCTL_VFE;
4722 E1000_WRITE_REG(hw, E1000_RCTL, reg);
4726 em_enable_intr(struct adapter *adapter)
4728 struct e1000_hw *hw = &adapter->hw;
4729 u32 ims_mask = IMS_ENABLE_MASK;
4731 if (hw->mac.type == e1000_82574) {
4732 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4733 ims_mask |= EM_MSIX_MASK;
4735 E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4739 em_disable_intr(struct adapter *adapter)
4741 struct e1000_hw *hw = &adapter->hw;
4743 if (hw->mac.type == e1000_82574)
4744 E1000_WRITE_REG(hw, EM_EIAC, 0);
4745 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4749 * Bit of a misnomer, what this really means is
4750 * to enable OS management of the system... aka
4751 * to disable special hardware management features
4754 em_init_manageability(struct adapter *adapter)
4756 /* A shared code workaround */
4757 #define E1000_82542_MANC2H E1000_MANC2H
4758 if (adapter->has_manage) {
4759 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4760 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4762 /* disable hardware interception of ARP */
4763 manc &= ~(E1000_MANC_ARP_EN);
4765 /* enable receiving management packets to the host */
4766 manc |= E1000_MANC_EN_MNG2HOST;
4767 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4768 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4769 manc2h |= E1000_MNG2HOST_PORT_623;
4770 manc2h |= E1000_MNG2HOST_PORT_664;
4771 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4772 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4777 * Give control back to hardware management
4778 * controller if there is one.
4781 em_release_manageability(struct adapter *adapter)
4783 if (adapter->has_manage) {
4784 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4786 /* re-enable hardware interception of ARP */
4787 manc |= E1000_MANC_ARP_EN;
4788 manc &= ~E1000_MANC_EN_MNG2HOST;
4790 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4795 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4796 * For ASF and Pass Through versions of f/w this means
4797 * that the driver is loaded. For AMT version type f/w
4798 * this means that the network i/f is open.
4801 em_get_hw_control(struct adapter *adapter)
4805 if (adapter->hw.mac.type == e1000_82573) {
4806 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4807 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4808 swsm | E1000_SWSM_DRV_LOAD);
4812 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4813 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4814 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4819 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4820 * For ASF and Pass Through versions of f/w this means that
4821 * the driver is no longer loaded. For AMT versions of the
4822 * f/w this means that the network i/f is closed.
4825 em_release_hw_control(struct adapter *adapter)
4829 if (!adapter->has_manage)
4832 if (adapter->hw.mac.type == e1000_82573) {
4833 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4834 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4835 swsm & ~E1000_SWSM_DRV_LOAD);
4839 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4840 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4841 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4846 em_is_valid_ether_addr(u8 *addr)
4848 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4850 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4858 ** Parse the interface capabilities with regard
4859 ** to both system management and wake-on-lan for
4863 em_get_wakeup(device_t dev)
4865 struct adapter *adapter = device_get_softc(dev);
4866 u16 eeprom_data = 0, device_id, apme_mask;
4868 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4869 apme_mask = EM_EEPROM_APME;
4871 switch (adapter->hw.mac.type) {
4874 adapter->has_amt = TRUE;
4878 case e1000_80003es2lan:
4879 if (adapter->hw.bus.func == 1) {
4880 e1000_read_nvm(&adapter->hw,
4881 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4884 e1000_read_nvm(&adapter->hw,
4885 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4889 case e1000_ich10lan:
4892 apme_mask = E1000_WUC_APME;
4893 adapter->has_amt = TRUE;
4894 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
4897 e1000_read_nvm(&adapter->hw,
4898 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4901 if (eeprom_data & apme_mask)
4902 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4904 * We have the eeprom settings, now apply the special cases
4905 * where the eeprom may be wrong or the board won't support
4906 * wake on lan on a particular port
4908 device_id = pci_get_device(dev);
4909 switch (device_id) {
4910 case E1000_DEV_ID_82571EB_FIBER:
4911 /* Wake events only supported on port A for dual fiber
4912 * regardless of eeprom setting */
4913 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4914 E1000_STATUS_FUNC_1)
4917 case E1000_DEV_ID_82571EB_QUAD_COPPER:
4918 case E1000_DEV_ID_82571EB_QUAD_FIBER:
4919 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
4920 /* if quad port adapter, disable WoL on all but port A */
4921 if (global_quad_port_a != 0)
4923 /* Reset for multiple quad port adapters */
4924 if (++global_quad_port_a == 4)
4925 global_quad_port_a = 0;
4933 * Enable PCI Wake On Lan capability
4936 em_enable_wakeup(device_t dev)
4938 struct adapter *adapter = device_get_softc(dev);
4939 struct ifnet *ifp = adapter->ifp;
4940 u32 pmc, ctrl, ctrl_ext, rctl;
4943 if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
4946 /* Advertise the wakeup capability */
4947 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4948 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4949 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4950 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4952 if ((adapter->hw.mac.type == e1000_ich8lan) ||
4953 (adapter->hw.mac.type == e1000_pchlan) ||
4954 (adapter->hw.mac.type == e1000_ich9lan) ||
4955 (adapter->hw.mac.type == e1000_ich10lan))
4956 e1000_suspend_workarounds_ich8lan(&adapter->hw);
4958 /* Keep the laser running on Fiber adapters */
4959 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4960 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4961 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4962 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4963 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4967 ** Determine type of Wakeup: note that wol
4968 ** is set with all bits on by default.
4970 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0)
4971 adapter->wol &= ~E1000_WUFC_MAG;
4973 if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0)
4974 adapter->wol &= ~E1000_WUFC_MC;
4976 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4977 rctl |= E1000_RCTL_MPE;
4978 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4981 if ((adapter->hw.mac.type == e1000_pchlan) ||
4982 (adapter->hw.mac.type == e1000_pch2lan)) {
4983 if (em_enable_phy_wakeup(adapter))
4986 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4987 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4990 if (adapter->hw.phy.type == e1000_phy_igp_3)
4991 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
4994 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4995 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4996 if (ifp->if_capenable & IFCAP_WOL)
4997 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4998 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
5004 ** WOL in the newer chipset interfaces (pchlan)
5005 ** require thing to be copied into the phy
5008 em_enable_phy_wakeup(struct adapter *adapter)
5010 struct e1000_hw *hw = &adapter->hw;
5014 /* copy MAC RARs to PHY RARs */
5015 e1000_copy_rx_addrs_to_phy_ich8lan(hw);
5017 /* copy MAC MTA to PHY MTA */
5018 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5019 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
5020 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
5021 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
5022 (u16)((mreg >> 16) & 0xFFFF));
5025 /* configure PHY Rx Control register */
5026 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
5027 mreg = E1000_READ_REG(hw, E1000_RCTL);
5028 if (mreg & E1000_RCTL_UPE)
5029 preg |= BM_RCTL_UPE;
5030 if (mreg & E1000_RCTL_MPE)
5031 preg |= BM_RCTL_MPE;
5032 preg &= ~(BM_RCTL_MO_MASK);
5033 if (mreg & E1000_RCTL_MO_3)
5034 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5035 << BM_RCTL_MO_SHIFT);
5036 if (mreg & E1000_RCTL_BAM)
5037 preg |= BM_RCTL_BAM;
5038 if (mreg & E1000_RCTL_PMCF)
5039 preg |= BM_RCTL_PMCF;
5040 mreg = E1000_READ_REG(hw, E1000_CTRL);
5041 if (mreg & E1000_CTRL_RFCE)
5042 preg |= BM_RCTL_RFCE;
5043 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
5045 /* enable PHY wakeup in MAC register */
5046 E1000_WRITE_REG(hw, E1000_WUC,
5047 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5048 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
5050 /* configure and enable PHY wakeup in PHY registers */
5051 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
5052 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5054 /* activate PHY wakeup */
5055 ret = hw->phy.ops.acquire(hw);
5057 printf("Could not acquire PHY\n");
5060 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
5061 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
5062 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
5064 printf("Could not read PHY page 769\n");
5067 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5068 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
5070 printf("Could not set PHY Host Wakeup bit\n");
5072 hw->phy.ops.release(hw);
5078 em_led_func(void *arg, int onoff)
5080 struct adapter *adapter = arg;
5082 EM_CORE_LOCK(adapter);
5084 e1000_setup_led(&adapter->hw);
5085 e1000_led_on(&adapter->hw);
5087 e1000_led_off(&adapter->hw);
5088 e1000_cleanup_led(&adapter->hw);
5090 EM_CORE_UNLOCK(adapter);
5094 ** Disable the L0S and L1 LINK states
5097 em_disable_aspm(struct adapter *adapter)
5100 u16 link_cap,link_ctrl;
5101 device_t dev = adapter->dev;
5103 switch (adapter->hw.mac.type) {
5111 if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0)
5113 reg = base + PCIER_LINK_CAP;
5114 link_cap = pci_read_config(dev, reg, 2);
5115 if ((link_cap & PCIEM_LINK_CAP_ASPM) == 0)
5117 reg = base + PCIER_LINK_CTL;
5118 link_ctrl = pci_read_config(dev, reg, 2);
5119 link_ctrl &= ~PCIEM_LINK_CTL_ASPMC;
5120 pci_write_config(dev, reg, link_ctrl, 2);
5124 /**********************************************************************
5126 * Update the board statistics counters.
5128 **********************************************************************/
5130 em_update_stats_counters(struct adapter *adapter)
5134 if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5135 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5136 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5137 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5139 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5140 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5141 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5142 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5144 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5145 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5146 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5147 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5148 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5149 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5150 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5152 ** For watchdog management we need to know if we have been
5153 ** paused during the last interval, so capture that here.
5155 adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5156 adapter->stats.xoffrxc += adapter->pause_frames;
5157 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5158 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5159 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5160 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5161 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5162 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5163 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5164 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5165 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5166 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5167 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5168 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5170 /* For the 64-bit byte counters the low dword must be read first. */
5171 /* Both registers clear on the read of the high dword */
5173 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
5174 ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
5175 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
5176 ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
5178 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5179 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5180 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5181 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5182 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5184 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5185 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5187 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5188 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5189 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5190 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5191 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5192 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5193 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5194 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5195 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5196 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5198 /* Interrupt Counts */
5200 adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC);
5201 adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC);
5202 adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC);
5203 adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC);
5204 adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC);
5205 adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC);
5206 adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC);
5207 adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC);
5208 adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC);
5210 if (adapter->hw.mac.type >= e1000_82543) {
5211 adapter->stats.algnerrc +=
5212 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5213 adapter->stats.rxerrc +=
5214 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5215 adapter->stats.tncrs +=
5216 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5217 adapter->stats.cexterr +=
5218 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5219 adapter->stats.tsctc +=
5220 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5221 adapter->stats.tsctfc +=
5222 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5226 ifp->if_collisions = adapter->stats.colc;
5229 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5230 adapter->stats.crcerrs + adapter->stats.algnerrc +
5231 adapter->stats.ruc + adapter->stats.roc +
5232 adapter->stats.mpc + adapter->stats.cexterr;
5235 ifp->if_oerrors = adapter->stats.ecol +
5236 adapter->stats.latecol + adapter->watchdog_events;
5239 /* Export a single 32-bit register via a read-only sysctl. */
5241 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5243 struct adapter *adapter;
5246 adapter = oidp->oid_arg1;
5247 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5248 return (sysctl_handle_int(oidp, &val, 0, req));
5252 * Add sysctl variables, one per statistic, to the system.
5255 em_add_hw_stats(struct adapter *adapter)
5257 device_t dev = adapter->dev;
5259 struct tx_ring *txr = adapter->tx_rings;
5260 struct rx_ring *rxr = adapter->rx_rings;
5262 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5263 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5264 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5265 struct e1000_hw_stats *stats = &adapter->stats;
5267 struct sysctl_oid *stat_node, *queue_node, *int_node;
5268 struct sysctl_oid_list *stat_list, *queue_list, *int_list;
5270 #define QUEUE_NAME_LEN 32
5271 char namebuf[QUEUE_NAME_LEN];
5273 /* Driver Statistics */
5274 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5275 CTLFLAG_RD, &adapter->link_irq,
5276 "Link MSIX IRQ Handled");
5277 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail",
5278 CTLFLAG_RD, &adapter->mbuf_alloc_failed,
5280 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail",
5281 CTLFLAG_RD, &adapter->mbuf_cluster_failed,
5282 "Std mbuf cluster failed");
5283 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5284 CTLFLAG_RD, &adapter->dropped_pkts,
5285 "Driver dropped packets");
5286 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5287 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5288 "Driver tx dma failure in xmit");
5289 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5290 CTLFLAG_RD, &adapter->rx_overruns,
5292 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5293 CTLFLAG_RD, &adapter->watchdog_events,
5294 "Watchdog timeouts");
5296 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
5297 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
5298 em_sysctl_reg_handler, "IU",
5299 "Device Control Register");
5300 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
5301 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
5302 em_sysctl_reg_handler, "IU",
5303 "Receiver Control Register");
5304 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5305 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5306 "Flow Control High Watermark");
5307 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5308 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5309 "Flow Control Low Watermark");
5311 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
5312 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5313 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5314 CTLFLAG_RD, NULL, "Queue Name");
5315 queue_list = SYSCTL_CHILDREN(queue_node);
5317 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5318 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5320 em_sysctl_reg_handler, "IU",
5321 "Transmit Descriptor Head");
5322 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5323 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5325 em_sysctl_reg_handler, "IU",
5326 "Transmit Descriptor Tail");
5327 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
5328 CTLFLAG_RD, &txr->tx_irq,
5329 "Queue MSI-X Transmit Interrupts");
5330 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail",
5331 CTLFLAG_RD, &txr->no_desc_avail,
5332 "Queue No Descriptor Available");
5334 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5335 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5337 em_sysctl_reg_handler, "IU",
5338 "Receive Descriptor Head");
5339 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5340 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5342 em_sysctl_reg_handler, "IU",
5343 "Receive Descriptor Tail");
5344 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
5345 CTLFLAG_RD, &rxr->rx_irq,
5346 "Queue MSI-X Receive Interrupts");
5349 /* MAC stats get their own sub node */
5351 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5352 CTLFLAG_RD, NULL, "Statistics");
5353 stat_list = SYSCTL_CHILDREN(stat_node);
5355 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5356 CTLFLAG_RD, &stats->ecol,
5357 "Excessive collisions");
5358 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
5359 CTLFLAG_RD, &stats->scc,
5360 "Single collisions");
5361 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5362 CTLFLAG_RD, &stats->mcc,
5363 "Multiple collisions");
5364 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
5365 CTLFLAG_RD, &stats->latecol,
5367 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
5368 CTLFLAG_RD, &stats->colc,
5370 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5371 CTLFLAG_RD, &adapter->stats.symerrs,
5373 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5374 CTLFLAG_RD, &adapter->stats.sec,
5376 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
5377 CTLFLAG_RD, &adapter->stats.dc,
5379 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5380 CTLFLAG_RD, &adapter->stats.mpc,
5382 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5383 CTLFLAG_RD, &adapter->stats.rnbc,
5384 "Receive No Buffers");
5385 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5386 CTLFLAG_RD, &adapter->stats.ruc,
5387 "Receive Undersize");
5388 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5389 CTLFLAG_RD, &adapter->stats.rfc,
5390 "Fragmented Packets Received ");
5391 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5392 CTLFLAG_RD, &adapter->stats.roc,
5393 "Oversized Packets Received");
5394 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5395 CTLFLAG_RD, &adapter->stats.rjc,
5397 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5398 CTLFLAG_RD, &adapter->stats.rxerrc,
5400 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5401 CTLFLAG_RD, &adapter->stats.crcerrs,
5403 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5404 CTLFLAG_RD, &adapter->stats.algnerrc,
5405 "Alignment Errors");
5406 /* On 82575 these are collision counts */
5407 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5408 CTLFLAG_RD, &adapter->stats.cexterr,
5409 "Collision/Carrier extension errors");
5410 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5411 CTLFLAG_RD, &adapter->stats.xonrxc,
5413 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5414 CTLFLAG_RD, &adapter->stats.xontxc,
5416 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5417 CTLFLAG_RD, &adapter->stats.xoffrxc,
5419 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5420 CTLFLAG_RD, &adapter->stats.xofftxc,
5421 "XOFF Transmitted");
5423 /* Packet Reception Stats */
5424 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5425 CTLFLAG_RD, &adapter->stats.tpr,
5426 "Total Packets Received ");
5427 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5428 CTLFLAG_RD, &adapter->stats.gprc,
5429 "Good Packets Received");
5430 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5431 CTLFLAG_RD, &adapter->stats.bprc,
5432 "Broadcast Packets Received");
5433 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5434 CTLFLAG_RD, &adapter->stats.mprc,
5435 "Multicast Packets Received");
5436 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5437 CTLFLAG_RD, &adapter->stats.prc64,
5438 "64 byte frames received ");
5439 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5440 CTLFLAG_RD, &adapter->stats.prc127,
5441 "65-127 byte frames received");
5442 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5443 CTLFLAG_RD, &adapter->stats.prc255,
5444 "128-255 byte frames received");
5445 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5446 CTLFLAG_RD, &adapter->stats.prc511,
5447 "256-511 byte frames received");
5448 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5449 CTLFLAG_RD, &adapter->stats.prc1023,
5450 "512-1023 byte frames received");
5451 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5452 CTLFLAG_RD, &adapter->stats.prc1522,
5453 "1023-1522 byte frames received");
5454 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5455 CTLFLAG_RD, &adapter->stats.gorc,
5456 "Good Octets Received");
5458 /* Packet Transmission Stats */
5459 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5460 CTLFLAG_RD, &adapter->stats.gotc,
5461 "Good Octets Transmitted");
5462 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5463 CTLFLAG_RD, &adapter->stats.tpt,
5464 "Total Packets Transmitted");
5465 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5466 CTLFLAG_RD, &adapter->stats.gptc,
5467 "Good Packets Transmitted");
5468 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5469 CTLFLAG_RD, &adapter->stats.bptc,
5470 "Broadcast Packets Transmitted");
5471 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5472 CTLFLAG_RD, &adapter->stats.mptc,
5473 "Multicast Packets Transmitted");
5474 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5475 CTLFLAG_RD, &adapter->stats.ptc64,
5476 "64 byte frames transmitted ");
5477 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5478 CTLFLAG_RD, &adapter->stats.ptc127,
5479 "65-127 byte frames transmitted");
5480 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5481 CTLFLAG_RD, &adapter->stats.ptc255,
5482 "128-255 byte frames transmitted");
5483 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5484 CTLFLAG_RD, &adapter->stats.ptc511,
5485 "256-511 byte frames transmitted");
5486 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5487 CTLFLAG_RD, &adapter->stats.ptc1023,
5488 "512-1023 byte frames transmitted");
5489 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5490 CTLFLAG_RD, &adapter->stats.ptc1522,
5491 "1024-1522 byte frames transmitted");
5492 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5493 CTLFLAG_RD, &adapter->stats.tsctc,
5494 "TSO Contexts Transmitted");
5495 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5496 CTLFLAG_RD, &adapter->stats.tsctfc,
5497 "TSO Contexts Failed");
5500 /* Interrupt Stats */
5502 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5503 CTLFLAG_RD, NULL, "Interrupt Statistics");
5504 int_list = SYSCTL_CHILDREN(int_node);
5506 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts",
5507 CTLFLAG_RD, &adapter->stats.iac,
5508 "Interrupt Assertion Count");
5510 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5511 CTLFLAG_RD, &adapter->stats.icrxptc,
5512 "Interrupt Cause Rx Pkt Timer Expire Count");
5514 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5515 CTLFLAG_RD, &adapter->stats.icrxatc,
5516 "Interrupt Cause Rx Abs Timer Expire Count");
5518 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5519 CTLFLAG_RD, &adapter->stats.ictxptc,
5520 "Interrupt Cause Tx Pkt Timer Expire Count");
5522 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5523 CTLFLAG_RD, &adapter->stats.ictxatc,
5524 "Interrupt Cause Tx Abs Timer Expire Count");
5526 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5527 CTLFLAG_RD, &adapter->stats.ictxqec,
5528 "Interrupt Cause Tx Queue Empty Count");
5530 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5531 CTLFLAG_RD, &adapter->stats.ictxqmtc,
5532 "Interrupt Cause Tx Queue Min Thresh Count");
5534 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5535 CTLFLAG_RD, &adapter->stats.icrxdmtc,
5536 "Interrupt Cause Rx Desc Min Thresh Count");
5538 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5539 CTLFLAG_RD, &adapter->stats.icrxoc,
5540 "Interrupt Cause Receiver Overrun Count");
5543 /**********************************************************************
5545 * This routine provides a way to dump out the adapter eeprom,
5546 * often a useful debug/service tool. This only dumps the first
5547 * 32 words, stuff that matters is in that extent.
5549 **********************************************************************/
5551 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5553 struct adapter *adapter = (struct adapter *)arg1;
5558 error = sysctl_handle_int(oidp, &result, 0, req);
5560 if (error || !req->newptr)
5564 * This value will cause a hex dump of the
5565 * first 32 16-bit words of the EEPROM to
5569 em_print_nvm_info(adapter);
5575 em_print_nvm_info(struct adapter *adapter)
5580 /* Its a bit crude, but it gets the job done */
5581 printf("\nInterface EEPROM Dump:\n");
5582 printf("Offset\n0x0000 ");
5583 for (i = 0, j = 0; i < 32; i++, j++) {
5584 if (j == 8) { /* Make the offset block */
5586 printf("\n0x00%x0 ",row);
5588 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5589 printf("%04x ", eeprom_data);
5595 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5597 struct em_int_delay_info *info;
5598 struct adapter *adapter;
5600 int error, usecs, ticks;
5602 info = (struct em_int_delay_info *)arg1;
5603 usecs = info->value;
5604 error = sysctl_handle_int(oidp, &usecs, 0, req);
5605 if (error != 0 || req->newptr == NULL)
5607 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5609 info->value = usecs;
5610 ticks = EM_USECS_TO_TICKS(usecs);
5612 adapter = info->adapter;
5614 EM_CORE_LOCK(adapter);
5615 regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5616 regval = (regval & ~0xffff) | (ticks & 0xffff);
5617 /* Handle a few special cases. */
5618 switch (info->offset) {
5623 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5624 /* Don't write 0 into the TIDV register. */
5627 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5630 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5631 EM_CORE_UNLOCK(adapter);
5636 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5637 const char *description, struct em_int_delay_info *info,
5638 int offset, int value)
5640 info->adapter = adapter;
5641 info->offset = offset;
5642 info->value = value;
5643 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5644 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5645 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5646 info, 0, em_sysctl_int_delay, "I", description);
5650 em_set_sysctl_value(struct adapter *adapter, const char *name,
5651 const char *description, int *limit, int value)
5654 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5655 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5656 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5661 ** Set flow control using sysctl:
5662 ** Flow control values:
5669 em_set_flowcntl(SYSCTL_HANDLER_ARGS)
5672 static int input = 3; /* default is full */
5673 struct adapter *adapter = (struct adapter *) arg1;
5675 error = sysctl_handle_int(oidp, &input, 0, req);
5677 if ((error) || (req->newptr == NULL))
5680 if (input == adapter->fc) /* no change? */
5684 case e1000_fc_rx_pause:
5685 case e1000_fc_tx_pause:
5688 adapter->hw.fc.requested_mode = input;
5689 adapter->fc = input;
5696 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
5697 e1000_force_mac_fc(&adapter->hw);
5702 ** Manage Energy Efficient Ethernet:
5704 ** 0/1 - enabled/disabled
5707 em_sysctl_eee(SYSCTL_HANDLER_ARGS)
5709 struct adapter *adapter = (struct adapter *) arg1;
5712 value = adapter->hw.dev_spec.ich8lan.eee_disable;
5713 error = sysctl_handle_int(oidp, &value, 0, req);
5714 if (error || req->newptr == NULL)
5716 EM_CORE_LOCK(adapter);
5717 adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0);
5718 em_init_locked(adapter);
5719 EM_CORE_UNLOCK(adapter);
5724 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5726 struct adapter *adapter;
5731 error = sysctl_handle_int(oidp, &result, 0, req);
5733 if (error || !req->newptr)
5737 adapter = (struct adapter *)arg1;
5738 em_print_debug_info(adapter);
5745 ** This routine is meant to be fluid, add whatever is
5746 ** needed for debugging a problem. -jfv
5749 em_print_debug_info(struct adapter *adapter)
5751 device_t dev = adapter->dev;
5752 struct tx_ring *txr = adapter->tx_rings;
5753 struct rx_ring *rxr = adapter->rx_rings;
5755 if (adapter->ifp->if_drv_flags & IFF_DRV_RUNNING)
5756 printf("Interface is RUNNING ");
5758 printf("Interface is NOT RUNNING\n");
5760 if (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE)
5761 printf("and INACTIVE\n");
5763 printf("and ACTIVE\n");
5765 device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5766 E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5767 E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5768 device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5769 E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5770 E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5771 device_printf(dev, "Tx Queue Status = %d\n", txr->queue_status);
5772 device_printf(dev, "TX descriptors avail = %d\n",
5774 device_printf(dev, "Tx Descriptors avail failure = %ld\n",
5775 txr->no_desc_avail);
5776 device_printf(dev, "RX discarded packets = %ld\n",
5778 device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check);
5779 device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh);