1 /******************************************************************************
3 Copyright (c) 2001-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
38 #include "opt_inet6.h"
40 #ifdef HAVE_KERNEL_OPTION_HEADERS
41 #include "opt_device_polling.h"
44 #include <sys/param.h>
45 #include <sys/systm.h>
47 #include <sys/types.h>
50 #if __FreeBSD_version >= 800000
51 #include <sys/buf_ring.h>
54 #include <sys/endian.h>
55 #include <sys/kernel.h>
56 #include <sys/kthread.h>
57 #include <sys/malloc.h>
59 #include <sys/module.h>
62 #include <sys/socket.h>
63 #include <sys/sockio.h>
64 #include <sys/sysctl.h>
65 #include <sys/taskqueue.h>
66 #include <sys/eventhandler.h>
67 #include <machine/bus.h>
68 #include <machine/resource.h>
71 #include <net/ethernet.h>
73 #include <net/if_var.h>
74 #include <net/if_arp.h>
75 #include <net/if_dl.h>
76 #include <net/if_media.h>
78 #include <net/if_types.h>
79 #include <net/if_vlan_var.h>
81 #include <netinet/in_systm.h>
82 #include <netinet/in.h>
83 #include <netinet/if_ether.h>
84 #include <netinet/ip.h>
85 #include <netinet/ip6.h>
86 #include <netinet/tcp.h>
87 #include <netinet/udp.h>
89 #include <machine/in_cksum.h>
90 #include <dev/led/led.h>
91 #include <dev/pci/pcivar.h>
92 #include <dev/pci/pcireg.h>
94 #include "e1000_api.h"
95 #include "e1000_82571.h"
98 /*********************************************************************
100 *********************************************************************/
101 char em_driver_version[] = "7.6.1-k";
103 /*********************************************************************
104 * PCI Device ID Table
106 * Used by probe to select devices to load on
107 * Last field stores an index into e1000_strings
108 * Last entry must be all 0s
110 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
111 *********************************************************************/
113 static em_vendor_info_t em_vendor_info_array[] =
115 /* Intel(R) PRO/1000 Network Connection */
116 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
117 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
118 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
119 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
120 PCI_ANY_ID, PCI_ANY_ID, 0},
121 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
122 PCI_ANY_ID, PCI_ANY_ID, 0},
123 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
124 PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
126 PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
128 PCI_ANY_ID, PCI_ANY_ID, 0},
129 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
130 PCI_ANY_ID, PCI_ANY_ID, 0},
131 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
132 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
133 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
134 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0},
136 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0},
137 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0},
138 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0},
139 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0},
140 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
141 PCI_ANY_ID, PCI_ANY_ID, 0},
142 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
143 PCI_ANY_ID, PCI_ANY_ID, 0},
144 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
145 PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
147 PCI_ANY_ID, PCI_ANY_ID, 0},
148 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
151 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
152 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
153 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
154 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
155 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0},
156 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
157 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
158 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
159 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
160 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0},
161 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
162 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
163 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
164 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0},
165 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0},
166 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0},
167 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
168 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
169 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
170 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
171 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
172 { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
173 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
174 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0},
175 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0},
176 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0},
177 { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
178 { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0},
179 { 0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
180 { 0x8086, E1000_DEV_ID_PCH_LPT_I217_V, PCI_ANY_ID, PCI_ANY_ID, 0},
181 { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM,
182 PCI_ANY_ID, PCI_ANY_ID, 0},
183 { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V,
184 PCI_ANY_ID, PCI_ANY_ID, 0},
185 { 0x8086, E1000_DEV_ID_PCH_I218_LM2, PCI_ANY_ID, PCI_ANY_ID, 0},
186 { 0x8086, E1000_DEV_ID_PCH_I218_V2, PCI_ANY_ID, PCI_ANY_ID, 0},
187 { 0x8086, E1000_DEV_ID_PCH_I218_LM3, PCI_ANY_ID, PCI_ANY_ID, 0},
188 { 0x8086, E1000_DEV_ID_PCH_I218_V3, PCI_ANY_ID, PCI_ANY_ID, 0},
189 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
190 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V, PCI_ANY_ID, PCI_ANY_ID, 0},
191 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2,
192 PCI_ANY_ID, PCI_ANY_ID, 0},
193 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, PCI_ANY_ID, PCI_ANY_ID, 0},
194 { 0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3,
195 PCI_ANY_ID, PCI_ANY_ID, 0},
196 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4,
197 PCI_ANY_ID, PCI_ANY_ID, 0},
198 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, PCI_ANY_ID, PCI_ANY_ID, 0},
199 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5,
200 PCI_ANY_ID, PCI_ANY_ID, 0},
201 { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, PCI_ANY_ID, PCI_ANY_ID, 0},
202 /* required last entry */
206 /*********************************************************************
207 * Table of branding strings for all supported NICs.
208 *********************************************************************/
210 static char *em_strings[] = {
211 "Intel(R) PRO/1000 Network Connection"
214 /*********************************************************************
215 * Function prototypes
216 *********************************************************************/
217 static int em_probe(device_t);
218 static int em_attach(device_t);
219 static int em_detach(device_t);
220 static int em_shutdown(device_t);
221 static int em_suspend(device_t);
222 static int em_resume(device_t);
224 static int em_mq_start(if_t, struct mbuf *);
225 static int em_mq_start_locked(if_t,
227 static void em_qflush(if_t);
229 static void em_start(if_t);
230 static void em_start_locked(if_t, struct tx_ring *);
232 static int em_ioctl(if_t, u_long, caddr_t);
233 static uint64_t em_get_counter(if_t, ift_counter);
234 static void em_init(void *);
235 static void em_init_locked(struct adapter *);
236 static void em_stop(void *);
237 static void em_media_status(if_t, struct ifmediareq *);
238 static int em_media_change(if_t);
239 static void em_identify_hardware(struct adapter *);
240 static int em_allocate_pci_resources(struct adapter *);
241 static int em_allocate_legacy(struct adapter *);
242 static int em_allocate_msix(struct adapter *);
243 static int em_allocate_queues(struct adapter *);
244 static int em_setup_msix(struct adapter *);
245 static void em_free_pci_resources(struct adapter *);
246 static void em_local_timer(void *);
247 static void em_reset(struct adapter *);
248 static int em_setup_interface(device_t, struct adapter *);
249 static void em_flush_desc_rings(struct adapter *);
251 static void em_setup_transmit_structures(struct adapter *);
252 static void em_initialize_transmit_unit(struct adapter *);
253 static int em_allocate_transmit_buffers(struct tx_ring *);
254 static void em_free_transmit_structures(struct adapter *);
255 static void em_free_transmit_buffers(struct tx_ring *);
257 static int em_setup_receive_structures(struct adapter *);
258 static int em_allocate_receive_buffers(struct rx_ring *);
259 static void em_initialize_receive_unit(struct adapter *);
260 static void em_free_receive_structures(struct adapter *);
261 static void em_free_receive_buffers(struct rx_ring *);
263 static void em_enable_intr(struct adapter *);
264 static void em_disable_intr(struct adapter *);
265 static void em_update_stats_counters(struct adapter *);
266 static void em_add_hw_stats(struct adapter *adapter);
267 static void em_txeof(struct tx_ring *);
268 static bool em_rxeof(struct rx_ring *, int, int *);
269 #ifndef __NO_STRICT_ALIGNMENT
270 static int em_fixup_rx(struct rx_ring *);
272 static void em_setup_rxdesc(union e1000_rx_desc_extended *,
273 const struct em_rxbuffer *rxbuf);
274 static void em_receive_checksum(uint32_t status, struct mbuf *);
275 static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int,
276 struct ip *, u32 *, u32 *);
277 static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *,
278 struct tcphdr *, u32 *, u32 *);
279 static void em_set_promisc(struct adapter *);
280 static void em_disable_promisc(struct adapter *);
281 static void em_set_multi(struct adapter *);
282 static void em_update_link_status(struct adapter *);
283 static void em_refresh_mbufs(struct rx_ring *, int);
284 static void em_register_vlan(void *, if_t, u16);
285 static void em_unregister_vlan(void *, if_t, u16);
286 static void em_setup_vlan_hw_support(struct adapter *);
287 static int em_xmit(struct tx_ring *, struct mbuf **);
288 static int em_dma_malloc(struct adapter *, bus_size_t,
289 struct em_dma_alloc *, int);
290 static void em_dma_free(struct adapter *, struct em_dma_alloc *);
291 static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
292 static void em_print_nvm_info(struct adapter *);
293 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
294 static void em_print_debug_info(struct adapter *);
295 static int em_is_valid_ether_addr(u8 *);
296 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
297 static void em_add_int_delay_sysctl(struct adapter *, const char *,
298 const char *, struct em_int_delay_info *, int, int);
299 /* Management and WOL Support */
300 static void em_init_manageability(struct adapter *);
301 static void em_release_manageability(struct adapter *);
302 static void em_get_hw_control(struct adapter *);
303 static void em_release_hw_control(struct adapter *);
304 static void em_get_wakeup(device_t);
305 static void em_enable_wakeup(device_t);
306 static int em_enable_phy_wakeup(struct adapter *);
307 static void em_led_func(void *, int);
308 static void em_disable_aspm(struct adapter *);
310 static int em_irq_fast(void *);
313 static void em_msix_tx(void *);
314 static void em_msix_rx(void *);
315 static void em_msix_link(void *);
316 static void em_handle_tx(void *context, int pending);
317 static void em_handle_rx(void *context, int pending);
318 static void em_handle_link(void *context, int pending);
321 static void em_enable_vectors_82574(struct adapter *);
324 static void em_set_sysctl_value(struct adapter *, const char *,
325 const char *, int *, int);
326 static int em_set_flowcntl(SYSCTL_HANDLER_ARGS);
327 static int em_sysctl_eee(SYSCTL_HANDLER_ARGS);
329 static __inline void em_rx_discard(struct rx_ring *, int);
331 #ifdef DEVICE_POLLING
332 static poll_handler_t em_poll;
335 /*********************************************************************
336 * FreeBSD Device Interface Entry Points
337 *********************************************************************/
339 static device_method_t em_methods[] = {
340 /* Device interface */
341 DEVMETHOD(device_probe, em_probe),
342 DEVMETHOD(device_attach, em_attach),
343 DEVMETHOD(device_detach, em_detach),
344 DEVMETHOD(device_shutdown, em_shutdown),
345 DEVMETHOD(device_suspend, em_suspend),
346 DEVMETHOD(device_resume, em_resume),
350 static driver_t em_driver = {
351 "em", em_methods, sizeof(struct adapter),
354 devclass_t em_devclass;
355 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
356 MODULE_DEPEND(em, pci, 1, 1, 1);
357 MODULE_DEPEND(em, ether, 1, 1, 1);
359 MODULE_DEPEND(em, netmap, 1, 1, 1);
360 #endif /* DEV_NETMAP */
362 /*********************************************************************
363 * Tunable default values.
364 *********************************************************************/
366 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000)
367 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024)
369 #define MAX_INTS_PER_SEC 8000
370 #define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256))
372 #define TSO_WORKAROUND 4
374 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
376 static int em_disable_crc_stripping = 0;
377 SYSCTL_INT(_hw_em, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN,
378 &em_disable_crc_stripping, 0, "Disable CRC Stripping");
380 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
381 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
382 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &em_tx_int_delay_dflt,
383 0, "Default transmit interrupt delay in usecs");
384 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt,
385 0, "Default receive interrupt delay in usecs");
387 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
388 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
389 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN,
390 &em_tx_abs_int_delay_dflt, 0,
391 "Default transmit interrupt delay limit in usecs");
392 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN,
393 &em_rx_abs_int_delay_dflt, 0,
394 "Default receive interrupt delay limit in usecs");
396 static int em_rxd = EM_DEFAULT_RXD;
397 static int em_txd = EM_DEFAULT_TXD;
398 SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0,
399 "Number of receive descriptors per queue");
400 SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0,
401 "Number of transmit descriptors per queue");
403 static int em_smart_pwr_down = FALSE;
404 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down,
405 0, "Set to true to leave smart power down enabled on newer adapters");
407 /* Controls whether promiscuous also shows bad packets */
408 static int em_debug_sbp = FALSE;
409 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0,
410 "Show bad packets in promiscuous mode");
412 static int em_enable_msix = TRUE;
413 SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0,
414 "Enable MSI-X interrupts");
417 static int em_num_queues = 1;
418 SYSCTL_INT(_hw_em, OID_AUTO, num_queues, CTLFLAG_RDTUN, &em_num_queues, 0,
419 "82574 only: Number of queues to configure, 0 indicates autoconfigure");
423 ** Global variable to store last used CPU when binding queues
424 ** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a
425 ** queue is bound to a cpu.
427 static int em_last_bind_cpu = -1;
429 /* How many packets rxeof tries to clean at a time */
430 static int em_rx_process_limit = 100;
431 SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
432 &em_rx_process_limit, 0,
433 "Maximum number of received packets to process "
434 "at a time, -1 means unlimited");
436 /* Energy efficient ethernet - default to OFF */
437 static int eee_setting = 1;
438 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0,
439 "Enable Energy Efficient Ethernet");
441 /* Global used in WOL setup with multiport cards */
442 static int global_quad_port_a = 0;
444 #ifdef DEV_NETMAP /* see ixgbe.c for details */
445 #include <dev/netmap/if_em_netmap.h>
446 #endif /* DEV_NETMAP */
448 /*********************************************************************
449 * Device identification routine
451 * em_probe determines if the driver should be loaded on
452 * adapter based on PCI vendor/device id of the adapter.
454 * return BUS_PROBE_DEFAULT on success, positive on failure
455 *********************************************************************/
458 em_probe(device_t dev)
460 char adapter_name[60];
461 uint16_t pci_vendor_id = 0;
462 uint16_t pci_device_id = 0;
463 uint16_t pci_subvendor_id = 0;
464 uint16_t pci_subdevice_id = 0;
465 em_vendor_info_t *ent;
467 INIT_DEBUGOUT("em_probe: begin");
469 pci_vendor_id = pci_get_vendor(dev);
470 if (pci_vendor_id != EM_VENDOR_ID)
473 pci_device_id = pci_get_device(dev);
474 pci_subvendor_id = pci_get_subvendor(dev);
475 pci_subdevice_id = pci_get_subdevice(dev);
477 ent = em_vendor_info_array;
478 while (ent->vendor_id != 0) {
479 if ((pci_vendor_id == ent->vendor_id) &&
480 (pci_device_id == ent->device_id) &&
482 ((pci_subvendor_id == ent->subvendor_id) ||
483 (ent->subvendor_id == PCI_ANY_ID)) &&
485 ((pci_subdevice_id == ent->subdevice_id) ||
486 (ent->subdevice_id == PCI_ANY_ID))) {
487 sprintf(adapter_name, "%s %s",
488 em_strings[ent->index],
490 device_set_desc_copy(dev, adapter_name);
491 return (BUS_PROBE_DEFAULT);
499 /*********************************************************************
500 * Device initialization routine
502 * The attach entry point is called when the driver is being loaded.
503 * This routine identifies the type of hardware, allocates all resources
504 * and initializes the hardware.
506 * return 0 on success, positive on failure
507 *********************************************************************/
510 em_attach(device_t dev)
512 struct adapter *adapter;
516 INIT_DEBUGOUT("em_attach: begin");
518 if (resource_disabled("em", device_get_unit(dev))) {
519 device_printf(dev, "Disabled by device hint\n");
523 adapter = device_get_softc(dev);
524 adapter->dev = adapter->osdep.dev = dev;
526 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
529 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
530 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
531 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
532 em_sysctl_nvm_info, "I", "NVM Information");
534 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
535 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
536 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
537 em_sysctl_debug_info, "I", "Debug Information");
539 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
540 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
541 OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
542 em_set_flowcntl, "I", "Flow Control");
544 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
546 /* Determine hardware and mac info */
547 em_identify_hardware(adapter);
549 /* Setup PCI resources */
550 if (em_allocate_pci_resources(adapter)) {
551 device_printf(dev, "Allocation of PCI resources failed\n");
557 ** For ICH8 and family we need to
558 ** map the flash memory, and this
559 ** must happen after the MAC is
562 if ((hw->mac.type == e1000_ich8lan) ||
563 (hw->mac.type == e1000_ich9lan) ||
564 (hw->mac.type == e1000_ich10lan) ||
565 (hw->mac.type == e1000_pchlan) ||
566 (hw->mac.type == e1000_pch2lan) ||
567 (hw->mac.type == e1000_pch_lpt)) {
568 int rid = EM_BAR_TYPE_FLASH;
569 adapter->flash = bus_alloc_resource_any(dev,
570 SYS_RES_MEMORY, &rid, RF_ACTIVE);
571 if (adapter->flash == NULL) {
572 device_printf(dev, "Mapping of Flash failed\n");
576 /* This is used in the shared code */
577 hw->flash_address = (u8 *)adapter->flash;
578 adapter->osdep.flash_bus_space_tag =
579 rman_get_bustag(adapter->flash);
580 adapter->osdep.flash_bus_space_handle =
581 rman_get_bushandle(adapter->flash);
584 ** In the new SPT device flash is not a
585 ** separate BAR, rather it is also in BAR0,
586 ** so use the same tag and an offset handle for the
587 ** FLASH read/write macros in the shared code.
589 else if (hw->mac.type == e1000_pch_spt) {
590 adapter->osdep.flash_bus_space_tag =
591 adapter->osdep.mem_bus_space_tag;
592 adapter->osdep.flash_bus_space_handle =
593 adapter->osdep.mem_bus_space_handle
594 + E1000_FLASH_BASE_ADDR;
597 /* Do Shared Code initialization */
598 error = e1000_setup_init_funcs(hw, TRUE);
600 device_printf(dev, "Setup of Shared code failed, error %d\n",
607 * Setup MSI/X or MSI if PCI Express
609 adapter->msix = em_setup_msix(adapter);
611 e1000_get_bus_info(hw);
613 /* Set up some sysctls for the tunable interrupt delays */
614 em_add_int_delay_sysctl(adapter, "rx_int_delay",
615 "receive interrupt delay in usecs", &adapter->rx_int_delay,
616 E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt);
617 em_add_int_delay_sysctl(adapter, "tx_int_delay",
618 "transmit interrupt delay in usecs", &adapter->tx_int_delay,
619 E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt);
620 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
621 "receive interrupt delay limit in usecs",
622 &adapter->rx_abs_int_delay,
623 E1000_REGISTER(hw, E1000_RADV),
624 em_rx_abs_int_delay_dflt);
625 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
626 "transmit interrupt delay limit in usecs",
627 &adapter->tx_abs_int_delay,
628 E1000_REGISTER(hw, E1000_TADV),
629 em_tx_abs_int_delay_dflt);
630 em_add_int_delay_sysctl(adapter, "itr",
631 "interrupt delay limit in usecs/4",
633 E1000_REGISTER(hw, E1000_ITR),
636 /* Sysctl for limiting the amount of work done in the taskqueue */
637 em_set_sysctl_value(adapter, "rx_processing_limit",
638 "max number of rx packets to process", &adapter->rx_process_limit,
639 em_rx_process_limit);
642 * Validate number of transmit and receive descriptors. It
643 * must not exceed hardware maximum, and must be multiple
644 * of E1000_DBA_ALIGN.
646 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
647 (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) {
648 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
649 EM_DEFAULT_TXD, em_txd);
650 adapter->num_tx_desc = EM_DEFAULT_TXD;
652 adapter->num_tx_desc = em_txd;
654 if (((em_rxd * sizeof(union e1000_rx_desc_extended)) % EM_DBA_ALIGN) != 0 ||
655 (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) {
656 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
657 EM_DEFAULT_RXD, em_rxd);
658 adapter->num_rx_desc = EM_DEFAULT_RXD;
660 adapter->num_rx_desc = em_rxd;
662 hw->mac.autoneg = DO_AUTO_NEG;
663 hw->phy.autoneg_wait_to_complete = FALSE;
664 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
667 if (hw->phy.media_type == e1000_media_type_copper) {
668 hw->phy.mdix = AUTO_ALL_MODES;
669 hw->phy.disable_polarity_correction = FALSE;
670 hw->phy.ms_type = EM_MASTER_SLAVE;
674 * Set the frame limits assuming
675 * standard ethernet sized frames.
677 adapter->hw.mac.max_frame_size =
678 ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
681 * This controls when hardware reports transmit completion
684 hw->mac.report_tx_early = 1;
687 ** Get queue/ring memory
689 if (em_allocate_queues(adapter)) {
694 /* Allocate multicast array memory. */
695 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
696 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
697 if (adapter->mta == NULL) {
698 device_printf(dev, "Can not allocate multicast setup array\n");
703 /* Check SOL/IDER usage */
704 if (e1000_check_reset_block(hw))
705 device_printf(dev, "PHY reset is blocked"
706 " due to SOL/IDER session.\n");
708 /* Sysctl for setting Energy Efficient Ethernet */
709 hw->dev_spec.ich8lan.eee_disable = eee_setting;
710 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
711 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
712 OID_AUTO, "eee_control", CTLTYPE_INT|CTLFLAG_RW,
713 adapter, 0, em_sysctl_eee, "I",
714 "Disable Energy Efficient Ethernet");
717 ** Start from a known state, this is
718 ** important in reading the nvm and
724 /* Make sure we have a good EEPROM before we read from it */
725 if (e1000_validate_nvm_checksum(hw) < 0) {
727 ** Some PCI-E parts fail the first check due to
728 ** the link being in sleep state, call it again,
729 ** if it fails a second time its a real issue.
731 if (e1000_validate_nvm_checksum(hw) < 0) {
733 "The EEPROM Checksum Is Not Valid\n");
739 /* Copy the permanent MAC address out of the EEPROM */
740 if (e1000_read_mac_addr(hw) < 0) {
741 device_printf(dev, "EEPROM read error while reading MAC"
747 if (!em_is_valid_ether_addr(hw->mac.addr)) {
748 device_printf(dev, "Invalid MAC address\n");
753 /* Disable ULP support */
754 e1000_disable_ulp_lpt_lp(hw, TRUE);
757 ** Do interrupt configuration
759 if (adapter->msix > 1) /* Do MSIX */
760 error = em_allocate_msix(adapter);
761 else /* MSI or Legacy */
762 error = em_allocate_legacy(adapter);
767 * Get Wake-on-Lan and Management info for later use
771 /* Setup OS specific network interface */
772 if (em_setup_interface(dev, adapter) != 0)
777 /* Initialize statistics */
778 em_update_stats_counters(adapter);
780 hw->mac.get_link_status = 1;
781 em_update_link_status(adapter);
783 /* Register for VLAN events */
784 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
785 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
786 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
787 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
789 em_add_hw_stats(adapter);
791 /* Non-AMT based hardware can now take control from firmware */
792 if (adapter->has_manage && !adapter->has_amt)
793 em_get_hw_control(adapter);
795 /* Tell the stack that the interface is not active */
796 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
798 adapter->led_dev = led_create(em_led_func, adapter,
799 device_get_nameunit(dev));
801 em_netmap_attach(adapter);
802 #endif /* DEV_NETMAP */
804 INIT_DEBUGOUT("em_attach: end");
809 em_free_transmit_structures(adapter);
810 em_free_receive_structures(adapter);
811 em_release_hw_control(adapter);
812 if (adapter->ifp != (void *)NULL)
813 if_free(adapter->ifp);
815 em_free_pci_resources(adapter);
816 free(adapter->mta, M_DEVBUF);
817 EM_CORE_LOCK_DESTROY(adapter);
822 /*********************************************************************
823 * Device removal routine
825 * The detach entry point is called when the driver is being removed.
826 * This routine stops the adapter and deallocates all the resources
827 * that were allocated for driver operation.
829 * return 0 on success, positive on failure
830 *********************************************************************/
833 em_detach(device_t dev)
835 struct adapter *adapter = device_get_softc(dev);
836 if_t ifp = adapter->ifp;
838 INIT_DEBUGOUT("em_detach: begin");
840 /* Make sure VLANS are not using driver */
841 if (if_vlantrunkinuse(ifp)) {
842 device_printf(dev,"Vlan in use, detach first\n");
846 #ifdef DEVICE_POLLING
847 if (if_getcapenable(ifp) & IFCAP_POLLING)
848 ether_poll_deregister(ifp);
851 if (adapter->led_dev != NULL)
852 led_destroy(adapter->led_dev);
854 EM_CORE_LOCK(adapter);
855 adapter->in_detach = 1;
857 EM_CORE_UNLOCK(adapter);
858 EM_CORE_LOCK_DESTROY(adapter);
860 e1000_phy_hw_reset(&adapter->hw);
862 em_release_manageability(adapter);
863 em_release_hw_control(adapter);
865 /* Unregister VLAN events */
866 if (adapter->vlan_attach != NULL)
867 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
868 if (adapter->vlan_detach != NULL)
869 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
871 ether_ifdetach(adapter->ifp);
872 callout_drain(&adapter->timer);
876 #endif /* DEV_NETMAP */
878 em_free_pci_resources(adapter);
879 bus_generic_detach(dev);
882 em_free_transmit_structures(adapter);
883 em_free_receive_structures(adapter);
885 em_release_hw_control(adapter);
886 free(adapter->mta, M_DEVBUF);
891 /*********************************************************************
893 * Shutdown entry point
895 **********************************************************************/
898 em_shutdown(device_t dev)
900 return em_suspend(dev);
904 * Suspend/resume device methods.
907 em_suspend(device_t dev)
909 struct adapter *adapter = device_get_softc(dev);
911 EM_CORE_LOCK(adapter);
913 em_release_manageability(adapter);
914 em_release_hw_control(adapter);
915 em_enable_wakeup(dev);
917 EM_CORE_UNLOCK(adapter);
919 return bus_generic_suspend(dev);
923 em_resume(device_t dev)
925 struct adapter *adapter = device_get_softc(dev);
926 struct tx_ring *txr = adapter->tx_rings;
927 if_t ifp = adapter->ifp;
929 EM_CORE_LOCK(adapter);
930 if (adapter->hw.mac.type == e1000_pch2lan)
931 e1000_resume_workarounds_pchlan(&adapter->hw);
932 em_init_locked(adapter);
933 em_init_manageability(adapter);
935 if ((if_getflags(ifp) & IFF_UP) &&
936 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) {
937 for (int i = 0; i < adapter->num_queues; i++, txr++) {
940 if (!drbr_empty(ifp, txr->br))
941 em_mq_start_locked(ifp, txr);
943 if (!if_sendq_empty(ifp))
944 em_start_locked(ifp, txr);
949 EM_CORE_UNLOCK(adapter);
951 return bus_generic_resume(dev);
955 #ifndef EM_MULTIQUEUE
957 em_start_locked(if_t ifp, struct tx_ring *txr)
959 struct adapter *adapter = if_getsoftc(ifp);
962 EM_TX_LOCK_ASSERT(txr);
964 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
968 if (!adapter->link_active)
971 while (!if_sendq_empty(ifp)) {
972 /* Call cleanup if number of TX descriptors low */
973 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD)
975 if (txr->tx_avail < EM_MAX_SCATTER) {
976 if_setdrvflagbits(ifp,IFF_DRV_OACTIVE, 0);
979 m_head = if_dequeue(ifp);
983 * Encapsulation can modify our pointer, and or make it
984 * NULL on failure. In that event, we can't requeue.
986 if (em_xmit(txr, &m_head)) {
989 if_sendq_prepend(ifp, m_head);
993 /* Mark the queue as having work */
994 if (txr->busy == EM_TX_IDLE)
995 txr->busy = EM_TX_BUSY;
997 /* Send a copy of the frame to the BPF listener */
998 ETHER_BPF_MTAP(ifp, m_head);
1008 struct adapter *adapter = if_getsoftc(ifp);
1009 struct tx_ring *txr = adapter->tx_rings;
1011 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1013 em_start_locked(ifp, txr);
1018 #else /* EM_MULTIQUEUE */
1019 /*********************************************************************
1020 * Multiqueue Transmit routines
1022 * em_mq_start is called by the stack to initiate a transmit.
1023 * however, if busy the driver can queue the request rather
1024 * than do an immediate send. It is this that is an advantage
1025 * in this driver, rather than also having multiple tx queues.
1026 **********************************************************************/
1028 ** Multiqueue capable stack interface
1031 em_mq_start(if_t ifp, struct mbuf *m)
1033 struct adapter *adapter = if_getsoftc(ifp);
1034 struct tx_ring *txr = adapter->tx_rings;
1035 unsigned int i, error;
1037 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1038 i = m->m_pkthdr.flowid % adapter->num_queues;
1040 i = curcpu % adapter->num_queues;
1042 txr = &adapter->tx_rings[i];
1044 error = drbr_enqueue(ifp, txr->br, m);
1048 if (EM_TX_TRYLOCK(txr)) {
1049 em_mq_start_locked(ifp, txr);
1052 taskqueue_enqueue(txr->tq, &txr->tx_task);
1058 em_mq_start_locked(if_t ifp, struct tx_ring *txr)
1060 struct adapter *adapter = txr->adapter;
1062 int err = 0, enq = 0;
1064 EM_TX_LOCK_ASSERT(txr);
1066 if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
1067 adapter->link_active == 0) {
1071 /* Process the queue */
1072 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
1073 if ((err = em_xmit(txr, &next)) != 0) {
1075 /* It was freed, move forward */
1076 drbr_advance(ifp, txr->br);
1079 * Still have one left, it may not be
1080 * the same since the transmit function
1081 * may have changed it.
1083 drbr_putback(ifp, txr->br, next);
1087 drbr_advance(ifp, txr->br);
1089 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
1090 if (next->m_flags & M_MCAST)
1091 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
1092 ETHER_BPF_MTAP(ifp, next);
1093 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
1097 /* Mark the queue as having work */
1098 if ((enq > 0) && (txr->busy == EM_TX_IDLE))
1099 txr->busy = EM_TX_BUSY;
1101 if (txr->tx_avail < EM_MAX_SCATTER)
1103 if (txr->tx_avail < EM_MAX_SCATTER) {
1104 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE,0);
1110 ** Flush all ring buffers
1115 struct adapter *adapter = if_getsoftc(ifp);
1116 struct tx_ring *txr = adapter->tx_rings;
1119 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1121 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
1127 #endif /* EM_MULTIQUEUE */
1129 /*********************************************************************
1132 * em_ioctl is called when the user wants to configure the
1135 * return 0 on success, positive on failure
1136 **********************************************************************/
1139 em_ioctl(if_t ifp, u_long command, caddr_t data)
1141 struct adapter *adapter = if_getsoftc(ifp);
1142 struct ifreq *ifr = (struct ifreq *)data;
1143 #if defined(INET) || defined(INET6)
1144 struct ifaddr *ifa = (struct ifaddr *)data;
1146 bool avoid_reset = FALSE;
1149 if (adapter->in_detach)
1155 if (ifa->ifa_addr->sa_family == AF_INET)
1159 if (ifa->ifa_addr->sa_family == AF_INET6)
1163 ** Calling init results in link renegotiation,
1164 ** so we avoid doing it when possible.
1167 if_setflagbits(ifp,IFF_UP,0);
1168 if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING))
1171 if (!(if_getflags(ifp) & IFF_NOARP))
1172 arp_ifinit(ifp, ifa);
1175 error = ether_ioctl(ifp, command, data);
1181 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1183 EM_CORE_LOCK(adapter);
1184 switch (adapter->hw.mac.type) {
1188 case e1000_ich10lan:
1194 case e1000_80003es2lan: /* 9K Jumbo Frame size */
1195 max_frame_size = 9234;
1198 max_frame_size = 4096;
1200 /* Adapters that do not support jumbo frames */
1202 max_frame_size = ETHER_MAX_LEN;
1205 max_frame_size = MAX_JUMBO_FRAME_SIZE;
1207 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1209 EM_CORE_UNLOCK(adapter);
1214 if_setmtu(ifp, ifr->ifr_mtu);
1215 adapter->hw.mac.max_frame_size =
1216 if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN;
1217 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
1218 em_init_locked(adapter);
1219 EM_CORE_UNLOCK(adapter);
1223 IOCTL_DEBUGOUT("ioctl rcv'd:\
1224 SIOCSIFFLAGS (Set Interface Flags)");
1225 EM_CORE_LOCK(adapter);
1226 if (if_getflags(ifp) & IFF_UP) {
1227 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1228 if ((if_getflags(ifp) ^ adapter->if_flags) &
1229 (IFF_PROMISC | IFF_ALLMULTI)) {
1230 em_disable_promisc(adapter);
1231 em_set_promisc(adapter);
1234 em_init_locked(adapter);
1236 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
1238 adapter->if_flags = if_getflags(ifp);
1239 EM_CORE_UNLOCK(adapter);
1243 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1244 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1245 EM_CORE_LOCK(adapter);
1246 em_disable_intr(adapter);
1247 em_set_multi(adapter);
1248 #ifdef DEVICE_POLLING
1249 if (!(if_getcapenable(ifp) & IFCAP_POLLING))
1251 em_enable_intr(adapter);
1252 EM_CORE_UNLOCK(adapter);
1256 /* Check SOL/IDER usage */
1257 EM_CORE_LOCK(adapter);
1258 if (e1000_check_reset_block(&adapter->hw)) {
1259 EM_CORE_UNLOCK(adapter);
1260 device_printf(adapter->dev, "Media change is"
1261 " blocked due to SOL/IDER session.\n");
1264 EM_CORE_UNLOCK(adapter);
1267 IOCTL_DEBUGOUT("ioctl rcv'd: \
1268 SIOCxIFMEDIA (Get/Set Interface Media)");
1269 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1275 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1277 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
1278 #ifdef DEVICE_POLLING
1279 if (mask & IFCAP_POLLING) {
1280 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1281 error = ether_poll_register(em_poll, ifp);
1284 EM_CORE_LOCK(adapter);
1285 em_disable_intr(adapter);
1286 if_setcapenablebit(ifp, IFCAP_POLLING, 0);
1287 EM_CORE_UNLOCK(adapter);
1289 error = ether_poll_deregister(ifp);
1290 /* Enable interrupt even in error case */
1291 EM_CORE_LOCK(adapter);
1292 em_enable_intr(adapter);
1293 if_setcapenablebit(ifp, 0, IFCAP_POLLING);
1294 EM_CORE_UNLOCK(adapter);
1298 if (mask & IFCAP_HWCSUM) {
1299 if_togglecapenable(ifp,IFCAP_HWCSUM);
1302 if (mask & IFCAP_TSO4) {
1303 if_togglecapenable(ifp,IFCAP_TSO4);
1306 if (mask & IFCAP_VLAN_HWTAGGING) {
1307 if_togglecapenable(ifp,IFCAP_VLAN_HWTAGGING);
1310 if (mask & IFCAP_VLAN_HWFILTER) {
1311 if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
1314 if (mask & IFCAP_VLAN_HWTSO) {
1315 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
1318 if ((mask & IFCAP_WOL) &&
1319 (if_getcapabilities(ifp) & IFCAP_WOL) != 0) {
1320 if (mask & IFCAP_WOL_MCAST)
1321 if_togglecapenable(ifp, IFCAP_WOL_MCAST);
1322 if (mask & IFCAP_WOL_MAGIC)
1323 if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
1325 if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1332 error = ether_ioctl(ifp, command, data);
1340 /*********************************************************************
1343 * This routine is used in two ways. It is used by the stack as
1344 * init entry point in network interface structure. It is also used
1345 * by the driver as a hw/sw initialization routine to get to a
1348 * return 0 on success, positive on failure
1349 **********************************************************************/
1352 em_init_locked(struct adapter *adapter)
1354 if_t ifp = adapter->ifp;
1355 device_t dev = adapter->dev;
1357 INIT_DEBUGOUT("em_init: begin");
1359 EM_CORE_LOCK_ASSERT(adapter);
1361 em_disable_intr(adapter);
1362 callout_stop(&adapter->timer);
1364 /* Get the latest mac address, User can use a LAA */
1365 bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr,
1368 /* Put the address into the Receive Address Array */
1369 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1372 * With the 82571 adapter, RAR[0] may be overwritten
1373 * when the other port is reset, we make a duplicate
1374 * in RAR[14] for that eventuality, this assures
1375 * the interface continues to function.
1377 if (adapter->hw.mac.type == e1000_82571) {
1378 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1379 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1380 E1000_RAR_ENTRIES - 1);
1383 /* Initialize the hardware */
1385 em_update_link_status(adapter);
1387 /* Setup VLAN support, basic and offload if available */
1388 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1390 /* Set hardware offload abilities */
1391 if_clearhwassist(ifp);
1392 if (if_getcapenable(ifp) & IFCAP_TXCSUM)
1393 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
1395 if (if_getcapenable(ifp) & IFCAP_TSO4)
1396 if_sethwassistbits(ifp, CSUM_TSO, 0);
1398 /* Configure for OS presence */
1399 em_init_manageability(adapter);
1401 /* Prepare transmit descriptors and buffers */
1402 em_setup_transmit_structures(adapter);
1403 em_initialize_transmit_unit(adapter);
1405 /* Setup Multicast table */
1406 em_set_multi(adapter);
1409 ** Figure out the desired mbuf
1410 ** pool for doing jumbos
1412 if (adapter->hw.mac.max_frame_size <= 2048)
1413 adapter->rx_mbuf_sz = MCLBYTES;
1414 else if (adapter->hw.mac.max_frame_size <= 4096)
1415 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1417 adapter->rx_mbuf_sz = MJUM9BYTES;
1419 /* Prepare receive descriptors and buffers */
1420 if (em_setup_receive_structures(adapter)) {
1421 device_printf(dev, "Could not setup receive structures\n");
1425 em_initialize_receive_unit(adapter);
1427 /* Use real VLAN Filter support? */
1428 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
1429 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
1430 /* Use real VLAN Filter support */
1431 em_setup_vlan_hw_support(adapter);
1434 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1435 ctrl |= E1000_CTRL_VME;
1436 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1440 /* Don't lose promiscuous settings */
1441 em_set_promisc(adapter);
1443 /* Set the interface as ACTIVE */
1444 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1446 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1447 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1449 /* MSI/X configuration for 82574 */
1450 if (adapter->hw.mac.type == e1000_82574) {
1452 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1453 tmp |= E1000_CTRL_EXT_PBA_CLR;
1454 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1455 /* Set the IVAR - interrupt vector routing. */
1456 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
1459 #ifdef DEVICE_POLLING
1461 * Only enable interrupts if we are not polling, make sure
1462 * they are off otherwise.
1464 if (if_getcapenable(ifp) & IFCAP_POLLING)
1465 em_disable_intr(adapter);
1467 #endif /* DEVICE_POLLING */
1468 em_enable_intr(adapter);
1470 /* AMT based hardware can now take control from firmware */
1471 if (adapter->has_manage && adapter->has_amt)
1472 em_get_hw_control(adapter);
1478 struct adapter *adapter = arg;
1480 EM_CORE_LOCK(adapter);
1481 em_init_locked(adapter);
1482 EM_CORE_UNLOCK(adapter);
1486 #ifdef DEVICE_POLLING
1487 /*********************************************************************
1489 * Legacy polling routine: note this only works with single queue
1491 *********************************************************************/
1493 em_poll(if_t ifp, enum poll_cmd cmd, int count)
1495 struct adapter *adapter = if_getsoftc(ifp);
1496 struct tx_ring *txr = adapter->tx_rings;
1497 struct rx_ring *rxr = adapter->rx_rings;
1501 EM_CORE_LOCK(adapter);
1502 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
1503 EM_CORE_UNLOCK(adapter);
1507 if (cmd == POLL_AND_CHECK_STATUS) {
1508 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1509 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1510 callout_stop(&adapter->timer);
1511 adapter->hw.mac.get_link_status = 1;
1512 em_update_link_status(adapter);
1513 callout_reset(&adapter->timer, hz,
1514 em_local_timer, adapter);
1517 EM_CORE_UNLOCK(adapter);
1519 em_rxeof(rxr, count, &rx_done);
1523 #ifdef EM_MULTIQUEUE
1524 if (!drbr_empty(ifp, txr->br))
1525 em_mq_start_locked(ifp, txr);
1527 if (!if_sendq_empty(ifp))
1528 em_start_locked(ifp, txr);
1534 #endif /* DEVICE_POLLING */
1537 /*********************************************************************
1539 * Fast Legacy/MSI Combined Interrupt Service routine
1541 *********************************************************************/
1543 em_irq_fast(void *arg)
1545 struct adapter *adapter = arg;
1551 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1554 if (reg_icr == 0xffffffff)
1555 return FILTER_STRAY;
1557 /* Definitely not our interrupt. */
1559 return FILTER_STRAY;
1562 * Starting with the 82571 chip, bit 31 should be used to
1563 * determine whether the interrupt belongs to us.
1565 if (adapter->hw.mac.type >= e1000_82571 &&
1566 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1567 return FILTER_STRAY;
1569 em_disable_intr(adapter);
1570 taskqueue_enqueue(adapter->tq, &adapter->que_task);
1572 /* Link status change */
1573 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1574 adapter->hw.mac.get_link_status = 1;
1575 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1578 if (reg_icr & E1000_ICR_RXO)
1579 adapter->rx_overruns++;
1580 return FILTER_HANDLED;
1583 /* Combined RX/TX handler, used by Legacy and MSI */
1585 em_handle_que(void *context, int pending)
1587 struct adapter *adapter = context;
1588 if_t ifp = adapter->ifp;
1589 struct tx_ring *txr = adapter->tx_rings;
1590 struct rx_ring *rxr = adapter->rx_rings;
1592 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1593 bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1597 #ifdef EM_MULTIQUEUE
1598 if (!drbr_empty(ifp, txr->br))
1599 em_mq_start_locked(ifp, txr);
1601 if (!if_sendq_empty(ifp))
1602 em_start_locked(ifp, txr);
1606 taskqueue_enqueue(adapter->tq, &adapter->que_task);
1611 em_enable_intr(adapter);
1616 /*********************************************************************
1618 * MSIX Interrupt Service Routines
1620 **********************************************************************/
1622 em_msix_tx(void *arg)
1624 struct tx_ring *txr = arg;
1625 struct adapter *adapter = txr->adapter;
1626 if_t ifp = adapter->ifp;
1631 #ifdef EM_MULTIQUEUE
1632 if (!drbr_empty(ifp, txr->br))
1633 em_mq_start_locked(ifp, txr);
1635 if (!if_sendq_empty(ifp))
1636 em_start_locked(ifp, txr);
1639 /* Reenable this interrupt */
1640 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1645 /*********************************************************************
1647 * MSIX RX Interrupt Service routine
1649 **********************************************************************/
1652 em_msix_rx(void *arg)
1654 struct rx_ring *rxr = arg;
1655 struct adapter *adapter = rxr->adapter;
1659 if (!(if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING))
1661 more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1663 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1665 /* Reenable this interrupt */
1666 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1671 /*********************************************************************
1673 * MSIX Link Fast Interrupt Service routine
1675 **********************************************************************/
1677 em_msix_link(void *arg)
1679 struct adapter *adapter = arg;
1682 ++adapter->link_irq;
1683 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1685 if (reg_icr & E1000_ICR_RXO)
1686 adapter->rx_overruns++;
1688 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1689 adapter->hw.mac.get_link_status = 1;
1690 em_handle_link(adapter, 0);
1692 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1693 EM_MSIX_LINK | E1000_IMS_LSC);
1695 ** Because we must read the ICR for this interrupt
1696 ** it may clear other causes using autoclear, for
1697 ** this reason we simply create a soft interrupt
1698 ** for all these vectors.
1701 E1000_WRITE_REG(&adapter->hw,
1702 E1000_ICS, adapter->ims);
1708 em_handle_rx(void *context, int pending)
1710 struct rx_ring *rxr = context;
1711 struct adapter *adapter = rxr->adapter;
1714 more = em_rxeof(rxr, adapter->rx_process_limit, NULL);
1716 taskqueue_enqueue(rxr->tq, &rxr->rx_task);
1718 /* Reenable this interrupt */
1719 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims);
1724 em_handle_tx(void *context, int pending)
1726 struct tx_ring *txr = context;
1727 struct adapter *adapter = txr->adapter;
1728 if_t ifp = adapter->ifp;
1732 #ifdef EM_MULTIQUEUE
1733 if (!drbr_empty(ifp, txr->br))
1734 em_mq_start_locked(ifp, txr);
1736 if (!if_sendq_empty(ifp))
1737 em_start_locked(ifp, txr);
1739 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims);
1744 em_handle_link(void *context, int pending)
1746 struct adapter *adapter = context;
1747 struct tx_ring *txr = adapter->tx_rings;
1748 if_t ifp = adapter->ifp;
1750 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1753 EM_CORE_LOCK(adapter);
1754 callout_stop(&adapter->timer);
1755 em_update_link_status(adapter);
1756 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1757 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1758 EM_MSIX_LINK | E1000_IMS_LSC);
1759 if (adapter->link_active) {
1760 for (int i = 0; i < adapter->num_queues; i++, txr++) {
1762 #ifdef EM_MULTIQUEUE
1763 if (!drbr_empty(ifp, txr->br))
1764 em_mq_start_locked(ifp, txr);
1766 if (if_sendq_empty(ifp))
1767 em_start_locked(ifp, txr);
1772 EM_CORE_UNLOCK(adapter);
1776 /*********************************************************************
1778 * Media Ioctl callback
1780 * This routine is called whenever the user queries the status of
1781 * the interface using ifconfig.
1783 **********************************************************************/
1785 em_media_status(if_t ifp, struct ifmediareq *ifmr)
1787 struct adapter *adapter = if_getsoftc(ifp);
1788 u_char fiber_type = IFM_1000_SX;
1790 INIT_DEBUGOUT("em_media_status: begin");
1792 EM_CORE_LOCK(adapter);
1793 em_update_link_status(adapter);
1795 ifmr->ifm_status = IFM_AVALID;
1796 ifmr->ifm_active = IFM_ETHER;
1798 if (!adapter->link_active) {
1799 EM_CORE_UNLOCK(adapter);
1803 ifmr->ifm_status |= IFM_ACTIVE;
1805 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1806 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1807 ifmr->ifm_active |= fiber_type | IFM_FDX;
1809 switch (adapter->link_speed) {
1811 ifmr->ifm_active |= IFM_10_T;
1814 ifmr->ifm_active |= IFM_100_TX;
1817 ifmr->ifm_active |= IFM_1000_T;
1820 if (adapter->link_duplex == FULL_DUPLEX)
1821 ifmr->ifm_active |= IFM_FDX;
1823 ifmr->ifm_active |= IFM_HDX;
1825 EM_CORE_UNLOCK(adapter);
1828 /*********************************************************************
1830 * Media Ioctl callback
1832 * This routine is called when the user changes speed/duplex using
1833 * media/mediopt option with ifconfig.
1835 **********************************************************************/
1837 em_media_change(if_t ifp)
1839 struct adapter *adapter = if_getsoftc(ifp);
1840 struct ifmedia *ifm = &adapter->media;
1842 INIT_DEBUGOUT("em_media_change: begin");
1844 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1847 EM_CORE_LOCK(adapter);
1848 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1850 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1851 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1856 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1857 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1860 adapter->hw.mac.autoneg = FALSE;
1861 adapter->hw.phy.autoneg_advertised = 0;
1862 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1863 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1865 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1868 adapter->hw.mac.autoneg = FALSE;
1869 adapter->hw.phy.autoneg_advertised = 0;
1870 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1871 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1873 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1876 device_printf(adapter->dev, "Unsupported media type\n");
1879 em_init_locked(adapter);
1880 EM_CORE_UNLOCK(adapter);
1885 /*********************************************************************
1887 * This routine maps the mbufs to tx descriptors.
1889 * return 0 on success, positive on failure
1890 **********************************************************************/
1893 em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1895 struct adapter *adapter = txr->adapter;
1896 bus_dma_segment_t segs[EM_MAX_SCATTER];
1898 struct em_txbuffer *tx_buffer, *tx_buffer_mapped;
1899 struct e1000_tx_desc *ctxd = NULL;
1900 struct mbuf *m_head;
1901 struct ether_header *eh;
1902 struct ip *ip = NULL;
1903 struct tcphdr *tp = NULL;
1904 u32 txd_upper = 0, txd_lower = 0;
1906 int nsegs, i, j, first, last = 0;
1908 bool do_tso, tso_desc, remap = TRUE;
1911 do_tso = (m_head->m_pkthdr.csum_flags & CSUM_TSO);
1916 * Intel recommends entire IP/TCP header length reside in a single
1917 * buffer. If multiple descriptors are used to describe the IP and
1918 * TCP header, each descriptor should describe one or more
1919 * complete headers; descriptors referencing only parts of headers
1920 * are not supported. If all layer headers are not coalesced into
1921 * a single buffer, each buffer should not cross a 4KB boundary,
1922 * or be larger than the maximum read request size.
1923 * Controller also requires modifing IP/TCP header to make TSO work
1924 * so we firstly get a writable mbuf chain then coalesce ethernet/
1925 * IP/TCP header into a single buffer to meet the requirement of
1926 * controller. This also simplifies IP/TCP/UDP checksum offloading
1927 * which also has similar restrictions.
1929 if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
1930 if (do_tso || (m_head->m_next != NULL &&
1931 m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) {
1932 if (M_WRITABLE(*m_headp) == 0) {
1933 m_head = m_dup(*m_headp, M_NOWAIT);
1935 if (m_head == NULL) {
1944 * Assume IPv4, we don't have TSO/checksum offload support
1947 ip_off = sizeof(struct ether_header);
1948 if (m_head->m_len < ip_off) {
1949 m_head = m_pullup(m_head, ip_off);
1950 if (m_head == NULL) {
1955 eh = mtod(m_head, struct ether_header *);
1956 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1957 ip_off = sizeof(struct ether_vlan_header);
1958 if (m_head->m_len < ip_off) {
1959 m_head = m_pullup(m_head, ip_off);
1960 if (m_head == NULL) {
1966 if (m_head->m_len < ip_off + sizeof(struct ip)) {
1967 m_head = m_pullup(m_head, ip_off + sizeof(struct ip));
1968 if (m_head == NULL) {
1973 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
1974 poff = ip_off + (ip->ip_hl << 2);
1976 if (do_tso || (m_head->m_pkthdr.csum_flags & CSUM_TCP)) {
1977 if (m_head->m_len < poff + sizeof(struct tcphdr)) {
1978 m_head = m_pullup(m_head, poff +
1979 sizeof(struct tcphdr));
1980 if (m_head == NULL) {
1985 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
1988 * pull 4 more bytes of data into it.
1990 if (m_head->m_len < poff + (tp->th_off << 2)) {
1991 m_head = m_pullup(m_head, poff +
1994 if (m_head == NULL) {
1999 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
2000 tp = (struct tcphdr *)(mtod(m_head, char *) + poff);
2002 ip->ip_len = htons(m_head->m_pkthdr.tso_segsz +
2007 * The pseudo TCP checksum does not include TCP
2008 * payload length so driver should recompute
2009 * the checksum here what hardware expect to
2010 * see. This is adherence of Microsoft's Large
2011 * Send specification.
2013 tp->th_sum = in_pseudo(ip->ip_src.s_addr,
2014 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
2016 } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) {
2017 if (m_head->m_len < poff + sizeof(struct udphdr)) {
2018 m_head = m_pullup(m_head, poff +
2019 sizeof(struct udphdr));
2020 if (m_head == NULL) {
2025 ip = (struct ip *)(mtod(m_head, char *) + ip_off);
2031 * Map the packet for DMA
2033 * Capture the first descriptor index,
2034 * this descriptor will have the index
2035 * of the EOP which is the only one that
2036 * now gets a DONE bit writeback.
2038 first = txr->next_avail_desc;
2039 tx_buffer = &txr->tx_buffers[first];
2040 tx_buffer_mapped = tx_buffer;
2041 map = tx_buffer->map;
2044 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
2045 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2048 * There are two types of errors we can (try) to handle:
2049 * - EFBIG means the mbuf chain was too long and bus_dma ran
2050 * out of segments. Defragment the mbuf chain and try again.
2051 * - ENOMEM means bus_dma could not obtain enough bounce buffers
2052 * at this point in time. Defer sending and try again later.
2053 * All other errors, in particular EINVAL, are fatal and prevent the
2054 * mbuf chain from ever going through. Drop it and report error.
2056 if (error == EFBIG && remap) {
2059 m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER);
2061 adapter->mbuf_defrag_failed++;
2068 /* Try it again, but only once */
2071 } else if (error != 0) {
2072 adapter->no_tx_dma_setup++;
2079 * TSO Hardware workaround, if this packet is not
2080 * TSO, and is only a single descriptor long, and
2081 * it follows a TSO burst, then we need to add a
2082 * sentinel descriptor to prevent premature writeback.
2084 if ((!do_tso) && (txr->tx_tso == TRUE)) {
2087 txr->tx_tso = FALSE;
2090 if (txr->tx_avail < (nsegs + EM_MAX_SCATTER)) {
2091 txr->no_desc_avail++;
2092 bus_dmamap_unload(txr->txtag, map);
2097 /* Do hardware assists */
2098 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2099 em_tso_setup(txr, m_head, ip_off, ip, tp,
2100 &txd_upper, &txd_lower);
2101 /* we need to make a final sentinel transmit desc */
2103 } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2104 em_transmit_checksum_setup(txr, m_head,
2105 ip_off, ip, &txd_upper, &txd_lower);
2107 if (m_head->m_flags & M_VLANTAG) {
2108 /* Set the vlan id. */
2109 txd_upper |= htole16(if_getvtag(m_head)) << 16;
2110 /* Tell hardware to add tag */
2111 txd_lower |= htole32(E1000_TXD_CMD_VLE);
2114 i = txr->next_avail_desc;
2116 /* Set up our transmit descriptors */
2117 for (j = 0; j < nsegs; j++) {
2119 bus_addr_t seg_addr;
2121 tx_buffer = &txr->tx_buffers[i];
2122 ctxd = &txr->tx_base[i];
2123 seg_addr = segs[j].ds_addr;
2124 seg_len = segs[j].ds_len;
2127 ** If this is the last descriptor, we want to
2128 ** split it so we have a small final sentinel
2130 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
2131 seg_len -= TSO_WORKAROUND;
2132 ctxd->buffer_addr = htole64(seg_addr);
2133 ctxd->lower.data = htole32(
2134 adapter->txd_cmd | txd_lower | seg_len);
2135 ctxd->upper.data = htole32(txd_upper);
2136 if (++i == adapter->num_tx_desc)
2139 /* Now make the sentinel */
2141 ctxd = &txr->tx_base[i];
2142 tx_buffer = &txr->tx_buffers[i];
2144 htole64(seg_addr + seg_len);
2145 ctxd->lower.data = htole32(
2146 adapter->txd_cmd | txd_lower | TSO_WORKAROUND);
2150 if (++i == adapter->num_tx_desc)
2153 ctxd->buffer_addr = htole64(seg_addr);
2154 ctxd->lower.data = htole32(
2155 adapter->txd_cmd | txd_lower | seg_len);
2156 ctxd->upper.data = htole32(txd_upper);
2158 if (++i == adapter->num_tx_desc)
2161 tx_buffer->m_head = NULL;
2162 tx_buffer->next_eop = -1;
2165 txr->next_avail_desc = i;
2166 txr->tx_avail -= nsegs;
2168 tx_buffer->m_head = m_head;
2170 ** Here we swap the map so the last descriptor,
2171 ** which gets the completion interrupt has the
2172 ** real map, and the first descriptor gets the
2173 ** unused map from this descriptor.
2175 tx_buffer_mapped->map = tx_buffer->map;
2176 tx_buffer->map = map;
2177 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
2180 * Last Descriptor of Packet
2181 * needs End Of Packet (EOP)
2182 * and Report Status (RS)
2185 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2187 * Keep track in the first buffer which
2188 * descriptor will be written back
2190 tx_buffer = &txr->tx_buffers[first];
2191 tx_buffer->next_eop = last;
2194 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2195 * that this frame is available to transmit.
2197 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2198 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2199 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
2205 em_set_promisc(struct adapter *adapter)
2207 if_t ifp = adapter->ifp;
2210 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2212 if (if_getflags(ifp) & IFF_PROMISC) {
2213 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2214 /* Turn this on if you want to see bad packets */
2216 reg_rctl |= E1000_RCTL_SBP;
2217 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2218 } else if (if_getflags(ifp) & IFF_ALLMULTI) {
2219 reg_rctl |= E1000_RCTL_MPE;
2220 reg_rctl &= ~E1000_RCTL_UPE;
2221 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2226 em_disable_promisc(struct adapter *adapter)
2228 if_t ifp = adapter->ifp;
2232 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2233 reg_rctl &= (~E1000_RCTL_UPE);
2234 if (if_getflags(ifp) & IFF_ALLMULTI)
2235 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2237 mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES);
2238 /* Don't disable if in MAX groups */
2239 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2240 reg_rctl &= (~E1000_RCTL_MPE);
2241 reg_rctl &= (~E1000_RCTL_SBP);
2242 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2246 /*********************************************************************
2249 * This routine is called whenever multicast address list is updated.
2251 **********************************************************************/
2254 em_set_multi(struct adapter *adapter)
2256 if_t ifp = adapter->ifp;
2258 u8 *mta; /* Multicast array memory */
2261 IOCTL_DEBUGOUT("em_set_multi: begin");
2264 bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
2266 if (adapter->hw.mac.type == e1000_82542 &&
2267 adapter->hw.revision_id == E1000_REVISION_2) {
2268 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2269 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2270 e1000_pci_clear_mwi(&adapter->hw);
2271 reg_rctl |= E1000_RCTL_RST;
2272 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2276 if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES);
2278 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2279 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2280 reg_rctl |= E1000_RCTL_MPE;
2281 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2283 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2285 if (adapter->hw.mac.type == e1000_82542 &&
2286 adapter->hw.revision_id == E1000_REVISION_2) {
2287 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2288 reg_rctl &= ~E1000_RCTL_RST;
2289 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2291 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2292 e1000_pci_set_mwi(&adapter->hw);
2297 /*********************************************************************
2300 * This routine checks for link status and updates statistics.
2302 **********************************************************************/
2305 em_local_timer(void *arg)
2307 struct adapter *adapter = arg;
2308 if_t ifp = adapter->ifp;
2309 struct tx_ring *txr = adapter->tx_rings;
2310 struct rx_ring *rxr = adapter->rx_rings;
2313 EM_CORE_LOCK_ASSERT(adapter);
2315 em_update_link_status(adapter);
2316 em_update_stats_counters(adapter);
2318 /* Reset LAA into RAR[0] on 82571 */
2319 if ((adapter->hw.mac.type == e1000_82571) &&
2320 e1000_get_laa_state_82571(&adapter->hw))
2321 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2323 /* Mask to use in the irq trigger */
2324 if (adapter->msix_mem) {
2325 for (int i = 0; i < adapter->num_queues; i++, rxr++)
2326 trigger |= rxr->ims;
2327 rxr = adapter->rx_rings;
2329 trigger = E1000_ICS_RXDMT0;
2332 ** Check on the state of the TX queue(s), this
2333 ** can be done without the lock because its RO
2334 ** and the HUNG state will be static if set.
2336 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2337 if (txr->busy == EM_TX_HUNG)
2339 if (txr->busy >= EM_TX_MAXTRIES)
2340 txr->busy = EM_TX_HUNG;
2341 /* Schedule a TX tasklet if needed */
2342 if (txr->tx_avail <= EM_MAX_SCATTER)
2343 taskqueue_enqueue(txr->tq, &txr->tx_task);
2346 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2347 #ifndef DEVICE_POLLING
2348 /* Trigger an RX interrupt to guarantee mbuf refresh */
2349 E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger);
2353 /* Looks like we're hung */
2354 device_printf(adapter->dev, "Watchdog timeout Queue[%d]-- resetting\n",
2356 em_print_debug_info(adapter);
2357 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
2358 adapter->watchdog_events++;
2359 em_init_locked(adapter);
2364 em_update_link_status(struct adapter *adapter)
2366 struct e1000_hw *hw = &adapter->hw;
2367 if_t ifp = adapter->ifp;
2368 device_t dev = adapter->dev;
2369 struct tx_ring *txr = adapter->tx_rings;
2372 /* Get the cached link value or read phy for real */
2373 switch (hw->phy.media_type) {
2374 case e1000_media_type_copper:
2375 if (hw->mac.get_link_status) {
2376 if (hw->mac.type == e1000_pch_spt)
2378 /* Do the work to read phy */
2379 e1000_check_for_link(hw);
2380 link_check = !hw->mac.get_link_status;
2381 if (link_check) /* ESB2 fix */
2382 e1000_cfg_on_link_up(hw);
2386 case e1000_media_type_fiber:
2387 e1000_check_for_link(hw);
2388 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2391 case e1000_media_type_internal_serdes:
2392 e1000_check_for_link(hw);
2393 link_check = adapter->hw.mac.serdes_has_link;
2396 case e1000_media_type_unknown:
2400 /* Now check for a transition */
2401 if (link_check && (adapter->link_active == 0)) {
2402 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2403 &adapter->link_duplex);
2405 ** There have proven to be problems with TSO when not
2406 ** at full gigabit speed, so disable the assist automatically
2407 ** when at lower speeds. -jfv
2409 if (adapter->link_speed != SPEED_1000) {
2410 if_sethwassistbits(ifp, 0, CSUM_TSO);
2411 if_setcapenablebit(ifp, 0, IFCAP_TSO4);
2412 if_setcapabilitiesbit(ifp, 0, IFCAP_TSO4);
2416 /* Check if we must disable SPEED_MODE bit on PCI-E */
2417 if ((adapter->link_speed != SPEED_1000) &&
2418 ((hw->mac.type == e1000_82571) ||
2419 (hw->mac.type == e1000_82572))) {
2421 tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2422 tarc0 &= ~TARC_SPEED_MODE_BIT;
2423 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2426 device_printf(dev, "Link is up %d Mbps %s\n",
2427 adapter->link_speed,
2428 ((adapter->link_duplex == FULL_DUPLEX) ?
2429 "Full Duplex" : "Half Duplex"));
2430 adapter->link_active = 1;
2431 adapter->smartspeed = 0;
2432 if_setbaudrate(ifp, adapter->link_speed * 1000000);
2433 if_link_state_change(ifp, LINK_STATE_UP);
2434 } else if (!link_check && (adapter->link_active == 1)) {
2435 if_setbaudrate(ifp, 0);
2436 adapter->link_speed = 0;
2437 adapter->link_duplex = 0;
2439 device_printf(dev, "Link is Down\n");
2440 adapter->link_active = 0;
2441 /* Link down, disable hang detection */
2442 for (int i = 0; i < adapter->num_queues; i++, txr++)
2443 txr->busy = EM_TX_IDLE;
2444 if_link_state_change(ifp, LINK_STATE_DOWN);
2448 /*********************************************************************
2450 * This routine disables all traffic on the adapter by issuing a
2451 * global reset on the MAC and deallocates TX/RX buffers.
2453 * This routine should always be called with BOTH the CORE
2455 **********************************************************************/
2460 struct adapter *adapter = arg;
2461 if_t ifp = adapter->ifp;
2462 struct tx_ring *txr = adapter->tx_rings;
2464 EM_CORE_LOCK_ASSERT(adapter);
2466 INIT_DEBUGOUT("em_stop: begin");
2468 em_disable_intr(adapter);
2469 callout_stop(&adapter->timer);
2471 /* Tell the stack that the interface is no longer active */
2472 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
2474 /* Disarm Hang Detection. */
2475 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2477 txr->busy = EM_TX_IDLE;
2481 /* I219 needs some special flushing to avoid hangs */
2482 if (adapter->hw.mac.type == e1000_pch_spt)
2483 em_flush_desc_rings(adapter);
2485 e1000_reset_hw(&adapter->hw);
2486 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2488 e1000_led_off(&adapter->hw);
2489 e1000_cleanup_led(&adapter->hw);
2493 /*********************************************************************
2495 * Determine hardware revision.
2497 **********************************************************************/
2499 em_identify_hardware(struct adapter *adapter)
2501 device_t dev = adapter->dev;
2503 /* Make sure our PCI config space has the necessary stuff set */
2504 pci_enable_busmaster(dev);
2505 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2507 /* Save off the information about this board */
2508 adapter->hw.vendor_id = pci_get_vendor(dev);
2509 adapter->hw.device_id = pci_get_device(dev);
2510 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2511 adapter->hw.subsystem_vendor_id =
2512 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2513 adapter->hw.subsystem_device_id =
2514 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2516 /* Do Shared Code Init and Setup */
2517 if (e1000_set_mac_type(&adapter->hw)) {
2518 device_printf(dev, "Setup init failure\n");
2524 em_allocate_pci_resources(struct adapter *adapter)
2526 device_t dev = adapter->dev;
2530 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2532 if (adapter->memory == NULL) {
2533 device_printf(dev, "Unable to allocate bus resource: memory\n");
2536 adapter->osdep.mem_bus_space_tag =
2537 rman_get_bustag(adapter->memory);
2538 adapter->osdep.mem_bus_space_handle =
2539 rman_get_bushandle(adapter->memory);
2540 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2542 adapter->hw.back = &adapter->osdep;
2547 /*********************************************************************
2549 * Setup the Legacy or MSI Interrupt handler
2551 **********************************************************************/
2553 em_allocate_legacy(struct adapter *adapter)
2555 device_t dev = adapter->dev;
2556 struct tx_ring *txr = adapter->tx_rings;
2559 /* Manually turn off all interrupts */
2560 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2562 if (adapter->msix == 1) /* using MSI */
2564 /* We allocate a single interrupt resource */
2565 adapter->res = bus_alloc_resource_any(dev,
2566 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2567 if (adapter->res == NULL) {
2568 device_printf(dev, "Unable to allocate bus resource: "
2574 * Allocate a fast interrupt and the associated
2575 * deferred processing contexts.
2577 TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter);
2578 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2579 taskqueue_thread_enqueue, &adapter->tq);
2580 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que",
2581 device_get_nameunit(adapter->dev));
2582 /* Use a TX only tasklet for local timer */
2583 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2584 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2585 taskqueue_thread_enqueue, &txr->tq);
2586 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq",
2587 device_get_nameunit(adapter->dev));
2588 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2589 if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET,
2590 em_irq_fast, NULL, adapter, &adapter->tag)) != 0) {
2591 device_printf(dev, "Failed to register fast interrupt "
2592 "handler: %d\n", error);
2593 taskqueue_free(adapter->tq);
2601 /*********************************************************************
2603 * Setup the MSIX Interrupt handlers
2604 * This is not really Multiqueue, rather
2605 * its just separate interrupt vectors
2606 * for TX, RX, and Link.
2608 **********************************************************************/
2610 em_allocate_msix(struct adapter *adapter)
2612 device_t dev = adapter->dev;
2613 struct tx_ring *txr = adapter->tx_rings;
2614 struct rx_ring *rxr = adapter->rx_rings;
2615 int error, rid, vector = 0;
2619 /* Make sure all interrupts are disabled */
2620 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2622 /* First set up ring resources */
2623 for (int i = 0; i < adapter->num_queues; i++, rxr++, vector++) {
2628 rxr->res = bus_alloc_resource_any(dev,
2629 SYS_RES_IRQ, &rid, RF_ACTIVE);
2630 if (rxr->res == NULL) {
2632 "Unable to allocate bus resource: "
2633 "RX MSIX Interrupt %d\n", i);
2636 if ((error = bus_setup_intr(dev, rxr->res,
2637 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx,
2638 rxr, &rxr->tag)) != 0) {
2639 device_printf(dev, "Failed to register RX handler");
2642 #if __FreeBSD_version >= 800504
2643 bus_describe_intr(dev, rxr->res, rxr->tag, "rx%d", i);
2647 if (em_last_bind_cpu < 0)
2648 em_last_bind_cpu = CPU_FIRST();
2649 cpu_id = em_last_bind_cpu;
2650 bus_bind_intr(dev, rxr->res, cpu_id);
2652 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr);
2653 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT,
2654 taskqueue_thread_enqueue, &rxr->tq);
2655 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq (cpuid %d)",
2656 device_get_nameunit(adapter->dev), cpu_id);
2658 ** Set the bit to enable interrupt
2659 ** in E1000_IMS -- bits 20 and 21
2660 ** are for RX0 and RX1, note this has
2661 ** NOTHING to do with the MSIX vector
2663 rxr->ims = 1 << (20 + i);
2664 adapter->ims |= rxr->ims;
2665 adapter->ivars |= (8 | rxr->msix) << (i * 4);
2667 em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu);
2670 for (int i = 0; i < adapter->num_queues; i++, txr++, vector++) {
2673 txr->res = bus_alloc_resource_any(dev,
2674 SYS_RES_IRQ, &rid, RF_ACTIVE);
2675 if (txr->res == NULL) {
2677 "Unable to allocate bus resource: "
2678 "TX MSIX Interrupt %d\n", i);
2681 if ((error = bus_setup_intr(dev, txr->res,
2682 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx,
2683 txr, &txr->tag)) != 0) {
2684 device_printf(dev, "Failed to register TX handler");
2687 #if __FreeBSD_version >= 800504
2688 bus_describe_intr(dev, txr->res, txr->tag, "tx%d", i);
2692 if (em_last_bind_cpu < 0)
2693 em_last_bind_cpu = CPU_FIRST();
2694 cpu_id = em_last_bind_cpu;
2695 bus_bind_intr(dev, txr->res, cpu_id);
2697 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr);
2698 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT,
2699 taskqueue_thread_enqueue, &txr->tq);
2700 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq (cpuid %d)",
2701 device_get_nameunit(adapter->dev), cpu_id);
2703 ** Set the bit to enable interrupt
2704 ** in E1000_IMS -- bits 22 and 23
2705 ** are for TX0 and TX1, note this has
2706 ** NOTHING to do with the MSIX vector
2708 txr->ims = 1 << (22 + i);
2709 adapter->ims |= txr->ims;
2710 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4));
2712 em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu);
2715 /* Link interrupt */
2717 adapter->res = bus_alloc_resource_any(dev,
2718 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2719 if (!adapter->res) {
2720 device_printf(dev,"Unable to allocate "
2721 "bus resource: Link interrupt [%d]\n", rid);
2724 /* Set the link handler function */
2725 error = bus_setup_intr(dev, adapter->res,
2726 INTR_TYPE_NET | INTR_MPSAFE, NULL,
2727 em_msix_link, adapter, &adapter->tag);
2729 adapter->res = NULL;
2730 device_printf(dev, "Failed to register LINK handler");
2733 #if __FreeBSD_version >= 800504
2734 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2736 adapter->linkvec = vector;
2737 adapter->ivars |= (8 | vector) << 16;
2738 adapter->ivars |= 0x80000000;
2745 em_free_pci_resources(struct adapter *adapter)
2747 device_t dev = adapter->dev;
2748 struct tx_ring *txr;
2749 struct rx_ring *rxr;
2754 ** Release all the queue interrupt resources:
2756 for (int i = 0; i < adapter->num_queues; i++) {
2757 txr = &adapter->tx_rings[i];
2758 /* an early abort? */
2762 if (txr->tag != NULL) {
2763 bus_teardown_intr(dev, txr->res, txr->tag);
2766 if (txr->res != NULL)
2767 bus_release_resource(dev, SYS_RES_IRQ,
2770 rxr = &adapter->rx_rings[i];
2771 /* an early abort? */
2775 if (rxr->tag != NULL) {
2776 bus_teardown_intr(dev, rxr->res, rxr->tag);
2779 if (rxr->res != NULL)
2780 bus_release_resource(dev, SYS_RES_IRQ,
2784 if (adapter->linkvec) /* we are doing MSIX */
2785 rid = adapter->linkvec + 1;
2787 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2789 if (adapter->tag != NULL) {
2790 bus_teardown_intr(dev, adapter->res, adapter->tag);
2791 adapter->tag = NULL;
2794 if (adapter->res != NULL)
2795 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2799 pci_release_msi(dev);
2801 if (adapter->msix_mem != NULL)
2802 bus_release_resource(dev, SYS_RES_MEMORY,
2803 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2805 if (adapter->memory != NULL)
2806 bus_release_resource(dev, SYS_RES_MEMORY,
2807 PCIR_BAR(0), adapter->memory);
2809 if (adapter->flash != NULL)
2810 bus_release_resource(dev, SYS_RES_MEMORY,
2811 EM_FLASH, adapter->flash);
2815 * Setup MSI or MSI/X
2818 em_setup_msix(struct adapter *adapter)
2820 device_t dev = adapter->dev;
2823 /* Nearly always going to use one queue */
2824 adapter->num_queues = 1;
2827 ** Try using MSI-X for Hartwell adapters
2829 if ((adapter->hw.mac.type == e1000_82574) &&
2830 (em_enable_msix == TRUE)) {
2831 #ifdef EM_MULTIQUEUE
2832 adapter->num_queues = (em_num_queues == 1) ? 1 : 2;
2833 if (adapter->num_queues > 1)
2834 em_enable_vectors_82574(adapter);
2836 /* Map the MSIX BAR */
2837 int rid = PCIR_BAR(EM_MSIX_BAR);
2838 adapter->msix_mem = bus_alloc_resource_any(dev,
2839 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2840 if (adapter->msix_mem == NULL) {
2841 /* May not be enabled */
2842 device_printf(adapter->dev,
2843 "Unable to map MSIX table \n");
2846 val = pci_msix_count(dev);
2848 #ifdef EM_MULTIQUEUE
2849 /* We need 5 vectors in the multiqueue case */
2850 if (adapter->num_queues > 1 ) {
2854 adapter->num_queues = 1;
2855 device_printf(adapter->dev,
2856 "Insufficient MSIX vectors for >1 queue, "
2857 "using single queue...\n");
2866 device_printf(adapter->dev,
2867 "Insufficient MSIX vectors, using MSI\n");
2870 #ifdef EM_MULTIQUEUE
2874 if ((pci_alloc_msix(dev, &val) == 0)) {
2875 device_printf(adapter->dev,
2876 "Using MSIX interrupts "
2877 "with %d vectors\n", val);
2882 ** If MSIX alloc failed or provided us with
2883 ** less than needed, free and fall through to MSI
2885 pci_release_msi(dev);
2888 if (adapter->msix_mem != NULL) {
2889 bus_release_resource(dev, SYS_RES_MEMORY,
2890 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem);
2891 adapter->msix_mem = NULL;
2894 if (pci_alloc_msi(dev, &val) == 0) {
2895 device_printf(adapter->dev, "Using an MSI interrupt\n");
2898 /* Should only happen due to manual configuration */
2899 device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n");
2905 ** The 3 following flush routines are used as a workaround in the
2906 ** I219 client parts and only for them.
2908 ** em_flush_tx_ring - remove all descriptors from the tx_ring
2910 ** We want to clear all pending descriptors from the TX ring.
2911 ** zeroing happens when the HW reads the regs. We assign the ring itself as
2912 ** the data of the next descriptor. We don't care about the data we are about
2916 em_flush_tx_ring(struct adapter *adapter)
2918 struct e1000_hw *hw = &adapter->hw;
2919 struct tx_ring *txr = adapter->tx_rings;
2920 struct e1000_tx_desc *txd;
2921 u32 tctl, txd_lower = E1000_TXD_CMD_IFCS;
2924 tctl = E1000_READ_REG(hw, E1000_TCTL);
2925 E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN);
2927 txd = &txr->tx_base[txr->next_avail_desc++];
2928 if (txr->next_avail_desc == adapter->num_tx_desc)
2929 txr->next_avail_desc = 0;
2931 /* Just use the ring as a dummy buffer addr */
2932 txd->buffer_addr = txr->txdma.dma_paddr;
2933 txd->lower.data = htole32(txd_lower | size);
2934 txd->upper.data = 0;
2936 /* flush descriptors to memory before notifying the HW */
2939 E1000_WRITE_REG(hw, E1000_TDT(0), txr->next_avail_desc);
2945 ** em_flush_rx_ring - remove all descriptors from the rx_ring
2947 ** Mark all descriptors in the RX ring as consumed and disable the rx ring
2950 em_flush_rx_ring(struct adapter *adapter)
2952 struct e1000_hw *hw = &adapter->hw;
2955 rctl = E1000_READ_REG(hw, E1000_RCTL);
2956 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
2957 E1000_WRITE_FLUSH(hw);
2960 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
2961 /* zero the lower 14 bits (prefetch and host thresholds) */
2962 rxdctl &= 0xffffc000;
2964 * update thresholds: prefetch threshold to 31, host threshold to 1
2965 * and make sure the granularity is "descriptors" and not "cache lines"
2967 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC);
2968 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl);
2970 /* momentarily enable the RX ring for the changes to take effect */
2971 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN);
2972 E1000_WRITE_FLUSH(hw);
2974 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
2978 ** em_flush_desc_rings - remove all descriptors from the descriptor rings
2980 ** In i219, the descriptor rings must be emptied before resetting the HW
2981 ** or before changing the device state to D3 during runtime (runtime PM).
2983 ** Failure to do this will cause the HW to enter a unit hang state which can
2984 ** only be released by PCI reset on the device
2988 em_flush_desc_rings(struct adapter *adapter)
2990 struct e1000_hw *hw = &adapter->hw;
2991 device_t dev = adapter->dev;
2993 u32 fext_nvm11, tdlen;
2995 /* First, disable MULR fix in FEXTNVM11 */
2996 fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11);
2997 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX;
2998 E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11);
3000 /* do nothing if we're not in faulty state, or if the queue is empty */
3001 tdlen = E1000_READ_REG(hw, E1000_TDLEN(0));
3002 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
3003 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen)
3005 em_flush_tx_ring(adapter);
3007 /* recheck, maybe the fault is caused by the rx ring */
3008 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
3009 if (hang_state & FLUSH_DESC_REQUIRED)
3010 em_flush_rx_ring(adapter);
3014 /*********************************************************************
3016 * Initialize the hardware to a configuration
3017 * as specified by the adapter structure.
3019 **********************************************************************/
3021 em_reset(struct adapter *adapter)
3023 device_t dev = adapter->dev;
3024 if_t ifp = adapter->ifp;
3025 struct e1000_hw *hw = &adapter->hw;
3029 INIT_DEBUGOUT("em_reset: begin");
3031 /* Set up smart power down as default off on newer adapters. */
3032 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
3033 hw->mac.type == e1000_82572)) {
3036 /* Speed up time to link by disabling smart power down. */
3037 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3038 phy_tmp &= ~IGP02E1000_PM_SPD;
3039 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3043 * Packet Buffer Allocation (PBA)
3044 * Writing PBA sets the receive portion of the buffer
3045 * the remainder is used for the transmit buffer.
3047 switch (hw->mac.type) {
3048 /* Total Packet Buffer on these is 48K */
3051 case e1000_80003es2lan:
3052 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
3054 case e1000_82573: /* 82573: Total Packet Buffer is 32K */
3055 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
3059 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
3065 case e1000_ich10lan:
3066 /* Boost Receive side for jumbo frames */
3067 if (adapter->hw.mac.max_frame_size > 4096)
3068 pba = E1000_PBA_14K;
3070 pba = E1000_PBA_10K;
3076 pba = E1000_PBA_26K;
3079 if (adapter->hw.mac.max_frame_size > 8192)
3080 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
3082 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
3084 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
3087 * These parameters control the automatic generation (Tx) and
3088 * response (Rx) to Ethernet PAUSE frames.
3089 * - High water mark should allow for at least two frames to be
3090 * received after sending an XOFF.
3091 * - Low water mark works best when it is very near the high water mark.
3092 * This allows the receiver to restart by sending XON when it has
3093 * drained a bit. Here we use an arbitrary value of 1500 which will
3094 * restart after one full frame is pulled from the buffer. There
3095 * could be several smaller frames in the buffer and if so they will
3096 * not trigger the XON until their total number reduces the buffer
3098 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3100 rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 );
3101 hw->fc.high_water = rx_buffer_size -
3102 roundup2(adapter->hw.mac.max_frame_size, 1024);
3103 hw->fc.low_water = hw->fc.high_water - 1500;
3105 if (adapter->fc) /* locally set flow control value? */
3106 hw->fc.requested_mode = adapter->fc;
3108 hw->fc.requested_mode = e1000_fc_full;
3110 if (hw->mac.type == e1000_80003es2lan)
3111 hw->fc.pause_time = 0xFFFF;
3113 hw->fc.pause_time = EM_FC_PAUSE_TIME;
3115 hw->fc.send_xon = TRUE;
3117 /* Device specific overrides/settings */
3118 switch (hw->mac.type) {
3120 /* Workaround: no TX flow ctrl for PCH */
3121 hw->fc.requested_mode = e1000_fc_rx_pause;
3122 hw->fc.pause_time = 0xFFFF; /* override */
3123 if (if_getmtu(ifp) > ETHERMTU) {
3124 hw->fc.high_water = 0x3500;
3125 hw->fc.low_water = 0x1500;
3127 hw->fc.high_water = 0x5000;
3128 hw->fc.low_water = 0x3000;
3130 hw->fc.refresh_time = 0x1000;
3135 hw->fc.high_water = 0x5C20;
3136 hw->fc.low_water = 0x5048;
3137 hw->fc.pause_time = 0x0650;
3138 hw->fc.refresh_time = 0x0400;
3139 /* Jumbos need adjusted PBA */
3140 if (if_getmtu(ifp) > ETHERMTU)
3141 E1000_WRITE_REG(hw, E1000_PBA, 12);
3143 E1000_WRITE_REG(hw, E1000_PBA, 26);
3146 case e1000_ich10lan:
3147 if (if_getmtu(ifp) > ETHERMTU) {
3148 hw->fc.high_water = 0x2800;
3149 hw->fc.low_water = hw->fc.high_water - 8;
3152 /* else fall thru */
3154 if (hw->mac.type == e1000_80003es2lan)
3155 hw->fc.pause_time = 0xFFFF;
3159 /* I219 needs some special flushing to avoid hangs */
3160 if (hw->mac.type == e1000_pch_spt)
3161 em_flush_desc_rings(adapter);
3163 /* Issue a global reset */
3165 E1000_WRITE_REG(hw, E1000_WUC, 0);
3166 em_disable_aspm(adapter);
3168 if (e1000_init_hw(hw) < 0) {
3169 device_printf(dev, "Hardware Initialization Failed\n");
3173 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
3174 e1000_get_phy_info(hw);
3175 e1000_check_for_link(hw);
3179 /*********************************************************************
3181 * Setup networking device structure and register an interface.
3183 **********************************************************************/
3185 em_setup_interface(device_t dev, struct adapter *adapter)
3189 INIT_DEBUGOUT("em_setup_interface: begin");
3191 ifp = adapter->ifp = if_gethandle(IFT_ETHER);
3193 device_printf(dev, "can not allocate ifnet structure\n");
3196 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3197 if_setdev(ifp, dev);
3198 if_setinitfn(ifp, em_init);
3199 if_setsoftc(ifp, adapter);
3200 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
3201 if_setioctlfn(ifp, em_ioctl);
3202 if_setgetcounterfn(ifp, em_get_counter);
3204 /* TSO parameters */
3205 ifp->if_hw_tsomax = IP_MAXPACKET;
3206 /* Take m_pullup(9)'s in em_xmit() w/ TSO into acount. */
3207 ifp->if_hw_tsomaxsegcount = EM_MAX_SCATTER - 5;
3208 ifp->if_hw_tsomaxsegsize = EM_TSO_SEG_SIZE;
3210 #ifdef EM_MULTIQUEUE
3211 /* Multiqueue stack interface */
3212 if_settransmitfn(ifp, em_mq_start);
3213 if_setqflushfn(ifp, em_qflush);
3215 if_setstartfn(ifp, em_start);
3216 if_setsendqlen(ifp, adapter->num_tx_desc - 1);
3217 if_setsendqready(ifp);
3220 ether_ifattach(ifp, adapter->hw.mac.addr);
3222 if_setcapabilities(ifp, 0);
3223 if_setcapenable(ifp, 0);
3226 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM |
3229 * Tell the upper layer(s) we
3230 * support full VLAN capability
3232 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
3233 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO |
3235 if_setcapenable(ifp, if_getcapabilities(ifp));
3238 ** Don't turn this on by default, if vlans are
3239 ** created on another pseudo device (eg. lagg)
3240 ** then vlan events are not passed thru, breaking
3241 ** operation, but with HW FILTER off it works. If
3242 ** using vlans directly on the em driver you can
3243 ** enable this and get full hardware tag filtering.
3245 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER,0);
3247 #ifdef DEVICE_POLLING
3248 if_setcapabilitiesbit(ifp, IFCAP_POLLING,0);
3251 /* Enable only WOL MAGIC by default */
3253 if_setcapabilitiesbit(ifp, IFCAP_WOL, 0);
3254 if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0);
3258 * Specify the media types supported by this adapter and register
3259 * callbacks to update media and link information
3261 ifmedia_init(&adapter->media, IFM_IMASK,
3262 em_media_change, em_media_status);
3263 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3264 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3265 u_char fiber_type = IFM_1000_SX; /* default type */
3267 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3269 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3271 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3272 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3274 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3276 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3278 if (adapter->hw.phy.type != e1000_phy_ife) {
3279 ifmedia_add(&adapter->media,
3280 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3281 ifmedia_add(&adapter->media,
3282 IFM_ETHER | IFM_1000_T, 0, NULL);
3285 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3286 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3292 * Manage DMA'able memory.
3295 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3299 *(bus_addr_t *) arg = segs[0].ds_addr;
3303 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3304 struct em_dma_alloc *dma, int mapflags)
3308 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3309 EM_DBA_ALIGN, 0, /* alignment, bounds */
3310 BUS_SPACE_MAXADDR, /* lowaddr */
3311 BUS_SPACE_MAXADDR, /* highaddr */
3312 NULL, NULL, /* filter, filterarg */
3315 size, /* maxsegsize */
3317 NULL, /* lockfunc */
3321 device_printf(adapter->dev,
3322 "%s: bus_dma_tag_create failed: %d\n",
3327 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3328 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3330 device_printf(adapter->dev,
3331 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3332 __func__, (uintmax_t)size, error);
3337 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3338 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3339 if (error || dma->dma_paddr == 0) {
3340 device_printf(adapter->dev,
3341 "%s: bus_dmamap_load failed: %d\n",
3349 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3351 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3352 bus_dma_tag_destroy(dma->dma_tag);
3354 dma->dma_tag = NULL;
3360 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3362 if (dma->dma_tag == NULL)
3364 if (dma->dma_paddr != 0) {
3365 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3366 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3367 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3370 if (dma->dma_vaddr != NULL) {
3371 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3372 dma->dma_vaddr = NULL;
3374 bus_dma_tag_destroy(dma->dma_tag);
3375 dma->dma_tag = NULL;
3379 /*********************************************************************
3381 * Allocate memory for the transmit and receive rings, and then
3382 * the descriptors associated with each, called only once at attach.
3384 **********************************************************************/
3386 em_allocate_queues(struct adapter *adapter)
3388 device_t dev = adapter->dev;
3389 struct tx_ring *txr = NULL;
3390 struct rx_ring *rxr = NULL;
3391 int rsize, tsize, error = E1000_SUCCESS;
3392 int txconf = 0, rxconf = 0;
3395 /* Allocate the TX ring struct memory */
3396 if (!(adapter->tx_rings =
3397 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
3398 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3399 device_printf(dev, "Unable to allocate TX ring memory\n");
3404 /* Now allocate the RX */
3405 if (!(adapter->rx_rings =
3406 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
3407 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3408 device_printf(dev, "Unable to allocate RX ring memory\n");
3413 tsize = roundup2(adapter->num_tx_desc *
3414 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
3416 * Now set up the TX queues, txconf is needed to handle the
3417 * possibility that things fail midcourse and we need to
3418 * undo memory gracefully
3420 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
3421 /* Set up some basics */
3422 txr = &adapter->tx_rings[i];
3423 txr->adapter = adapter;
3426 /* Initialize the TX lock */
3427 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3428 device_get_nameunit(dev), txr->me);
3429 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
3431 if (em_dma_malloc(adapter, tsize,
3432 &txr->txdma, BUS_DMA_NOWAIT)) {
3434 "Unable to allocate TX Descriptor memory\n");
3438 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
3439 bzero((void *)txr->tx_base, tsize);
3441 if (em_allocate_transmit_buffers(txr)) {
3443 "Critical Failure setting up transmit buffers\n");
3447 #if __FreeBSD_version >= 800000
3448 /* Allocate a buf ring */
3449 txr->br = buf_ring_alloc(4096, M_DEVBUF,
3450 M_WAITOK, &txr->tx_mtx);
3455 * Next the RX queues...
3457 rsize = roundup2(adapter->num_rx_desc *
3458 sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN);
3459 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
3460 rxr = &adapter->rx_rings[i];
3461 rxr->adapter = adapter;
3464 /* Initialize the RX lock */
3465 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3466 device_get_nameunit(dev), txr->me);
3467 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
3469 if (em_dma_malloc(adapter, rsize,
3470 &rxr->rxdma, BUS_DMA_NOWAIT)) {
3472 "Unable to allocate RxDescriptor memory\n");
3476 rxr->rx_base = (union e1000_rx_desc_extended *)rxr->rxdma.dma_vaddr;
3477 bzero((void *)rxr->rx_base, rsize);
3479 /* Allocate receive buffers for the ring*/
3480 if (em_allocate_receive_buffers(rxr)) {
3482 "Critical Failure setting up receive buffers\n");
3491 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
3492 em_dma_free(adapter, &rxr->rxdma);
3494 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
3495 em_dma_free(adapter, &txr->txdma);
3496 free(adapter->rx_rings, M_DEVBUF);
3498 #if __FreeBSD_version >= 800000
3499 buf_ring_free(txr->br, M_DEVBUF);
3501 free(adapter->tx_rings, M_DEVBUF);
3507 /*********************************************************************
3509 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3510 * the information needed to transmit a packet on the wire. This is
3511 * called only once at attach, setup is done every reset.
3513 **********************************************************************/
3515 em_allocate_transmit_buffers(struct tx_ring *txr)
3517 struct adapter *adapter = txr->adapter;
3518 device_t dev = adapter->dev;
3519 struct em_txbuffer *txbuf;
3523 * Setup DMA descriptor areas.
3525 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3526 1, 0, /* alignment, bounds */
3527 BUS_SPACE_MAXADDR, /* lowaddr */
3528 BUS_SPACE_MAXADDR, /* highaddr */
3529 NULL, NULL, /* filter, filterarg */
3530 EM_TSO_SIZE, /* maxsize */
3531 EM_MAX_SCATTER, /* nsegments */
3532 PAGE_SIZE, /* maxsegsize */
3534 NULL, /* lockfunc */
3535 NULL, /* lockfuncarg */
3537 device_printf(dev,"Unable to allocate TX DMA tag\n");
3541 if (!(txr->tx_buffers =
3542 (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) *
3543 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3544 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3549 /* Create the descriptor buffer dma maps */
3550 txbuf = txr->tx_buffers;
3551 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3552 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3554 device_printf(dev, "Unable to create TX DMA map\n");
3561 /* We free all, it handles case where we are in the middle */
3562 em_free_transmit_structures(adapter);
3566 /*********************************************************************
3568 * Initialize a transmit ring.
3570 **********************************************************************/
3572 em_setup_transmit_ring(struct tx_ring *txr)
3574 struct adapter *adapter = txr->adapter;
3575 struct em_txbuffer *txbuf;
3578 struct netmap_slot *slot;
3579 struct netmap_adapter *na = netmap_getna(adapter->ifp);
3580 #endif /* DEV_NETMAP */
3582 /* Clear the old descriptor contents */
3585 slot = netmap_reset(na, NR_TX, txr->me, 0);
3586 #endif /* DEV_NETMAP */
3588 bzero((void *)txr->tx_base,
3589 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3591 txr->next_avail_desc = 0;
3592 txr->next_to_clean = 0;
3594 /* Free any existing tx buffers. */
3595 txbuf = txr->tx_buffers;
3596 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3597 if (txbuf->m_head != NULL) {
3598 bus_dmamap_sync(txr->txtag, txbuf->map,
3599 BUS_DMASYNC_POSTWRITE);
3600 bus_dmamap_unload(txr->txtag, txbuf->map);
3601 m_freem(txbuf->m_head);
3602 txbuf->m_head = NULL;
3606 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3610 addr = PNMB(na, slot + si, &paddr);
3611 txr->tx_base[i].buffer_addr = htole64(paddr);
3612 /* reload the map for netmap mode */
3613 netmap_load_map(na, txr->txtag, txbuf->map, addr);
3615 #endif /* DEV_NETMAP */
3617 /* clear the watch index */
3618 txbuf->next_eop = -1;
3621 /* Set number of descriptors available */
3622 txr->tx_avail = adapter->num_tx_desc;
3623 txr->busy = EM_TX_IDLE;
3625 /* Clear checksum offload context. */
3626 txr->last_hw_offload = 0;
3627 txr->last_hw_ipcss = 0;
3628 txr->last_hw_ipcso = 0;
3629 txr->last_hw_tucss = 0;
3630 txr->last_hw_tucso = 0;
3632 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3633 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3637 /*********************************************************************
3639 * Initialize all transmit rings.
3641 **********************************************************************/
3643 em_setup_transmit_structures(struct adapter *adapter)
3645 struct tx_ring *txr = adapter->tx_rings;
3647 for (int i = 0; i < adapter->num_queues; i++, txr++)
3648 em_setup_transmit_ring(txr);
3653 /*********************************************************************
3655 * Enable transmit unit.
3657 **********************************************************************/
3659 em_initialize_transmit_unit(struct adapter *adapter)
3661 struct tx_ring *txr = adapter->tx_rings;
3662 struct e1000_hw *hw = &adapter->hw;
3663 u32 tctl, txdctl = 0, tarc, tipg = 0;
3665 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3667 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3668 u64 bus_addr = txr->txdma.dma_paddr;
3669 /* Base and Len of TX Ring */
3670 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3671 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3672 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3673 (u32)(bus_addr >> 32));
3674 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3676 /* Init the HEAD/TAIL indices */
3677 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3678 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3680 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3681 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
3682 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
3684 txr->busy = EM_TX_IDLE;
3685 txdctl = 0; /* clear txdctl */
3686 txdctl |= 0x1f; /* PTHRESH */
3687 txdctl |= 1 << 8; /* HTHRESH */
3688 txdctl |= 1 << 16;/* WTHRESH */
3689 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */
3690 txdctl |= E1000_TXDCTL_GRAN;
3691 txdctl |= 1 << 25; /* LWTHRESH */
3693 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3696 /* Set the default values for the Tx Inter Packet Gap timer */
3697 switch (adapter->hw.mac.type) {
3698 case e1000_80003es2lan:
3699 tipg = DEFAULT_82543_TIPG_IPGR1;
3700 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3701 E1000_TIPG_IPGR2_SHIFT;
3704 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3705 (adapter->hw.phy.media_type ==
3706 e1000_media_type_internal_serdes))
3707 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3709 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3710 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3711 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3714 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3715 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3717 if(adapter->hw.mac.type >= e1000_82540)
3718 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3719 adapter->tx_abs_int_delay.value);
3721 if ((adapter->hw.mac.type == e1000_82571) ||
3722 (adapter->hw.mac.type == e1000_82572)) {
3723 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3724 tarc |= TARC_SPEED_MODE_BIT;
3725 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3726 } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3727 /* errata: program both queues to unweighted RR */
3728 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3730 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3731 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3733 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3734 } else if (adapter->hw.mac.type == e1000_82574) {
3735 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3736 tarc |= TARC_ERRATA_BIT;
3737 if ( adapter->num_queues > 1) {
3738 tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX);
3739 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3740 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3742 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3745 adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3746 if (adapter->tx_int_delay.value > 0)
3747 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3749 /* Program the Transmit Control Register */
3750 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3751 tctl &= ~E1000_TCTL_CT;
3752 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3753 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3755 if (adapter->hw.mac.type >= e1000_82571)
3756 tctl |= E1000_TCTL_MULR;
3758 /* This write will effectively turn on the transmit unit. */
3759 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3761 if (hw->mac.type == e1000_pch_spt) {
3763 reg = E1000_READ_REG(hw, E1000_IOSFPC);
3764 reg |= E1000_RCTL_RDMTS_HEX;
3765 E1000_WRITE_REG(hw, E1000_IOSFPC, reg);
3766 reg = E1000_READ_REG(hw, E1000_TARC(0));
3767 reg |= E1000_TARC0_CB_MULTIQ_3_REQ;
3768 E1000_WRITE_REG(hw, E1000_TARC(0), reg);
3773 /*********************************************************************
3775 * Free all transmit rings.
3777 **********************************************************************/
3779 em_free_transmit_structures(struct adapter *adapter)
3781 struct tx_ring *txr = adapter->tx_rings;
3783 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3785 em_free_transmit_buffers(txr);
3786 em_dma_free(adapter, &txr->txdma);
3788 EM_TX_LOCK_DESTROY(txr);
3791 free(adapter->tx_rings, M_DEVBUF);
3794 /*********************************************************************
3796 * Free transmit ring related data structures.
3798 **********************************************************************/
3800 em_free_transmit_buffers(struct tx_ring *txr)
3802 struct adapter *adapter = txr->adapter;
3803 struct em_txbuffer *txbuf;
3805 INIT_DEBUGOUT("free_transmit_ring: begin");
3807 if (txr->tx_buffers == NULL)
3810 for (int i = 0; i < adapter->num_tx_desc; i++) {
3811 txbuf = &txr->tx_buffers[i];
3812 if (txbuf->m_head != NULL) {
3813 bus_dmamap_sync(txr->txtag, txbuf->map,
3814 BUS_DMASYNC_POSTWRITE);
3815 bus_dmamap_unload(txr->txtag,
3817 m_freem(txbuf->m_head);
3818 txbuf->m_head = NULL;
3819 if (txbuf->map != NULL) {
3820 bus_dmamap_destroy(txr->txtag,
3824 } else if (txbuf->map != NULL) {
3825 bus_dmamap_unload(txr->txtag,
3827 bus_dmamap_destroy(txr->txtag,
3832 #if __FreeBSD_version >= 800000
3833 if (txr->br != NULL)
3834 buf_ring_free(txr->br, M_DEVBUF);
3836 if (txr->tx_buffers != NULL) {
3837 free(txr->tx_buffers, M_DEVBUF);
3838 txr->tx_buffers = NULL;
3840 if (txr->txtag != NULL) {
3841 bus_dma_tag_destroy(txr->txtag);
3848 /*********************************************************************
3849 * The offload context is protocol specific (TCP/UDP) and thus
3850 * only needs to be set when the protocol changes. The occasion
3851 * of a context change can be a performance detriment, and
3852 * might be better just disabled. The reason arises in the way
3853 * in which the controller supports pipelined requests from the
3854 * Tx data DMA. Up to four requests can be pipelined, and they may
3855 * belong to the same packet or to multiple packets. However all
3856 * requests for one packet are issued before a request is issued
3857 * for a subsequent packet and if a request for the next packet
3858 * requires a context change, that request will be stalled
3859 * until the previous request completes. This means setting up
3860 * a new context effectively disables pipelined Tx data DMA which
3861 * in turn greatly slow down performance to send small sized
3863 **********************************************************************/
3865 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
3866 struct ip *ip, u32 *txd_upper, u32 *txd_lower)
3868 struct adapter *adapter = txr->adapter;
3869 struct e1000_context_desc *TXD = NULL;
3870 struct em_txbuffer *tx_buffer;
3874 u8 ipcso, ipcss, tucso, tucss;
3876 ipcss = ipcso = tucss = tucso = 0;
3877 hdr_len = ip_off + (ip->ip_hl << 2);
3878 cur = txr->next_avail_desc;
3880 /* Setup of IP header checksum. */
3881 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3882 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3885 ipcso = ip_off + offsetof(struct ip, ip_sum);
3887 * Start offset for header checksum calculation.
3888 * End offset for header checksum calculation.
3889 * Offset of place to put the checksum.
3891 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3892 TXD->lower_setup.ip_fields.ipcss = ipcss;
3893 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
3894 TXD->lower_setup.ip_fields.ipcso = ipcso;
3895 cmd |= E1000_TXD_CMD_IP;
3898 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3899 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3900 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3901 offload |= CSUM_TCP;
3903 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
3905 * The 82574L can only remember the *last* context used
3906 * regardless of queue that it was use for. We cannot reuse
3907 * contexts on this hardware platform and must generate a new
3908 * context every time. 82574L hardware spec, section 7.2.6,
3911 if (adapter->num_queues < 2) {
3913 * Setting up new checksum offload context for every
3914 * frames takes a lot of processing time for hardware.
3915 * This also reduces performance a lot for small sized
3916 * frames so avoid it if driver can use previously
3917 * configured checksum offload context.
3919 if (txr->last_hw_offload == offload) {
3920 if (offload & CSUM_IP) {
3921 if (txr->last_hw_ipcss == ipcss &&
3922 txr->last_hw_ipcso == ipcso &&
3923 txr->last_hw_tucss == tucss &&
3924 txr->last_hw_tucso == tucso)
3927 if (txr->last_hw_tucss == tucss &&
3928 txr->last_hw_tucso == tucso)
3932 txr->last_hw_offload = offload;
3933 txr->last_hw_tucss = tucss;
3934 txr->last_hw_tucso = tucso;
3937 * Start offset for payload checksum calculation.
3938 * End offset for payload checksum calculation.
3939 * Offset of place to put the checksum.
3941 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3942 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3943 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3944 TXD->upper_setup.tcp_fields.tucso = tucso;
3945 cmd |= E1000_TXD_CMD_TCP;
3946 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3947 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3948 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3950 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
3952 * The 82574L can only remember the *last* context used
3953 * regardless of queue that it was use for. We cannot reuse
3954 * contexts on this hardware platform and must generate a new
3955 * context every time. 82574L hardware spec, section 7.2.6,
3958 if (adapter->num_queues < 2) {
3960 * Setting up new checksum offload context for every
3961 * frames takes a lot of processing time for hardware.
3962 * This also reduces performance a lot for small sized
3963 * frames so avoid it if driver can use previously
3964 * configured checksum offload context.
3966 if (txr->last_hw_offload == offload) {
3967 if (offload & CSUM_IP) {
3968 if (txr->last_hw_ipcss == ipcss &&
3969 txr->last_hw_ipcso == ipcso &&
3970 txr->last_hw_tucss == tucss &&
3971 txr->last_hw_tucso == tucso)
3974 if (txr->last_hw_tucss == tucss &&
3975 txr->last_hw_tucso == tucso)
3979 txr->last_hw_offload = offload;
3980 txr->last_hw_tucss = tucss;
3981 txr->last_hw_tucso = tucso;
3984 * Start offset for header checksum calculation.
3985 * End offset for header checksum calculation.
3986 * Offset of place to put the checksum.
3988 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
3989 TXD->upper_setup.tcp_fields.tucss = tucss;
3990 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3991 TXD->upper_setup.tcp_fields.tucso = tucso;
3994 if (offload & CSUM_IP) {
3995 txr->last_hw_ipcss = ipcss;
3996 txr->last_hw_ipcso = ipcso;
3999 TXD->tcp_seg_setup.data = htole32(0);
4000 TXD->cmd_and_length =
4001 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
4002 tx_buffer = &txr->tx_buffers[cur];
4003 tx_buffer->m_head = NULL;
4004 tx_buffer->next_eop = -1;
4006 if (++cur == adapter->num_tx_desc)
4010 txr->next_avail_desc = cur;
4014 /**********************************************************************
4016 * Setup work for hardware segmentation offload (TSO)
4018 **********************************************************************/
4020 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off,
4021 struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower)
4023 struct adapter *adapter = txr->adapter;
4024 struct e1000_context_desc *TXD;
4025 struct em_txbuffer *tx_buffer;
4029 * In theory we can use the same TSO context if and only if
4030 * frame is the same type(IP/TCP) and the same MSS. However
4031 * checking whether a frame has the same IP/TCP structure is
4032 * hard thing so just ignore that and always restablish a
4035 hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2);
4036 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
4037 E1000_TXD_DTYP_D | /* Data descr type */
4038 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
4040 /* IP and/or TCP header checksum calculation and insertion. */
4041 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
4043 cur = txr->next_avail_desc;
4044 tx_buffer = &txr->tx_buffers[cur];
4045 TXD = (struct e1000_context_desc *) &txr->tx_base[cur];
4048 * Start offset for header checksum calculation.
4049 * End offset for header checksum calculation.
4050 * Offset of place put the checksum.
4052 TXD->lower_setup.ip_fields.ipcss = ip_off;
4053 TXD->lower_setup.ip_fields.ipcse =
4054 htole16(ip_off + (ip->ip_hl << 2) - 1);
4055 TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum);
4057 * Start offset for payload checksum calculation.
4058 * End offset for payload checksum calculation.
4059 * Offset of place to put the checksum.
4061 TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2);
4062 TXD->upper_setup.tcp_fields.tucse = 0;
4063 TXD->upper_setup.tcp_fields.tucso =
4064 ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum);
4066 * Payload size per packet w/o any headers.
4067 * Length of all headers up to payload.
4069 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
4070 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
4072 TXD->cmd_and_length = htole32(adapter->txd_cmd |
4073 E1000_TXD_CMD_DEXT | /* Extended descr */
4074 E1000_TXD_CMD_TSE | /* TSE context */
4075 E1000_TXD_CMD_IP | /* Do IP csum */
4076 E1000_TXD_CMD_TCP | /* Do TCP checksum */
4077 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
4079 tx_buffer->m_head = NULL;
4080 tx_buffer->next_eop = -1;
4082 if (++cur == adapter->num_tx_desc)
4086 txr->next_avail_desc = cur;
4091 /**********************************************************************
4093 * Examine each tx_buffer in the used queue. If the hardware is done
4094 * processing the packet then free associated resources. The
4095 * tx_buffer is put back on the free queue.
4097 **********************************************************************/
4099 em_txeof(struct tx_ring *txr)
4101 struct adapter *adapter = txr->adapter;
4102 int first, last, done, processed;
4103 struct em_txbuffer *tx_buffer;
4104 struct e1000_tx_desc *tx_desc, *eop_desc;
4105 if_t ifp = adapter->ifp;
4107 EM_TX_LOCK_ASSERT(txr);
4109 if (netmap_tx_irq(ifp, txr->me))
4111 #endif /* DEV_NETMAP */
4113 /* No work, make sure hang detection is disabled */
4114 if (txr->tx_avail == adapter->num_tx_desc) {
4115 txr->busy = EM_TX_IDLE;
4120 first = txr->next_to_clean;
4121 tx_desc = &txr->tx_base[first];
4122 tx_buffer = &txr->tx_buffers[first];
4123 last = tx_buffer->next_eop;
4124 eop_desc = &txr->tx_base[last];
4127 * What this does is get the index of the
4128 * first descriptor AFTER the EOP of the
4129 * first packet, that way we can do the
4130 * simple comparison on the inner while loop.
4132 if (++last == adapter->num_tx_desc)
4136 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4137 BUS_DMASYNC_POSTREAD);
4139 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
4140 /* We clean the range of the packet */
4141 while (first != done) {
4142 tx_desc->upper.data = 0;
4143 tx_desc->lower.data = 0;
4144 tx_desc->buffer_addr = 0;
4148 if (tx_buffer->m_head) {
4149 bus_dmamap_sync(txr->txtag,
4151 BUS_DMASYNC_POSTWRITE);
4152 bus_dmamap_unload(txr->txtag,
4154 m_freem(tx_buffer->m_head);
4155 tx_buffer->m_head = NULL;
4157 tx_buffer->next_eop = -1;
4159 if (++first == adapter->num_tx_desc)
4162 tx_buffer = &txr->tx_buffers[first];
4163 tx_desc = &txr->tx_base[first];
4165 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
4166 /* See if we can continue to the next packet */
4167 last = tx_buffer->next_eop;
4169 eop_desc = &txr->tx_base[last];
4170 /* Get new done point */
4171 if (++last == adapter->num_tx_desc) last = 0;
4176 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
4177 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4179 txr->next_to_clean = first;
4182 ** Hang detection: we know there's work outstanding
4183 ** or the entry return would have been taken, so no
4184 ** descriptor processed here indicates a potential hang.
4185 ** The local timer will examine this and do a reset if needed.
4187 if (processed == 0) {
4188 if (txr->busy != EM_TX_HUNG)
4190 } else /* At least one descriptor was cleaned */
4191 txr->busy = EM_TX_BUSY; /* note this clears HUNG */
4194 * If we have a minimum free, clear IFF_DRV_OACTIVE
4195 * to tell the stack that it is OK to send packets.
4196 * Notice that all writes of OACTIVE happen under the
4197 * TX lock which, with a single queue, guarantees
4200 if (txr->tx_avail >= EM_MAX_SCATTER) {
4201 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
4204 /* Disable hang detection if all clean */
4205 if (txr->tx_avail == adapter->num_tx_desc)
4206 txr->busy = EM_TX_IDLE;
4209 /*********************************************************************
4211 * Refresh RX descriptor mbufs from system mbuf buffer pool.
4213 **********************************************************************/
4215 em_refresh_mbufs(struct rx_ring *rxr, int limit)
4217 struct adapter *adapter = rxr->adapter;
4219 bus_dma_segment_t segs;
4220 struct em_rxbuffer *rxbuf;
4221 int i, j, error, nsegs;
4222 bool cleaned = FALSE;
4224 i = j = rxr->next_to_refresh;
4226 ** Get one descriptor beyond
4227 ** our work mark to control
4230 if (++j == adapter->num_rx_desc)
4233 while (j != limit) {
4234 rxbuf = &rxr->rx_buffers[i];
4235 if (rxbuf->m_head == NULL) {
4236 m = m_getjcl(M_NOWAIT, MT_DATA,
4237 M_PKTHDR, adapter->rx_mbuf_sz);
4239 ** If we have a temporary resource shortage
4240 ** that causes a failure, just abort refresh
4241 ** for now, we will return to this point when
4242 ** reinvoked from em_rxeof.
4249 m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz;
4250 m->m_flags |= M_PKTHDR;
4251 m->m_data = m->m_ext.ext_buf;
4253 /* Use bus_dma machinery to setup the memory mapping */
4254 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map,
4255 m, &segs, &nsegs, BUS_DMA_NOWAIT);
4257 printf("Refresh mbufs: hdr dmamap load"
4258 " failure - %d\n", error);
4260 rxbuf->m_head = NULL;
4264 rxbuf->paddr = segs.ds_addr;
4265 bus_dmamap_sync(rxr->rxtag,
4266 rxbuf->map, BUS_DMASYNC_PREREAD);
4267 em_setup_rxdesc(&rxr->rx_base[i], rxbuf);
4270 i = j; /* Next is precalulated for us */
4271 rxr->next_to_refresh = i;
4272 /* Calculate next controlling index */
4273 if (++j == adapter->num_rx_desc)
4278 ** Update the tail pointer only if,
4279 ** and as far as we have refreshed.
4282 E1000_WRITE_REG(&adapter->hw,
4283 E1000_RDT(rxr->me), rxr->next_to_refresh);
4289 /*********************************************************************
4291 * Allocate memory for rx_buffer structures. Since we use one
4292 * rx_buffer per received packet, the maximum number of rx_buffer's
4293 * that we'll need is equal to the number of receive descriptors
4294 * that we've allocated.
4296 **********************************************************************/
4298 em_allocate_receive_buffers(struct rx_ring *rxr)
4300 struct adapter *adapter = rxr->adapter;
4301 device_t dev = adapter->dev;
4302 struct em_rxbuffer *rxbuf;
4305 rxr->rx_buffers = malloc(sizeof(struct em_rxbuffer) *
4306 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
4307 if (rxr->rx_buffers == NULL) {
4308 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4312 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
4313 1, 0, /* alignment, bounds */
4314 BUS_SPACE_MAXADDR, /* lowaddr */
4315 BUS_SPACE_MAXADDR, /* highaddr */
4316 NULL, NULL, /* filter, filterarg */
4317 MJUM9BYTES, /* maxsize */
4319 MJUM9BYTES, /* maxsegsize */
4321 NULL, /* lockfunc */
4325 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
4330 rxbuf = rxr->rx_buffers;
4331 for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
4332 rxbuf = &rxr->rx_buffers[i];
4333 error = bus_dmamap_create(rxr->rxtag, 0, &rxbuf->map);
4335 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4344 em_free_receive_structures(adapter);
4349 /*********************************************************************
4351 * Initialize a receive ring and its buffers.
4353 **********************************************************************/
4355 em_setup_receive_ring(struct rx_ring *rxr)
4357 struct adapter *adapter = rxr->adapter;
4358 struct em_rxbuffer *rxbuf;
4359 bus_dma_segment_t seg[1];
4360 int rsize, nsegs, error = 0;
4362 struct netmap_slot *slot;
4363 struct netmap_adapter *na = netmap_getna(adapter->ifp);
4367 /* Clear the ring contents */
4369 rsize = roundup2(adapter->num_rx_desc *
4370 sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN);
4371 bzero((void *)rxr->rx_base, rsize);
4373 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4377 ** Free current RX buffer structs and their mbufs
4379 for (int i = 0; i < adapter->num_rx_desc; i++) {
4380 rxbuf = &rxr->rx_buffers[i];
4381 if (rxbuf->m_head != NULL) {
4382 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4383 BUS_DMASYNC_POSTREAD);
4384 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4385 m_freem(rxbuf->m_head);
4386 rxbuf->m_head = NULL; /* mark as freed */
4390 /* Now replenish the mbufs */
4391 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4392 rxbuf = &rxr->rx_buffers[j];
4395 int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4399 addr = PNMB(na, slot + si, &paddr);
4400 netmap_load_map(na, rxr->rxtag, rxbuf->map, addr);
4401 rxbuf->paddr = paddr;
4402 em_setup_rxdesc(&rxr->rx_base[j], rxbuf);
4405 #endif /* DEV_NETMAP */
4406 rxbuf->m_head = m_getjcl(M_NOWAIT, MT_DATA,
4407 M_PKTHDR, adapter->rx_mbuf_sz);
4408 if (rxbuf->m_head == NULL) {
4412 rxbuf->m_head->m_len = adapter->rx_mbuf_sz;
4413 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */
4414 rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz;
4416 /* Get the memory mapping */
4417 error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
4418 rxbuf->map, rxbuf->m_head, seg,
4419 &nsegs, BUS_DMA_NOWAIT);
4421 m_freem(rxbuf->m_head);
4422 rxbuf->m_head = NULL;
4425 bus_dmamap_sync(rxr->rxtag,
4426 rxbuf->map, BUS_DMASYNC_PREREAD);
4428 rxbuf->paddr = seg[0].ds_addr;
4429 em_setup_rxdesc(&rxr->rx_base[j], rxbuf);
4431 rxr->next_to_check = 0;
4432 rxr->next_to_refresh = 0;
4433 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4434 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4441 /*********************************************************************
4443 * Initialize all receive rings.
4445 **********************************************************************/
4447 em_setup_receive_structures(struct adapter *adapter)
4449 struct rx_ring *rxr = adapter->rx_rings;
4452 for (q = 0; q < adapter->num_queues; q++, rxr++)
4453 if (em_setup_receive_ring(rxr))
4459 * Free RX buffers allocated so far, we will only handle
4460 * the rings that completed, the failing case will have
4461 * cleaned up for itself. 'q' failed, so its the terminus.
4463 for (int i = 0; i < q; ++i) {
4464 rxr = &adapter->rx_rings[i];
4465 for (int n = 0; n < adapter->num_rx_desc; n++) {
4466 struct em_rxbuffer *rxbuf;
4467 rxbuf = &rxr->rx_buffers[n];
4468 if (rxbuf->m_head != NULL) {
4469 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4470 BUS_DMASYNC_POSTREAD);
4471 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4472 m_freem(rxbuf->m_head);
4473 rxbuf->m_head = NULL;
4476 rxr->next_to_check = 0;
4477 rxr->next_to_refresh = 0;
4483 /*********************************************************************
4485 * Free all receive rings.
4487 **********************************************************************/
4489 em_free_receive_structures(struct adapter *adapter)
4491 struct rx_ring *rxr = adapter->rx_rings;
4493 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4494 em_free_receive_buffers(rxr);
4495 /* Free the ring memory as well */
4496 em_dma_free(adapter, &rxr->rxdma);
4497 EM_RX_LOCK_DESTROY(rxr);
4500 free(adapter->rx_rings, M_DEVBUF);
4504 /*********************************************************************
4506 * Free receive ring data structures
4508 **********************************************************************/
4510 em_free_receive_buffers(struct rx_ring *rxr)
4512 struct adapter *adapter = rxr->adapter;
4513 struct em_rxbuffer *rxbuf = NULL;
4515 INIT_DEBUGOUT("free_receive_buffers: begin");
4517 if (rxr->rx_buffers != NULL) {
4518 for (int i = 0; i < adapter->num_rx_desc; i++) {
4519 rxbuf = &rxr->rx_buffers[i];
4520 if (rxbuf->map != NULL) {
4521 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
4522 BUS_DMASYNC_POSTREAD);
4523 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
4524 bus_dmamap_destroy(rxr->rxtag, rxbuf->map);
4526 if (rxbuf->m_head != NULL) {
4527 m_freem(rxbuf->m_head);
4528 rxbuf->m_head = NULL;
4531 free(rxr->rx_buffers, M_DEVBUF);
4532 rxr->rx_buffers = NULL;
4533 rxr->next_to_check = 0;
4534 rxr->next_to_refresh = 0;
4537 if (rxr->rxtag != NULL) {
4538 bus_dma_tag_destroy(rxr->rxtag);
4546 /*********************************************************************
4548 * Enable receive unit.
4550 **********************************************************************/
4553 em_initialize_receive_unit(struct adapter *adapter)
4555 struct rx_ring *rxr = adapter->rx_rings;
4556 if_t ifp = adapter->ifp;
4557 struct e1000_hw *hw = &adapter->hw;
4558 u32 rctl, rxcsum, rfctl;
4560 INIT_DEBUGOUT("em_initialize_receive_units: begin");
4563 * Make sure receives are disabled while setting
4564 * up the descriptor ring
4566 rctl = E1000_READ_REG(hw, E1000_RCTL);
4567 /* Do not disable if ever enabled on this hardware */
4568 if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583))
4569 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4571 /* Setup the Receive Control Register */
4572 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4573 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
4574 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
4575 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4577 /* Do not store bad packets */
4578 rctl &= ~E1000_RCTL_SBP;
4580 /* Enable Long Packet receive */
4581 if (if_getmtu(ifp) > ETHERMTU)
4582 rctl |= E1000_RCTL_LPE;
4584 rctl &= ~E1000_RCTL_LPE;
4587 if (!em_disable_crc_stripping)
4588 rctl |= E1000_RCTL_SECRC;
4590 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4591 adapter->rx_abs_int_delay.value);
4593 E1000_WRITE_REG(&adapter->hw, E1000_RDTR,
4594 adapter->rx_int_delay.value);
4596 * Set the interrupt throttling rate. Value is calculated
4597 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4599 E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
4601 /* Use extended rx descriptor formats */
4602 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
4603 rfctl |= E1000_RFCTL_EXTEN;
4605 ** When using MSIX interrupts we need to throttle
4606 ** using the EITR register (82574 only)
4608 if (hw->mac.type == e1000_82574) {
4609 for (int i = 0; i < 4; i++)
4610 E1000_WRITE_REG(hw, E1000_EITR_82574(i),
4612 /* Disable accelerated acknowledge */
4613 rfctl |= E1000_RFCTL_ACK_DIS;
4615 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
4617 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
4618 if (if_getcapenable(ifp) & IFCAP_RXCSUM) {
4619 #ifdef EM_MULTIQUEUE
4620 rxcsum |= E1000_RXCSUM_TUOFL |
4621 E1000_RXCSUM_IPOFL |
4624 rxcsum |= E1000_RXCSUM_TUOFL;
4627 rxcsum &= ~E1000_RXCSUM_TUOFL;
4629 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4631 #ifdef EM_MULTIQUEUE
4632 #define RSSKEYLEN 10
4633 if (adapter->num_queues > 1) {
4634 uint8_t rss_key[4 * RSSKEYLEN];
4641 arc4rand(rss_key, sizeof(rss_key), 0);
4642 for (i = 0; i < RSSKEYLEN; ++i) {
4645 rssrk = EM_RSSRK_VAL(rss_key, i);
4646 E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk);
4650 * Configure RSS redirect table in following fashion:
4651 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
4653 for (i = 0; i < sizeof(reta); ++i) {
4656 q = (i % adapter->num_queues) << 7;
4657 reta |= q << (8 * i);
4660 for (i = 0; i < 32; ++i) {
4661 E1000_WRITE_REG(hw, E1000_RETA(i), reta);
4664 E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q |
4665 E1000_MRQC_RSS_FIELD_IPV4_TCP |
4666 E1000_MRQC_RSS_FIELD_IPV4 |
4667 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX |
4668 E1000_MRQC_RSS_FIELD_IPV6_EX |
4669 E1000_MRQC_RSS_FIELD_IPV6);
4673 ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4674 ** long latencies are observed, like Lenovo X60. This
4675 ** change eliminates the problem, but since having positive
4676 ** values in RDTR is a known source of problems on other
4677 ** platforms another solution is being sought.
4679 if (hw->mac.type == e1000_82573)
4680 E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
4682 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4683 /* Setup the Base and Length of the Rx Descriptor Ring */
4684 u64 bus_addr = rxr->rxdma.dma_paddr;
4685 u32 rdt = adapter->num_rx_desc - 1; /* default */
4687 E1000_WRITE_REG(hw, E1000_RDLEN(i),
4688 adapter->num_rx_desc * sizeof(union e1000_rx_desc_extended));
4689 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
4690 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
4691 /* Setup the Head and Tail Descriptor Pointers */
4692 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4695 * an init() while a netmap client is active must
4696 * preserve the rx buffers passed to userspace.
4698 if (if_getcapenable(ifp) & IFCAP_NETMAP) {
4699 struct netmap_adapter *na = netmap_getna(adapter->ifp);
4700 rdt -= nm_kr_rxspace(&na->rx_rings[i]);
4702 #endif /* DEV_NETMAP */
4703 E1000_WRITE_REG(hw, E1000_RDT(i), rdt);
4707 * Set PTHRESH for improved jumbo performance
4708 * According to 10.2.5.11 of Intel 82574 Datasheet,
4709 * RXDCTL(1) is written whenever RXDCTL(0) is written.
4710 * Only write to RXDCTL(1) if there is a need for different
4713 if (((adapter->hw.mac.type == e1000_ich9lan) ||
4714 (adapter->hw.mac.type == e1000_pch2lan) ||
4715 (adapter->hw.mac.type == e1000_ich10lan)) &&
4716 (if_getmtu(ifp) > ETHERMTU)) {
4717 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
4718 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3);
4719 } else if (adapter->hw.mac.type == e1000_82574) {
4720 for (int i = 0; i < adapter->num_queues; i++) {
4721 u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
4723 rxdctl |= 0x20; /* PTHRESH */
4724 rxdctl |= 4 << 8; /* HTHRESH */
4725 rxdctl |= 4 << 16;/* WTHRESH */
4726 rxdctl |= 1 << 24; /* Switch to granularity */
4727 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
4731 if (adapter->hw.mac.type >= e1000_pch2lan) {
4732 if (if_getmtu(ifp) > ETHERMTU)
4733 e1000_lv_jumbo_workaround_ich8lan(hw, TRUE);
4735 e1000_lv_jumbo_workaround_ich8lan(hw, FALSE);
4738 /* Make sure VLAN Filters are off */
4739 rctl &= ~E1000_RCTL_VFE;
4741 if (adapter->rx_mbuf_sz == MCLBYTES)
4742 rctl |= E1000_RCTL_SZ_2048;
4743 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
4744 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
4745 else if (adapter->rx_mbuf_sz > MJUMPAGESIZE)
4746 rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
4748 /* ensure we clear use DTYPE of 00 here */
4749 rctl &= ~0x00000C00;
4750 /* Write out the settings */
4751 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4757 /*********************************************************************
4759 * This routine executes in interrupt context. It replenishes
4760 * the mbufs in the descriptor and sends data which has been
4761 * dma'ed into host memory to upper layer.
4763 * We loop at most count times if count is > 0, or until done if
4766 * For polling we also now return the number of cleaned packets
4767 *********************************************************************/
4769 em_rxeof(struct rx_ring *rxr, int count, int *done)
4771 struct adapter *adapter = rxr->adapter;
4772 if_t ifp = adapter->ifp;
4773 struct mbuf *mp, *sendmp;
4776 int i, processed, rxdone = 0;
4778 union e1000_rx_desc_extended *cur;
4783 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4784 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4788 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
4792 #endif /* DEV_NETMAP */
4794 for (i = rxr->next_to_check, processed = 0; count != 0;) {
4795 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
4798 cur = &rxr->rx_base[i];
4799 status = le32toh(cur->wb.upper.status_error);
4802 if ((status & E1000_RXD_STAT_DD) == 0)
4805 len = le16toh(cur->wb.upper.length);
4806 eop = (status & E1000_RXD_STAT_EOP) != 0;
4808 if ((status & E1000_RXDEXT_ERR_FRAME_ERR_MASK) ||
4809 (rxr->discard == TRUE)) {
4810 adapter->dropped_pkts++;
4811 ++rxr->rx_discarded;
4812 if (!eop) /* Catch subsequent segs */
4813 rxr->discard = TRUE;
4815 rxr->discard = FALSE;
4816 em_rx_discard(rxr, i);
4819 bus_dmamap_unload(rxr->rxtag, rxr->rx_buffers[i].map);
4821 /* Assign correct length to the current fragment */
4822 mp = rxr->rx_buffers[i].m_head;
4825 /* Trigger for refresh */
4826 rxr->rx_buffers[i].m_head = NULL;
4828 /* First segment? */
4829 if (rxr->fmp == NULL) {
4830 mp->m_pkthdr.len = len;
4831 rxr->fmp = rxr->lmp = mp;
4833 /* Chain mbuf's together */
4834 mp->m_flags &= ~M_PKTHDR;
4835 rxr->lmp->m_next = mp;
4837 rxr->fmp->m_pkthdr.len += len;
4843 if_setrcvif(sendmp, ifp);
4844 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
4845 em_receive_checksum(status, sendmp);
4846 #ifndef __NO_STRICT_ALIGNMENT
4847 if (adapter->hw.mac.max_frame_size >
4848 (MCLBYTES - ETHER_ALIGN) &&
4849 em_fixup_rx(rxr) != 0)
4852 if (status & E1000_RXD_STAT_VP) {
4854 le16toh(cur->wb.upper.vlan));
4855 sendmp->m_flags |= M_VLANTAG;
4857 #ifndef __NO_STRICT_ALIGNMENT
4860 rxr->fmp = rxr->lmp = NULL;
4864 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4865 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4867 /* Zero out the receive descriptors status. */
4868 cur->wb.upper.status_error &= htole32(~0xFF);
4869 ++rxdone; /* cumulative for POLL */
4872 /* Advance our pointers to the next descriptor. */
4873 if (++i == adapter->num_rx_desc)
4876 /* Send to the stack */
4877 if (sendmp != NULL) {
4878 rxr->next_to_check = i;
4880 if_input(ifp, sendmp);
4882 i = rxr->next_to_check;
4885 /* Only refresh mbufs every 8 descriptors */
4886 if (processed == 8) {
4887 em_refresh_mbufs(rxr, i);
4892 /* Catch any remaining refresh work */
4893 if (e1000_rx_unrefreshed(rxr))
4894 em_refresh_mbufs(rxr, i);
4896 rxr->next_to_check = i;
4901 return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
4904 static __inline void
4905 em_rx_discard(struct rx_ring *rxr, int i)
4907 struct em_rxbuffer *rbuf;
4909 rbuf = &rxr->rx_buffers[i];
4910 bus_dmamap_unload(rxr->rxtag, rbuf->map);
4912 /* Free any previous pieces */
4913 if (rxr->fmp != NULL) {
4914 rxr->fmp->m_flags |= M_PKTHDR;
4920 ** Free buffer and allow em_refresh_mbufs()
4921 ** to clean up and recharge buffer.
4924 m_free(rbuf->m_head);
4925 rbuf->m_head = NULL;
4930 #ifndef __NO_STRICT_ALIGNMENT
4932 * When jumbo frames are enabled we should realign entire payload on
4933 * architecures with strict alignment. This is serious design mistake of 8254x
4934 * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4935 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4936 * payload. On architecures without strict alignment restrictions 8254x still
4937 * performs unaligned memory access which would reduce the performance too.
4938 * To avoid copying over an entire frame to align, we allocate a new mbuf and
4939 * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4940 * existing mbuf chain.
4942 * Be aware, best performance of the 8254x is achived only when jumbo frame is
4943 * not used at all on architectures with strict alignment.
4946 em_fixup_rx(struct rx_ring *rxr)
4948 struct adapter *adapter = rxr->adapter;
4954 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4955 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4956 m->m_data += ETHER_HDR_LEN;
4958 MGETHDR(n, M_NOWAIT, MT_DATA);
4960 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4961 m->m_data += ETHER_HDR_LEN;
4962 m->m_len -= ETHER_HDR_LEN;
4963 n->m_len = ETHER_HDR_LEN;
4964 M_MOVE_PKTHDR(n, m);
4968 adapter->dropped_pkts++;
4980 em_setup_rxdesc(union e1000_rx_desc_extended *rxd, const struct em_rxbuffer *rxbuf)
4982 rxd->read.buffer_addr = htole64(rxbuf->paddr);
4983 /* DD bits must be cleared */
4984 rxd->wb.upper.status_error= 0;
4987 /*********************************************************************
4989 * Verify that the hardware indicated that the checksum is valid.
4990 * Inform the stack about the status of checksum so that stack
4991 * doesn't spend time verifying the checksum.
4993 *********************************************************************/
4995 em_receive_checksum(uint32_t status, struct mbuf *mp)
4997 mp->m_pkthdr.csum_flags = 0;
4999 /* Ignore Checksum bit is set */
5000 if (status & E1000_RXD_STAT_IXSM)
5003 /* If the IP checksum exists and there is no IP Checksum error */
5004 if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) ==
5005 E1000_RXD_STAT_IPCS) {
5006 mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
5009 /* TCP or UDP checksum */
5010 if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) ==
5011 E1000_RXD_STAT_TCPCS) {
5012 mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5013 mp->m_pkthdr.csum_data = htons(0xffff);
5015 if (status & E1000_RXD_STAT_UDPCS) {
5016 mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
5017 mp->m_pkthdr.csum_data = htons(0xffff);
5022 * This routine is run via an vlan
5026 em_register_vlan(void *arg, if_t ifp, u16 vtag)
5028 struct adapter *adapter = if_getsoftc(ifp);
5031 if ((void*)adapter != arg) /* Not our event */
5034 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */
5037 EM_CORE_LOCK(adapter);
5038 index = (vtag >> 5) & 0x7F;
5040 adapter->shadow_vfta[index] |= (1 << bit);
5041 ++adapter->num_vlans;
5042 /* Re-init to load the changes */
5043 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
5044 em_init_locked(adapter);
5045 EM_CORE_UNLOCK(adapter);
5049 * This routine is run via an vlan
5053 em_unregister_vlan(void *arg, if_t ifp, u16 vtag)
5055 struct adapter *adapter = if_getsoftc(ifp);
5061 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
5064 EM_CORE_LOCK(adapter);
5065 index = (vtag >> 5) & 0x7F;
5067 adapter->shadow_vfta[index] &= ~(1 << bit);
5068 --adapter->num_vlans;
5069 /* Re-init to load the changes */
5070 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
5071 em_init_locked(adapter);
5072 EM_CORE_UNLOCK(adapter);
5076 em_setup_vlan_hw_support(struct adapter *adapter)
5078 struct e1000_hw *hw = &adapter->hw;
5082 ** We get here thru init_locked, meaning
5083 ** a soft reset, this has already cleared
5084 ** the VFTA and other state, so if there
5085 ** have been no vlan's registered do nothing.
5087 if (adapter->num_vlans == 0)
5091 ** A soft reset zero's out the VFTA, so
5092 ** we need to repopulate it now.
5094 for (int i = 0; i < EM_VFTA_SIZE; i++)
5095 if (adapter->shadow_vfta[i] != 0)
5096 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
5097 i, adapter->shadow_vfta[i]);
5099 reg = E1000_READ_REG(hw, E1000_CTRL);
5100 reg |= E1000_CTRL_VME;
5101 E1000_WRITE_REG(hw, E1000_CTRL, reg);
5103 /* Enable the Filter Table */
5104 reg = E1000_READ_REG(hw, E1000_RCTL);
5105 reg &= ~E1000_RCTL_CFIEN;
5106 reg |= E1000_RCTL_VFE;
5107 E1000_WRITE_REG(hw, E1000_RCTL, reg);
5111 em_enable_intr(struct adapter *adapter)
5113 struct e1000_hw *hw = &adapter->hw;
5114 u32 ims_mask = IMS_ENABLE_MASK;
5116 if (hw->mac.type == e1000_82574) {
5117 E1000_WRITE_REG(hw, EM_EIAC, adapter->ims);
5118 ims_mask |= adapter->ims;
5120 E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
5124 em_disable_intr(struct adapter *adapter)
5126 struct e1000_hw *hw = &adapter->hw;
5128 if (hw->mac.type == e1000_82574)
5129 E1000_WRITE_REG(hw, EM_EIAC, 0);
5130 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
5134 * Bit of a misnomer, what this really means is
5135 * to enable OS management of the system... aka
5136 * to disable special hardware management features
5139 em_init_manageability(struct adapter *adapter)
5141 /* A shared code workaround */
5142 #define E1000_82542_MANC2H E1000_MANC2H
5143 if (adapter->has_manage) {
5144 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
5145 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5147 /* disable hardware interception of ARP */
5148 manc &= ~(E1000_MANC_ARP_EN);
5150 /* enable receiving management packets to the host */
5151 manc |= E1000_MANC_EN_MNG2HOST;
5152 #define E1000_MNG2HOST_PORT_623 (1 << 5)
5153 #define E1000_MNG2HOST_PORT_664 (1 << 6)
5154 manc2h |= E1000_MNG2HOST_PORT_623;
5155 manc2h |= E1000_MNG2HOST_PORT_664;
5156 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
5157 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5162 * Give control back to hardware management
5163 * controller if there is one.
5166 em_release_manageability(struct adapter *adapter)
5168 if (adapter->has_manage) {
5169 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
5171 /* re-enable hardware interception of ARP */
5172 manc |= E1000_MANC_ARP_EN;
5173 manc &= ~E1000_MANC_EN_MNG2HOST;
5175 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
5180 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
5181 * For ASF and Pass Through versions of f/w this means
5182 * that the driver is loaded. For AMT version type f/w
5183 * this means that the network i/f is open.
5186 em_get_hw_control(struct adapter *adapter)
5190 if (adapter->hw.mac.type == e1000_82573) {
5191 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
5192 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
5193 swsm | E1000_SWSM_DRV_LOAD);
5197 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5198 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5199 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
5204 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
5205 * For ASF and Pass Through versions of f/w this means that
5206 * the driver is no longer loaded. For AMT versions of the
5207 * f/w this means that the network i/f is closed.
5210 em_release_hw_control(struct adapter *adapter)
5214 if (!adapter->has_manage)
5217 if (adapter->hw.mac.type == e1000_82573) {
5218 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
5219 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
5220 swsm & ~E1000_SWSM_DRV_LOAD);
5224 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5225 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
5226 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
5231 em_is_valid_ether_addr(u8 *addr)
5233 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
5235 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
5243 ** Parse the interface capabilities with regard
5244 ** to both system management and wake-on-lan for
5248 em_get_wakeup(device_t dev)
5250 struct adapter *adapter = device_get_softc(dev);
5251 u16 eeprom_data = 0, device_id, apme_mask;
5253 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
5254 apme_mask = EM_EEPROM_APME;
5256 switch (adapter->hw.mac.type) {
5259 adapter->has_amt = TRUE;
5263 case e1000_80003es2lan:
5264 if (adapter->hw.bus.func == 1) {
5265 e1000_read_nvm(&adapter->hw,
5266 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
5269 e1000_read_nvm(&adapter->hw,
5270 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
5274 case e1000_ich10lan:
5279 apme_mask = E1000_WUC_APME;
5280 adapter->has_amt = TRUE;
5281 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
5284 e1000_read_nvm(&adapter->hw,
5285 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
5288 if (eeprom_data & apme_mask)
5289 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
5291 * We have the eeprom settings, now apply the special cases
5292 * where the eeprom may be wrong or the board won't support
5293 * wake on lan on a particular port
5295 device_id = pci_get_device(dev);
5296 switch (device_id) {
5297 case E1000_DEV_ID_82571EB_FIBER:
5298 /* Wake events only supported on port A for dual fiber
5299 * regardless of eeprom setting */
5300 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
5301 E1000_STATUS_FUNC_1)
5304 case E1000_DEV_ID_82571EB_QUAD_COPPER:
5305 case E1000_DEV_ID_82571EB_QUAD_FIBER:
5306 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
5307 /* if quad port adapter, disable WoL on all but port A */
5308 if (global_quad_port_a != 0)
5310 /* Reset for multiple quad port adapters */
5311 if (++global_quad_port_a == 4)
5312 global_quad_port_a = 0;
5320 * Enable PCI Wake On Lan capability
5323 em_enable_wakeup(device_t dev)
5325 struct adapter *adapter = device_get_softc(dev);
5326 if_t ifp = adapter->ifp;
5327 u32 pmc, ctrl, ctrl_ext, rctl, wuc;
5330 if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
5333 /* Advertise the wakeup capability */
5334 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
5335 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
5336 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
5337 wuc = E1000_READ_REG(&adapter->hw, E1000_WUC);
5338 wuc |= E1000_WUC_PME_EN;
5339 E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc);
5341 if ((adapter->hw.mac.type == e1000_ich8lan) ||
5342 (adapter->hw.mac.type == e1000_pchlan) ||
5343 (adapter->hw.mac.type == e1000_ich9lan) ||
5344 (adapter->hw.mac.type == e1000_ich10lan))
5345 e1000_suspend_workarounds_ich8lan(&adapter->hw);
5347 /* Keep the laser running on Fiber adapters */
5348 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
5349 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
5350 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
5351 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
5352 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
5356 ** Determine type of Wakeup: note that wol
5357 ** is set with all bits on by default.
5359 if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0)
5360 adapter->wol &= ~E1000_WUFC_MAG;
5362 if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0)
5363 adapter->wol &= ~E1000_WUFC_MC;
5365 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
5366 rctl |= E1000_RCTL_MPE;
5367 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
5370 if ((adapter->hw.mac.type == e1000_pchlan) ||
5371 (adapter->hw.mac.type == e1000_pch2lan) ||
5372 (adapter->hw.mac.type == e1000_pch_lpt) ||
5373 (adapter->hw.mac.type == e1000_pch_spt)) {
5374 if (em_enable_phy_wakeup(adapter))
5377 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
5378 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
5381 if (adapter->hw.phy.type == e1000_phy_igp_3)
5382 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5385 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
5386 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
5387 if (if_getcapenable(ifp) & IFCAP_WOL)
5388 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5389 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
5395 ** WOL in the newer chipset interfaces (pchlan)
5396 ** require thing to be copied into the phy
5399 em_enable_phy_wakeup(struct adapter *adapter)
5401 struct e1000_hw *hw = &adapter->hw;
5405 /* copy MAC RARs to PHY RARs */
5406 e1000_copy_rx_addrs_to_phy_ich8lan(hw);
5408 /* copy MAC MTA to PHY MTA */
5409 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5410 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
5411 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
5412 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
5413 (u16)((mreg >> 16) & 0xFFFF));
5416 /* configure PHY Rx Control register */
5417 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
5418 mreg = E1000_READ_REG(hw, E1000_RCTL);
5419 if (mreg & E1000_RCTL_UPE)
5420 preg |= BM_RCTL_UPE;
5421 if (mreg & E1000_RCTL_MPE)
5422 preg |= BM_RCTL_MPE;
5423 preg &= ~(BM_RCTL_MO_MASK);
5424 if (mreg & E1000_RCTL_MO_3)
5425 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5426 << BM_RCTL_MO_SHIFT);
5427 if (mreg & E1000_RCTL_BAM)
5428 preg |= BM_RCTL_BAM;
5429 if (mreg & E1000_RCTL_PMCF)
5430 preg |= BM_RCTL_PMCF;
5431 mreg = E1000_READ_REG(hw, E1000_CTRL);
5432 if (mreg & E1000_CTRL_RFCE)
5433 preg |= BM_RCTL_RFCE;
5434 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
5436 /* enable PHY wakeup in MAC register */
5437 E1000_WRITE_REG(hw, E1000_WUC,
5438 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5439 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
5441 /* configure and enable PHY wakeup in PHY registers */
5442 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
5443 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5445 /* activate PHY wakeup */
5446 ret = hw->phy.ops.acquire(hw);
5448 printf("Could not acquire PHY\n");
5451 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
5452 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
5453 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
5455 printf("Could not read PHY page 769\n");
5458 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5459 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
5461 printf("Could not set PHY Host Wakeup bit\n");
5463 hw->phy.ops.release(hw);
5469 em_led_func(void *arg, int onoff)
5471 struct adapter *adapter = arg;
5473 EM_CORE_LOCK(adapter);
5475 e1000_setup_led(&adapter->hw);
5476 e1000_led_on(&adapter->hw);
5478 e1000_led_off(&adapter->hw);
5479 e1000_cleanup_led(&adapter->hw);
5481 EM_CORE_UNLOCK(adapter);
5485 ** Disable the L0S and L1 LINK states
5488 em_disable_aspm(struct adapter *adapter)
5491 u16 link_cap,link_ctrl;
5492 device_t dev = adapter->dev;
5494 switch (adapter->hw.mac.type) {
5502 if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0)
5504 reg = base + PCIER_LINK_CAP;
5505 link_cap = pci_read_config(dev, reg, 2);
5506 if ((link_cap & PCIEM_LINK_CAP_ASPM) == 0)
5508 reg = base + PCIER_LINK_CTL;
5509 link_ctrl = pci_read_config(dev, reg, 2);
5510 link_ctrl &= ~PCIEM_LINK_CTL_ASPMC;
5511 pci_write_config(dev, reg, link_ctrl, 2);
5515 /**********************************************************************
5517 * Update the board statistics counters.
5519 **********************************************************************/
5521 em_update_stats_counters(struct adapter *adapter)
5524 if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5525 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5526 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5527 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5529 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5530 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5531 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5532 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5534 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5535 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5536 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5537 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5538 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5539 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5540 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5541 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5542 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5543 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5544 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5545 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5546 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5547 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5548 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5549 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5550 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5551 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5552 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5553 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5555 /* For the 64-bit byte counters the low dword must be read first. */
5556 /* Both registers clear on the read of the high dword */
5558 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
5559 ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
5560 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
5561 ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
5563 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5564 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5565 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5566 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5567 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5569 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5570 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5572 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5573 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5574 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5575 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5576 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5577 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5578 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5579 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5580 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5581 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5583 /* Interrupt Counts */
5585 adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC);
5586 adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC);
5587 adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC);
5588 adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC);
5589 adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC);
5590 adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC);
5591 adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC);
5592 adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC);
5593 adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC);
5595 if (adapter->hw.mac.type >= e1000_82543) {
5596 adapter->stats.algnerrc +=
5597 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5598 adapter->stats.rxerrc +=
5599 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5600 adapter->stats.tncrs +=
5601 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5602 adapter->stats.cexterr +=
5603 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5604 adapter->stats.tsctc +=
5605 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5606 adapter->stats.tsctfc +=
5607 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5612 em_get_counter(if_t ifp, ift_counter cnt)
5614 struct adapter *adapter;
5616 adapter = if_getsoftc(ifp);
5619 case IFCOUNTER_COLLISIONS:
5620 return (adapter->stats.colc);
5621 case IFCOUNTER_IERRORS:
5622 return (adapter->dropped_pkts + adapter->stats.rxerrc +
5623 adapter->stats.crcerrs + adapter->stats.algnerrc +
5624 adapter->stats.ruc + adapter->stats.roc +
5625 adapter->stats.mpc + adapter->stats.cexterr);
5626 case IFCOUNTER_OERRORS:
5627 return (adapter->stats.ecol + adapter->stats.latecol +
5628 adapter->watchdog_events);
5630 return (if_get_counter_default(ifp, cnt));
5634 /* Export a single 32-bit register via a read-only sysctl. */
5636 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
5638 struct adapter *adapter;
5641 adapter = oidp->oid_arg1;
5642 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
5643 return (sysctl_handle_int(oidp, &val, 0, req));
5647 * Add sysctl variables, one per statistic, to the system.
5650 em_add_hw_stats(struct adapter *adapter)
5652 device_t dev = adapter->dev;
5654 struct tx_ring *txr = adapter->tx_rings;
5655 struct rx_ring *rxr = adapter->rx_rings;
5657 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
5658 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
5659 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5660 struct e1000_hw_stats *stats = &adapter->stats;
5662 struct sysctl_oid *stat_node, *queue_node, *int_node;
5663 struct sysctl_oid_list *stat_list, *queue_list, *int_list;
5665 #define QUEUE_NAME_LEN 32
5666 char namebuf[QUEUE_NAME_LEN];
5668 /* Driver Statistics */
5669 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5670 CTLFLAG_RD, &adapter->dropped_pkts,
5671 "Driver dropped packets");
5672 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5673 CTLFLAG_RD, &adapter->link_irq,
5674 "Link MSIX IRQ Handled");
5675 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
5676 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5677 "Defragmenting mbuf chain failed");
5678 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
5679 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5680 "Driver tx dma failure in xmit");
5681 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
5682 CTLFLAG_RD, &adapter->rx_overruns,
5684 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
5685 CTLFLAG_RD, &adapter->watchdog_events,
5686 "Watchdog timeouts");
5688 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
5689 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
5690 em_sysctl_reg_handler, "IU",
5691 "Device Control Register");
5692 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
5693 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
5694 em_sysctl_reg_handler, "IU",
5695 "Receiver Control Register");
5696 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
5697 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
5698 "Flow Control High Watermark");
5699 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
5700 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
5701 "Flow Control Low Watermark");
5703 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
5704 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i);
5705 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5706 CTLFLAG_RD, NULL, "TX Queue Name");
5707 queue_list = SYSCTL_CHILDREN(queue_node);
5709 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5710 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5712 em_sysctl_reg_handler, "IU",
5713 "Transmit Descriptor Head");
5714 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5715 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5717 em_sysctl_reg_handler, "IU",
5718 "Transmit Descriptor Tail");
5719 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
5720 CTLFLAG_RD, &txr->tx_irq,
5721 "Queue MSI-X Transmit Interrupts");
5722 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail",
5723 CTLFLAG_RD, &txr->no_desc_avail,
5724 "Queue No Descriptor Available");
5726 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", i);
5727 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5728 CTLFLAG_RD, NULL, "RX Queue Name");
5729 queue_list = SYSCTL_CHILDREN(queue_node);
5731 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5732 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5734 em_sysctl_reg_handler, "IU",
5735 "Receive Descriptor Head");
5736 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5737 CTLTYPE_UINT | CTLFLAG_RD, adapter,
5739 em_sysctl_reg_handler, "IU",
5740 "Receive Descriptor Tail");
5741 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
5742 CTLFLAG_RD, &rxr->rx_irq,
5743 "Queue MSI-X Receive Interrupts");
5746 /* MAC stats get their own sub node */
5748 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5749 CTLFLAG_RD, NULL, "Statistics");
5750 stat_list = SYSCTL_CHILDREN(stat_node);
5752 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
5753 CTLFLAG_RD, &stats->ecol,
5754 "Excessive collisions");
5755 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
5756 CTLFLAG_RD, &stats->scc,
5757 "Single collisions");
5758 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
5759 CTLFLAG_RD, &stats->mcc,
5760 "Multiple collisions");
5761 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
5762 CTLFLAG_RD, &stats->latecol,
5764 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
5765 CTLFLAG_RD, &stats->colc,
5767 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
5768 CTLFLAG_RD, &adapter->stats.symerrs,
5770 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
5771 CTLFLAG_RD, &adapter->stats.sec,
5773 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
5774 CTLFLAG_RD, &adapter->stats.dc,
5776 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
5777 CTLFLAG_RD, &adapter->stats.mpc,
5779 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
5780 CTLFLAG_RD, &adapter->stats.rnbc,
5781 "Receive No Buffers");
5782 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
5783 CTLFLAG_RD, &adapter->stats.ruc,
5784 "Receive Undersize");
5785 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5786 CTLFLAG_RD, &adapter->stats.rfc,
5787 "Fragmented Packets Received ");
5788 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
5789 CTLFLAG_RD, &adapter->stats.roc,
5790 "Oversized Packets Received");
5791 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
5792 CTLFLAG_RD, &adapter->stats.rjc,
5794 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
5795 CTLFLAG_RD, &adapter->stats.rxerrc,
5797 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5798 CTLFLAG_RD, &adapter->stats.crcerrs,
5800 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
5801 CTLFLAG_RD, &adapter->stats.algnerrc,
5802 "Alignment Errors");
5803 /* On 82575 these are collision counts */
5804 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
5805 CTLFLAG_RD, &adapter->stats.cexterr,
5806 "Collision/Carrier extension errors");
5807 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
5808 CTLFLAG_RD, &adapter->stats.xonrxc,
5810 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
5811 CTLFLAG_RD, &adapter->stats.xontxc,
5813 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
5814 CTLFLAG_RD, &adapter->stats.xoffrxc,
5816 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
5817 CTLFLAG_RD, &adapter->stats.xofftxc,
5818 "XOFF Transmitted");
5820 /* Packet Reception Stats */
5821 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
5822 CTLFLAG_RD, &adapter->stats.tpr,
5823 "Total Packets Received ");
5824 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
5825 CTLFLAG_RD, &adapter->stats.gprc,
5826 "Good Packets Received");
5827 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
5828 CTLFLAG_RD, &adapter->stats.bprc,
5829 "Broadcast Packets Received");
5830 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
5831 CTLFLAG_RD, &adapter->stats.mprc,
5832 "Multicast Packets Received");
5833 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5834 CTLFLAG_RD, &adapter->stats.prc64,
5835 "64 byte frames received ");
5836 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5837 CTLFLAG_RD, &adapter->stats.prc127,
5838 "65-127 byte frames received");
5839 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5840 CTLFLAG_RD, &adapter->stats.prc255,
5841 "128-255 byte frames received");
5842 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5843 CTLFLAG_RD, &adapter->stats.prc511,
5844 "256-511 byte frames received");
5845 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5846 CTLFLAG_RD, &adapter->stats.prc1023,
5847 "512-1023 byte frames received");
5848 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5849 CTLFLAG_RD, &adapter->stats.prc1522,
5850 "1023-1522 byte frames received");
5851 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
5852 CTLFLAG_RD, &adapter->stats.gorc,
5853 "Good Octets Received");
5855 /* Packet Transmission Stats */
5856 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5857 CTLFLAG_RD, &adapter->stats.gotc,
5858 "Good Octets Transmitted");
5859 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5860 CTLFLAG_RD, &adapter->stats.tpt,
5861 "Total Packets Transmitted");
5862 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5863 CTLFLAG_RD, &adapter->stats.gptc,
5864 "Good Packets Transmitted");
5865 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5866 CTLFLAG_RD, &adapter->stats.bptc,
5867 "Broadcast Packets Transmitted");
5868 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5869 CTLFLAG_RD, &adapter->stats.mptc,
5870 "Multicast Packets Transmitted");
5871 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5872 CTLFLAG_RD, &adapter->stats.ptc64,
5873 "64 byte frames transmitted ");
5874 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5875 CTLFLAG_RD, &adapter->stats.ptc127,
5876 "65-127 byte frames transmitted");
5877 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5878 CTLFLAG_RD, &adapter->stats.ptc255,
5879 "128-255 byte frames transmitted");
5880 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5881 CTLFLAG_RD, &adapter->stats.ptc511,
5882 "256-511 byte frames transmitted");
5883 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5884 CTLFLAG_RD, &adapter->stats.ptc1023,
5885 "512-1023 byte frames transmitted");
5886 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5887 CTLFLAG_RD, &adapter->stats.ptc1522,
5888 "1024-1522 byte frames transmitted");
5889 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
5890 CTLFLAG_RD, &adapter->stats.tsctc,
5891 "TSO Contexts Transmitted");
5892 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
5893 CTLFLAG_RD, &adapter->stats.tsctfc,
5894 "TSO Contexts Failed");
5897 /* Interrupt Stats */
5899 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
5900 CTLFLAG_RD, NULL, "Interrupt Statistics");
5901 int_list = SYSCTL_CHILDREN(int_node);
5903 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts",
5904 CTLFLAG_RD, &adapter->stats.iac,
5905 "Interrupt Assertion Count");
5907 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
5908 CTLFLAG_RD, &adapter->stats.icrxptc,
5909 "Interrupt Cause Rx Pkt Timer Expire Count");
5911 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
5912 CTLFLAG_RD, &adapter->stats.icrxatc,
5913 "Interrupt Cause Rx Abs Timer Expire Count");
5915 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
5916 CTLFLAG_RD, &adapter->stats.ictxptc,
5917 "Interrupt Cause Tx Pkt Timer Expire Count");
5919 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
5920 CTLFLAG_RD, &adapter->stats.ictxatc,
5921 "Interrupt Cause Tx Abs Timer Expire Count");
5923 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
5924 CTLFLAG_RD, &adapter->stats.ictxqec,
5925 "Interrupt Cause Tx Queue Empty Count");
5927 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
5928 CTLFLAG_RD, &adapter->stats.ictxqmtc,
5929 "Interrupt Cause Tx Queue Min Thresh Count");
5931 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
5932 CTLFLAG_RD, &adapter->stats.icrxdmtc,
5933 "Interrupt Cause Rx Desc Min Thresh Count");
5935 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun",
5936 CTLFLAG_RD, &adapter->stats.icrxoc,
5937 "Interrupt Cause Receiver Overrun Count");
5940 /**********************************************************************
5942 * This routine provides a way to dump out the adapter eeprom,
5943 * often a useful debug/service tool. This only dumps the first
5944 * 32 words, stuff that matters is in that extent.
5946 **********************************************************************/
5948 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
5950 struct adapter *adapter = (struct adapter *)arg1;
5955 error = sysctl_handle_int(oidp, &result, 0, req);
5957 if (error || !req->newptr)
5961 * This value will cause a hex dump of the
5962 * first 32 16-bit words of the EEPROM to
5966 em_print_nvm_info(adapter);
5972 em_print_nvm_info(struct adapter *adapter)
5977 /* Its a bit crude, but it gets the job done */
5978 printf("\nInterface EEPROM Dump:\n");
5979 printf("Offset\n0x0000 ");
5980 for (i = 0, j = 0; i < 32; i++, j++) {
5981 if (j == 8) { /* Make the offset block */
5983 printf("\n0x00%x0 ",row);
5985 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5986 printf("%04x ", eeprom_data);
5992 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5994 struct em_int_delay_info *info;
5995 struct adapter *adapter;
5997 int error, usecs, ticks;
5999 info = (struct em_int_delay_info *)arg1;
6000 usecs = info->value;
6001 error = sysctl_handle_int(oidp, &usecs, 0, req);
6002 if (error != 0 || req->newptr == NULL)
6004 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
6006 info->value = usecs;
6007 ticks = EM_USECS_TO_TICKS(usecs);
6008 if (info->offset == E1000_ITR) /* units are 256ns here */
6011 adapter = info->adapter;
6013 EM_CORE_LOCK(adapter);
6014 regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
6015 regval = (regval & ~0xffff) | (ticks & 0xffff);
6016 /* Handle a few special cases. */
6017 switch (info->offset) {
6022 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
6023 /* Don't write 0 into the TIDV register. */
6026 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
6029 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
6030 EM_CORE_UNLOCK(adapter);
6035 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
6036 const char *description, struct em_int_delay_info *info,
6037 int offset, int value)
6039 info->adapter = adapter;
6040 info->offset = offset;
6041 info->value = value;
6042 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
6043 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
6044 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
6045 info, 0, em_sysctl_int_delay, "I", description);
6049 em_set_sysctl_value(struct adapter *adapter, const char *name,
6050 const char *description, int *limit, int value)
6053 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
6054 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
6055 OID_AUTO, name, CTLFLAG_RW, limit, value, description);
6060 ** Set flow control using sysctl:
6061 ** Flow control values:
6068 em_set_flowcntl(SYSCTL_HANDLER_ARGS)
6071 static int input = 3; /* default is full */
6072 struct adapter *adapter = (struct adapter *) arg1;
6074 error = sysctl_handle_int(oidp, &input, 0, req);
6076 if ((error) || (req->newptr == NULL))
6079 if (input == adapter->fc) /* no change? */
6083 case e1000_fc_rx_pause:
6084 case e1000_fc_tx_pause:
6087 adapter->hw.fc.requested_mode = input;
6088 adapter->fc = input;
6095 adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
6096 e1000_force_mac_fc(&adapter->hw);
6101 ** Manage Energy Efficient Ethernet:
6103 ** 0/1 - enabled/disabled
6106 em_sysctl_eee(SYSCTL_HANDLER_ARGS)
6108 struct adapter *adapter = (struct adapter *) arg1;
6111 value = adapter->hw.dev_spec.ich8lan.eee_disable;
6112 error = sysctl_handle_int(oidp, &value, 0, req);
6113 if (error || req->newptr == NULL)
6115 EM_CORE_LOCK(adapter);
6116 adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0);
6117 em_init_locked(adapter);
6118 EM_CORE_UNLOCK(adapter);
6123 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
6125 struct adapter *adapter;
6130 error = sysctl_handle_int(oidp, &result, 0, req);
6132 if (error || !req->newptr)
6136 adapter = (struct adapter *)arg1;
6137 em_print_debug_info(adapter);
6144 ** This routine is meant to be fluid, add whatever is
6145 ** needed for debugging a problem. -jfv
6148 em_print_debug_info(struct adapter *adapter)
6150 device_t dev = adapter->dev;
6151 struct tx_ring *txr = adapter->tx_rings;
6152 struct rx_ring *rxr = adapter->rx_rings;
6154 if (if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING)
6155 printf("Interface is RUNNING ");
6157 printf("Interface is NOT RUNNING\n");
6159 if (if_getdrvflags(adapter->ifp) & IFF_DRV_OACTIVE)
6160 printf("and INACTIVE\n");
6162 printf("and ACTIVE\n");
6164 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) {
6165 device_printf(dev, "TX Queue %d ------\n", i);
6166 device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
6167 E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
6168 E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
6169 device_printf(dev, "Tx Queue Status = %d\n", txr->busy);
6170 device_printf(dev, "TX descriptors avail = %d\n",
6172 device_printf(dev, "Tx Descriptors avail failure = %ld\n",
6173 txr->no_desc_avail);
6174 device_printf(dev, "RX Queue %d ------\n", i);
6175 device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
6176 E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
6177 E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
6178 device_printf(dev, "RX discarded packets = %ld\n",
6180 device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check);
6181 device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh);
6185 #ifdef EM_MULTIQUEUE
6188 * Write a new value to the EEPROM increasing the number of MSIX
6189 * vectors from 3 to 5, for proper multiqueue support.
6192 em_enable_vectors_82574(struct adapter *adapter)
6194 struct e1000_hw *hw = &adapter->hw;
6195 device_t dev = adapter->dev;
6198 e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata);
6199 printf("Current cap: %#06x\n", edata);
6200 if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) {
6201 device_printf(dev, "Writing to eeprom: increasing "
6202 "reported MSIX vectors from 3 to 5...\n");
6203 edata &= ~(EM_NVM_MSIX_N_MASK);
6204 edata |= 4 << EM_NVM_MSIX_N_SHIFT;
6205 e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata);
6206 e1000_update_nvm_checksum(hw);
6207 device_printf(dev, "Writing to eeprom: done\n");
6213 DB_COMMAND(em_reset_dev, em_ddb_reset_dev)
6218 dc = devclass_find("em");
6219 max_em = devclass_get_maxunit(dc);
6221 for (int index = 0; index < (max_em - 1); index++) {
6223 dev = devclass_get_device(dc, index);
6224 if (device_get_driver(dev) == &em_driver) {
6225 struct adapter *adapter = device_get_softc(dev);
6226 EM_CORE_LOCK(adapter);
6227 em_init_locked(adapter);
6228 EM_CORE_UNLOCK(adapter);
6232 DB_COMMAND(em_dump_queue, em_ddb_dump_queue)
6237 dc = devclass_find("em");
6238 max_em = devclass_get_maxunit(dc);
6240 for (int index = 0; index < (max_em - 1); index++) {
6242 dev = devclass_get_device(dc, index);
6243 if (device_get_driver(dev) == &em_driver)
6244 em_print_debug_info(device_get_softc(dev));
projects / FreeBSD / FreeBSD.git / blob