1 /******************************************************************************
3 Copyright (c) 2013-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
40 #include "opt_inet6.h"
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/buf_ring.h>
48 #include <sys/protosw.h>
49 #include <sys/socket.h>
50 #include <sys/malloc.h>
51 #include <sys/kernel.h>
52 #include <sys/module.h>
53 #include <sys/sockio.h>
54 #include <sys/eventhandler.h>
55 #include <sys/syslog.h>
58 #include <net/if_var.h>
59 #include <net/if_arp.h>
61 #include <net/ethernet.h>
62 #include <net/if_dl.h>
63 #include <net/if_media.h>
66 #include <net/if_types.h>
67 #include <net/if_vlan_var.h>
69 #include <netinet/in_systm.h>
70 #include <netinet/in.h>
71 #include <netinet/if_ether.h>
72 #include <netinet/ip.h>
73 #include <netinet/ip6.h>
74 #include <netinet/tcp.h>
75 #include <netinet/tcp_lro.h>
76 #include <netinet/udp.h>
77 #include <netinet/sctp.h>
79 #include <machine/in_cksum.h>
82 #include <machine/bus.h>
84 #include <machine/resource.h>
87 #include <machine/clock.h>
88 #include <dev/pci/pcivar.h>
89 #include <dev/pci/pcireg.h>
91 #include <sys/sysctl.h>
92 #include <sys/endian.h>
93 #include <sys/taskqueue.h>
97 #include <machine/smp.h>
98 #include <machine/stdarg.h>
101 #include <net/rss_config.h>
102 #include <netinet/in_rss.h>
105 #include "i40e_type.h"
106 #include "i40e_prototype.h"
108 #define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x"
109 #define MAC_FORMAT_ARGS(mac_addr) \
110 (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \
111 (mac_addr)[4], (mac_addr)[5]
112 #define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off")
116 #define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__)
117 #define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__)
118 #define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__)
120 /* Defines for printing generic debug information */
121 #define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__)
122 #define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__)
123 #define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__)
125 /* Defines for printing specific debug information */
127 #define DEBUG_IOCTL 1
130 #define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__)
131 #define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__)
132 #define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__)
134 #define IOCTL_DEBUGOUT(...) if (DEBUG_IOCTL) _DBG_PRINTF(__VA_ARGS__)
135 #define IOCTL_DBG_IF2(ifp, S, ...) if (DEBUG_IOCTL) \
136 if_printf(ifp, S "\n", ##__VA_ARGS__)
137 #define IOCTL_DBG_IF(...) if (DEBUG_IOCTL) _IF_DBG_PRINTF(__VA_ARGS__)
139 #define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__)
141 #else /* no IXL_DEBUG */
143 #define DEBUG_IOCTL 0
147 #define DDPRINTF(...)
148 #define IDPRINTF(...)
150 #define INIT_DEBUGOUT(...)
151 #define INIT_DBG_DEV(...)
152 #define INIT_DBG_IF(...)
153 #define IOCTL_DEBUGOUT(...)
154 #define IOCTL_DBG_IF2(...)
155 #define IOCTL_DBG_IF(...)
156 #define HW_DEBUGOUT(...)
157 #endif /* IXL_DEBUG */
160 IXL_DBG_INFO = 0x00000001,
161 IXL_DBG_EN_DIS = 0x00000002,
162 IXL_DBG_AQ = 0x00000004,
163 IXL_DBG_NVMUPD = 0x00000008,
165 IXL_DBG_IOCTL_KNOWN = 0x00000010,
166 IXL_DBG_IOCTL_UNKNOWN = 0x00000020,
167 IXL_DBG_IOCTL_ALL = 0x00000030,
169 I40E_DEBUG_RSS = 0x00000100,
171 IXL_DBG_IOV = 0x00001000,
172 IXL_DBG_IOV_VC = 0x00002000,
174 IXL_DBG_SWITCH_INFO = 0x00010000,
175 IXL_DBG_I2C = 0x00020000,
177 IXL_DBG_ALL = 0xFFFFFFFF
183 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the
184 * number of tx/rx descriptors allocated by the driver. Increasing this
185 * value allows the driver to queue more operations.
187 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes.
188 * The driver currently always uses 32 byte Rx descriptors.
190 #define IXL_DEFAULT_RING 1024
191 #define IXL_MAX_RING 8160
192 #define IXL_MIN_RING 32
193 #define IXL_RING_INCREMENT 32
195 #define IXL_AQ_LEN 256
196 #define IXL_AQ_LEN_MAX 1024
199 ** Default number of entries in Tx queue buf_ring.
201 #define DEFAULT_TXBRSZ 4096
203 /* Alignment for rings */
204 #define DBA_ALIGN 128
207 * This is the max watchdog interval, ie. the time that can
208 * pass between any two TX clean operations, such only happening
209 * when the TX hardware is functioning.
211 #define IXL_WATCHDOG (10 * hz)
214 * This parameters control when the driver calls the routine to reclaim
215 * transmit descriptors.
217 #define IXL_TX_CLEANUP_THRESHOLD (que->num_desc / 8)
218 #define IXL_TX_OP_THRESHOLD (que->num_desc / 32)
220 #define MAX_MULTICAST_ADDR 128
222 #define IXL_MSIX_BAR 3
223 #define IXL_ADM_LIMIT 2
224 #define IXL_TSO_SIZE 65535
225 #define IXL_AQ_BUF_SZ ((u32) 4096)
226 #define IXL_RX_HDR 128
227 #define IXL_RX_LIMIT 512
230 #define IXL_ITR_NONE 3
231 #define IXL_QUEUE_EOL 0x7FF
232 #define IXL_MAX_FRAME 9728
233 #define IXL_MAX_TX_SEGS 8
234 #define IXL_MAX_TSO_SEGS 128
235 #define IXL_SPARSE_CHAIN 6
236 #define IXL_QUEUE_HUNG 0x80000000
237 #define IXL_MIN_TSO_MSS 64
239 #define IXL_RSS_KEY_SIZE_REG 13
240 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4)
241 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */
242 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F
243 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF
245 #define IXL_VF_MAX_BUFFER 0x3F80
246 #define IXL_VF_MAX_HDR_BUFFER 0x840
247 #define IXL_VF_MAX_FRAME 0x3FFF
249 /* ERJ: hardware can support ~2k (SW5+) filters between all functions */
250 #define IXL_MAX_FILTERS 256
251 #define IXL_MAX_TX_BUSY 10
253 #define IXL_NVM_VERSION_LO_SHIFT 0
254 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT)
255 #define IXL_NVM_VERSION_HI_SHIFT 12
256 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT)
259 * Interrupt Moderation parameters
260 * Multiply ITR values by 2 for real ITR value
262 #define IXL_MAX_ITR 0x0FF0
263 #define IXL_ITR_100K 0x0005
264 #define IXL_ITR_20K 0x0019
265 #define IXL_ITR_8K 0x003E
266 #define IXL_ITR_4K 0x007A
267 #define IXL_ITR_1K 0x01F4
268 #define IXL_ITR_DYNAMIC 0x8000
269 #define IXL_LOW_LATENCY 0
270 #define IXL_AVE_LATENCY 1
271 #define IXL_BULK_LATENCY 2
274 #define IXL_FILTER_USED (u16)(1 << 0)
275 #define IXL_FILTER_VLAN (u16)(1 << 1)
276 #define IXL_FILTER_ADD (u16)(1 << 2)
277 #define IXL_FILTER_DEL (u16)(1 << 3)
278 #define IXL_FILTER_MC (u16)(1 << 4)
280 /* used in the vlan field of the filter when not a vlan */
281 #define IXL_VLAN_ANY -1
283 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP)
284 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6)
285 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO)
287 /* Misc flags for ixl_vsi.flags */
288 #define IXL_FLAGS_KEEP_TSO4 (1 << 0)
289 #define IXL_FLAGS_KEEP_TSO6 (1 << 1)
291 #define IXL_VF_RESET_TIMEOUT 100
293 #define IXL_VSI_DATA_PORT 0x01
295 #define IXLV_MAX_QUEUES 16
296 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1))
298 #define IXL_RX_CTX_BASE_UNITS 128
299 #define IXL_TX_CTX_BASE_UNITS 128
301 #define IXL_VPINT_LNKLSTN_REG(hw, vector, vf_num) \
302 I40E_VPINT_LNKLSTN(((vector) - 1) + \
303 (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num)))
305 #define IXL_VFINT_DYN_CTLN_REG(hw, vector, vf_num) \
306 I40E_VFINT_DYN_CTLN(((vector) - 1) + \
307 (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num)))
309 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA
311 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20
313 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32)
314 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32))
316 #define IXL_MAX_ITR_IDX 3
318 #define IXL_END_OF_INTR_LNKLST 0x7FF
320 #define IXL_DEFAULT_RSS_HENA_BASE (\
321 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \
322 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \
323 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \
324 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \
325 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \
326 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \
327 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \
328 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \
329 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \
330 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \
331 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD))
333 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE
335 #define IXL_DEFAULT_RSS_HENA_X722 (\
336 IXL_DEFAULT_RSS_HENA_BASE | \
337 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \
338 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \
339 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \
340 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \
341 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \
342 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK))
344 #define IXL_TX_LOCK(_sc) mtx_lock(&(_sc)->mtx)
345 #define IXL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx)
346 #define IXL_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx)
347 #define IXL_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->mtx)
348 #define IXL_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->mtx, MA_OWNED)
350 #define IXL_RX_LOCK(_sc) mtx_lock(&(_sc)->mtx)
351 #define IXL_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx)
352 #define IXL_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx)
354 /* Pre-11 counter(9) compatibility */
355 #if __FreeBSD_version >= 1100036
356 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count)
357 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count)
358 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count)
359 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count)
360 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */
361 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count)
362 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count)
363 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count)
364 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count)
365 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count)
366 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count)
367 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count)
369 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count)
370 #define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count)
371 #define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count)
372 #define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count)
373 #define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count)
374 #define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count)
375 #define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count)
376 #define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count)
377 #define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count)
378 #define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count)
379 #define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops)
380 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count)
384 *****************************************************************************
387 * This array contains the list of Subvendor/Subdevice IDs on which the driver
390 *****************************************************************************
392 typedef struct _ixl_vendor_info_t {
393 unsigned int vendor_id;
394 unsigned int device_id;
395 unsigned int subvendor_id;
396 unsigned int subdevice_id;
417 ** This struct has multiple uses, multicast
418 ** addresses, vlans, and mac filters all use it.
420 struct ixl_mac_filter {
421 SLIST_ENTRY(ixl_mac_filter) next;
422 u8 macaddr[ETHER_ADDR_LEN];
428 * The Transmit ring control struct
431 struct ixl_queue *que;
434 struct i40e_tx_desc *base;
435 struct i40e_dma_mem dma;
442 struct ixl_tx_buf *buffers;
445 bus_dma_tag_t tx_tag;
446 bus_dma_tag_t tso_tag;
451 /* Used for Dynamic ITR calculation */
463 * The Receive ring control struct
466 struct ixl_queue *que;
468 union i40e_rx_desc *base;
469 struct i40e_dma_mem dma;
479 struct ixl_rx_buf *buffers;
485 /* Used for Dynamic ITR calculation */
498 ** Driver queue struct: this is the interrupt container
499 ** for the associated tx and rx ring pair.
504 u32 msix; /* This queue's MSIX vector */
505 u32 eims; /* This queue's EIMS bit */
506 struct resource *res;
508 int num_desc; /* both tx and rx */
513 struct taskqueue *tq;
518 u64 mbuf_defrag_failed;
521 u64 tx_dmamap_failed;
527 ** Virtual Station Interface
529 SLIST_HEAD(ixl_ftl_head, ixl_mac_filter);
535 struct ifmedia media;
536 enum i40e_vsi_type type;
543 struct ixl_queue *queues; /* head of queues */
551 /* MAC/VLAN Filter list */
552 struct ixl_ftl_head ftl;
555 /* Contains readylist & stat counter id */
556 struct i40e_aqc_vsi_properties_data info;
558 eventhandler_tag vlan_attach;
559 eventhandler_tag vlan_detach;
562 /* Per-VSI stats from hardware */
563 struct i40e_eth_stats eth_stats;
564 struct i40e_eth_stats eth_stats_offsets;
565 bool stat_offsets_loaded;
566 /* VSI stat counters */
579 /* Driver statistics */
585 struct sysctl_oid *vsi_node;
589 ** Find the number of unrefreshed RX descriptors
592 ixl_rx_unrefreshed(struct ixl_queue *que)
594 struct rx_ring *rxr = &que->rxr;
596 if (rxr->next_check > rxr->next_refresh)
597 return (rxr->next_check - rxr->next_refresh - 1);
599 return ((que->num_desc + rxr->next_check) -
600 rxr->next_refresh - 1);
604 ** Find the next available unused filter
606 static inline struct ixl_mac_filter *
607 ixl_get_filter(struct ixl_vsi *vsi)
609 struct ixl_mac_filter *f;
611 /* create a new empty filter */
612 f = malloc(sizeof(struct ixl_mac_filter),
613 M_DEVBUF, M_NOWAIT | M_ZERO);
615 SLIST_INSERT_HEAD(&vsi->ftl, f, next);
621 ** Compare two ethernet addresses
624 cmp_etheraddr(const u8 *ea1, const u8 *ea2)
628 if ((ea1[0] == ea2[0]) && (ea1[1] == ea2[1]) &&
629 (ea1[2] == ea2[2]) && (ea1[3] == ea2[3]) &&
630 (ea1[4] == ea2[4]) && (ea1[5] == ea2[5]))
637 * Return next largest power of 2, unsigned
639 * Public domain, from Bit Twiddling Hacks
642 next_power_of_two(u32 n)
652 /* Next power of two > 0 is 1 */
659 * Info for stats sysctls
661 struct ixl_sysctl_info {
667 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN];
669 /*********************************************************************
670 * TXRX Function prototypes
671 *********************************************************************/
672 int ixl_allocate_tx_data(struct ixl_queue *);
673 int ixl_allocate_rx_data(struct ixl_queue *);
674 void ixl_init_tx_ring(struct ixl_queue *);
675 int ixl_init_rx_ring(struct ixl_queue *);
676 bool ixl_rxeof(struct ixl_queue *, int);
677 bool ixl_txeof(struct ixl_queue *);
678 void ixl_free_que_tx(struct ixl_queue *);
679 void ixl_free_que_rx(struct ixl_queue *);
681 int ixl_mq_start(struct ifnet *, struct mbuf *);
682 int ixl_mq_start_locked(struct ifnet *, struct tx_ring *);
683 void ixl_deferred_mq_start(void *, int);
684 void ixl_free_vsi(struct ixl_vsi *);
685 void ixl_qflush(struct ifnet *);
687 /* Common function prototypes between PF/VF driver */
688 #if __FreeBSD_version >= 1100000
689 uint64_t ixl_get_counter(if_t ifp, ift_counter cnt);
691 void ixl_get_default_rss_key(u32 *);