1 /******************************************************************************
3 Copyright (c) 2013-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
36 ** IXL driver TX/RX Routines:
37 ** This was seperated to allow usage by
38 ** both the PF and VF drivers.
41 #ifndef IXL_STANDALONE_BUILD
43 #include "opt_inet6.h"
50 #include <net/rss_config.h>
53 /* Local Prototypes */
54 static void ixl_rx_checksum(struct mbuf *, u32, u32, u8);
55 static void ixl_refresh_mbufs(struct ixl_queue *, int);
56 static int ixl_xmit(struct ixl_queue *, struct mbuf **);
57 static int ixl_tx_setup_offload(struct ixl_queue *,
58 struct mbuf *, u32 *, u32 *);
59 static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *);
61 static inline void ixl_rx_discard(struct rx_ring *, int);
62 static inline void ixl_rx_input(struct rx_ring *, struct ifnet *,
65 static inline bool ixl_tso_detect_sparse(struct mbuf *mp);
66 static inline u32 ixl_get_tx_head(struct ixl_queue *que);
69 #include <dev/netmap/if_ixl_netmap.h>
70 int ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip = 1;
71 #endif /* DEV_NETMAP */
74 * @key key is saved into this parameter
77 ixl_get_default_rss_key(u32 *key)
81 u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687,
82 0x183cfd8c, 0xce880440, 0x580cbc3c,
83 0x35897377, 0x328b25e1, 0x4fa98922,
84 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1,
87 bcopy(rss_seed, key, IXL_RSS_KEY_SIZE);
91 ** Multiqueue Transmit driver
94 ixl_mq_start(struct ifnet *ifp, struct mbuf *m)
96 struct ixl_vsi *vsi = ifp->if_softc;
97 struct ixl_queue *que;
105 ** Which queue to use:
107 ** When doing RSS, map it to the same outbound
108 ** queue as the incoming flow would be mapped to.
109 ** If everything is setup correctly, it should be
110 ** the same bucket that the current CPU we're on is.
112 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
114 if (rss_hash2bucket(m->m_pkthdr.flowid,
115 M_HASHTYPE_GET(m), &bucket_id) == 0) {
116 i = bucket_id % vsi->num_queues;
119 i = m->m_pkthdr.flowid % vsi->num_queues;
121 i = curcpu % vsi->num_queues;
123 que = &vsi->queues[i];
126 err = drbr_enqueue(ifp, txr->br, m);
129 if (IXL_TX_TRYLOCK(txr)) {
130 ixl_mq_start_locked(ifp, txr);
133 taskqueue_enqueue(que->tq, &que->tx_task);
139 ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
141 struct ixl_queue *que = txr->que;
142 struct ixl_vsi *vsi = que->vsi;
147 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
148 vsi->link_active == 0)
151 /* Process the transmit queue */
152 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
153 if ((err = ixl_xmit(que, &next)) != 0) {
155 drbr_advance(ifp, txr->br);
157 drbr_putback(ifp, txr->br, next);
160 drbr_advance(ifp, txr->br);
161 /* Send a copy of the frame to the BPF listener */
162 ETHER_BPF_MTAP(ifp, next);
163 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
167 if (txr->avail < IXL_TX_CLEANUP_THRESHOLD)
174 * Called from a taskqueue to drain queued transmit packets.
177 ixl_deferred_mq_start(void *arg, int pending)
179 struct ixl_queue *que = arg;
180 struct tx_ring *txr = &que->txr;
181 struct ixl_vsi *vsi = que->vsi;
182 struct ifnet *ifp = vsi->ifp;
185 if (!drbr_empty(ifp, txr->br))
186 ixl_mq_start_locked(ifp, txr);
191 ** Flush all queue ring buffers
194 ixl_qflush(struct ifnet *ifp)
196 struct ixl_vsi *vsi = ifp->if_softc;
198 for (int i = 0; i < vsi->num_queues; i++) {
199 struct ixl_queue *que = &vsi->queues[i];
200 struct tx_ring *txr = &que->txr;
203 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
211 ** Find mbuf chains passed to the driver
212 ** that are 'sparse', using more than 8
213 ** mbufs to deliver an mss-size chunk of data
216 ixl_tso_detect_sparse(struct mbuf *mp)
222 mss = mp->m_pkthdr.tso_segsz;
224 /* Exclude first mbuf; assume it contains all headers */
225 for (m = mp->m_next; m != NULL; m = m->m_next) {
229 mss -= m->m_len % mp->m_pkthdr.tso_segsz;
232 if (num > IXL_SPARSE_CHAIN)
234 num = (mss == 0) ? 0 : 1;
235 mss += mp->m_pkthdr.tso_segsz;
243 /*********************************************************************
245 * This routine maps the mbufs to tx descriptors, allowing the
246 * TX engine to transmit the packets.
247 * - return 0 on success, positive on failure
249 **********************************************************************/
250 #define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
253 ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp)
255 struct ixl_vsi *vsi = que->vsi;
256 struct i40e_hw *hw = vsi->hw;
257 struct tx_ring *txr = &que->txr;
258 struct ixl_tx_buf *buf;
259 struct i40e_tx_desc *txd = NULL;
260 struct mbuf *m_head, *m;
261 int i, j, error, nsegs;
267 bus_dma_segment_t segs[IXL_MAX_TSO_SEGS];
273 * Important to capture the first descriptor
274 * used because it will contain the index of
275 * the one we tell the hardware to report back
277 first = txr->next_avail;
278 buf = &txr->buffers[first];
282 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
283 /* Use larger mapping for TSO */
285 if (ixl_tso_detect_sparse(m_head)) {
286 m = m_defrag(m_head, M_NOWAIT);
297 * Map the packet for DMA.
299 error = bus_dmamap_load_mbuf_sg(tag, map,
300 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
302 if (error == EFBIG) {
305 m = m_defrag(*m_headp, M_NOWAIT);
307 que->mbuf_defrag_failed++;
315 error = bus_dmamap_load_mbuf_sg(tag, map,
316 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
319 que->tx_dmamap_failed++;
324 } else if (error != 0) {
325 que->tx_dmamap_failed++;
331 /* Make certain there are enough descriptors */
332 if (nsegs > txr->avail - 2) {
339 /* Set up the TSO/CSUM offload */
340 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
341 error = ixl_tx_setup_offload(que, m_head, &cmd, &off);
346 cmd |= I40E_TX_DESC_CMD_ICRC;
347 /* Grab the VLAN tag */
348 if (m_head->m_flags & M_VLANTAG) {
349 cmd |= I40E_TX_DESC_CMD_IL2TAG1;
350 vtag = htole16(m_head->m_pkthdr.ether_vtag);
354 for (j = 0; j < nsegs; j++) {
357 buf = &txr->buffers[i];
358 buf->tag = tag; /* Keep track of the type tag */
360 seglen = segs[j].ds_len;
362 txd->buffer_addr = htole64(segs[j].ds_addr);
363 txd->cmd_type_offset_bsz =
364 htole64(I40E_TX_DESC_DTYPE_DATA
365 | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT)
366 | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT)
367 | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)
368 | ((u64)vtag << I40E_TXD_QW1_L2TAG1_SHIFT));
370 last = i; /* descriptor that will get completion IRQ */
372 if (++i == que->num_desc)
378 /* Set the last descriptor for report */
379 txd->cmd_type_offset_bsz |=
380 htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT));
384 buf->m_head = m_head;
385 /* Swap the dma map between the first and last descriptor */
386 txr->buffers[first].map = buf->map;
388 bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE);
390 /* Set the index of the descriptor that will be marked done */
391 buf = &txr->buffers[first];
392 buf->eop_index = last;
394 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
395 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
397 * Advance the Transmit Descriptor Tail (Tdt), this tells the
398 * hardware that this frame is available to transmit.
400 ++txr->total_packets;
401 wr32(hw, txr->tail, i);
403 /* Mark outstanding work */
404 atomic_store_rel_32(&txr->watchdog_timer, IXL_WATCHDOG);
408 bus_dmamap_unload(tag, buf->map);
413 /*********************************************************************
415 * Allocate memory for tx_buffer structures. The tx_buffer stores all
416 * the information needed to transmit a packet on the wire. This is
417 * called only once at attach, setup is done every reset.
419 **********************************************************************/
421 ixl_allocate_tx_data(struct ixl_queue *que)
423 struct tx_ring *txr = &que->txr;
424 struct ixl_vsi *vsi = que->vsi;
425 device_t dev = vsi->dev;
426 struct ixl_tx_buf *buf;
430 * Setup DMA descriptor areas.
432 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
433 1, 0, /* alignment, bounds */
434 BUS_SPACE_MAXADDR, /* lowaddr */
435 BUS_SPACE_MAXADDR, /* highaddr */
436 NULL, NULL, /* filter, filterarg */
437 IXL_TSO_SIZE, /* maxsize */
438 IXL_MAX_TX_SEGS, /* nsegments */
439 PAGE_SIZE, /* maxsegsize */
442 NULL, /* lockfuncarg */
444 device_printf(dev,"Unable to allocate TX DMA tag\n");
448 /* Make a special tag for TSO */
449 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
450 1, 0, /* alignment, bounds */
451 BUS_SPACE_MAXADDR, /* lowaddr */
452 BUS_SPACE_MAXADDR, /* highaddr */
453 NULL, NULL, /* filter, filterarg */
454 IXL_TSO_SIZE, /* maxsize */
455 IXL_MAX_TSO_SEGS, /* nsegments */
456 PAGE_SIZE, /* maxsegsize */
459 NULL, /* lockfuncarg */
461 device_printf(dev,"Unable to allocate TX TSO DMA tag\n");
466 (struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) *
467 que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
468 device_printf(dev, "Unable to allocate tx_buffer memory\n");
473 /* Create the descriptor buffer default dma maps */
475 for (int i = 0; i < que->num_desc; i++, buf++) {
476 buf->tag = txr->tx_tag;
477 error = bus_dmamap_create(buf->tag, 0, &buf->map);
479 device_printf(dev, "Unable to create TX DMA map\n");
488 /*********************************************************************
490 * (Re)Initialize a queue transmit ring.
491 * - called by init, it clears the descriptor ring,
492 * and frees any stale mbufs
494 **********************************************************************/
496 ixl_init_tx_ring(struct ixl_queue *que)
499 struct netmap_adapter *na = NA(que->vsi->ifp);
500 struct netmap_slot *slot;
501 #endif /* DEV_NETMAP */
502 struct tx_ring *txr = &que->txr;
503 struct ixl_tx_buf *buf;
505 /* Clear the old ring contents */
510 * (under lock): if in netmap mode, do some consistency
511 * checks and set slot to entry 0 of the netmap ring.
513 slot = netmap_reset(na, NR_TX, que->me, 0);
514 #endif /* DEV_NETMAP */
516 bzero((void *)txr->base,
517 (sizeof(struct i40e_tx_desc)) * que->num_desc);
521 txr->next_to_clean = 0;
523 /* Reset watchdog status */
524 txr->watchdog_timer = 0;
527 /* Initialize flow director */
528 txr->atr_rate = ixl_atr_rate;
531 /* Free any existing tx mbufs. */
533 for (int i = 0; i < que->num_desc; i++, buf++) {
534 if (buf->m_head != NULL) {
535 bus_dmamap_sync(buf->tag, buf->map,
536 BUS_DMASYNC_POSTWRITE);
537 bus_dmamap_unload(buf->tag, buf->map);
538 m_freem(buf->m_head);
543 * In netmap mode, set the map for the packet buffer.
544 * NOTE: Some drivers (not this one) also need to set
545 * the physical buffer address in the NIC ring.
546 * netmap_idx_n2k() maps a nic index, i, into the corresponding
547 * netmap slot index, si
550 int si = netmap_idx_n2k(&na->tx_rings[que->me], i);
551 netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si));
553 #endif /* DEV_NETMAP */
554 /* Clear the EOP index */
558 /* Set number of descriptors available */
559 txr->avail = que->num_desc;
561 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
562 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
567 /*********************************************************************
569 * Free transmit ring related data structures.
571 **********************************************************************/
573 ixl_free_que_tx(struct ixl_queue *que)
575 struct tx_ring *txr = &que->txr;
576 struct ixl_tx_buf *buf;
578 INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
580 for (int i = 0; i < que->num_desc; i++) {
581 buf = &txr->buffers[i];
582 if (buf->m_head != NULL) {
583 bus_dmamap_sync(buf->tag, buf->map,
584 BUS_DMASYNC_POSTWRITE);
585 bus_dmamap_unload(buf->tag,
587 m_freem(buf->m_head);
589 if (buf->map != NULL) {
590 bus_dmamap_destroy(buf->tag,
594 } else if (buf->map != NULL) {
595 bus_dmamap_unload(buf->tag,
597 bus_dmamap_destroy(buf->tag,
603 buf_ring_free(txr->br, M_DEVBUF);
604 if (txr->buffers != NULL) {
605 free(txr->buffers, M_DEVBUF);
608 if (txr->tx_tag != NULL) {
609 bus_dma_tag_destroy(txr->tx_tag);
612 if (txr->tso_tag != NULL) {
613 bus_dma_tag_destroy(txr->tso_tag);
617 INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
621 /*********************************************************************
623 * Setup descriptor for hw offloads
625 **********************************************************************/
628 ixl_tx_setup_offload(struct ixl_queue *que,
629 struct mbuf *mp, u32 *cmd, u32 *off)
631 struct ether_vlan_header *eh;
633 struct ip *ip = NULL;
635 struct tcphdr *th = NULL;
639 int elen, ip_hlen = 0, tcp_hlen;
644 /* Set up the TSO context descriptor if required */
645 if (mp->m_pkthdr.csum_flags & CSUM_TSO) {
646 tso = ixl_tso_setup(que, mp);
654 * Determine where frame payload starts.
655 * Jump over vlan headers if already present,
656 * helpful for QinQ too.
658 eh = mtod(mp, struct ether_vlan_header *);
659 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
660 etype = ntohs(eh->evl_proto);
661 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
663 etype = ntohs(eh->evl_encap_proto);
664 elen = ETHER_HDR_LEN;
670 ip = (struct ip *)(mp->m_data + elen);
671 ip_hlen = ip->ip_hl << 2;
673 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
674 /* The IP checksum must be recalculated with TSO */
676 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
678 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
683 ip6 = (struct ip6_hdr *)(mp->m_data + elen);
684 ip_hlen = sizeof(struct ip6_hdr);
685 ipproto = ip6->ip6_nxt;
686 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
687 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
694 *off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
695 *off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
699 tcp_hlen = th->th_off << 2;
700 if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) {
701 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
702 *off |= (tcp_hlen >> 2) <<
703 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
706 ixl_atr(que, th, etype);
710 if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) {
711 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
712 *off |= (sizeof(struct udphdr) >> 2) <<
713 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
718 if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) {
719 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
720 *off |= (sizeof(struct sctphdr) >> 2) <<
721 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
732 /**********************************************************************
734 * Setup context for hardware segmentation offload (TSO)
736 **********************************************************************/
738 ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp)
740 struct tx_ring *txr = &que->txr;
741 struct i40e_tx_context_desc *TXD;
742 struct ixl_tx_buf *buf;
743 u32 cmd, mss, type, tsolen;
745 int idx, elen, ip_hlen, tcp_hlen;
746 struct ether_vlan_header *eh;
753 #if defined(INET6) || defined(INET)
756 u64 type_cmd_tso_mss;
759 * Determine where frame payload starts.
760 * Jump over vlan headers if already present
762 eh = mtod(mp, struct ether_vlan_header *);
763 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
764 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
765 etype = eh->evl_proto;
767 elen = ETHER_HDR_LEN;
768 etype = eh->evl_encap_proto;
771 switch (ntohs(etype)) {
774 ip6 = (struct ip6_hdr *)(mp->m_data + elen);
775 if (ip6->ip6_nxt != IPPROTO_TCP)
777 ip_hlen = sizeof(struct ip6_hdr);
778 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
779 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
780 tcp_hlen = th->th_off << 2;
782 * The corresponding flag is set by the stack in the IPv4
783 * TSO case, but not in IPv6 (at least in FreeBSD 10.2).
784 * So, set it here because the rest of the flow requires it.
786 mp->m_pkthdr.csum_flags |= CSUM_TCP_IPV6;
791 ip = (struct ip *)(mp->m_data + elen);
792 if (ip->ip_p != IPPROTO_TCP)
795 ip_hlen = ip->ip_hl << 2;
796 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
797 th->th_sum = in_pseudo(ip->ip_src.s_addr,
798 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
799 tcp_hlen = th->th_off << 2;
803 printf("%s: CSUM_TSO but no supported IP version (0x%04x)",
804 __func__, ntohs(etype));
808 /* Ensure we have at least the IP+TCP header in the first mbuf. */
809 if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr))
812 idx = txr->next_avail;
813 buf = &txr->buffers[idx];
814 TXD = (struct i40e_tx_context_desc *) &txr->base[idx];
815 tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen);
817 type = I40E_TX_DESC_DTYPE_CONTEXT;
818 cmd = I40E_TX_CTX_DESC_TSO;
819 /* TSO MSS must not be less than 64 */
820 if (mp->m_pkthdr.tso_segsz < IXL_MIN_TSO_MSS) {
821 que->mss_too_small++;
822 mp->m_pkthdr.tso_segsz = IXL_MIN_TSO_MSS;
824 mss = mp->m_pkthdr.tso_segsz;
826 type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) |
827 ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
828 ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
829 ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
830 TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss);
832 TXD->tunneling_params = htole32(0);
836 if (++idx == que->num_desc)
840 txr->next_avail = idx;
846 ** ixl_get_tx_head - Retrieve the value from the
847 ** location the HW records its HEAD index
850 ixl_get_tx_head(struct ixl_queue *que)
852 struct tx_ring *txr = &que->txr;
853 void *head = &txr->base[que->num_desc];
854 return LE32_TO_CPU(*(volatile __le32 *)head);
857 /**********************************************************************
859 * Examine each tx_buffer in the used queue. If the hardware is done
860 * processing the packet then free associated resources. The
861 * tx_buffer is put back on the free queue.
863 **********************************************************************/
865 ixl_txeof(struct ixl_queue *que)
867 struct tx_ring *txr = &que->txr;
868 u32 first, last, head, done, processed;
869 struct ixl_tx_buf *buf;
870 struct i40e_tx_desc *tx_desc, *eop_desc;
873 mtx_assert(&txr->mtx, MA_OWNED);
876 // XXX todo: implement moderation
877 if (netmap_tx_irq(que->vsi->ifp, que->me))
879 #endif /* DEF_NETMAP */
881 /* These are not the descriptors you seek, move along :) */
882 if (txr->avail == que->num_desc) {
883 atomic_store_rel_32(&txr->watchdog_timer, 0);
888 first = txr->next_to_clean;
889 buf = &txr->buffers[first];
890 tx_desc = (struct i40e_tx_desc *)&txr->base[first];
891 last = buf->eop_index;
894 eop_desc = (struct i40e_tx_desc *)&txr->base[last];
896 /* Get the Head WB value */
897 head = ixl_get_tx_head(que);
900 ** Get the index of the first descriptor
901 ** BEYOND the EOP and call that 'done'.
902 ** I do this so the comparison in the
903 ** inner while loop below can be simple
905 if (++last == que->num_desc) last = 0;
908 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
909 BUS_DMASYNC_POSTREAD);
911 ** The HEAD index of the ring is written in a
912 ** defined location, this rather than a done bit
913 ** is what is used to keep track of what must be
916 while (first != head) {
917 /* We clean the range of the packet */
918 while (first != done) {
923 txr->bytes += /* for ITR adjustment */
924 buf->m_head->m_pkthdr.len;
925 txr->tx_bytes += /* for TX stats */
926 buf->m_head->m_pkthdr.len;
927 bus_dmamap_sync(buf->tag,
929 BUS_DMASYNC_POSTWRITE);
930 bus_dmamap_unload(buf->tag,
932 m_freem(buf->m_head);
937 if (++first == que->num_desc)
940 buf = &txr->buffers[first];
941 tx_desc = &txr->base[first];
944 /* See if there is more work now */
945 last = buf->eop_index;
947 eop_desc = &txr->base[last];
948 /* Get next done point */
949 if (++last == que->num_desc) last = 0;
954 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
955 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
957 txr->next_to_clean = first;
961 * If there are no pending descriptors, clear the timeout.
963 if (txr->avail == que->num_desc) {
964 atomic_store_rel_32(&txr->watchdog_timer, 0);
971 /*********************************************************************
973 * Refresh mbuf buffers for RX descriptor rings
974 * - now keeps its own state so discards due to resource
975 * exhaustion are unnecessary, if an mbuf cannot be obtained
976 * it just returns, keeping its placeholder, thus it can simply
977 * be recalled to try again.
979 **********************************************************************/
981 ixl_refresh_mbufs(struct ixl_queue *que, int limit)
983 struct ixl_vsi *vsi = que->vsi;
984 struct rx_ring *rxr = &que->rxr;
985 bus_dma_segment_t hseg[1];
986 bus_dma_segment_t pseg[1];
987 struct ixl_rx_buf *buf;
988 struct mbuf *mh, *mp;
989 int i, j, nsegs, error;
990 bool refreshed = FALSE;
992 i = j = rxr->next_refresh;
993 /* Control the loop with one beyond */
994 if (++j == que->num_desc)
998 buf = &rxr->buffers[i];
999 if (rxr->hdr_split == FALSE)
1002 if (buf->m_head == NULL) {
1003 mh = m_gethdr(M_NOWAIT, MT_DATA);
1009 mh->m_pkthdr.len = mh->m_len = MHLEN;
1011 mh->m_flags |= M_PKTHDR;
1012 /* Get the memory mapping */
1013 error = bus_dmamap_load_mbuf_sg(rxr->htag,
1014 buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
1016 printf("Refresh mbufs: hdr dmamap load"
1017 " failure - %d\n", error);
1023 bus_dmamap_sync(rxr->htag, buf->hmap,
1024 BUS_DMASYNC_PREREAD);
1025 rxr->base[i].read.hdr_addr =
1026 htole64(hseg[0].ds_addr);
1029 if (buf->m_pack == NULL) {
1030 mp = m_getjcl(M_NOWAIT, MT_DATA,
1031 M_PKTHDR, rxr->mbuf_sz);
1037 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1038 /* Get the memory mapping */
1039 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1040 buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
1042 printf("Refresh mbufs: payload dmamap load"
1043 " failure - %d\n", error);
1049 bus_dmamap_sync(rxr->ptag, buf->pmap,
1050 BUS_DMASYNC_PREREAD);
1051 rxr->base[i].read.pkt_addr =
1052 htole64(pseg[0].ds_addr);
1053 /* Used only when doing header split */
1054 rxr->base[i].read.hdr_addr = 0;
1057 /* Next is precalculated */
1059 rxr->next_refresh = i;
1060 if (++j == que->num_desc)
1064 if (refreshed) /* Update hardware tail index */
1065 wr32(vsi->hw, rxr->tail, rxr->next_refresh);
1070 /*********************************************************************
1072 * Allocate memory for rx_buffer structures. Since we use one
1073 * rx_buffer per descriptor, the maximum number of rx_buffer's
1074 * that we'll need is equal to the number of receive descriptors
1075 * that we've defined.
1077 **********************************************************************/
1079 ixl_allocate_rx_data(struct ixl_queue *que)
1081 struct rx_ring *rxr = &que->rxr;
1082 struct ixl_vsi *vsi = que->vsi;
1083 device_t dev = vsi->dev;
1084 struct ixl_rx_buf *buf;
1085 int i, bsize, error;
1087 bsize = sizeof(struct ixl_rx_buf) * que->num_desc;
1088 if (!(rxr->buffers =
1089 (struct ixl_rx_buf *) malloc(bsize,
1090 M_DEVBUF, M_NOWAIT | M_ZERO))) {
1091 device_printf(dev, "Unable to allocate rx_buffer memory\n");
1096 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1097 1, 0, /* alignment, bounds */
1098 BUS_SPACE_MAXADDR, /* lowaddr */
1099 BUS_SPACE_MAXADDR, /* highaddr */
1100 NULL, NULL, /* filter, filterarg */
1101 MSIZE, /* maxsize */
1103 MSIZE, /* maxsegsize */
1105 NULL, /* lockfunc */
1106 NULL, /* lockfuncarg */
1108 device_printf(dev, "Unable to create RX DMA htag\n");
1112 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1113 1, 0, /* alignment, bounds */
1114 BUS_SPACE_MAXADDR, /* lowaddr */
1115 BUS_SPACE_MAXADDR, /* highaddr */
1116 NULL, NULL, /* filter, filterarg */
1117 MJUM16BYTES, /* maxsize */
1119 MJUM16BYTES, /* maxsegsize */
1121 NULL, /* lockfunc */
1122 NULL, /* lockfuncarg */
1124 device_printf(dev, "Unable to create RX DMA ptag\n");
1128 for (i = 0; i < que->num_desc; i++) {
1129 buf = &rxr->buffers[i];
1130 error = bus_dmamap_create(rxr->htag,
1131 BUS_DMA_NOWAIT, &buf->hmap);
1133 device_printf(dev, "Unable to create RX head map\n");
1136 error = bus_dmamap_create(rxr->ptag,
1137 BUS_DMA_NOWAIT, &buf->pmap);
1139 device_printf(dev, "Unable to create RX pkt map\n");
1148 /*********************************************************************
1150 * (Re)Initialize the queue receive ring and its buffers.
1152 **********************************************************************/
1154 ixl_init_rx_ring(struct ixl_queue *que)
1156 struct rx_ring *rxr = &que->rxr;
1157 struct ixl_vsi *vsi = que->vsi;
1158 #if defined(INET6) || defined(INET)
1159 struct ifnet *ifp = vsi->ifp;
1160 struct lro_ctrl *lro = &rxr->lro;
1162 struct ixl_rx_buf *buf;
1163 bus_dma_segment_t pseg[1], hseg[1];
1164 int rsize, nsegs, error = 0;
1166 struct netmap_adapter *na = NA(que->vsi->ifp);
1167 struct netmap_slot *slot;
1168 #endif /* DEV_NETMAP */
1172 /* same as in ixl_init_tx_ring() */
1173 slot = netmap_reset(na, NR_RX, que->me, 0);
1174 #endif /* DEV_NETMAP */
1175 /* Clear the ring contents */
1176 rsize = roundup2(que->num_desc *
1177 sizeof(union i40e_rx_desc), DBA_ALIGN);
1178 bzero((void *)rxr->base, rsize);
1179 /* Cleanup any existing buffers */
1180 for (int i = 0; i < que->num_desc; i++) {
1181 buf = &rxr->buffers[i];
1182 if (buf->m_head != NULL) {
1183 bus_dmamap_sync(rxr->htag, buf->hmap,
1184 BUS_DMASYNC_POSTREAD);
1185 bus_dmamap_unload(rxr->htag, buf->hmap);
1186 buf->m_head->m_flags |= M_PKTHDR;
1187 m_freem(buf->m_head);
1189 if (buf->m_pack != NULL) {
1190 bus_dmamap_sync(rxr->ptag, buf->pmap,
1191 BUS_DMASYNC_POSTREAD);
1192 bus_dmamap_unload(rxr->ptag, buf->pmap);
1193 buf->m_pack->m_flags |= M_PKTHDR;
1194 m_freem(buf->m_pack);
1200 /* header split is off */
1201 rxr->hdr_split = FALSE;
1203 /* Now replenish the mbufs */
1204 for (int j = 0; j != que->num_desc; ++j) {
1205 struct mbuf *mh, *mp;
1207 buf = &rxr->buffers[j];
1210 * In netmap mode, fill the map and set the buffer
1211 * address in the NIC ring, considering the offset
1212 * between the netmap and NIC rings (see comment in
1213 * ixgbe_setup_transmit_ring() ). No need to allocate
1214 * an mbuf, so end the block with a continue;
1217 int sj = netmap_idx_n2k(&na->rx_rings[que->me], j);
1221 addr = PNMB(na, slot + sj, &paddr);
1222 netmap_load_map(na, rxr->dma.tag, buf->pmap, addr);
1223 /* Update descriptor and the cached value */
1224 rxr->base[j].read.pkt_addr = htole64(paddr);
1225 rxr->base[j].read.hdr_addr = 0;
1228 #endif /* DEV_NETMAP */
1230 ** Don't allocate mbufs if not
1231 ** doing header split, its wasteful
1233 if (rxr->hdr_split == FALSE)
1236 /* First the header */
1237 buf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
1238 if (buf->m_head == NULL) {
1242 m_adj(buf->m_head, ETHER_ALIGN);
1244 mh->m_len = mh->m_pkthdr.len = MHLEN;
1245 mh->m_flags |= M_PKTHDR;
1246 /* Get the memory mapping */
1247 error = bus_dmamap_load_mbuf_sg(rxr->htag,
1248 buf->hmap, buf->m_head, hseg,
1249 &nsegs, BUS_DMA_NOWAIT);
1250 if (error != 0) /* Nothing elegant to do here */
1252 bus_dmamap_sync(rxr->htag,
1253 buf->hmap, BUS_DMASYNC_PREREAD);
1254 /* Update descriptor */
1255 rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
1258 /* Now the payload cluster */
1259 buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
1260 M_PKTHDR, rxr->mbuf_sz);
1261 if (buf->m_pack == NULL) {
1266 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1267 /* Get the memory mapping */
1268 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1269 buf->pmap, mp, pseg,
1270 &nsegs, BUS_DMA_NOWAIT);
1273 bus_dmamap_sync(rxr->ptag,
1274 buf->pmap, BUS_DMASYNC_PREREAD);
1275 /* Update descriptor */
1276 rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
1277 rxr->base[j].read.hdr_addr = 0;
1281 /* Setup our descriptor indices */
1282 rxr->next_check = 0;
1283 rxr->next_refresh = 0;
1284 rxr->lro_enabled = FALSE;
1287 rxr->discard = FALSE;
1289 wr32(vsi->hw, rxr->tail, que->num_desc - 1);
1292 #if defined(INET6) || defined(INET)
1294 ** Now set up the LRO interface:
1296 if (ifp->if_capenable & IFCAP_LRO) {
1297 int err = tcp_lro_init(lro);
1299 if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me);
1302 INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me);
1303 rxr->lro_enabled = TRUE;
1304 lro->ifp = vsi->ifp;
1308 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1309 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1317 /*********************************************************************
1319 * Free station receive ring data structures
1321 **********************************************************************/
1323 ixl_free_que_rx(struct ixl_queue *que)
1325 struct rx_ring *rxr = &que->rxr;
1326 struct ixl_rx_buf *buf;
1328 INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
1330 /* Cleanup any existing buffers */
1331 if (rxr->buffers != NULL) {
1332 for (int i = 0; i < que->num_desc; i++) {
1333 buf = &rxr->buffers[i];
1334 if (buf->m_head != NULL) {
1335 bus_dmamap_sync(rxr->htag, buf->hmap,
1336 BUS_DMASYNC_POSTREAD);
1337 bus_dmamap_unload(rxr->htag, buf->hmap);
1338 buf->m_head->m_flags |= M_PKTHDR;
1339 m_freem(buf->m_head);
1341 if (buf->m_pack != NULL) {
1342 bus_dmamap_sync(rxr->ptag, buf->pmap,
1343 BUS_DMASYNC_POSTREAD);
1344 bus_dmamap_unload(rxr->ptag, buf->pmap);
1345 buf->m_pack->m_flags |= M_PKTHDR;
1346 m_freem(buf->m_pack);
1350 if (buf->hmap != NULL) {
1351 bus_dmamap_destroy(rxr->htag, buf->hmap);
1354 if (buf->pmap != NULL) {
1355 bus_dmamap_destroy(rxr->ptag, buf->pmap);
1359 if (rxr->buffers != NULL) {
1360 free(rxr->buffers, M_DEVBUF);
1361 rxr->buffers = NULL;
1365 if (rxr->htag != NULL) {
1366 bus_dma_tag_destroy(rxr->htag);
1369 if (rxr->ptag != NULL) {
1370 bus_dma_tag_destroy(rxr->ptag);
1374 INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
1379 ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype)
1382 #if defined(INET6) || defined(INET)
1384 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
1385 * should be computed by hardware. Also it should not have VLAN tag in
1388 if (rxr->lro_enabled &&
1389 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
1390 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
1391 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
1393 * Send to the stack if:
1394 ** - LRO not enabled, or
1395 ** - no LRO resources, or
1396 ** - lro enqueue fails
1398 if (rxr->lro.lro_cnt != 0)
1399 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
1403 (*ifp->if_input)(ifp, m);
1408 ixl_rx_discard(struct rx_ring *rxr, int i)
1410 struct ixl_rx_buf *rbuf;
1412 rbuf = &rxr->buffers[i];
1414 if (rbuf->fmp != NULL) {/* Partial chain ? */
1415 rbuf->fmp->m_flags |= M_PKTHDR;
1421 ** With advanced descriptors the writeback
1422 ** clobbers the buffer addrs, so its easier
1423 ** to just free the existing mbufs and take
1424 ** the normal refresh path to get new buffers
1428 m_free(rbuf->m_head);
1429 rbuf->m_head = NULL;
1433 m_free(rbuf->m_pack);
1434 rbuf->m_pack = NULL;
1442 ** i40e_ptype_to_hash: parse the packet type
1443 ** to determine the appropriate hash.
1446 ixl_ptype_to_hash(u8 ptype)
1448 struct i40e_rx_ptype_decoded decoded;
1450 decoded = decode_rx_desc_ptype(ptype);
1453 return M_HASHTYPE_OPAQUE_HASH;
1455 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2)
1456 return M_HASHTYPE_OPAQUE_HASH;
1458 /* Note: anything that gets to this point is IP */
1459 if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) {
1460 switch (decoded.inner_prot) {
1461 case I40E_RX_PTYPE_INNER_PROT_TCP:
1462 return M_HASHTYPE_RSS_TCP_IPV6;
1463 case I40E_RX_PTYPE_INNER_PROT_UDP:
1464 return M_HASHTYPE_RSS_UDP_IPV6;
1466 return M_HASHTYPE_RSS_IPV6;
1469 if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) {
1470 switch (decoded.inner_prot) {
1471 case I40E_RX_PTYPE_INNER_PROT_TCP:
1472 return M_HASHTYPE_RSS_TCP_IPV4;
1473 case I40E_RX_PTYPE_INNER_PROT_UDP:
1474 return M_HASHTYPE_RSS_UDP_IPV4;
1476 return M_HASHTYPE_RSS_IPV4;
1479 /* We should never get here!! */
1480 return M_HASHTYPE_OPAQUE_HASH;
1484 /*********************************************************************
1486 * This routine executes in interrupt context. It replenishes
1487 * the mbufs in the descriptor and sends data which has been
1488 * dma'ed into host memory to upper layer.
1490 * We loop at most count times if count is > 0, or until done if
1493 * Return TRUE for more work, FALSE for all clean.
1494 *********************************************************************/
1496 ixl_rxeof(struct ixl_queue *que, int count)
1498 struct ixl_vsi *vsi = que->vsi;
1499 struct rx_ring *rxr = &que->rxr;
1500 struct ifnet *ifp = vsi->ifp;
1501 #if defined(INET6) || defined(INET)
1502 struct lro_ctrl *lro = &rxr->lro;
1504 int i, nextp, processed = 0;
1505 union i40e_rx_desc *cur;
1506 struct ixl_rx_buf *rbuf, *nbuf;
1512 if (netmap_rx_irq(ifp, que->me, &count)) {
1516 #endif /* DEV_NETMAP */
1518 for (i = rxr->next_check; count != 0;) {
1519 struct mbuf *sendmp, *mh, *mp;
1521 u16 hlen, plen, vtag;
1526 /* Sync the ring. */
1527 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1528 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1530 cur = &rxr->base[i];
1531 qword = le64toh(cur->wb.qword1.status_error_len);
1532 status = (qword & I40E_RXD_QW1_STATUS_MASK)
1533 >> I40E_RXD_QW1_STATUS_SHIFT;
1534 error = (qword & I40E_RXD_QW1_ERROR_MASK)
1535 >> I40E_RXD_QW1_ERROR_SHIFT;
1536 plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK)
1537 >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1538 hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK)
1539 >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1540 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK)
1541 >> I40E_RXD_QW1_PTYPE_SHIFT;
1543 if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) {
1547 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1553 cur->wb.qword1.status_error_len = 0;
1554 rbuf = &rxr->buffers[i];
1557 eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT));
1558 if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT))
1559 vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1);
1563 /* Remove device access to the rx buffers. */
1564 if (rbuf->m_head != NULL) {
1565 bus_dmamap_sync(rxr->htag, rbuf->hmap,
1566 BUS_DMASYNC_POSTREAD);
1567 bus_dmamap_unload(rxr->htag, rbuf->hmap);
1569 if (rbuf->m_pack != NULL) {
1570 bus_dmamap_sync(rxr->ptag, rbuf->pmap,
1571 BUS_DMASYNC_POSTREAD);
1572 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
1576 ** Make sure bad packets are discarded,
1577 ** note that only EOP descriptor has valid
1580 if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1582 ixl_rx_discard(rxr, i);
1586 /* Prefetch the next buffer */
1589 if (nextp == que->num_desc)
1591 nbuf = &rxr->buffers[nextp];
1596 ** The header mbuf is ONLY used when header
1597 ** split is enabled, otherwise we get normal
1598 ** behavior, ie, both header and payload
1599 ** are DMA'd into the payload buffer.
1601 ** Rather than using the fmp/lmp global pointers
1602 ** we now keep the head of a packet chain in the
1603 ** buffer struct and pass this along from one
1604 ** descriptor to the next, until we get EOP.
1606 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
1607 if (hlen > IXL_RX_HDR)
1610 mh->m_flags |= M_PKTHDR;
1612 mh->m_pkthdr.len = mh->m_len;
1613 /* Null buf pointer so it is refreshed */
1614 rbuf->m_head = NULL;
1616 ** Check the payload length, this
1617 ** could be zero if its a small
1623 mp->m_flags &= ~M_PKTHDR;
1625 mh->m_pkthdr.len += mp->m_len;
1626 /* Null buf pointer so it is refreshed */
1627 rbuf->m_pack = NULL;
1631 ** Now create the forward
1632 ** chain so when complete
1636 /* stash the chain head */
1638 /* Make forward chain */
1640 mp->m_next = nbuf->m_pack;
1642 mh->m_next = nbuf->m_pack;
1644 /* Singlet, prepare to send */
1647 sendmp->m_pkthdr.ether_vtag = vtag;
1648 sendmp->m_flags |= M_VLANTAG;
1653 ** Either no header split, or a
1654 ** secondary piece of a fragmented
1659 ** See if there is a stored head
1660 ** that determines what we are
1663 rbuf->m_pack = rbuf->fmp = NULL;
1665 if (sendmp != NULL) /* secondary frag */
1666 sendmp->m_pkthdr.len += mp->m_len;
1668 /* first desc of a non-ps chain */
1670 sendmp->m_flags |= M_PKTHDR;
1671 sendmp->m_pkthdr.len = mp->m_len;
1673 /* Pass the head pointer on */
1677 mp->m_next = nbuf->m_pack;
1681 /* Sending this frame? */
1683 sendmp->m_pkthdr.rcvif = ifp;
1686 rxr->rx_bytes += sendmp->m_pkthdr.len;
1687 /* capture data for dynamic ITR adjustment */
1689 rxr->bytes += sendmp->m_pkthdr.len;
1690 /* Set VLAN tag (field only valid in eop desc) */
1692 sendmp->m_pkthdr.ether_vtag = vtag;
1693 sendmp->m_flags |= M_VLANTAG;
1695 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
1696 ixl_rx_checksum(sendmp, status, error, ptype);
1698 sendmp->m_pkthdr.flowid =
1699 le32toh(cur->wb.qword0.hi_dword.rss);
1700 M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype));
1702 sendmp->m_pkthdr.flowid = que->msix;
1703 M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
1707 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1708 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1710 /* Advance our pointers to the next descriptor. */
1711 if (++i == que->num_desc)
1714 /* Now send to the stack or do LRO */
1715 if (sendmp != NULL) {
1716 rxr->next_check = i;
1718 ixl_rx_input(rxr, ifp, sendmp, ptype);
1720 i = rxr->next_check;
1723 /* Every 8 descriptors we go to refresh mbufs */
1724 if (processed == 8) {
1725 ixl_refresh_mbufs(que, i);
1730 /* Refresh any remaining buf structs */
1731 if (ixl_rx_unrefreshed(que))
1732 ixl_refresh_mbufs(que, i);
1734 rxr->next_check = i;
1738 #if defined(INET6) || defined(INET)
1740 * Flush any outstanding LRO work
1742 #if __FreeBSD_version >= 1100105
1743 tcp_lro_flush_all(lro);
1745 struct lro_entry *queued;
1746 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
1747 SLIST_REMOVE_HEAD(&lro->lro_active, next);
1748 tcp_lro_flush(lro, queued);
1751 #endif /* defined(INET6) || defined(INET) */
1757 /*********************************************************************
1759 * Verify that the hardware indicated that the checksum is valid.
1760 * Inform the stack about the status of checksum so that stack
1761 * doesn't spend time verifying the checksum.
1763 *********************************************************************/
1765 ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype)
1767 struct i40e_rx_ptype_decoded decoded;
1769 decoded = decode_rx_desc_ptype(ptype);
1772 if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1773 (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) {
1774 mp->m_pkthdr.csum_flags = 0;
1778 /* IPv6 with extension headers likely have bad csum */
1779 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1780 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1782 (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) {
1783 mp->m_pkthdr.csum_flags = 0;
1788 /* IP Checksum Good */
1789 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
1790 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1792 if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) {
1793 mp->m_pkthdr.csum_flags |=
1794 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1795 mp->m_pkthdr.csum_data |= htons(0xffff);
1800 #if __FreeBSD_version >= 1100000
1802 ixl_get_counter(if_t ifp, ift_counter cnt)
1804 struct ixl_vsi *vsi;
1806 vsi = if_getsoftc(ifp);
1809 case IFCOUNTER_IPACKETS:
1810 return (vsi->ipackets);
1811 case IFCOUNTER_IERRORS:
1812 return (vsi->ierrors);
1813 case IFCOUNTER_OPACKETS:
1814 return (vsi->opackets);
1815 case IFCOUNTER_OERRORS:
1816 return (vsi->oerrors);
1817 case IFCOUNTER_COLLISIONS:
1818 /* Collisions are by standard impossible in 40G/10G Ethernet */
1820 case IFCOUNTER_IBYTES:
1821 return (vsi->ibytes);
1822 case IFCOUNTER_OBYTES:
1823 return (vsi->obytes);
1824 case IFCOUNTER_IMCASTS:
1825 return (vsi->imcasts);
1826 case IFCOUNTER_OMCASTS:
1827 return (vsi->omcasts);
1828 case IFCOUNTER_IQDROPS:
1829 return (vsi->iqdrops);
1830 case IFCOUNTER_OQDROPS:
1831 return (vsi->oqdrops);
1832 case IFCOUNTER_NOPROTO:
1833 return (vsi->noproto);
1835 return (if_get_counter_default(ifp, cnt));