1 /******************************************************************************
3 Copyright (c) 2013-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
36 ** IXL driver TX/RX Routines:
37 ** This was seperated to allow usage by
38 ** both the PF and VF drivers.
41 #ifndef IXL_STANDALONE_BUILD
43 #include "opt_inet6.h"
50 #include <net/rss_config.h>
53 /* Local Prototypes */
54 static void ixl_rx_checksum(struct mbuf *, u32, u32, u8);
55 static void ixl_refresh_mbufs(struct ixl_queue *, int);
56 static int ixl_xmit(struct ixl_queue *, struct mbuf **);
57 static int ixl_tx_setup_offload(struct ixl_queue *,
58 struct mbuf *, u32 *, u32 *);
59 static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *);
61 static inline void ixl_rx_discard(struct rx_ring *, int);
62 static inline void ixl_rx_input(struct rx_ring *, struct ifnet *,
65 static inline bool ixl_tso_detect_sparse(struct mbuf *mp);
66 static int ixl_tx_setup_offload(struct ixl_queue *que,
67 struct mbuf *mp, u32 *cmd, u32 *off);
68 static inline u32 ixl_get_tx_head(struct ixl_queue *que);
71 #include <dev/netmap/if_ixl_netmap.h>
72 int ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip = 1;
73 #endif /* DEV_NETMAP */
76 * @key key is saved into this parameter
79 ixl_get_default_rss_key(u32 *key)
83 u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687,
84 0x183cfd8c, 0xce880440, 0x580cbc3c,
85 0x35897377, 0x328b25e1, 0x4fa98922,
86 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1,
89 bcopy(rss_seed, key, IXL_RSS_KEY_SIZE);
93 ** Multiqueue Transmit driver
96 ixl_mq_start(struct ifnet *ifp, struct mbuf *m)
98 struct ixl_vsi *vsi = ifp->if_softc;
99 struct ixl_queue *que;
107 ** Which queue to use:
109 ** When doing RSS, map it to the same outbound
110 ** queue as the incoming flow would be mapped to.
111 ** If everything is setup correctly, it should be
112 ** the same bucket that the current CPU we're on is.
114 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
116 if (rss_hash2bucket(m->m_pkthdr.flowid,
117 M_HASHTYPE_GET(m), &bucket_id) == 0) {
118 i = bucket_id % vsi->num_queues;
121 i = m->m_pkthdr.flowid % vsi->num_queues;
123 i = curcpu % vsi->num_queues;
125 que = &vsi->queues[i];
128 err = drbr_enqueue(ifp, txr->br, m);
131 if (IXL_TX_TRYLOCK(txr)) {
132 ixl_mq_start_locked(ifp, txr);
135 taskqueue_enqueue(que->tq, &que->tx_task);
141 ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
143 struct ixl_queue *que = txr->que;
144 struct ixl_vsi *vsi = que->vsi;
149 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
150 vsi->link_active == 0)
153 /* Process the transmit queue */
154 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
155 if ((err = ixl_xmit(que, &next)) != 0) {
157 drbr_advance(ifp, txr->br);
159 drbr_putback(ifp, txr->br, next);
162 drbr_advance(ifp, txr->br);
163 /* Send a copy of the frame to the BPF listener */
164 ETHER_BPF_MTAP(ifp, next);
165 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
169 if (txr->avail < IXL_TX_CLEANUP_THRESHOLD)
176 * Called from a taskqueue to drain queued transmit packets.
179 ixl_deferred_mq_start(void *arg, int pending)
181 struct ixl_queue *que = arg;
182 struct tx_ring *txr = &que->txr;
183 struct ixl_vsi *vsi = que->vsi;
184 struct ifnet *ifp = vsi->ifp;
187 if (!drbr_empty(ifp, txr->br))
188 ixl_mq_start_locked(ifp, txr);
193 ** Flush all queue ring buffers
196 ixl_qflush(struct ifnet *ifp)
198 struct ixl_vsi *vsi = ifp->if_softc;
200 for (int i = 0; i < vsi->num_queues; i++) {
201 struct ixl_queue *que = &vsi->queues[i];
202 struct tx_ring *txr = &que->txr;
205 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
213 ** Find mbuf chains passed to the driver
214 ** that are 'sparse', using more than 8
215 ** mbufs to deliver an mss-size chunk of data
218 ixl_tso_detect_sparse(struct mbuf *mp)
224 mss = mp->m_pkthdr.tso_segsz;
226 /* Exclude first mbuf; assume it contains all headers */
227 for (m = mp->m_next; m != NULL; m = m->m_next) {
231 mss -= m->m_len % mp->m_pkthdr.tso_segsz;
234 if (num > IXL_SPARSE_CHAIN)
236 num = (mss == 0) ? 0 : 1;
237 mss += mp->m_pkthdr.tso_segsz;
245 /*********************************************************************
247 * This routine maps the mbufs to tx descriptors, allowing the
248 * TX engine to transmit the packets.
249 * - return 0 on success, positive on failure
251 **********************************************************************/
252 #define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
255 ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp)
257 struct ixl_vsi *vsi = que->vsi;
258 struct i40e_hw *hw = vsi->hw;
259 struct tx_ring *txr = &que->txr;
260 struct ixl_tx_buf *buf;
261 struct i40e_tx_desc *txd = NULL;
262 struct mbuf *m_head, *m;
263 int i, j, error, nsegs;
269 bus_dma_segment_t segs[IXL_MAX_TSO_SEGS];
275 * Important to capture the first descriptor
276 * used because it will contain the index of
277 * the one we tell the hardware to report back
279 first = txr->next_avail;
280 buf = &txr->buffers[first];
284 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
285 /* Use larger mapping for TSO */
287 if (ixl_tso_detect_sparse(m_head)) {
288 m = m_defrag(m_head, M_NOWAIT);
299 * Map the packet for DMA.
301 error = bus_dmamap_load_mbuf_sg(tag, map,
302 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
304 if (error == EFBIG) {
307 m = m_defrag(*m_headp, M_NOWAIT);
309 que->mbuf_defrag_failed++;
317 error = bus_dmamap_load_mbuf_sg(tag, map,
318 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
321 que->tx_dmamap_failed++;
326 } else if (error != 0) {
327 que->tx_dmamap_failed++;
333 /* Make certain there are enough descriptors */
334 if (nsegs > txr->avail - 2) {
341 /* Set up the TSO/CSUM offload */
342 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
343 error = ixl_tx_setup_offload(que, m_head, &cmd, &off);
348 cmd |= I40E_TX_DESC_CMD_ICRC;
349 /* Grab the VLAN tag */
350 if (m_head->m_flags & M_VLANTAG) {
351 cmd |= I40E_TX_DESC_CMD_IL2TAG1;
352 vtag = htole16(m_head->m_pkthdr.ether_vtag);
356 for (j = 0; j < nsegs; j++) {
359 buf = &txr->buffers[i];
360 buf->tag = tag; /* Keep track of the type tag */
362 seglen = segs[j].ds_len;
364 txd->buffer_addr = htole64(segs[j].ds_addr);
365 txd->cmd_type_offset_bsz =
366 htole64(I40E_TX_DESC_DTYPE_DATA
367 | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT)
368 | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT)
369 | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)
370 | ((u64)vtag << I40E_TXD_QW1_L2TAG1_SHIFT));
372 last = i; /* descriptor that will get completion IRQ */
374 if (++i == que->num_desc)
380 /* Set the last descriptor for report */
381 txd->cmd_type_offset_bsz |=
382 htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT));
386 buf->m_head = m_head;
387 /* Swap the dma map between the first and last descriptor */
388 txr->buffers[first].map = buf->map;
390 bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE);
392 /* Set the index of the descriptor that will be marked done */
393 buf = &txr->buffers[first];
394 buf->eop_index = last;
396 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
397 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
399 * Advance the Transmit Descriptor Tail (Tdt), this tells the
400 * hardware that this frame is available to transmit.
402 ++txr->total_packets;
403 wr32(hw, txr->tail, i);
405 /* Mark outstanding work */
406 atomic_store_rel_32(&txr->watchdog_timer, IXL_WATCHDOG);
410 bus_dmamap_unload(tag, buf->map);
415 /*********************************************************************
417 * Allocate memory for tx_buffer structures. The tx_buffer stores all
418 * the information needed to transmit a packet on the wire. This is
419 * called only once at attach, setup is done every reset.
421 **********************************************************************/
423 ixl_allocate_tx_data(struct ixl_queue *que)
425 struct tx_ring *txr = &que->txr;
426 struct ixl_vsi *vsi = que->vsi;
427 device_t dev = vsi->dev;
428 struct ixl_tx_buf *buf;
432 * Setup DMA descriptor areas.
434 if ((error = bus_dma_tag_create(NULL, /* parent */
435 1, 0, /* alignment, bounds */
436 BUS_SPACE_MAXADDR, /* lowaddr */
437 BUS_SPACE_MAXADDR, /* highaddr */
438 NULL, NULL, /* filter, filterarg */
439 IXL_TSO_SIZE, /* maxsize */
440 IXL_MAX_TX_SEGS, /* nsegments */
441 PAGE_SIZE, /* maxsegsize */
444 NULL, /* lockfuncarg */
446 device_printf(dev,"Unable to allocate TX DMA tag\n");
450 /* Make a special tag for TSO */
451 if ((error = bus_dma_tag_create(NULL, /* parent */
452 1, 0, /* alignment, bounds */
453 BUS_SPACE_MAXADDR, /* lowaddr */
454 BUS_SPACE_MAXADDR, /* highaddr */
455 NULL, NULL, /* filter, filterarg */
456 IXL_TSO_SIZE, /* maxsize */
457 IXL_MAX_TSO_SEGS, /* nsegments */
458 PAGE_SIZE, /* maxsegsize */
461 NULL, /* lockfuncarg */
463 device_printf(dev,"Unable to allocate TX TSO DMA tag\n");
468 (struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) *
469 que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
470 device_printf(dev, "Unable to allocate tx_buffer memory\n");
475 /* Create the descriptor buffer default dma maps */
477 for (int i = 0; i < que->num_desc; i++, buf++) {
478 buf->tag = txr->tx_tag;
479 error = bus_dmamap_create(buf->tag, 0, &buf->map);
481 device_printf(dev, "Unable to create TX DMA map\n");
490 /*********************************************************************
492 * (Re)Initialize a queue transmit ring.
493 * - called by init, it clears the descriptor ring,
494 * and frees any stale mbufs
496 **********************************************************************/
498 ixl_init_tx_ring(struct ixl_queue *que)
501 struct netmap_adapter *na = NA(que->vsi->ifp);
502 struct netmap_slot *slot;
503 #endif /* DEV_NETMAP */
504 struct tx_ring *txr = &que->txr;
505 struct ixl_tx_buf *buf;
507 /* Clear the old ring contents */
512 * (under lock): if in netmap mode, do some consistency
513 * checks and set slot to entry 0 of the netmap ring.
515 slot = netmap_reset(na, NR_TX, que->me, 0);
516 #endif /* DEV_NETMAP */
518 bzero((void *)txr->base,
519 (sizeof(struct i40e_tx_desc)) * que->num_desc);
523 txr->next_to_clean = 0;
525 /* Reset watchdog status */
526 txr->watchdog_timer = 0;
529 /* Initialize flow director */
530 txr->atr_rate = ixl_atr_rate;
533 /* Free any existing tx mbufs. */
535 for (int i = 0; i < que->num_desc; i++, buf++) {
536 if (buf->m_head != NULL) {
537 bus_dmamap_sync(buf->tag, buf->map,
538 BUS_DMASYNC_POSTWRITE);
539 bus_dmamap_unload(buf->tag, buf->map);
540 m_freem(buf->m_head);
545 * In netmap mode, set the map for the packet buffer.
546 * NOTE: Some drivers (not this one) also need to set
547 * the physical buffer address in the NIC ring.
548 * netmap_idx_n2k() maps a nic index, i, into the corresponding
549 * netmap slot index, si
552 int si = netmap_idx_n2k(&na->tx_rings[que->me], i);
553 netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si));
555 #endif /* DEV_NETMAP */
556 /* Clear the EOP index */
560 /* Set number of descriptors available */
561 txr->avail = que->num_desc;
563 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
564 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
569 /*********************************************************************
571 * Free transmit ring related data structures.
573 **********************************************************************/
575 ixl_free_que_tx(struct ixl_queue *que)
577 struct tx_ring *txr = &que->txr;
578 struct ixl_tx_buf *buf;
580 INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
582 for (int i = 0; i < que->num_desc; i++) {
583 buf = &txr->buffers[i];
584 if (buf->m_head != NULL) {
585 bus_dmamap_sync(buf->tag, buf->map,
586 BUS_DMASYNC_POSTWRITE);
587 bus_dmamap_unload(buf->tag,
589 m_freem(buf->m_head);
591 if (buf->map != NULL) {
592 bus_dmamap_destroy(buf->tag,
596 } else if (buf->map != NULL) {
597 bus_dmamap_unload(buf->tag,
599 bus_dmamap_destroy(buf->tag,
605 buf_ring_free(txr->br, M_DEVBUF);
606 if (txr->buffers != NULL) {
607 free(txr->buffers, M_DEVBUF);
610 if (txr->tx_tag != NULL) {
611 bus_dma_tag_destroy(txr->tx_tag);
614 if (txr->tso_tag != NULL) {
615 bus_dma_tag_destroy(txr->tso_tag);
619 INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
623 /*********************************************************************
625 * Setup descriptor for hw offloads
627 **********************************************************************/
630 ixl_tx_setup_offload(struct ixl_queue *que,
631 struct mbuf *mp, u32 *cmd, u32 *off)
633 struct ether_vlan_header *eh;
635 struct ip *ip = NULL;
637 struct tcphdr *th = NULL;
641 int elen, ip_hlen = 0, tcp_hlen;
646 /* Set up the TSO context descriptor if required */
647 if (mp->m_pkthdr.csum_flags & CSUM_TSO) {
648 tso = ixl_tso_setup(que, mp);
656 * Determine where frame payload starts.
657 * Jump over vlan headers if already present,
658 * helpful for QinQ too.
660 eh = mtod(mp, struct ether_vlan_header *);
661 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
662 etype = ntohs(eh->evl_proto);
663 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
665 etype = ntohs(eh->evl_encap_proto);
666 elen = ETHER_HDR_LEN;
672 ip = (struct ip *)(mp->m_data + elen);
673 ip_hlen = ip->ip_hl << 2;
675 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
676 /* The IP checksum must be recalculated with TSO */
678 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
680 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
685 ip6 = (struct ip6_hdr *)(mp->m_data + elen);
686 ip_hlen = sizeof(struct ip6_hdr);
687 ipproto = ip6->ip6_nxt;
688 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
689 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
696 *off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
697 *off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
701 tcp_hlen = th->th_off << 2;
702 if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) {
703 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
704 *off |= (tcp_hlen >> 2) <<
705 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
708 ixl_atr(que, th, etype);
712 if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) {
713 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
714 *off |= (sizeof(struct udphdr) >> 2) <<
715 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
720 if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) {
721 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
722 *off |= (sizeof(struct sctphdr) >> 2) <<
723 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
734 /**********************************************************************
736 * Setup context for hardware segmentation offload (TSO)
738 **********************************************************************/
740 ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp)
742 struct tx_ring *txr = &que->txr;
743 struct i40e_tx_context_desc *TXD;
744 struct ixl_tx_buf *buf;
745 u32 cmd, mss, type, tsolen;
747 int idx, elen, ip_hlen, tcp_hlen;
748 struct ether_vlan_header *eh;
755 #if defined(INET6) || defined(INET)
758 u64 type_cmd_tso_mss;
761 * Determine where frame payload starts.
762 * Jump over vlan headers if already present
764 eh = mtod(mp, struct ether_vlan_header *);
765 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
766 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
767 etype = eh->evl_proto;
769 elen = ETHER_HDR_LEN;
770 etype = eh->evl_encap_proto;
773 switch (ntohs(etype)) {
776 ip6 = (struct ip6_hdr *)(mp->m_data + elen);
777 if (ip6->ip6_nxt != IPPROTO_TCP)
779 ip_hlen = sizeof(struct ip6_hdr);
780 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
781 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
782 tcp_hlen = th->th_off << 2;
784 * The corresponding flag is set by the stack in the IPv4
785 * TSO case, but not in IPv6 (at least in FreeBSD 10.2).
786 * So, set it here because the rest of the flow requires it.
788 mp->m_pkthdr.csum_flags |= CSUM_TCP_IPV6;
793 ip = (struct ip *)(mp->m_data + elen);
794 if (ip->ip_p != IPPROTO_TCP)
797 ip_hlen = ip->ip_hl << 2;
798 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
799 th->th_sum = in_pseudo(ip->ip_src.s_addr,
800 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
801 tcp_hlen = th->th_off << 2;
805 printf("%s: CSUM_TSO but no supported IP version (0x%04x)",
806 __func__, ntohs(etype));
810 /* Ensure we have at least the IP+TCP header in the first mbuf. */
811 if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr))
814 idx = txr->next_avail;
815 buf = &txr->buffers[idx];
816 TXD = (struct i40e_tx_context_desc *) &txr->base[idx];
817 tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen);
819 type = I40E_TX_DESC_DTYPE_CONTEXT;
820 cmd = I40E_TX_CTX_DESC_TSO;
821 /* TSO MSS must not be less than 64 */
822 if (mp->m_pkthdr.tso_segsz < IXL_MIN_TSO_MSS) {
823 que->mss_too_small++;
824 mp->m_pkthdr.tso_segsz = IXL_MIN_TSO_MSS;
826 mss = mp->m_pkthdr.tso_segsz;
828 type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) |
829 ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
830 ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
831 ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
832 TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss);
834 TXD->tunneling_params = htole32(0);
838 if (++idx == que->num_desc)
842 txr->next_avail = idx;
848 ** ixl_get_tx_head - Retrieve the value from the
849 ** location the HW records its HEAD index
852 ixl_get_tx_head(struct ixl_queue *que)
854 struct tx_ring *txr = &que->txr;
855 void *head = &txr->base[que->num_desc];
856 return LE32_TO_CPU(*(volatile __le32 *)head);
859 /**********************************************************************
861 * Examine each tx_buffer in the used queue. If the hardware is done
862 * processing the packet then free associated resources. The
863 * tx_buffer is put back on the free queue.
865 **********************************************************************/
867 ixl_txeof(struct ixl_queue *que)
869 struct tx_ring *txr = &que->txr;
870 u32 first, last, head, done, processed;
871 struct ixl_tx_buf *buf;
872 struct i40e_tx_desc *tx_desc, *eop_desc;
875 mtx_assert(&txr->mtx, MA_OWNED);
878 // XXX todo: implement moderation
879 if (netmap_tx_irq(que->vsi->ifp, que->me))
881 #endif /* DEF_NETMAP */
883 /* These are not the descriptors you seek, move along :) */
884 if (txr->avail == que->num_desc) {
885 atomic_store_rel_32(&txr->watchdog_timer, 0);
890 first = txr->next_to_clean;
891 buf = &txr->buffers[first];
892 tx_desc = (struct i40e_tx_desc *)&txr->base[first];
893 last = buf->eop_index;
896 eop_desc = (struct i40e_tx_desc *)&txr->base[last];
898 /* Get the Head WB value */
899 head = ixl_get_tx_head(que);
902 ** Get the index of the first descriptor
903 ** BEYOND the EOP and call that 'done'.
904 ** I do this so the comparison in the
905 ** inner while loop below can be simple
907 if (++last == que->num_desc) last = 0;
910 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
911 BUS_DMASYNC_POSTREAD);
913 ** The HEAD index of the ring is written in a
914 ** defined location, this rather than a done bit
915 ** is what is used to keep track of what must be
918 while (first != head) {
919 /* We clean the range of the packet */
920 while (first != done) {
925 txr->bytes += /* for ITR adjustment */
926 buf->m_head->m_pkthdr.len;
927 txr->tx_bytes += /* for TX stats */
928 buf->m_head->m_pkthdr.len;
929 bus_dmamap_sync(buf->tag,
931 BUS_DMASYNC_POSTWRITE);
932 bus_dmamap_unload(buf->tag,
934 m_freem(buf->m_head);
940 if (++first == que->num_desc)
943 buf = &txr->buffers[first];
944 tx_desc = &txr->base[first];
947 /* See if there is more work now */
948 last = buf->eop_index;
950 eop_desc = &txr->base[last];
951 /* Get next done point */
952 if (++last == que->num_desc) last = 0;
957 bus_dmamap_sync(txr->dma.tag, txr->dma.map,
958 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
960 txr->next_to_clean = first;
964 * If there are no pending descriptors, clear the timeout.
966 if (txr->avail == que->num_desc) {
967 atomic_store_rel_32(&txr->watchdog_timer, 0);
974 /*********************************************************************
976 * Refresh mbuf buffers for RX descriptor rings
977 * - now keeps its own state so discards due to resource
978 * exhaustion are unnecessary, if an mbuf cannot be obtained
979 * it just returns, keeping its placeholder, thus it can simply
980 * be recalled to try again.
982 **********************************************************************/
984 ixl_refresh_mbufs(struct ixl_queue *que, int limit)
986 struct ixl_vsi *vsi = que->vsi;
987 struct rx_ring *rxr = &que->rxr;
988 bus_dma_segment_t hseg[1];
989 bus_dma_segment_t pseg[1];
990 struct ixl_rx_buf *buf;
991 struct mbuf *mh, *mp;
992 int i, j, nsegs, error;
993 bool refreshed = FALSE;
995 i = j = rxr->next_refresh;
996 /* Control the loop with one beyond */
997 if (++j == que->num_desc)
1000 while (j != limit) {
1001 buf = &rxr->buffers[i];
1002 if (rxr->hdr_split == FALSE)
1005 if (buf->m_head == NULL) {
1006 mh = m_gethdr(M_NOWAIT, MT_DATA);
1012 mh->m_pkthdr.len = mh->m_len = MHLEN;
1014 mh->m_flags |= M_PKTHDR;
1015 /* Get the memory mapping */
1016 error = bus_dmamap_load_mbuf_sg(rxr->htag,
1017 buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
1019 printf("Refresh mbufs: hdr dmamap load"
1020 " failure - %d\n", error);
1026 bus_dmamap_sync(rxr->htag, buf->hmap,
1027 BUS_DMASYNC_PREREAD);
1028 rxr->base[i].read.hdr_addr =
1029 htole64(hseg[0].ds_addr);
1032 if (buf->m_pack == NULL) {
1033 mp = m_getjcl(M_NOWAIT, MT_DATA,
1034 M_PKTHDR, rxr->mbuf_sz);
1040 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1041 /* Get the memory mapping */
1042 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1043 buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
1045 printf("Refresh mbufs: payload dmamap load"
1046 " failure - %d\n", error);
1052 bus_dmamap_sync(rxr->ptag, buf->pmap,
1053 BUS_DMASYNC_PREREAD);
1054 rxr->base[i].read.pkt_addr =
1055 htole64(pseg[0].ds_addr);
1056 /* Used only when doing header split */
1057 rxr->base[i].read.hdr_addr = 0;
1060 /* Next is precalculated */
1062 rxr->next_refresh = i;
1063 if (++j == que->num_desc)
1067 if (refreshed) /* Update hardware tail index */
1068 wr32(vsi->hw, rxr->tail, rxr->next_refresh);
1073 /*********************************************************************
1075 * Allocate memory for rx_buffer structures. Since we use one
1076 * rx_buffer per descriptor, the maximum number of rx_buffer's
1077 * that we'll need is equal to the number of receive descriptors
1078 * that we've defined.
1080 **********************************************************************/
1082 ixl_allocate_rx_data(struct ixl_queue *que)
1084 struct rx_ring *rxr = &que->rxr;
1085 struct ixl_vsi *vsi = que->vsi;
1086 device_t dev = vsi->dev;
1087 struct ixl_rx_buf *buf;
1088 int i, bsize, error;
1090 bsize = sizeof(struct ixl_rx_buf) * que->num_desc;
1091 if (!(rxr->buffers =
1092 (struct ixl_rx_buf *) malloc(bsize,
1093 M_DEVBUF, M_NOWAIT | M_ZERO))) {
1094 device_printf(dev, "Unable to allocate rx_buffer memory\n");
1099 if ((error = bus_dma_tag_create(NULL, /* parent */
1100 1, 0, /* alignment, bounds */
1101 BUS_SPACE_MAXADDR, /* lowaddr */
1102 BUS_SPACE_MAXADDR, /* highaddr */
1103 NULL, NULL, /* filter, filterarg */
1104 MSIZE, /* maxsize */
1106 MSIZE, /* maxsegsize */
1108 NULL, /* lockfunc */
1109 NULL, /* lockfuncarg */
1111 device_printf(dev, "Unable to create RX DMA htag\n");
1115 if ((error = bus_dma_tag_create(NULL, /* parent */
1116 1, 0, /* alignment, bounds */
1117 BUS_SPACE_MAXADDR, /* lowaddr */
1118 BUS_SPACE_MAXADDR, /* highaddr */
1119 NULL, NULL, /* filter, filterarg */
1120 MJUM16BYTES, /* maxsize */
1122 MJUM16BYTES, /* maxsegsize */
1124 NULL, /* lockfunc */
1125 NULL, /* lockfuncarg */
1127 device_printf(dev, "Unable to create RX DMA ptag\n");
1131 for (i = 0; i < que->num_desc; i++) {
1132 buf = &rxr->buffers[i];
1133 error = bus_dmamap_create(rxr->htag,
1134 BUS_DMA_NOWAIT, &buf->hmap);
1136 device_printf(dev, "Unable to create RX head map\n");
1139 error = bus_dmamap_create(rxr->ptag,
1140 BUS_DMA_NOWAIT, &buf->pmap);
1142 device_printf(dev, "Unable to create RX pkt map\n");
1151 /*********************************************************************
1153 * (Re)Initialize the queue receive ring and its buffers.
1155 **********************************************************************/
1157 ixl_init_rx_ring(struct ixl_queue *que)
1159 struct rx_ring *rxr = &que->rxr;
1160 struct ixl_vsi *vsi = que->vsi;
1161 #if defined(INET6) || defined(INET)
1162 struct ifnet *ifp = vsi->ifp;
1163 struct lro_ctrl *lro = &rxr->lro;
1165 struct ixl_rx_buf *buf;
1166 bus_dma_segment_t pseg[1], hseg[1];
1167 int rsize, nsegs, error = 0;
1169 struct netmap_adapter *na = NA(que->vsi->ifp);
1170 struct netmap_slot *slot;
1171 #endif /* DEV_NETMAP */
1175 /* same as in ixl_init_tx_ring() */
1176 slot = netmap_reset(na, NR_RX, que->me, 0);
1177 #endif /* DEV_NETMAP */
1178 /* Clear the ring contents */
1179 rsize = roundup2(que->num_desc *
1180 sizeof(union i40e_rx_desc), DBA_ALIGN);
1181 bzero((void *)rxr->base, rsize);
1182 /* Cleanup any existing buffers */
1183 for (int i = 0; i < que->num_desc; i++) {
1184 buf = &rxr->buffers[i];
1185 if (buf->m_head != NULL) {
1186 bus_dmamap_sync(rxr->htag, buf->hmap,
1187 BUS_DMASYNC_POSTREAD);
1188 bus_dmamap_unload(rxr->htag, buf->hmap);
1189 buf->m_head->m_flags |= M_PKTHDR;
1190 m_freem(buf->m_head);
1192 if (buf->m_pack != NULL) {
1193 bus_dmamap_sync(rxr->ptag, buf->pmap,
1194 BUS_DMASYNC_POSTREAD);
1195 bus_dmamap_unload(rxr->ptag, buf->pmap);
1196 buf->m_pack->m_flags |= M_PKTHDR;
1197 m_freem(buf->m_pack);
1203 /* header split is off */
1204 rxr->hdr_split = FALSE;
1206 /* Now replenish the mbufs */
1207 for (int j = 0; j != que->num_desc; ++j) {
1208 struct mbuf *mh, *mp;
1210 buf = &rxr->buffers[j];
1213 * In netmap mode, fill the map and set the buffer
1214 * address in the NIC ring, considering the offset
1215 * between the netmap and NIC rings (see comment in
1216 * ixgbe_setup_transmit_ring() ). No need to allocate
1217 * an mbuf, so end the block with a continue;
1220 int sj = netmap_idx_n2k(&na->rx_rings[que->me], j);
1224 addr = PNMB(na, slot + sj, &paddr);
1225 netmap_load_map(na, rxr->dma.tag, buf->pmap, addr);
1226 /* Update descriptor and the cached value */
1227 rxr->base[j].read.pkt_addr = htole64(paddr);
1228 rxr->base[j].read.hdr_addr = 0;
1231 #endif /* DEV_NETMAP */
1233 ** Don't allocate mbufs if not
1234 ** doing header split, its wasteful
1236 if (rxr->hdr_split == FALSE)
1239 /* First the header */
1240 buf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
1241 if (buf->m_head == NULL) {
1245 m_adj(buf->m_head, ETHER_ALIGN);
1247 mh->m_len = mh->m_pkthdr.len = MHLEN;
1248 mh->m_flags |= M_PKTHDR;
1249 /* Get the memory mapping */
1250 error = bus_dmamap_load_mbuf_sg(rxr->htag,
1251 buf->hmap, buf->m_head, hseg,
1252 &nsegs, BUS_DMA_NOWAIT);
1253 if (error != 0) /* Nothing elegant to do here */
1255 bus_dmamap_sync(rxr->htag,
1256 buf->hmap, BUS_DMASYNC_PREREAD);
1257 /* Update descriptor */
1258 rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
1261 /* Now the payload cluster */
1262 buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
1263 M_PKTHDR, rxr->mbuf_sz);
1264 if (buf->m_pack == NULL) {
1269 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1270 /* Get the memory mapping */
1271 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1272 buf->pmap, mp, pseg,
1273 &nsegs, BUS_DMA_NOWAIT);
1276 bus_dmamap_sync(rxr->ptag,
1277 buf->pmap, BUS_DMASYNC_PREREAD);
1278 /* Update descriptor */
1279 rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
1280 rxr->base[j].read.hdr_addr = 0;
1284 /* Setup our descriptor indices */
1285 rxr->next_check = 0;
1286 rxr->next_refresh = 0;
1287 rxr->lro_enabled = FALSE;
1290 rxr->discard = FALSE;
1292 wr32(vsi->hw, rxr->tail, que->num_desc - 1);
1295 #if defined(INET6) || defined(INET)
1297 ** Now set up the LRO interface:
1299 if (ifp->if_capenable & IFCAP_LRO) {
1300 int err = tcp_lro_init(lro);
1302 if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me);
1305 INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me);
1306 rxr->lro_enabled = TRUE;
1307 lro->ifp = vsi->ifp;
1311 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1312 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1320 /*********************************************************************
1322 * Free station receive ring data structures
1324 **********************************************************************/
1326 ixl_free_que_rx(struct ixl_queue *que)
1328 struct rx_ring *rxr = &que->rxr;
1329 struct ixl_rx_buf *buf;
1331 INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
1333 /* Cleanup any existing buffers */
1334 if (rxr->buffers != NULL) {
1335 for (int i = 0; i < que->num_desc; i++) {
1336 buf = &rxr->buffers[i];
1337 if (buf->m_head != NULL) {
1338 bus_dmamap_sync(rxr->htag, buf->hmap,
1339 BUS_DMASYNC_POSTREAD);
1340 bus_dmamap_unload(rxr->htag, buf->hmap);
1341 buf->m_head->m_flags |= M_PKTHDR;
1342 m_freem(buf->m_head);
1344 if (buf->m_pack != NULL) {
1345 bus_dmamap_sync(rxr->ptag, buf->pmap,
1346 BUS_DMASYNC_POSTREAD);
1347 bus_dmamap_unload(rxr->ptag, buf->pmap);
1348 buf->m_pack->m_flags |= M_PKTHDR;
1349 m_freem(buf->m_pack);
1353 if (buf->hmap != NULL) {
1354 bus_dmamap_destroy(rxr->htag, buf->hmap);
1357 if (buf->pmap != NULL) {
1358 bus_dmamap_destroy(rxr->ptag, buf->pmap);
1362 if (rxr->buffers != NULL) {
1363 free(rxr->buffers, M_DEVBUF);
1364 rxr->buffers = NULL;
1368 if (rxr->htag != NULL) {
1369 bus_dma_tag_destroy(rxr->htag);
1372 if (rxr->ptag != NULL) {
1373 bus_dma_tag_destroy(rxr->ptag);
1377 INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
1382 ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype)
1385 #if defined(INET6) || defined(INET)
1387 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
1388 * should be computed by hardware. Also it should not have VLAN tag in
1391 if (rxr->lro_enabled &&
1392 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
1393 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
1394 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
1396 * Send to the stack if:
1397 ** - LRO not enabled, or
1398 ** - no LRO resources, or
1399 ** - lro enqueue fails
1401 if (rxr->lro.lro_cnt != 0)
1402 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
1407 (*ifp->if_input)(ifp, m);
1413 ixl_rx_discard(struct rx_ring *rxr, int i)
1415 struct ixl_rx_buf *rbuf;
1417 rbuf = &rxr->buffers[i];
1419 if (rbuf->fmp != NULL) {/* Partial chain ? */
1420 rbuf->fmp->m_flags |= M_PKTHDR;
1426 ** With advanced descriptors the writeback
1427 ** clobbers the buffer addrs, so its easier
1428 ** to just free the existing mbufs and take
1429 ** the normal refresh path to get new buffers
1433 m_free(rbuf->m_head);
1434 rbuf->m_head = NULL;
1438 m_free(rbuf->m_pack);
1439 rbuf->m_pack = NULL;
1447 ** i40e_ptype_to_hash: parse the packet type
1448 ** to determine the appropriate hash.
1451 ixl_ptype_to_hash(u8 ptype)
1453 struct i40e_rx_ptype_decoded decoded;
1456 decoded = decode_rx_desc_ptype(ptype);
1457 ex = decoded.outer_frag;
1460 return M_HASHTYPE_OPAQUE_HASH;
1462 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2)
1463 return M_HASHTYPE_OPAQUE_HASH;
1465 /* Note: anything that gets to this point is IP */
1466 if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) {
1467 switch (decoded.inner_prot) {
1468 case I40E_RX_PTYPE_INNER_PROT_TCP:
1470 return M_HASHTYPE_RSS_TCP_IPV6_EX;
1472 return M_HASHTYPE_RSS_TCP_IPV6;
1473 case I40E_RX_PTYPE_INNER_PROT_UDP:
1475 return M_HASHTYPE_RSS_UDP_IPV6_EX;
1477 return M_HASHTYPE_RSS_UDP_IPV6;
1480 return M_HASHTYPE_RSS_IPV6_EX;
1482 return M_HASHTYPE_RSS_IPV6;
1485 if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) {
1486 switch (decoded.inner_prot) {
1487 case I40E_RX_PTYPE_INNER_PROT_TCP:
1488 return M_HASHTYPE_RSS_TCP_IPV4;
1489 case I40E_RX_PTYPE_INNER_PROT_UDP:
1491 return M_HASHTYPE_RSS_UDP_IPV4_EX;
1493 return M_HASHTYPE_RSS_UDP_IPV4;
1495 return M_HASHTYPE_RSS_IPV4;
1498 /* We should never get here!! */
1499 return M_HASHTYPE_OPAQUE_HASH;
1503 /*********************************************************************
1505 * This routine executes in interrupt context. It replenishes
1506 * the mbufs in the descriptor and sends data which has been
1507 * dma'ed into host memory to upper layer.
1509 * We loop at most count times if count is > 0, or until done if
1512 * Return TRUE for more work, FALSE for all clean.
1513 *********************************************************************/
1515 ixl_rxeof(struct ixl_queue *que, int count)
1517 struct ixl_vsi *vsi = que->vsi;
1518 struct rx_ring *rxr = &que->rxr;
1519 struct ifnet *ifp = vsi->ifp;
1520 #if defined(INET6) || defined(INET)
1521 struct lro_ctrl *lro = &rxr->lro;
1523 int i, nextp, processed = 0;
1524 union i40e_rx_desc *cur;
1525 struct ixl_rx_buf *rbuf, *nbuf;
1531 if (netmap_rx_irq(ifp, que->me, &count)) {
1535 #endif /* DEV_NETMAP */
1537 for (i = rxr->next_check; count != 0;) {
1538 struct mbuf *sendmp, *mh, *mp;
1540 u16 hlen, plen, vtag;
1545 /* Sync the ring. */
1546 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1547 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1549 cur = &rxr->base[i];
1550 qword = le64toh(cur->wb.qword1.status_error_len);
1551 status = (qword & I40E_RXD_QW1_STATUS_MASK)
1552 >> I40E_RXD_QW1_STATUS_SHIFT;
1553 error = (qword & I40E_RXD_QW1_ERROR_MASK)
1554 >> I40E_RXD_QW1_ERROR_SHIFT;
1555 plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK)
1556 >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1557 hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK)
1558 >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1559 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK)
1560 >> I40E_RXD_QW1_PTYPE_SHIFT;
1562 if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) {
1566 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1572 cur->wb.qword1.status_error_len = 0;
1573 rbuf = &rxr->buffers[i];
1576 eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT));
1577 if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT))
1578 vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1);
1583 ** Make sure bad packets are discarded,
1584 ** note that only EOP descriptor has valid
1587 if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1589 ixl_rx_discard(rxr, i);
1593 /* Prefetch the next buffer */
1596 if (nextp == que->num_desc)
1598 nbuf = &rxr->buffers[nextp];
1603 ** The header mbuf is ONLY used when header
1604 ** split is enabled, otherwise we get normal
1605 ** behavior, ie, both header and payload
1606 ** are DMA'd into the payload buffer.
1608 ** Rather than using the fmp/lmp global pointers
1609 ** we now keep the head of a packet chain in the
1610 ** buffer struct and pass this along from one
1611 ** descriptor to the next, until we get EOP.
1613 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
1614 if (hlen > IXL_RX_HDR)
1617 mh->m_flags |= M_PKTHDR;
1619 mh->m_pkthdr.len = mh->m_len;
1620 /* Null buf pointer so it is refreshed */
1621 rbuf->m_head = NULL;
1623 ** Check the payload length, this
1624 ** could be zero if its a small
1630 mp->m_flags &= ~M_PKTHDR;
1632 mh->m_pkthdr.len += mp->m_len;
1633 /* Null buf pointer so it is refreshed */
1634 rbuf->m_pack = NULL;
1638 ** Now create the forward
1639 ** chain so when complete
1643 /* stash the chain head */
1645 /* Make forward chain */
1647 mp->m_next = nbuf->m_pack;
1649 mh->m_next = nbuf->m_pack;
1651 /* Singlet, prepare to send */
1654 sendmp->m_pkthdr.ether_vtag = vtag;
1655 sendmp->m_flags |= M_VLANTAG;
1660 ** Either no header split, or a
1661 ** secondary piece of a fragmented
1666 ** See if there is a stored head
1667 ** that determines what we are
1670 rbuf->m_pack = rbuf->fmp = NULL;
1672 if (sendmp != NULL) /* secondary frag */
1673 sendmp->m_pkthdr.len += mp->m_len;
1675 /* first desc of a non-ps chain */
1677 sendmp->m_flags |= M_PKTHDR;
1678 sendmp->m_pkthdr.len = mp->m_len;
1680 /* Pass the head pointer on */
1684 mp->m_next = nbuf->m_pack;
1688 /* Sending this frame? */
1690 sendmp->m_pkthdr.rcvif = ifp;
1693 rxr->rx_bytes += sendmp->m_pkthdr.len;
1694 /* capture data for dynamic ITR adjustment */
1696 rxr->bytes += sendmp->m_pkthdr.len;
1697 /* Set VLAN tag (field only valid in eop desc) */
1699 sendmp->m_pkthdr.ether_vtag = vtag;
1700 sendmp->m_flags |= M_VLANTAG;
1702 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
1703 ixl_rx_checksum(sendmp, status, error, ptype);
1705 sendmp->m_pkthdr.flowid =
1706 le32toh(cur->wb.qword0.hi_dword.rss);
1707 M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype));
1709 sendmp->m_pkthdr.flowid = que->msix;
1710 M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
1714 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1715 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1717 /* Advance our pointers to the next descriptor. */
1718 if (++i == que->num_desc)
1721 /* Now send to the stack or do LRO */
1722 if (sendmp != NULL) {
1723 rxr->next_check = i;
1724 ixl_rx_input(rxr, ifp, sendmp, ptype);
1725 i = rxr->next_check;
1728 /* Every 8 descriptors we go to refresh mbufs */
1729 if (processed == 8) {
1730 ixl_refresh_mbufs(que, i);
1735 /* Refresh any remaining buf structs */
1736 if (ixl_rx_unrefreshed(que))
1737 ixl_refresh_mbufs(que, i);
1739 rxr->next_check = i;
1741 #if defined(INET6) || defined(INET)
1743 * Flush any outstanding LRO work
1745 #if __FreeBSD_version >= 1100105
1746 tcp_lro_flush_all(lro);
1748 struct lro_entry *queued;
1749 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
1750 SLIST_REMOVE_HEAD(&lro->lro_active, next);
1751 tcp_lro_flush(lro, queued);
1754 #endif /* defined(INET6) || defined(INET) */
1761 /*********************************************************************
1763 * Verify that the hardware indicated that the checksum is valid.
1764 * Inform the stack about the status of checksum so that stack
1765 * doesn't spend time verifying the checksum.
1767 *********************************************************************/
1769 ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype)
1771 struct i40e_rx_ptype_decoded decoded;
1773 decoded = decode_rx_desc_ptype(ptype);
1776 if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1777 (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) {
1778 mp->m_pkthdr.csum_flags = 0;
1782 /* IPv6 with extension headers likely have bad csum */
1783 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1784 decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1786 (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) {
1787 mp->m_pkthdr.csum_flags = 0;
1792 /* IP Checksum Good */
1793 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
1794 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1796 if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) {
1797 mp->m_pkthdr.csum_flags |=
1798 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1799 mp->m_pkthdr.csum_data |= htons(0xffff);
1804 #if __FreeBSD_version >= 1100000
1806 ixl_get_counter(if_t ifp, ift_counter cnt)
1808 struct ixl_vsi *vsi;
1810 vsi = if_getsoftc(ifp);
1813 case IFCOUNTER_IPACKETS:
1814 return (vsi->ipackets);
1815 case IFCOUNTER_IERRORS:
1816 return (vsi->ierrors);
1817 case IFCOUNTER_OPACKETS:
1818 return (vsi->opackets);
1819 case IFCOUNTER_OERRORS:
1820 return (vsi->oerrors);
1821 case IFCOUNTER_COLLISIONS:
1822 /* Collisions are by standard impossible in 40G/10G Ethernet */
1824 case IFCOUNTER_IBYTES:
1825 return (vsi->ibytes);
1826 case IFCOUNTER_OBYTES:
1827 return (vsi->obytes);
1828 case IFCOUNTER_IMCASTS:
1829 return (vsi->imcasts);
1830 case IFCOUNTER_OMCASTS:
1831 return (vsi->omcasts);
1832 case IFCOUNTER_IQDROPS:
1833 return (vsi->iqdrops);
1834 case IFCOUNTER_OQDROPS:
1835 return (vsi->oqdrops);
1836 case IFCOUNTER_NOPROTO:
1837 return (vsi->noproto);
1839 return (if_get_counter_default(ifp, cnt));