1 /******************************************************************************
3 Copyright (c) 2001-2015, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
36 #ifndef IXGBE_STANDALONE_BUILD
38 #include "opt_inet6.h"
45 #include <net/rss_config.h>
46 #include <netinet/in_rss.h>
50 #include <net/netmap.h>
51 #include <sys/selinfo.h>
52 #include <dev/netmap/netmap_kern.h>
54 extern int ix_crcstrip;
59 ** this feature only works with
60 ** IPv4, and only on 82599 and later.
61 ** Also this will cause IP forwarding to
62 ** fail and that can't be controlled by
63 ** the stack as LRO can. For all these
64 ** reasons I've deemed it best to leave
65 ** this off and not bother with a tuneable
66 ** interface, this would need to be compiled
69 static bool ixgbe_rsc_enable = FALSE;
73 ** For Flow Director: this is the
74 ** number of TX packets we sample
75 ** for the filter pool, this means
76 ** every 20th packet will be probed.
78 ** This feature can be disabled by
81 static int atr_sample_rate = 20;
84 /* Shared PCI config read/write */
86 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
90 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
97 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
99 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
105 /*********************************************************************
106 * Local Function prototypes
107 *********************************************************************/
108 static void ixgbe_setup_transmit_ring(struct tx_ring *);
109 static void ixgbe_free_transmit_buffers(struct tx_ring *);
110 static int ixgbe_setup_receive_ring(struct rx_ring *);
111 static void ixgbe_free_receive_buffers(struct rx_ring *);
113 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
114 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
115 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
116 static int ixgbe_tx_ctx_setup(struct tx_ring *,
117 struct mbuf *, u32 *, u32 *);
118 static int ixgbe_tso_setup(struct tx_ring *,
119 struct mbuf *, u32 *, u32 *);
121 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
123 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
124 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
127 #ifdef IXGBE_LEGACY_TX
128 /*********************************************************************
129 * Transmit entry point
131 * ixgbe_start is called by the stack to initiate a transmit.
132 * The driver will remain in this routine as long as there are
133 * packets to transmit and transmit resources are available.
134 * In case resources are not available stack is notified and
135 * the packet is requeued.
136 **********************************************************************/
139 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
142 struct adapter *adapter = txr->adapter;
144 IXGBE_TX_LOCK_ASSERT(txr);
146 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
148 if (!adapter->link_active)
151 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
152 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
155 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
159 if (ixgbe_xmit(txr, &m_head)) {
161 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
164 /* Send a copy of the frame to the BPF listener */
165 ETHER_BPF_MTAP(ifp, m_head);
171 * Legacy TX start - called by the stack, this
172 * always uses the first tx ring, and should
173 * not be used with multiqueue tx enabled.
176 ixgbe_start(struct ifnet *ifp)
178 struct adapter *adapter = ifp->if_softc;
179 struct tx_ring *txr = adapter->tx_rings;
181 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
183 ixgbe_start_locked(txr, ifp);
184 IXGBE_TX_UNLOCK(txr);
189 #else /* ! IXGBE_LEGACY_TX */
192 ** Multiqueue Transmit driver
196 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
198 struct adapter *adapter = ifp->if_softc;
199 struct ix_queue *que;
207 * When doing RSS, map it to the same outbound queue
208 * as the incoming flow would be mapped to.
210 * If everything is setup correctly, it should be the
211 * same bucket that the current CPU we're on is.
213 #if __FreeBSD_version < 1100054
214 if (m->m_flags & M_FLOWID) {
216 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
219 if (rss_hash2bucket(m->m_pkthdr.flowid,
220 M_HASHTYPE_GET(m), &bucket_id) == 0)
221 /* TODO: spit out something if bucket_id > num_queues? */
222 i = bucket_id % adapter->num_queues;
225 i = m->m_pkthdr.flowid % adapter->num_queues;
227 i = curcpu % adapter->num_queues;
229 /* Check for a hung queue and pick alternative */
230 if (((1 << i) & adapter->active_queues) == 0)
231 i = ffsl(adapter->active_queues);
233 txr = &adapter->tx_rings[i];
234 que = &adapter->queues[i];
236 err = drbr_enqueue(ifp, txr->br, m);
239 if (IXGBE_TX_TRYLOCK(txr)) {
240 ixgbe_mq_start_locked(ifp, txr);
241 IXGBE_TX_UNLOCK(txr);
243 taskqueue_enqueue(que->tq, &txr->txq_task);
249 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
251 struct adapter *adapter = txr->adapter;
253 int enqueued = 0, err = 0;
255 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
256 adapter->link_active == 0)
259 /* Process the queue */
260 #if __FreeBSD_version < 901504
261 next = drbr_dequeue(ifp, txr->br);
262 while (next != NULL) {
263 if ((err = ixgbe_xmit(txr, &next)) != 0) {
265 err = drbr_enqueue(ifp, txr->br, next);
267 while ((next = drbr_peek(ifp, txr->br)) != NULL) {
268 if ((err = ixgbe_xmit(txr, &next)) != 0) {
270 drbr_advance(ifp, txr->br);
272 drbr_putback(ifp, txr->br, next);
277 #if __FreeBSD_version >= 901504
278 drbr_advance(ifp, txr->br);
281 #if 0 // this is VF-only
282 #if __FreeBSD_version >= 1100036
284 * Since we're looking at the tx ring, we can check
285 * to see if we're a VF by examing our tail register
288 if (txr->tail < IXGBE_TDT(0) && next->m_flags & M_MCAST)
289 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
292 /* Send a copy of the frame to the BPF listener */
293 ETHER_BPF_MTAP(ifp, next);
294 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
296 #if __FreeBSD_version < 901504
297 next = drbr_dequeue(ifp, txr->br);
301 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
308 * Called from a taskqueue to drain queued transmit packets.
311 ixgbe_deferred_mq_start(void *arg, int pending)
313 struct tx_ring *txr = arg;
314 struct adapter *adapter = txr->adapter;
315 struct ifnet *ifp = adapter->ifp;
318 if (!drbr_empty(ifp, txr->br))
319 ixgbe_mq_start_locked(ifp, txr);
320 IXGBE_TX_UNLOCK(txr);
324 * Flush all ring buffers
327 ixgbe_qflush(struct ifnet *ifp)
329 struct adapter *adapter = ifp->if_softc;
330 struct tx_ring *txr = adapter->tx_rings;
333 for (int i = 0; i < adapter->num_queues; i++, txr++) {
335 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
337 IXGBE_TX_UNLOCK(txr);
341 #endif /* IXGBE_LEGACY_TX */
344 /*********************************************************************
346 * This routine maps the mbufs to tx descriptors, allowing the
347 * TX engine to transmit the packets.
348 * - return 0 on success, positive on failure
350 **********************************************************************/
353 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
355 struct adapter *adapter = txr->adapter;
356 u32 olinfo_status = 0, cmd_type_len;
357 int i, j, error, nsegs;
361 bus_dma_segment_t segs[adapter->num_segs];
363 struct ixgbe_tx_buf *txbuf;
364 union ixgbe_adv_tx_desc *txd = NULL;
368 /* Basic descriptor defines */
369 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
370 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
372 if (m_head->m_flags & M_VLANTAG)
373 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
376 * Important to capture the first descriptor
377 * used because it will contain the index of
378 * the one we tell the hardware to report back
380 first = txr->next_avail_desc;
381 txbuf = &txr->tx_buffers[first];
385 * Map the packet for DMA.
388 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
389 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
391 if (__predict_false(error)) {
396 /* Try it again? - one try */
400 * XXX: m_defrag will choke on
401 * non-MCLBYTES-sized clusters
403 m = m_defrag(*m_headp, M_NOWAIT);
405 adapter->mbuf_defrag_failed++;
415 txr->no_tx_dma_setup++;
418 txr->no_tx_dma_setup++;
425 /* Make certain there are enough descriptors */
426 if (nsegs > txr->tx_avail - 2) {
427 txr->no_desc_avail++;
428 bus_dmamap_unload(txr->txtag, map);
434 * Set up the appropriate offload context
435 * this will consume the first descriptor
437 error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
438 if (__predict_false(error)) {
439 if (error == ENOBUFS)
445 /* Do the flow director magic */
446 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
448 if (txr->atr_count >= atr_sample_rate) {
449 ixgbe_atr(txr, m_head);
455 i = txr->next_avail_desc;
456 for (j = 0; j < nsegs; j++) {
460 txbuf = &txr->tx_buffers[i];
461 txd = &txr->tx_base[i];
462 seglen = segs[j].ds_len;
463 segaddr = htole64(segs[j].ds_addr);
465 txd->read.buffer_addr = segaddr;
466 txd->read.cmd_type_len = htole32(txr->txd_cmd |
467 cmd_type_len |seglen);
468 txd->read.olinfo_status = htole32(olinfo_status);
470 if (++i == txr->num_desc)
474 txd->read.cmd_type_len |=
475 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
476 txr->tx_avail -= nsegs;
477 txr->next_avail_desc = i;
479 txbuf->m_head = m_head;
481 * Here we swap the map so the last descriptor,
482 * which gets the completion interrupt has the
483 * real map, and the first descriptor gets the
484 * unused map from this descriptor.
486 txr->tx_buffers[first].map = txbuf->map;
488 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
490 /* Set the EOP descriptor that will be marked done */
491 txbuf = &txr->tx_buffers[first];
494 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
495 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
497 * Advance the Transmit Descriptor Tail (Tdt), this tells the
498 * hardware that this frame is available to transmit.
500 ++txr->total_packets;
501 IXGBE_WRITE_REG(&adapter->hw, txr->tail, i);
503 /* Mark queue as having work */
511 /*********************************************************************
513 * Allocate memory for tx_buffer structures. The tx_buffer stores all
514 * the information needed to transmit a packet on the wire. This is
515 * called only once at attach, setup is done every reset.
517 **********************************************************************/
519 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
521 struct adapter *adapter = txr->adapter;
522 device_t dev = adapter->dev;
523 struct ixgbe_tx_buf *txbuf;
527 * Setup DMA descriptor areas.
529 if ((error = bus_dma_tag_create(
530 bus_get_dma_tag(adapter->dev), /* parent */
531 1, 0, /* alignment, bounds */
532 BUS_SPACE_MAXADDR, /* lowaddr */
533 BUS_SPACE_MAXADDR, /* highaddr */
534 NULL, NULL, /* filter, filterarg */
535 IXGBE_TSO_SIZE, /* maxsize */
536 adapter->num_segs, /* nsegments */
537 PAGE_SIZE, /* maxsegsize */
540 NULL, /* lockfuncarg */
542 device_printf(dev,"Unable to allocate TX DMA tag\n");
546 if (!(txr->tx_buffers =
547 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) *
548 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
549 device_printf(dev, "Unable to allocate tx_buffer memory\n");
554 /* Create the descriptor buffer dma maps */
555 txbuf = txr->tx_buffers;
556 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
557 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
559 device_printf(dev, "Unable to create TX DMA map\n");
566 /* We free all, it handles case where we are in the middle */
567 ixgbe_free_transmit_structures(adapter);
571 /*********************************************************************
573 * Initialize a transmit ring.
575 **********************************************************************/
577 ixgbe_setup_transmit_ring(struct tx_ring *txr)
579 struct adapter *adapter = txr->adapter;
580 struct ixgbe_tx_buf *txbuf;
583 struct netmap_adapter *na = NA(adapter->ifp);
584 struct netmap_slot *slot;
585 #endif /* DEV_NETMAP */
587 /* Clear the old ring contents */
591 * (under lock): if in netmap mode, do some consistency
592 * checks and set slot to entry 0 of the netmap ring.
594 slot = netmap_reset(na, NR_TX, txr->me, 0);
595 #endif /* DEV_NETMAP */
596 bzero((void *)txr->tx_base,
597 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
599 txr->next_avail_desc = 0;
600 txr->next_to_clean = 0;
602 /* Free any existing tx buffers. */
603 txbuf = txr->tx_buffers;
604 for (i = 0; i < txr->num_desc; i++, txbuf++) {
605 if (txbuf->m_head != NULL) {
606 bus_dmamap_sync(txr->txtag, txbuf->map,
607 BUS_DMASYNC_POSTWRITE);
608 bus_dmamap_unload(txr->txtag, txbuf->map);
609 m_freem(txbuf->m_head);
610 txbuf->m_head = NULL;
614 * In netmap mode, set the map for the packet buffer.
615 * NOTE: Some drivers (not this one) also need to set
616 * the physical buffer address in the NIC ring.
617 * Slots in the netmap ring (indexed by "si") are
618 * kring->nkr_hwofs positions "ahead" wrt the
619 * corresponding slot in the NIC ring. In some drivers
620 * (not here) nkr_hwofs can be negative. Function
621 * netmap_idx_n2k() handles wraparounds properly.
624 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
625 netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
627 #endif /* DEV_NETMAP */
628 /* Clear the EOP descriptor pointer */
633 /* Set the rate at which we sample packets */
634 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
635 txr->atr_sample = atr_sample_rate;
638 /* Set number of descriptors available */
639 txr->tx_avail = adapter->num_tx_desc;
641 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
642 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
643 IXGBE_TX_UNLOCK(txr);
646 /*********************************************************************
648 * Initialize all transmit rings.
650 **********************************************************************/
652 ixgbe_setup_transmit_structures(struct adapter *adapter)
654 struct tx_ring *txr = adapter->tx_rings;
656 for (int i = 0; i < adapter->num_queues; i++, txr++)
657 ixgbe_setup_transmit_ring(txr);
662 /*********************************************************************
664 * Free all transmit rings.
666 **********************************************************************/
668 ixgbe_free_transmit_structures(struct adapter *adapter)
670 struct tx_ring *txr = adapter->tx_rings;
672 for (int i = 0; i < adapter->num_queues; i++, txr++) {
674 ixgbe_free_transmit_buffers(txr);
675 ixgbe_dma_free(adapter, &txr->txdma);
676 IXGBE_TX_UNLOCK(txr);
677 IXGBE_TX_LOCK_DESTROY(txr);
679 free(adapter->tx_rings, M_DEVBUF);
682 /*********************************************************************
684 * Free transmit ring related data structures.
686 **********************************************************************/
688 ixgbe_free_transmit_buffers(struct tx_ring *txr)
690 struct adapter *adapter = txr->adapter;
691 struct ixgbe_tx_buf *tx_buffer;
694 INIT_DEBUGOUT("ixgbe_free_transmit_ring: begin");
696 if (txr->tx_buffers == NULL)
699 tx_buffer = txr->tx_buffers;
700 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
701 if (tx_buffer->m_head != NULL) {
702 bus_dmamap_sync(txr->txtag, tx_buffer->map,
703 BUS_DMASYNC_POSTWRITE);
704 bus_dmamap_unload(txr->txtag,
706 m_freem(tx_buffer->m_head);
707 tx_buffer->m_head = NULL;
708 if (tx_buffer->map != NULL) {
709 bus_dmamap_destroy(txr->txtag,
711 tx_buffer->map = NULL;
713 } else if (tx_buffer->map != NULL) {
714 bus_dmamap_unload(txr->txtag,
716 bus_dmamap_destroy(txr->txtag,
718 tx_buffer->map = NULL;
721 #ifdef IXGBE_LEGACY_TX
723 buf_ring_free(txr->br, M_DEVBUF);
725 if (txr->tx_buffers != NULL) {
726 free(txr->tx_buffers, M_DEVBUF);
727 txr->tx_buffers = NULL;
729 if (txr->txtag != NULL) {
730 bus_dma_tag_destroy(txr->txtag);
736 /*********************************************************************
738 * Advanced Context Descriptor setup for VLAN, CSUM or TSO
740 **********************************************************************/
743 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
744 u32 *cmd_type_len, u32 *olinfo_status)
746 struct adapter *adapter = txr->adapter;
747 struct ixgbe_adv_tx_context_desc *TXD;
748 struct ether_vlan_header *eh;
751 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
752 int ehdrlen, ip_hlen = 0;
756 int ctxd = txr->next_avail_desc;
759 /* First check if TSO is to be used */
760 if (mp->m_pkthdr.csum_flags & CSUM_TSO)
761 return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status));
763 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
766 /* Indicate the whole packet as payload when not doing TSO */
767 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
769 /* Now ready a context descriptor */
770 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
773 ** In advanced descriptors the vlan tag must
774 ** be placed into the context descriptor. Hence
775 ** we need to make one even if not doing offloads.
777 if (mp->m_flags & M_VLANTAG) {
778 vtag = htole16(mp->m_pkthdr.ether_vtag);
779 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
781 else if (!IXGBE_IS_X550VF(adapter) && (offload == FALSE))
785 * Determine where frame payload starts.
786 * Jump over vlan headers if already present,
787 * helpful for QinQ too.
789 eh = mtod(mp, struct ether_vlan_header *);
790 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
791 etype = ntohs(eh->evl_proto);
792 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
794 etype = ntohs(eh->evl_encap_proto);
795 ehdrlen = ETHER_HDR_LEN;
798 /* Set the ether header length */
799 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
801 if (offload == FALSE)
806 ip = (struct ip *)(mp->m_data + ehdrlen);
807 ip_hlen = ip->ip_hl << 2;
809 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
812 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
813 ip_hlen = sizeof(struct ip6_hdr);
814 /* XXX-BZ this will go badly in case of ext hdrs. */
815 ipproto = ip6->ip6_nxt;
816 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
823 vlan_macip_lens |= ip_hlen;
827 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
828 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
832 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
833 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
836 #if __FreeBSD_version >= 800000
838 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
839 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
847 if (offload) /* For the TX descriptor setup */
848 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
851 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
853 /* Now copy bits into descriptor */
854 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
855 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
856 TXD->seqnum_seed = htole32(0);
857 TXD->mss_l4len_idx = htole32(0);
859 /* We've consumed the first desc, adjust counters */
860 if (++ctxd == txr->num_desc)
862 txr->next_avail_desc = ctxd;
868 /**********************************************************************
870 * Setup work for hardware segmentation offload (TSO) on
871 * adapters using advanced tx descriptors
873 **********************************************************************/
875 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp,
876 u32 *cmd_type_len, u32 *olinfo_status)
878 struct ixgbe_adv_tx_context_desc *TXD;
879 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
880 u32 mss_l4len_idx = 0, paylen;
881 u16 vtag = 0, eh_type;
882 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
883 struct ether_vlan_header *eh;
894 * Determine where frame payload starts.
895 * Jump over vlan headers if already present
897 eh = mtod(mp, struct ether_vlan_header *);
898 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
899 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
900 eh_type = eh->evl_proto;
902 ehdrlen = ETHER_HDR_LEN;
903 eh_type = eh->evl_encap_proto;
906 switch (ntohs(eh_type)) {
909 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
910 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
911 if (ip6->ip6_nxt != IPPROTO_TCP)
913 ip_hlen = sizeof(struct ip6_hdr);
914 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
915 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
916 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
917 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
922 ip = (struct ip *)(mp->m_data + ehdrlen);
923 if (ip->ip_p != IPPROTO_TCP)
926 ip_hlen = ip->ip_hl << 2;
927 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
928 th->th_sum = in_pseudo(ip->ip_src.s_addr,
929 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
930 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
931 /* Tell transmit desc to also do IPv4 checksum. */
932 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
936 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
937 __func__, ntohs(eh_type));
941 ctxd = txr->next_avail_desc;
942 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
944 tcp_hlen = th->th_off << 2;
946 /* This is used in the transmit desc in encap */
947 paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
949 /* VLAN MACLEN IPLEN */
950 if (mp->m_flags & M_VLANTAG) {
951 vtag = htole16(mp->m_pkthdr.ether_vtag);
952 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
955 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
956 vlan_macip_lens |= ip_hlen;
957 TXD->vlan_macip_lens = htole32(vlan_macip_lens);
959 /* ADV DTYPE TUCMD */
960 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
961 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
962 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
965 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
966 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
967 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
969 TXD->seqnum_seed = htole32(0);
971 if (++ctxd == txr->num_desc)
975 txr->next_avail_desc = ctxd;
976 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
977 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
978 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
984 /**********************************************************************
986 * Examine each tx_buffer in the used queue. If the hardware is done
987 * processing the packet then free associated resources. The
988 * tx_buffer is put back on the free queue.
990 **********************************************************************/
992 ixgbe_txeof(struct tx_ring *txr)
995 struct adapter *adapter = txr->adapter;
996 struct ifnet *ifp = adapter->ifp;
998 u32 work, processed = 0;
999 u16 limit = txr->process_limit;
1000 struct ixgbe_tx_buf *buf;
1001 union ixgbe_adv_tx_desc *txd;
1003 mtx_assert(&txr->tx_mtx, MA_OWNED);
1006 if (ifp->if_capenable & IFCAP_NETMAP) {
1007 struct netmap_adapter *na = NA(ifp);
1008 struct netmap_kring *kring = &na->tx_rings[txr->me];
1010 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1011 BUS_DMASYNC_POSTREAD);
1013 * In netmap mode, all the work is done in the context
1014 * of the client thread. Interrupt handlers only wake up
1015 * clients, which may be sleeping on individual rings
1016 * or on a global resource for all rings.
1017 * To implement tx interrupt mitigation, we wake up the client
1018 * thread roughly every half ring, even if the NIC interrupts
1019 * more frequently. This is implemented as follows:
1020 * - ixgbe_txsync() sets kring->nr_kflags with the index of
1021 * the slot that should wake up the thread (nkr_num_slots
1022 * means the user thread should not be woken up);
1023 * - the driver ignores tx interrupts unless netmap_mitigate=0
1024 * or the slot has the DD bit set.
1026 if (!netmap_mitigate ||
1027 (kring->nr_kflags < kring->nkr_num_slots &&
1028 txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) {
1029 netmap_tx_irq(ifp, txr->me);
1033 #endif /* DEV_NETMAP */
1035 if (txr->tx_avail == txr->num_desc) {
1040 /* Get work starting point */
1041 work = txr->next_to_clean;
1042 buf = &txr->tx_buffers[work];
1043 txd = &txr->tx_base[work];
1044 work -= txr->num_desc; /* The distance to ring end */
1045 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1046 BUS_DMASYNC_POSTREAD);
1049 union ixgbe_adv_tx_desc *eop= buf->eop;
1050 if (eop == NULL) /* No work */
1053 if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0)
1054 break; /* I/O not complete */
1058 buf->m_head->m_pkthdr.len;
1059 bus_dmamap_sync(txr->txtag,
1061 BUS_DMASYNC_POSTWRITE);
1062 bus_dmamap_unload(txr->txtag,
1064 m_freem(buf->m_head);
1070 /* We clean the range if multi segment */
1071 while (txd != eop) {
1075 /* wrap the ring? */
1076 if (__predict_false(!work)) {
1077 work -= txr->num_desc;
1078 buf = txr->tx_buffers;
1083 buf->m_head->m_pkthdr.len;
1084 bus_dmamap_sync(txr->txtag,
1086 BUS_DMASYNC_POSTWRITE);
1087 bus_dmamap_unload(txr->txtag,
1089 m_freem(buf->m_head);
1099 /* Try the next packet */
1103 /* reset with a wrap */
1104 if (__predict_false(!work)) {
1105 work -= txr->num_desc;
1106 buf = txr->tx_buffers;
1110 } while (__predict_true(--limit));
1112 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1113 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1115 work += txr->num_desc;
1116 txr->next_to_clean = work;
1119 ** Queue Hang detection, we know there's
1120 ** work outstanding or the first return
1121 ** would have been taken, so increment busy
1122 ** if nothing managed to get cleaned, then
1123 ** in local_timer it will be checked and
1124 ** marked as HUNG if it exceeds a MAX attempt.
1126 if ((processed == 0) && (txr->busy != IXGBE_QUEUE_HUNG))
1129 ** If anything gets cleaned we reset state to 1,
1130 ** note this will turn off HUNG if its set.
1135 if (txr->tx_avail == txr->num_desc)
1144 ** This routine parses packet headers so that Flow
1145 ** Director can make a hashed filter table entry
1146 ** allowing traffic flows to be identified and kept
1147 ** on the same cpu. This would be a performance
1148 ** hit, but we only do it at IXGBE_FDIR_RATE of
1152 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
1154 struct adapter *adapter = txr->adapter;
1155 struct ix_queue *que;
1159 struct ether_vlan_header *eh;
1160 union ixgbe_atr_hash_dword input = {.dword = 0};
1161 union ixgbe_atr_hash_dword common = {.dword = 0};
1162 int ehdrlen, ip_hlen;
1165 eh = mtod(mp, struct ether_vlan_header *);
1166 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1167 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1168 etype = eh->evl_proto;
1170 ehdrlen = ETHER_HDR_LEN;
1171 etype = eh->evl_encap_proto;
1174 /* Only handling IPv4 */
1175 if (etype != htons(ETHERTYPE_IP))
1178 ip = (struct ip *)(mp->m_data + ehdrlen);
1179 ip_hlen = ip->ip_hl << 2;
1181 /* check if we're UDP or TCP */
1184 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
1185 /* src and dst are inverted */
1186 common.port.dst ^= th->th_sport;
1187 common.port.src ^= th->th_dport;
1188 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
1191 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
1192 /* src and dst are inverted */
1193 common.port.dst ^= uh->uh_sport;
1194 common.port.src ^= uh->uh_dport;
1195 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
1201 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
1202 if (mp->m_pkthdr.ether_vtag)
1203 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
1205 common.flex_bytes ^= etype;
1206 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
1208 que = &adapter->queues[txr->me];
1210 ** This assumes the Rx queue and Tx
1211 ** queue are bound to the same CPU
1213 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
1214 input, common, que->msix);
1216 #endif /* IXGBE_FDIR */
1219 ** Used to detect a descriptor that has
1220 ** been merged by Hardware RSC.
1223 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
1225 return (le32toh(rx->wb.lower.lo_dword.data) &
1226 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
1229 /*********************************************************************
1231 * Initialize Hardware RSC (LRO) feature on 82599
1232 * for an RX ring, this is toggled by the LRO capability
1233 * even though it is transparent to the stack.
1235 * NOTE: since this HW feature only works with IPV4 and
1236 * our testing has shown soft LRO to be as effective
1237 * I have decided to disable this by default.
1239 **********************************************************************/
1241 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
1243 struct adapter *adapter = rxr->adapter;
1244 struct ixgbe_hw *hw = &adapter->hw;
1245 u32 rscctrl, rdrxctl;
1247 /* If turning LRO/RSC off we need to disable it */
1248 if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) {
1249 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
1250 rscctrl &= ~IXGBE_RSCCTL_RSCEN;
1254 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
1255 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
1256 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
1257 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
1258 #endif /* DEV_NETMAP */
1259 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
1260 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
1261 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
1263 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
1264 rscctrl |= IXGBE_RSCCTL_RSCEN;
1266 ** Limit the total number of descriptors that
1267 ** can be combined, so it does not exceed 64K
1269 if (rxr->mbuf_sz == MCLBYTES)
1270 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
1271 else if (rxr->mbuf_sz == MJUMPAGESIZE)
1272 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
1273 else if (rxr->mbuf_sz == MJUM9BYTES)
1274 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
1275 else /* Using 16K cluster */
1276 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
1278 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
1280 /* Enable TCP header recognition */
1281 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
1282 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
1283 IXGBE_PSRTYPE_TCPHDR));
1285 /* Disable RSC for ACK packets */
1286 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
1287 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
1291 /*********************************************************************
1293 * Refresh mbuf buffers for RX descriptor rings
1294 * - now keeps its own state so discards due to resource
1295 * exhaustion are unnecessary, if an mbuf cannot be obtained
1296 * it just returns, keeping its placeholder, thus it can simply
1297 * be recalled to try again.
1299 **********************************************************************/
1301 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
1303 struct adapter *adapter = rxr->adapter;
1304 bus_dma_segment_t seg[1];
1305 struct ixgbe_rx_buf *rxbuf;
1307 int i, j, nsegs, error;
1308 bool refreshed = FALSE;
1310 i = j = rxr->next_to_refresh;
1311 /* Control the loop with one beyond */
1312 if (++j == rxr->num_desc)
1315 while (j != limit) {
1316 rxbuf = &rxr->rx_buffers[i];
1317 if (rxbuf->buf == NULL) {
1318 mp = m_getjcl(M_NOWAIT, MT_DATA,
1319 M_PKTHDR, rxr->mbuf_sz);
1322 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
1323 m_adj(mp, ETHER_ALIGN);
1327 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1329 /* If we're dealing with an mbuf that was copied rather
1330 * than replaced, there's no need to go through busdma.
1332 if ((rxbuf->flags & IXGBE_RX_COPY) == 0) {
1333 /* Get the memory mapping */
1334 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
1335 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1336 rxbuf->pmap, mp, seg, &nsegs, BUS_DMA_NOWAIT);
1338 printf("Refresh mbufs: payload dmamap load"
1339 " failure - %d\n", error);
1345 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
1346 BUS_DMASYNC_PREREAD);
1347 rxbuf->addr = rxr->rx_base[i].read.pkt_addr =
1348 htole64(seg[0].ds_addr);
1350 rxr->rx_base[i].read.pkt_addr = rxbuf->addr;
1351 rxbuf->flags &= ~IXGBE_RX_COPY;
1355 /* Next is precalculated */
1357 rxr->next_to_refresh = i;
1358 if (++j == rxr->num_desc)
1362 if (refreshed) /* Update hardware tail index */
1363 IXGBE_WRITE_REG(&adapter->hw,
1364 rxr->tail, rxr->next_to_refresh);
1368 /*********************************************************************
1370 * Allocate memory for rx_buffer structures. Since we use one
1371 * rx_buffer per received packet, the maximum number of rx_buffer's
1372 * that we'll need is equal to the number of receive descriptors
1373 * that we've allocated.
1375 **********************************************************************/
1377 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
1379 struct adapter *adapter = rxr->adapter;
1380 device_t dev = adapter->dev;
1381 struct ixgbe_rx_buf *rxbuf;
1382 int i, bsize, error;
1384 bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc;
1385 if (!(rxr->rx_buffers =
1386 (struct ixgbe_rx_buf *) malloc(bsize,
1387 M_DEVBUF, M_NOWAIT | M_ZERO))) {
1388 device_printf(dev, "Unable to allocate rx_buffer memory\n");
1393 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1394 1, 0, /* alignment, bounds */
1395 BUS_SPACE_MAXADDR, /* lowaddr */
1396 BUS_SPACE_MAXADDR, /* highaddr */
1397 NULL, NULL, /* filter, filterarg */
1398 MJUM16BYTES, /* maxsize */
1400 MJUM16BYTES, /* maxsegsize */
1402 NULL, /* lockfunc */
1403 NULL, /* lockfuncarg */
1405 device_printf(dev, "Unable to create RX DMA tag\n");
1409 for (i = 0; i < rxr->num_desc; i++, rxbuf++) {
1410 rxbuf = &rxr->rx_buffers[i];
1411 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
1413 device_printf(dev, "Unable to create RX dma map\n");
1421 /* Frees all, but can handle partial completion */
1422 ixgbe_free_receive_structures(adapter);
1428 ixgbe_free_receive_ring(struct rx_ring *rxr)
1430 struct ixgbe_rx_buf *rxbuf;
1433 for (i = 0; i < rxr->num_desc; i++) {
1434 rxbuf = &rxr->rx_buffers[i];
1435 if (rxbuf->buf != NULL) {
1436 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
1437 BUS_DMASYNC_POSTREAD);
1438 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
1439 rxbuf->buf->m_flags |= M_PKTHDR;
1440 m_freem(rxbuf->buf);
1448 /*********************************************************************
1450 * Initialize a receive ring and its buffers.
1452 **********************************************************************/
1454 ixgbe_setup_receive_ring(struct rx_ring *rxr)
1456 struct adapter *adapter;
1459 struct ixgbe_rx_buf *rxbuf;
1460 bus_dma_segment_t seg[1];
1461 struct lro_ctrl *lro = &rxr->lro;
1462 int rsize, nsegs, error = 0;
1464 struct netmap_adapter *na = NA(rxr->adapter->ifp);
1465 struct netmap_slot *slot;
1466 #endif /* DEV_NETMAP */
1468 adapter = rxr->adapter;
1472 /* Clear the ring contents */
1475 /* same as in ixgbe_setup_transmit_ring() */
1476 slot = netmap_reset(na, NR_RX, rxr->me, 0);
1477 #endif /* DEV_NETMAP */
1478 rsize = roundup2(adapter->num_rx_desc *
1479 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
1480 bzero((void *)rxr->rx_base, rsize);
1481 /* Cache the size */
1482 rxr->mbuf_sz = adapter->rx_mbuf_sz;
1484 /* Free current RX buffer structs and their mbufs */
1485 ixgbe_free_receive_ring(rxr);
1487 /* Now replenish the mbufs */
1488 for (int j = 0; j != rxr->num_desc; ++j) {
1491 rxbuf = &rxr->rx_buffers[j];
1494 * In netmap mode, fill the map and set the buffer
1495 * address in the NIC ring, considering the offset
1496 * between the netmap and NIC rings (see comment in
1497 * ixgbe_setup_transmit_ring() ). No need to allocate
1498 * an mbuf, so end the block with a continue;
1501 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
1505 addr = PNMB(na, slot + sj, &paddr);
1506 netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
1507 /* Update descriptor and the cached value */
1508 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
1509 rxbuf->addr = htole64(paddr);
1512 #endif /* DEV_NETMAP */
1514 rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA,
1515 M_PKTHDR, adapter->rx_mbuf_sz);
1516 if (rxbuf->buf == NULL) {
1521 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1522 /* Get the memory mapping */
1523 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1524 rxbuf->pmap, mp, seg,
1525 &nsegs, BUS_DMA_NOWAIT);
1528 bus_dmamap_sync(rxr->ptag,
1529 rxbuf->pmap, BUS_DMASYNC_PREREAD);
1530 /* Update the descriptor and the cached value */
1531 rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr);
1532 rxbuf->addr = htole64(seg[0].ds_addr);
1536 /* Setup our descriptor indices */
1537 rxr->next_to_check = 0;
1538 rxr->next_to_refresh = 0;
1539 rxr->lro_enabled = FALSE;
1542 rxr->vtag_strip = FALSE;
1544 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
1545 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1548 ** Now set up the LRO interface:
1550 if (ixgbe_rsc_enable)
1551 ixgbe_setup_hw_rsc(rxr);
1552 else if (ifp->if_capenable & IFCAP_LRO) {
1553 int err = tcp_lro_init(lro);
1555 device_printf(dev, "LRO Initialization failed!\n");
1558 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
1559 rxr->lro_enabled = TRUE;
1560 lro->ifp = adapter->ifp;
1563 IXGBE_RX_UNLOCK(rxr);
1567 ixgbe_free_receive_ring(rxr);
1568 IXGBE_RX_UNLOCK(rxr);
1572 /*********************************************************************
1574 * Initialize all receive rings.
1576 **********************************************************************/
1578 ixgbe_setup_receive_structures(struct adapter *adapter)
1580 struct rx_ring *rxr = adapter->rx_rings;
1583 for (j = 0; j < adapter->num_queues; j++, rxr++)
1584 if (ixgbe_setup_receive_ring(rxr))
1590 * Free RX buffers allocated so far, we will only handle
1591 * the rings that completed, the failing case will have
1592 * cleaned up for itself. 'j' failed, so its the terminus.
1594 for (int i = 0; i < j; ++i) {
1595 rxr = &adapter->rx_rings[i];
1596 ixgbe_free_receive_ring(rxr);
1603 /*********************************************************************
1605 * Free all receive rings.
1607 **********************************************************************/
1609 ixgbe_free_receive_structures(struct adapter *adapter)
1611 struct rx_ring *rxr = adapter->rx_rings;
1613 INIT_DEBUGOUT("ixgbe_free_receive_structures: begin");
1615 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
1616 struct lro_ctrl *lro = &rxr->lro;
1617 ixgbe_free_receive_buffers(rxr);
1618 /* Free LRO memory */
1620 /* Free the ring memory as well */
1621 ixgbe_dma_free(adapter, &rxr->rxdma);
1624 free(adapter->rx_rings, M_DEVBUF);
1628 /*********************************************************************
1630 * Free receive ring data structures
1632 **********************************************************************/
1634 ixgbe_free_receive_buffers(struct rx_ring *rxr)
1636 struct adapter *adapter = rxr->adapter;
1637 struct ixgbe_rx_buf *rxbuf;
1639 INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin");
1641 /* Cleanup any existing buffers */
1642 if (rxr->rx_buffers != NULL) {
1643 for (int i = 0; i < adapter->num_rx_desc; i++) {
1644 rxbuf = &rxr->rx_buffers[i];
1645 if (rxbuf->buf != NULL) {
1646 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
1647 BUS_DMASYNC_POSTREAD);
1648 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
1649 rxbuf->buf->m_flags |= M_PKTHDR;
1650 m_freem(rxbuf->buf);
1653 if (rxbuf->pmap != NULL) {
1654 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
1658 if (rxr->rx_buffers != NULL) {
1659 free(rxr->rx_buffers, M_DEVBUF);
1660 rxr->rx_buffers = NULL;
1664 if (rxr->ptag != NULL) {
1665 bus_dma_tag_destroy(rxr->ptag);
1672 static __inline void
1673 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
1677 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
1678 * should be computed by hardware. Also it should not have VLAN tag in
1679 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
1681 if (rxr->lro_enabled &&
1682 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
1683 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
1684 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
1685 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
1686 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
1687 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
1688 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
1689 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
1691 * Send to the stack if:
1692 ** - LRO not enabled, or
1693 ** - no LRO resources, or
1694 ** - lro enqueue fails
1696 if (rxr->lro.lro_cnt != 0)
1697 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
1700 IXGBE_RX_UNLOCK(rxr);
1701 (*ifp->if_input)(ifp, m);
1705 static __inline void
1706 ixgbe_rx_discard(struct rx_ring *rxr, int i)
1708 struct ixgbe_rx_buf *rbuf;
1710 rbuf = &rxr->rx_buffers[i];
1714 ** With advanced descriptors the writeback
1715 ** clobbers the buffer addrs, so its easier
1716 ** to just free the existing mbufs and take
1717 ** the normal refresh path to get new buffers
1721 if (rbuf->fmp != NULL) {/* Partial chain ? */
1722 rbuf->fmp->m_flags |= M_PKTHDR;
1725 rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */
1726 } else if (rbuf->buf) {
1730 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
1738 /*********************************************************************
1740 * This routine executes in interrupt context. It replenishes
1741 * the mbufs in the descriptor and sends data which has been
1742 * dma'ed into host memory to upper layer.
1744 * Return TRUE for more work, FALSE for all clean.
1745 *********************************************************************/
1747 ixgbe_rxeof(struct ix_queue *que)
1749 struct adapter *adapter = que->adapter;
1750 struct rx_ring *rxr = que->rxr;
1751 struct ifnet *ifp = adapter->ifp;
1752 struct lro_ctrl *lro = &rxr->lro;
1753 struct lro_entry *queued;
1754 int i, nextp, processed = 0;
1756 u16 count = rxr->process_limit;
1757 union ixgbe_adv_rx_desc *cur;
1758 struct ixgbe_rx_buf *rbuf, *nbuf;
1764 /* Same as the txeof routine: wakeup clients on intr. */
1765 if (netmap_rx_irq(ifp, rxr->me, &processed)) {
1766 IXGBE_RX_UNLOCK(rxr);
1769 #endif /* DEV_NETMAP */
1771 for (i = rxr->next_to_check; count != 0;) {
1772 struct mbuf *sendmp, *mp;
1778 /* Sync the ring. */
1779 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
1780 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1782 cur = &rxr->rx_base[i];
1783 staterr = le32toh(cur->wb.upper.status_error);
1784 pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
1786 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
1788 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1795 cur->wb.upper.status_error = 0;
1796 rbuf = &rxr->rx_buffers[i];
1799 len = le16toh(cur->wb.upper.length);
1800 ptype = le32toh(cur->wb.lower.lo_dword.data) &
1801 IXGBE_RXDADV_PKTTYPE_MASK;
1802 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
1804 /* Make sure bad packets are discarded */
1805 if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) {
1806 #if __FreeBSD_version >= 1100036
1807 if (IXGBE_IS_VF(adapter))
1808 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
1810 rxr->rx_discarded++;
1811 ixgbe_rx_discard(rxr, i);
1816 ** On 82599 which supports a hardware
1817 ** LRO (called HW RSC), packets need
1818 ** not be fragmented across sequential
1819 ** descriptors, rather the next descriptor
1820 ** is indicated in bits of the descriptor.
1821 ** This also means that we might proceses
1822 ** more than one packet at a time, something
1823 ** that has never been true before, it
1824 ** required eliminating global chain pointers
1825 ** in favor of what we are doing here. -jfv
1829 ** Figure out the next descriptor
1832 if (rxr->hw_rsc == TRUE) {
1833 rsc = ixgbe_rsc_count(cur);
1834 rxr->rsc_num += (rsc - 1);
1836 if (rsc) { /* Get hardware index */
1838 IXGBE_RXDADV_NEXTP_MASK) >>
1839 IXGBE_RXDADV_NEXTP_SHIFT);
1840 } else { /* Just sequential */
1842 if (nextp == adapter->num_rx_desc)
1845 nbuf = &rxr->rx_buffers[nextp];
1849 ** Rather than using the fmp/lmp global pointers
1850 ** we now keep the head of a packet chain in the
1851 ** buffer struct and pass this along from one
1852 ** descriptor to the next, until we get EOP.
1856 ** See if there is a stored head
1857 ** that determines what we are
1860 if (sendmp != NULL) { /* secondary frag */
1861 rbuf->buf = rbuf->fmp = NULL;
1862 mp->m_flags &= ~M_PKTHDR;
1863 sendmp->m_pkthdr.len += mp->m_len;
1866 * Optimize. This might be a small packet,
1867 * maybe just a TCP ACK. Do a fast copy that
1868 * is cache aligned into a new mbuf, and
1869 * leave the old mbuf+cluster for re-use.
1871 if (eop && len <= IXGBE_RX_COPY_LEN) {
1872 sendmp = m_gethdr(M_NOWAIT, MT_DATA);
1873 if (sendmp != NULL) {
1875 IXGBE_RX_COPY_ALIGN;
1876 ixgbe_bcopy(mp->m_data,
1877 sendmp->m_data, len);
1878 sendmp->m_len = len;
1880 rbuf->flags |= IXGBE_RX_COPY;
1883 if (sendmp == NULL) {
1884 rbuf->buf = rbuf->fmp = NULL;
1888 /* first desc of a non-ps chain */
1889 sendmp->m_flags |= M_PKTHDR;
1890 sendmp->m_pkthdr.len = mp->m_len;
1894 /* Pass the head pointer on */
1898 mp->m_next = nbuf->buf;
1899 } else { /* Sending this frame */
1900 sendmp->m_pkthdr.rcvif = ifp;
1902 /* capture data for AIM */
1903 rxr->bytes += sendmp->m_pkthdr.len;
1904 rxr->rx_bytes += sendmp->m_pkthdr.len;
1905 /* Process vlan info */
1906 if ((rxr->vtag_strip) &&
1907 (staterr & IXGBE_RXD_STAT_VP))
1908 vtag = le16toh(cur->wb.upper.vlan);
1910 sendmp->m_pkthdr.ether_vtag = vtag;
1911 sendmp->m_flags |= M_VLANTAG;
1913 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
1914 ixgbe_rx_checksum(staterr, sendmp, ptype);
1915 #if __FreeBSD_version >= 800000
1917 sendmp->m_pkthdr.flowid =
1918 le32toh(cur->wb.lower.hi_dword.rss);
1919 #if __FreeBSD_version < 1100054
1920 sendmp->m_flags |= M_FLOWID;
1922 switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) {
1923 case IXGBE_RXDADV_RSSTYPE_IPV4_TCP:
1924 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV4);
1926 case IXGBE_RXDADV_RSSTYPE_IPV4:
1927 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV4);
1929 case IXGBE_RXDADV_RSSTYPE_IPV6_TCP:
1930 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6);
1932 case IXGBE_RXDADV_RSSTYPE_IPV6_EX:
1933 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6_EX);
1935 case IXGBE_RXDADV_RSSTYPE_IPV6:
1936 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_IPV6);
1938 case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX:
1939 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_TCP_IPV6_EX);
1941 case IXGBE_RXDADV_RSSTYPE_IPV4_UDP:
1942 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV4);
1944 case IXGBE_RXDADV_RSSTYPE_IPV6_UDP:
1945 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6);
1947 case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX:
1948 M_HASHTYPE_SET(sendmp, M_HASHTYPE_RSS_UDP_IPV6_EX);
1951 M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
1954 sendmp->m_pkthdr.flowid = que->msix;
1955 #if __FreeBSD_version >= 1100054
1956 M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
1958 sendmp->m_flags |= M_FLOWID;
1961 #endif /* FreeBSD_version */
1964 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
1965 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1967 /* Advance our pointers to the next descriptor. */
1968 if (++i == rxr->num_desc)
1971 /* Now send to the stack or do LRO */
1972 if (sendmp != NULL) {
1973 rxr->next_to_check = i;
1974 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
1975 i = rxr->next_to_check;
1978 /* Every 8 descriptors we go to refresh mbufs */
1979 if (processed == 8) {
1980 ixgbe_refresh_mbufs(rxr, i);
1985 /* Refresh any remaining buf structs */
1986 if (ixgbe_rx_unrefreshed(rxr))
1987 ixgbe_refresh_mbufs(rxr, i);
1989 rxr->next_to_check = i;
1992 * Flush any outstanding LRO work
1994 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
1995 SLIST_REMOVE_HEAD(&lro->lro_active, next);
1996 tcp_lro_flush(lro, queued);
1999 IXGBE_RX_UNLOCK(rxr);
2002 ** Still have cleaning to do?
2004 if ((staterr & IXGBE_RXD_STAT_DD) != 0)
2011 /*********************************************************************
2013 * Verify that the hardware indicated that the checksum is valid.
2014 * Inform the stack about the status of checksum so that stack
2015 * doesn't spend time verifying the checksum.
2017 *********************************************************************/
2019 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
2021 u16 status = (u16) staterr;
2022 u8 errors = (u8) (staterr >> 24);
2025 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
2026 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
2029 if (status & IXGBE_RXD_STAT_IPCS) {
2030 if (!(errors & IXGBE_RXD_ERR_IPE)) {
2031 /* IP Checksum Good */
2032 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2033 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2036 mp->m_pkthdr.csum_flags = 0;
2038 if (status & IXGBE_RXD_STAT_L4CS) {
2039 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2040 #if __FreeBSD_version >= 800000
2042 type = CSUM_SCTP_VALID;
2044 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
2045 mp->m_pkthdr.csum_flags |= type;
2047 mp->m_pkthdr.csum_data = htons(0xffff);
2053 /********************************************************************
2054 * Manage DMA'able memory.
2055 *******************************************************************/
2057 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2061 *(bus_addr_t *) arg = segs->ds_addr;
2066 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2067 struct ixgbe_dma_alloc *dma, int mapflags)
2069 device_t dev = adapter->dev;
2072 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2073 DBA_ALIGN, 0, /* alignment, bounds */
2074 BUS_SPACE_MAXADDR, /* lowaddr */
2075 BUS_SPACE_MAXADDR, /* highaddr */
2076 NULL, NULL, /* filter, filterarg */
2079 size, /* maxsegsize */
2080 BUS_DMA_ALLOCNOW, /* flags */
2081 NULL, /* lockfunc */
2082 NULL, /* lockfuncarg */
2085 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2089 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2090 BUS_DMA_NOWAIT, &dma->dma_map);
2092 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2096 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2100 mapflags | BUS_DMA_NOWAIT);
2102 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2106 dma->dma_size = size;
2109 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2111 bus_dma_tag_destroy(dma->dma_tag);
2113 dma->dma_tag = NULL;
2118 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2120 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2121 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2122 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2123 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2124 bus_dma_tag_destroy(dma->dma_tag);
2128 /*********************************************************************
2130 * Allocate memory for the transmit and receive rings, and then
2131 * the descriptors associated with each, called only once at attach.
2133 **********************************************************************/
2135 ixgbe_allocate_queues(struct adapter *adapter)
2137 device_t dev = adapter->dev;
2138 struct ix_queue *que;
2139 struct tx_ring *txr;
2140 struct rx_ring *rxr;
2141 int rsize, tsize, error = IXGBE_SUCCESS;
2142 int txconf = 0, rxconf = 0;
2144 /* First allocate the top level queue structs */
2145 if (!(adapter->queues =
2146 (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2147 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2148 device_printf(dev, "Unable to allocate queue memory\n");
2153 /* First allocate the TX ring struct memory */
2154 if (!(adapter->tx_rings =
2155 (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2156 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2157 device_printf(dev, "Unable to allocate TX ring memory\n");
2162 /* Next allocate the RX */
2163 if (!(adapter->rx_rings =
2164 (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2165 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2166 device_printf(dev, "Unable to allocate RX ring memory\n");
2171 /* For the ring itself */
2172 tsize = roundup2(adapter->num_tx_desc *
2173 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2176 * Now set up the TX queues, txconf is needed to handle the
2177 * possibility that things fail midcourse and we need to
2178 * undo memory gracefully
2180 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2181 /* Set up some basics */
2182 txr = &adapter->tx_rings[i];
2183 txr->adapter = adapter;
2185 txr->num_desc = adapter->num_tx_desc;
2187 /* Initialize the TX side lock */
2188 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2189 device_get_nameunit(dev), txr->me);
2190 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2192 if (ixgbe_dma_malloc(adapter, tsize,
2193 &txr->txdma, BUS_DMA_NOWAIT)) {
2195 "Unable to allocate TX Descriptor memory\n");
2199 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2200 bzero((void *)txr->tx_base, tsize);
2202 /* Now allocate transmit buffers for the ring */
2203 if (ixgbe_allocate_transmit_buffers(txr)) {
2205 "Critical Failure setting up transmit buffers\n");
2209 #ifndef IXGBE_LEGACY_TX
2210 /* Allocate a buf ring */
2211 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2212 M_WAITOK, &txr->tx_mtx);
2213 if (txr->br == NULL) {
2215 "Critical Failure setting up buf ring\n");
2223 * Next the RX queues...
2225 rsize = roundup2(adapter->num_rx_desc *
2226 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2227 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2228 rxr = &adapter->rx_rings[i];
2229 /* Set up some basics */
2230 rxr->adapter = adapter;
2232 rxr->num_desc = adapter->num_rx_desc;
2234 /* Initialize the RX side lock */
2235 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2236 device_get_nameunit(dev), rxr->me);
2237 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2239 if (ixgbe_dma_malloc(adapter, rsize,
2240 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2242 "Unable to allocate RxDescriptor memory\n");
2246 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2247 bzero((void *)rxr->rx_base, rsize);
2249 /* Allocate receive buffers for the ring*/
2250 if (ixgbe_allocate_receive_buffers(rxr)) {
2252 "Critical Failure setting up receive buffers\n");
2259 ** Finally set up the queue holding structs
2261 for (int i = 0; i < adapter->num_queues; i++) {
2262 que = &adapter->queues[i];
2263 que->adapter = adapter;
2265 que->txr = &adapter->tx_rings[i];
2266 que->rxr = &adapter->rx_rings[i];
2272 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2273 ixgbe_dma_free(adapter, &rxr->rxdma);
2275 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2276 ixgbe_dma_free(adapter, &txr->txdma);
2277 free(adapter->rx_rings, M_DEVBUF);
2279 free(adapter->tx_rings, M_DEVBUF);
2281 free(adapter->queues, M_DEVBUF);