4 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
34 #include "ena_datapath.h"
36 #include "ena_netmap.h"
37 #endif /* DEV_NETMAP */
39 /*********************************************************************
40 * Static functions prototypes
41 *********************************************************************/
43 static int ena_tx_cleanup(struct ena_ring *);
44 static int ena_rx_cleanup(struct ena_ring *);
45 static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
46 static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
48 static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
49 struct ena_com_rx_ctx *, uint16_t *);
50 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
52 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *);
53 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
55 static void ena_dmamap_llq(void *, bus_dma_segment_t *, int, int);
56 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
57 static void ena_start_xmit(struct ena_ring *);
59 /*********************************************************************
61 *********************************************************************/
64 ena_cleanup(void *arg, int pending)
66 struct ena_que *que = arg;
67 struct ena_adapter *adapter = que->adapter;
68 if_t ifp = adapter->ifp;
69 struct ena_ring *tx_ring;
70 struct ena_ring *rx_ring;
71 struct ena_com_io_cq* io_cq;
72 struct ena_eth_io_intr_reg intr_reg;
76 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
79 ena_trace(ENA_DBG, "MSI-X TX/RX routine\n");
81 tx_ring = que->tx_ring;
82 rx_ring = que->rx_ring;
84 ena_qid = ENA_IO_TXQ_IDX(qid);
85 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
87 tx_ring->first_interrupt = true;
88 rx_ring->first_interrupt = true;
90 for (i = 0; i < CLEAN_BUDGET; ++i) {
91 rxc = ena_rx_cleanup(rx_ring);
92 txc = ena_tx_cleanup(tx_ring);
94 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
97 if ((txc != TX_BUDGET) && (rxc != RX_BUDGET))
101 /* Signal that work is done and unmask interrupt */
102 ena_com_update_intr_reg(&intr_reg,
106 ena_com_unmask_intr(io_cq, &intr_reg);
110 ena_deferred_mq_start(void *arg, int pending)
112 struct ena_ring *tx_ring = (struct ena_ring *)arg;
113 struct ifnet *ifp = tx_ring->adapter->ifp;
115 while (!drbr_empty(ifp, tx_ring->br) &&
117 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
118 ENA_RING_MTX_LOCK(tx_ring);
119 ena_start_xmit(tx_ring);
120 ENA_RING_MTX_UNLOCK(tx_ring);
125 ena_mq_start(if_t ifp, struct mbuf *m)
127 struct ena_adapter *adapter = ifp->if_softc;
128 struct ena_ring *tx_ring;
129 int ret, is_drbr_empty;
132 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
135 /* Which queue to use */
137 * If everything is setup correctly, it should be the
138 * same bucket that the current CPU we're on is.
139 * It should improve performance.
141 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
142 i = m->m_pkthdr.flowid % adapter->num_queues;
144 i = curcpu % adapter->num_queues;
146 tx_ring = &adapter->tx_ring[i];
148 /* Check if drbr is empty before putting packet */
149 is_drbr_empty = drbr_empty(ifp, tx_ring->br);
150 ret = drbr_enqueue(ifp, tx_ring->br, m);
151 if (unlikely(ret != 0)) {
152 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
156 if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) {
157 ena_start_xmit(tx_ring);
158 ENA_RING_MTX_UNLOCK(tx_ring);
160 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
169 struct ena_adapter *adapter = ifp->if_softc;
170 struct ena_ring *tx_ring = adapter->tx_ring;
173 for(i = 0; i < adapter->num_queues; ++i, ++tx_ring)
174 if (!drbr_empty(ifp, tx_ring->br)) {
175 ENA_RING_MTX_LOCK(tx_ring);
176 drbr_flush(ifp, tx_ring->br);
177 ENA_RING_MTX_UNLOCK(tx_ring);
183 /*********************************************************************
185 *********************************************************************/
188 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
190 struct ena_adapter *adapter = tx_ring->adapter;
191 struct ena_tx_buffer *tx_info = NULL;
193 if (likely(req_id < tx_ring->ring_size)) {
194 tx_info = &tx_ring->tx_buffer_info[req_id];
195 if (tx_info->mbuf != NULL)
197 device_printf(adapter->pdev,
198 "tx_info doesn't have valid mbuf\n");
201 device_printf(adapter->pdev, "Invalid req_id: %hu\n", req_id);
202 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
204 /* Trigger device reset */
205 adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
206 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
212 * ena_tx_cleanup - clear sent packets and corresponding descriptors
213 * @tx_ring: ring for which we want to clean packets
215 * Once packets are sent, we ask the device in a loop for no longer used
216 * descriptors. We find the related mbuf chain in a map (index in an array)
217 * and free it, then update ring state.
218 * This is performed in "endless" loop, updating ring pointers every
219 * TX_COMMIT. The first check of free descriptor is performed before the actual
220 * loop, then repeated at the loop end.
223 ena_tx_cleanup(struct ena_ring *tx_ring)
225 struct ena_adapter *adapter;
226 struct ena_com_io_cq* io_cq;
227 uint16_t next_to_clean;
230 unsigned int total_done = 0;
232 int commit = TX_COMMIT;
233 int budget = TX_BUDGET;
237 adapter = tx_ring->que->adapter;
238 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
239 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
240 next_to_clean = tx_ring->next_to_clean;
243 if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS)
245 #endif /* DEV_NETMAP */
248 struct ena_tx_buffer *tx_info;
251 rc = ena_com_tx_comp_req_id_get(io_cq, &req_id);
252 if (unlikely(rc != 0))
255 rc = validate_tx_req_id(tx_ring, req_id);
256 if (unlikely(rc != 0))
259 tx_info = &tx_ring->tx_buffer_info[req_id];
261 mbuf = tx_info->mbuf;
263 tx_info->mbuf = NULL;
264 bintime_clear(&tx_info->timestamp);
266 /* Map is no longer required */
267 if (tx_info->head_mapped == true) {
268 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head,
269 BUS_DMASYNC_POSTWRITE);
270 bus_dmamap_unload(adapter->tx_buf_tag,
272 tx_info->head_mapped = false;
274 if (tx_info->seg_mapped == true) {
275 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg,
276 BUS_DMASYNC_POSTWRITE);
277 bus_dmamap_unload(adapter->tx_buf_tag,
279 tx_info->seg_mapped = false;
282 ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed\n",
287 total_done += tx_info->tx_descs;
289 tx_ring->free_tx_ids[next_to_clean] = req_id;
290 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
293 if (unlikely(--commit == 0)) {
295 /* update ring state every TX_COMMIT descriptor */
296 tx_ring->next_to_clean = next_to_clean;
298 &adapter->ena_dev->io_sq_queues[ena_qid],
300 ena_com_update_dev_comp_head(io_cq);
303 } while (likely(--budget));
305 work_done = TX_BUDGET - budget;
307 ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d\n",
308 tx_ring->qid, work_done);
310 /* If there is still something to commit update ring state */
311 if (likely(commit != TX_COMMIT)) {
312 tx_ring->next_to_clean = next_to_clean;
313 ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid],
315 ena_com_update_dev_comp_head(io_cq);
319 * Need to make the rings circular update visible to
320 * ena_xmit_mbuf() before checking for tx_ring->running.
324 above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
325 ENA_TX_RESUME_THRESH);
326 if (unlikely(!tx_ring->running && above_thresh)) {
327 ENA_RING_MTX_LOCK(tx_ring);
329 ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
330 ENA_TX_RESUME_THRESH);
331 if (!tx_ring->running && above_thresh) {
332 tx_ring->running = true;
333 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
334 taskqueue_enqueue(tx_ring->enqueue_tq,
335 &tx_ring->enqueue_task);
337 ENA_RING_MTX_UNLOCK(tx_ring);
344 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
347 struct ena_adapter *adapter = rx_ring->adapter;
349 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
350 mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
352 if (ena_rx_ctx->frag &&
353 (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) {
354 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
358 switch (ena_rx_ctx->l3_proto) {
359 case ENA_ETH_IO_L3_PROTO_IPV4:
360 switch (ena_rx_ctx->l4_proto) {
361 case ENA_ETH_IO_L4_PROTO_TCP:
362 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
364 case ENA_ETH_IO_L4_PROTO_UDP:
365 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
368 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
371 case ENA_ETH_IO_L3_PROTO_IPV6:
372 switch (ena_rx_ctx->l4_proto) {
373 case ENA_ETH_IO_L4_PROTO_TCP:
374 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
376 case ENA_ETH_IO_L4_PROTO_UDP:
377 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
380 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
383 case ENA_ETH_IO_L3_PROTO_UNKNOWN:
384 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
387 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
390 mbuf->m_pkthdr.flowid = rx_ring->qid;
391 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
396 * ena_rx_mbuf - assemble mbuf from descriptors
397 * @rx_ring: ring for which we want to clean packets
398 * @ena_bufs: buffer info
399 * @ena_rx_ctx: metadata for this packet(s)
400 * @next_to_clean: ring pointer, will be updated only upon success
404 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
405 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
408 struct ena_rx_buffer *rx_info;
409 struct ena_adapter *adapter;
410 unsigned int descs = ena_rx_ctx->descs;
412 uint16_t ntc, len, req_id, buf = 0;
414 ntc = *next_to_clean;
415 adapter = rx_ring->adapter;
417 len = ena_bufs[buf].len;
418 req_id = ena_bufs[buf].req_id;
419 rc = validate_rx_req_id(rx_ring, req_id);
420 if (unlikely(rc != 0))
423 rx_info = &rx_ring->rx_buffer_info[req_id];
424 if (unlikely(rx_info->mbuf == NULL)) {
425 device_printf(adapter->pdev, "NULL mbuf in rx_info");
429 ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx\n",
430 rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
432 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
433 BUS_DMASYNC_POSTREAD);
434 mbuf = rx_info->mbuf;
435 mbuf->m_flags |= M_PKTHDR;
436 mbuf->m_pkthdr.len = len;
438 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
440 /* Fill mbuf with hash key and it's interpretation for optimization */
441 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
443 ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d\n",
444 mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
446 /* DMA address is not needed anymore, unmap it */
447 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
449 rx_info->mbuf = NULL;
450 rx_ring->free_rx_ids[ntc] = req_id;
451 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
454 * While we have more than 1 descriptors for one rcvd packet, append
455 * other mbufs to the main one
459 len = ena_bufs[buf].len;
460 req_id = ena_bufs[buf].req_id;
461 rc = validate_rx_req_id(rx_ring, req_id);
462 if (unlikely(rc != 0)) {
464 * If the req_id is invalid, then the device will be
465 * reset. In that case we must free all mbufs that
466 * were already gathered.
471 rx_info = &rx_ring->rx_buffer_info[req_id];
473 if (unlikely(rx_info->mbuf == NULL)) {
474 device_printf(adapter->pdev, "NULL mbuf in rx_info");
476 * If one of the required mbufs was not allocated yet,
477 * we can break there.
478 * All earlier used descriptors will be reallocated
479 * later and not used mbufs can be reused.
480 * The next_to_clean pointer will not be updated in case
481 * of an error, so caller should advance it manually
482 * in error handling routine to keep it up to date
489 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
490 BUS_DMASYNC_POSTREAD);
491 if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
492 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
493 ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p\n",
497 ena_trace(ENA_DBG | ENA_RXPTH,
498 "rx mbuf updated. len %d\n", mbuf->m_pkthdr.len);
500 /* Free already appended mbuf, it won't be useful anymore */
501 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
502 m_freem(rx_info->mbuf);
503 rx_info->mbuf = NULL;
505 rx_ring->free_rx_ids[ntc] = req_id;
506 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
509 *next_to_clean = ntc;
515 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
518 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
522 /* if IP and error */
523 if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
524 ena_rx_ctx->l3_csum_err)) {
525 /* ipv4 checksum error */
526 mbuf->m_pkthdr.csum_flags = 0;
527 counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
528 ena_trace(ENA_DBG, "RX IPv4 header checksum error\n");
533 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
534 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
535 if (ena_rx_ctx->l4_csum_err) {
536 /* TCP/UDP checksum error */
537 mbuf->m_pkthdr.csum_flags = 0;
538 counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
539 ena_trace(ENA_DBG, "RX L4 checksum error\n");
541 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
542 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
548 * ena_rx_cleanup - handle rx irq
549 * @arg: ring for which irq is being handled
552 ena_rx_cleanup(struct ena_ring *rx_ring)
554 struct ena_adapter *adapter;
556 struct ena_com_rx_ctx ena_rx_ctx;
557 struct ena_com_io_cq* io_cq;
558 struct ena_com_io_sq* io_sq;
561 uint16_t next_to_clean;
562 uint32_t refill_required;
563 uint32_t refill_threshold;
564 uint32_t do_if_input = 0;
567 int budget = RX_BUDGET;
570 #endif /* DEV_NETMAP */
572 adapter = rx_ring->que->adapter;
574 qid = rx_ring->que->id;
575 ena_qid = ENA_IO_RXQ_IDX(qid);
576 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
577 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
578 next_to_clean = rx_ring->next_to_clean;
581 if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS)
583 #endif /* DEV_NETMAP */
585 ena_trace(ENA_DBG, "rx: qid %d\n", qid);
588 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
589 ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
590 ena_rx_ctx.descs = 0;
591 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
592 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
593 rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
595 if (unlikely(rc != 0))
598 if (unlikely(ena_rx_ctx.descs == 0))
601 ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. "
602 "descs #: %d l3 proto %d l4 proto %d hash: %x\n",
603 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
604 ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
606 /* Receive mbuf from the ring */
607 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
608 &ena_rx_ctx, &next_to_clean);
609 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
610 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
611 /* Exit if we failed to retrieve a buffer */
612 if (unlikely(mbuf == NULL)) {
613 for (i = 0; i < ena_rx_ctx.descs; ++i) {
614 rx_ring->free_rx_ids[next_to_clean] =
615 rx_ring->ena_bufs[i].req_id;
617 ENA_RX_RING_IDX_NEXT(next_to_clean,
624 if (((ifp->if_capenable & IFCAP_RXCSUM) != 0) ||
625 ((ifp->if_capenable & IFCAP_RXCSUM_IPV6) != 0)) {
626 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
630 counter_u64_add_protected(rx_ring->rx_stats.bytes,
632 counter_u64_add_protected(adapter->hw_stats.rx_bytes,
636 * LRO is only for IP/TCP packets and TCP checksum of the packet
637 * should be computed by hardware.
640 if (((ifp->if_capenable & IFCAP_LRO) != 0) &&
641 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
642 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
644 * Send to the stack if:
645 * - LRO not enabled, or
646 * - no LRO resources, or
647 * - lro enqueue fails
649 if ((rx_ring->lro.lro_cnt != 0) &&
650 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
653 if (do_if_input != 0) {
654 ena_trace(ENA_DBG | ENA_RXPTH,
655 "calling if_input() with mbuf %p\n", mbuf);
656 (*ifp->if_input)(ifp, mbuf);
660 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
661 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
665 rx_ring->next_to_clean = next_to_clean;
667 refill_required = ena_com_free_desc(io_sq);
668 refill_threshold = min_t(int,
669 rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
670 ENA_RX_REFILL_THRESH_PACKET);
672 if (refill_required > refill_threshold) {
673 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
674 ena_refill_rx_bufs(rx_ring, refill_required);
677 tcp_lro_flush_all(&rx_ring->lro);
679 return (RX_BUDGET - budget);
682 counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1);
684 /* Too many desc from the device. Trigger reset */
685 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
686 adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
687 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
694 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf)
696 struct ena_com_tx_meta *ena_meta;
697 struct ether_vlan_header *eh;
698 struct mbuf *mbuf_next;
709 ena_meta = &ena_tx_ctx->ena_meta;
710 mss = mbuf->m_pkthdr.tso_segsz;
715 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)
718 if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)
722 ena_tx_ctx->meta_valid = 0;
726 /* Determine where frame payload starts. */
727 eh = mtod(mbuf, struct ether_vlan_header *);
728 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
729 etype = ntohs(eh->evl_proto);
730 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
732 etype = ntohs(eh->evl_encap_proto);
733 ehdrlen = ETHER_HDR_LEN;
736 mbuf_next = m_getptr(mbuf, ehdrlen, &offset);
737 ip = (struct ip *)(mtodo(mbuf_next, offset));
738 iphlen = ip->ip_hl << 2;
740 mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset);
741 th = (struct tcphdr *)(mtodo(mbuf_next, offset));
743 if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) {
744 ena_tx_ctx->l3_csum_enable = 1;
746 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
747 ena_tx_ctx->tso_enable = 1;
748 ena_meta->l4_hdr_len = (th->th_off);
753 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
754 if ((ip->ip_off & htons(IP_DF)) != 0)
758 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
764 if (ip->ip_p == IPPROTO_TCP) {
765 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
766 if ((mbuf->m_pkthdr.csum_flags &
767 (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0)
768 ena_tx_ctx->l4_csum_enable = 1;
770 ena_tx_ctx->l4_csum_enable = 0;
771 } else if (ip->ip_p == IPPROTO_UDP) {
772 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
773 if ((mbuf->m_pkthdr.csum_flags &
774 (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0)
775 ena_tx_ctx->l4_csum_enable = 1;
777 ena_tx_ctx->l4_csum_enable = 0;
779 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
780 ena_tx_ctx->l4_csum_enable = 0;
784 ena_meta->l3_hdr_len = iphlen;
785 ena_meta->l3_hdr_offset = ehdrlen;
786 ena_tx_ctx->meta_valid = 1;
790 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
792 struct ena_adapter *adapter;
793 struct mbuf *collapsed_mbuf;
796 adapter = tx_ring->adapter;
797 num_frags = ena_mbuf_count(*mbuf);
799 /* One segment must be reserved for configuration descriptor. */
800 if (num_frags < adapter->max_tx_sgl_size)
802 counter_u64_add(tx_ring->tx_stats.collapse, 1);
804 collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT,
805 adapter->max_tx_sgl_size - 1);
806 if (unlikely(collapsed_mbuf == NULL)) {
807 counter_u64_add(tx_ring->tx_stats.collapse_err, 1);
811 /* If mbuf was collapsed succesfully, original mbuf is released. */
812 *mbuf = collapsed_mbuf;
818 ena_dmamap_llq(void *arg, bus_dma_segment_t *segs, int nseg, int error)
820 struct ena_com_buf *ena_buf = arg;
822 if (unlikely(error != 0)) {
827 KASSERT(nseg == 1, ("Invalid num of segments for LLQ dma"));
829 ena_buf->paddr = segs->ds_addr;
830 ena_buf->len = segs->ds_len;
834 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info,
835 struct mbuf *mbuf, void **push_hdr, u16 *header_len)
837 struct ena_adapter *adapter = tx_ring->adapter;
838 struct ena_com_buf *ena_buf;
839 bus_dma_segment_t segs[ENA_BUS_DMA_SEGS];
840 uint32_t mbuf_head_len, frag_len;
841 uint16_t push_len = 0;
845 mbuf_head_len = mbuf->m_len;
846 tx_info->mbuf = mbuf;
847 ena_buf = tx_info->bufs;
849 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
851 * When the device is LLQ mode, the driver will copy
852 * the header into the device memory space.
853 * the ena_com layer assumes the header is in a linear
855 * This assumption might be wrong since part of the header
856 * can be in the fragmented buffers.
857 * First check if header fits in the mbuf. If not, copy it to
858 * separate buffer that will be holding linearized data.
860 push_len = min_t(uint32_t, mbuf->m_pkthdr.len,
861 tx_ring->tx_max_header_size);
862 *header_len = push_len;
863 /* If header is in linear space, just point into mbuf's data. */
864 if (likely(push_len <= mbuf_head_len)) {
865 *push_hdr = mbuf->m_data;
867 * Otherwise, copy whole portion of header from multiple mbufs
868 * to intermediate buffer.
871 m_copydata(mbuf, 0, push_len,
872 tx_ring->push_buf_intermediate_buf);
873 *push_hdr = tx_ring->push_buf_intermediate_buf;
875 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
876 delta = push_len - mbuf_head_len;
879 ena_trace(ENA_DBG | ENA_TXPTH,
880 "mbuf: %p header_buf->vaddr: %p push_len: %d\n",
881 mbuf, *push_hdr, push_len);
884 * If header was in linear memory space, map for the dma rest of the data
885 * in the first mbuf of the mbuf chain.
887 if (mbuf_head_len > push_len) {
888 rc = bus_dmamap_load(adapter->tx_buf_tag,
890 mbuf->m_data + push_len, mbuf_head_len - push_len,
891 ena_dmamap_llq, ena_buf, BUS_DMA_NOWAIT);
892 if (unlikely((rc != 0) || (ena_buf->paddr == 0)))
893 goto single_dma_error;
896 tx_info->num_of_bufs++;
898 tx_info->head_mapped = true;
904 * header_len is just a hint for the device. Because FreeBSD is not
905 * giving us information about packet header length and it is not
906 * guaranteed that all packet headers will be in the 1st mbuf, setting
907 * header_len to 0 is making the device ignore this value and resolve
908 * header on it's own.
914 * If header is in non linear space (delta > 0), then skip mbufs
915 * containing header and map the last one containing both header and the
917 * The first segment is already counted in.
918 * If LLQ is not supported, the loop will be skipped.
921 frag_len = mbuf->m_len;
924 * If whole segment contains header just move to the
925 * next one and reduce delta.
927 if (unlikely(delta >= frag_len)) {
931 * Map rest of the packet data that was contained in
934 rc = bus_dmamap_load(adapter->tx_buf_tag,
935 tx_info->map_head, mbuf->m_data + delta,
936 frag_len - delta, ena_dmamap_llq, ena_buf,
938 if (unlikely((rc != 0) || (ena_buf->paddr == 0)))
939 goto single_dma_error;
942 tx_info->num_of_bufs++;
943 tx_info->head_mapped = true;
955 /* Map rest of the mbufs */
956 rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->map_seg, mbuf,
957 segs, &nsegs, BUS_DMA_NOWAIT);
958 if (unlikely((rc != 0) || (nsegs == 0))) {
959 ena_trace(ENA_WARNING,
960 "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs);
964 for (i = 0; i < nsegs; i++) {
965 ena_buf->len = segs[i].ds_len;
966 ena_buf->paddr = segs[i].ds_addr;
969 tx_info->num_of_bufs += nsegs;
970 tx_info->seg_mapped = true;
975 if (tx_info->head_mapped == true)
976 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map_head);
978 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1);
979 tx_info->mbuf = NULL;
984 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
986 struct ena_adapter *adapter;
987 struct ena_tx_buffer *tx_info;
988 struct ena_com_tx_ctx ena_tx_ctx;
989 struct ena_com_dev *ena_dev;
990 struct ena_com_io_sq* io_sq;
992 uint16_t next_to_use;
999 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
1000 adapter = tx_ring->que->adapter;
1001 ena_dev = adapter->ena_dev;
1002 io_sq = &ena_dev->io_sq_queues[ena_qid];
1004 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);
1005 if (unlikely(rc != 0)) {
1006 ena_trace(ENA_WARNING,
1007 "Failed to collapse mbuf! err: %d\n", rc);
1011 ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len);
1013 next_to_use = tx_ring->next_to_use;
1014 req_id = tx_ring->free_tx_ids[next_to_use];
1015 tx_info = &tx_ring->tx_buffer_info[req_id];
1016 tx_info->num_of_bufs = 0;
1018 rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len);
1019 if (unlikely(rc != 0)) {
1020 ena_trace(ENA_WARNING, "Failed to map TX mbuf\n");
1023 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
1024 ena_tx_ctx.ena_bufs = tx_info->bufs;
1025 ena_tx_ctx.push_header = push_hdr;
1026 ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
1027 ena_tx_ctx.req_id = req_id;
1028 ena_tx_ctx.header_len = header_len;
1030 /* Set flags and meta data */
1031 ena_tx_csum(&ena_tx_ctx, *mbuf);
1033 if (tx_ring->acum_pkts == DB_THRESHOLD ||
1034 ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) {
1035 ena_trace(ENA_DBG | ENA_TXPTH,
1036 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
1039 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
1040 counter_u64_add(tx_ring->tx_stats.doorbells, 1);
1041 tx_ring->acum_pkts = 0;
1044 /* Prepare the packet's descriptors and send them to device */
1045 rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc);
1046 if (unlikely(rc != 0)) {
1047 if (likely(rc == ENA_COM_NO_MEM)) {
1048 ena_trace(ENA_DBG | ENA_TXPTH,
1049 "tx ring[%d] if out of space\n", tx_ring->que->id);
1051 device_printf(adapter->pdev,
1052 "failed to prepare tx bufs\n");
1054 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
1059 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
1060 counter_u64_add_protected(tx_ring->tx_stats.bytes,
1061 (*mbuf)->m_pkthdr.len);
1063 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
1064 counter_u64_add_protected(adapter->hw_stats.tx_bytes,
1065 (*mbuf)->m_pkthdr.len);
1068 tx_info->tx_descs = nb_hw_desc;
1069 getbinuptime(&tx_info->timestamp);
1070 tx_info->print_once = true;
1072 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
1073 tx_ring->ring_size);
1075 /* stop the queue when no more space available, the packet can have up
1076 * to sgl_size + 2. one for the meta descriptor and one for header
1077 * (if the header is larger than tx_max_header_size).
1079 if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1080 adapter->max_tx_sgl_size + 2))) {
1081 ena_trace(ENA_DBG | ENA_TXPTH, "Stop queue %d\n",
1084 tx_ring->running = false;
1085 counter_u64_add(tx_ring->tx_stats.queue_stop, 1);
1087 /* There is a rare condition where this function decides to
1088 * stop the queue but meanwhile tx_cleanup() updates
1089 * next_to_completion and terminates.
1090 * The queue will remain stopped forever.
1091 * To solve this issue this function performs mb(), checks
1092 * the wakeup condition and wakes up the queue if needed.
1096 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1097 ENA_TX_RESUME_THRESH)) {
1098 tx_ring->running = true;
1099 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
1103 if (tx_info->head_mapped == true)
1104 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head,
1105 BUS_DMASYNC_PREWRITE);
1106 if (tx_info->seg_mapped == true)
1107 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg,
1108 BUS_DMASYNC_PREWRITE);
1113 tx_info->mbuf = NULL;
1114 if (tx_info->seg_mapped == true) {
1115 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map_seg);
1116 tx_info->seg_mapped = false;
1118 if (tx_info->head_mapped == true) {
1119 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map_head);
1120 tx_info->head_mapped = false;
1127 ena_start_xmit(struct ena_ring *tx_ring)
1130 struct ena_adapter *adapter = tx_ring->adapter;
1131 struct ena_com_io_sq* io_sq;
1135 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
1138 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)))
1141 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
1142 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
1144 while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {
1145 ena_trace(ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and"
1146 " header csum flags %#jx\n",
1147 mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags);
1149 if (unlikely(!tx_ring->running)) {
1150 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1154 if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) {
1155 if (ret == ENA_COM_NO_MEM) {
1156 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1157 } else if (ret == ENA_COM_NO_SPACE) {
1158 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1161 drbr_advance(adapter->ifp, tx_ring->br);
1167 drbr_advance(adapter->ifp, tx_ring->br);
1169 if (unlikely((if_getdrvflags(adapter->ifp) &
1170 IFF_DRV_RUNNING) == 0))
1173 tx_ring->acum_pkts++;
1175 BPF_MTAP(adapter->ifp, mbuf);
1178 if (likely(tx_ring->acum_pkts != 0)) {
1180 /* Trigger the dma engine */
1181 ena_com_write_sq_doorbell(io_sq);
1182 counter_u64_add(tx_ring->tx_stats.doorbells, 1);
1183 tx_ring->acum_pkts = 0;
1186 if (unlikely(!tx_ring->running))
1187 taskqueue_enqueue(tx_ring->que->cleanup_tq,
1188 &tx_ring->que->cleanup_task);