4 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
36 #include "ena_netmap.h"
38 #define ENA_NETMAP_MORE_FRAMES 1
39 #define ENA_NETMAP_NO_MORE_FRAMES 0
40 #define ENA_MAX_FRAMES 16384
42 struct ena_netmap_ctx {
43 struct netmap_kring *kring;
44 struct ena_adapter *adapter;
45 struct netmap_adapter *na;
46 struct netmap_slot *slots;
47 struct ena_ring *ring;
48 struct ena_com_io_cq *io_cq;
49 struct ena_com_io_sq *io_sq;
55 /* Netmap callbacks */
56 static int ena_netmap_reg(struct netmap_adapter *, int);
57 static int ena_netmap_txsync(struct netmap_kring *, int);
58 static int ena_netmap_rxsync(struct netmap_kring *, int);
60 /* Helper functions */
61 static int ena_netmap_tx_frames(struct ena_netmap_ctx *);
62 static int ena_netmap_tx_frame(struct ena_netmap_ctx *);
63 static inline uint16_t ena_netmap_count_slots(struct ena_netmap_ctx *);
64 static inline uint16_t ena_netmap_packet_len(struct netmap_slot *, u_int,
66 static int ena_netmap_copy_data(struct netmap_adapter *,
67 struct netmap_slot *, u_int, uint16_t, uint16_t, void *);
68 static int ena_netmap_map_single_slot(struct netmap_adapter *,
69 struct netmap_slot *, bus_dma_tag_t, bus_dmamap_t, void **, uint64_t *);
70 static int ena_netmap_tx_map_slots(struct ena_netmap_ctx *,
71 struct ena_tx_buffer *, void **, uint16_t *, uint16_t *);
72 static void ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *,
73 struct ena_tx_buffer *);
74 static void ena_netmap_tx_cleanup(struct ena_netmap_ctx *);
75 static uint16_t ena_netmap_tx_clean_one(struct ena_netmap_ctx *,
77 static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
78 static int ena_netmap_rx_frames(struct ena_netmap_ctx *);
79 static int ena_netmap_rx_frame(struct ena_netmap_ctx *);
80 static int ena_netmap_rx_load_desc(struct ena_netmap_ctx *, uint16_t,
82 static void ena_netmap_rx_cleanup(struct ena_netmap_ctx *);
83 static void ena_netmap_fill_ctx(struct netmap_kring *,
84 struct ena_netmap_ctx *, uint16_t);
87 ena_netmap_attach(struct ena_adapter *adapter)
89 struct netmap_adapter na;
91 ena_trace(ENA_NETMAP, "netmap attach\n");
93 bzero(&na, sizeof(na));
94 na.na_flags = NAF_MOREFRAG;
95 na.ifp = adapter->ifp;
96 na.num_tx_desc = adapter->tx_ring_size;
97 na.num_rx_desc = adapter->rx_ring_size;
98 na.num_tx_rings = adapter->num_queues;
99 na.num_rx_rings = adapter->num_queues;
100 na.rx_buf_maxsize = adapter->buf_ring_size;
101 na.nm_txsync = ena_netmap_txsync;
102 na.nm_rxsync = ena_netmap_rxsync;
103 na.nm_register = ena_netmap_reg;
105 return (netmap_attach(&na));
109 ena_netmap_alloc_rx_slot(struct ena_adapter *adapter,
110 struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info)
112 struct netmap_adapter *na = NA(adapter->ifp);
113 struct netmap_kring *kring;
114 struct netmap_ring *ring;
115 struct netmap_slot *slot;
118 int nm_i, qid, head, lim, rc;
120 /* if previously allocated frag is not used */
121 if (unlikely(rx_info->netmap_buf_idx != 0))
125 kring = na->rx_rings[qid];
126 nm_i = kring->nr_hwcur;
129 ena_trace(ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, "
130 "rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur,
131 kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail);
133 if ((nm_i == head) && rx_ring->initialized) {
134 ena_trace(ENA_NETMAP, "No free slots in netmap ring\n");
140 device_printf(adapter->pdev, "Rx ring %d is NULL\n", qid);
143 slot = &ring->slot[nm_i];
145 addr = PNMB(na, slot, &paddr);
146 if (addr == NETMAP_BUF_BASE(na)) {
147 device_printf(adapter->pdev, "Bad buff in slot\n");
151 rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr);
153 ena_trace(ENA_WARNING, "DMA mapping error\n");
156 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
158 rx_info->ena_buf.paddr = paddr;
159 rx_info->ena_buf.len = ring->nr_buf_size;
160 rx_info->mbuf = NULL;
161 rx_info->netmap_buf_idx = slot->buf_idx;
165 lim = kring->nkr_num_slots - 1;
166 kring->nr_hwcur = nm_next(nm_i, lim);
172 ena_netmap_free_rx_slot(struct ena_adapter *adapter,
173 struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info)
175 struct netmap_adapter *na;
176 struct netmap_kring *kring;
177 struct netmap_slot *slot;
180 na = NA(adapter->ifp);
182 device_printf(adapter->pdev, "netmap adapter is NULL\n");
186 if (na->rx_rings == NULL) {
187 device_printf(adapter->pdev, "netmap rings are NULL\n");
192 kring = na->rx_rings[qid];
194 device_printf(adapter->pdev,
195 "netmap kernel ring %d is NULL\n", qid);
199 lim = kring->nkr_num_slots - 1;
200 nm_i = nm_prev(kring->nr_hwcur, lim);
202 if (kring->nr_mode != NKR_NETMAP_ON)
205 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
206 BUS_DMASYNC_POSTREAD);
207 netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map);
209 KASSERT(kring->ring == NULL, ("Netmap Rx ring is NULL\n"));
211 slot = &kring->ring->slot[nm_i];
213 ENA_ASSERT(slot->buf_idx == 0, "Overwrite slot buf\n");
214 slot->buf_idx = rx_info->netmap_buf_idx;
215 slot->flags = NS_BUF_CHANGED;
217 rx_info->netmap_buf_idx = 0;
218 kring->nr_hwcur = nm_i;
222 ena_ring_in_netmap(struct ena_adapter *adapter, int qid, enum txrx x)
224 struct netmap_adapter *na;
225 struct netmap_kring *kring;
227 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
228 na = NA(adapter->ifp);
229 kring = (x == NR_RX) ? na->rx_rings[qid] : na->tx_rings[qid];
230 if (kring->nr_mode == NKR_NETMAP_ON)
237 ena_tx_ring_in_netmap(struct ena_adapter *adapter, int qid)
239 return ena_ring_in_netmap(adapter, qid, NR_TX);
243 ena_rx_ring_in_netmap(struct ena_adapter *adapter, int qid)
245 return ena_ring_in_netmap(adapter, qid, NR_RX);
249 ena_netmap_reset_ring(struct ena_adapter *adapter, int qid, enum txrx x)
251 if (!ena_ring_in_netmap(adapter, qid, x))
254 netmap_reset(NA(adapter->ifp), x, qid, 0);
255 ena_trace(ENA_NETMAP, "%s ring %d is in netmap mode\n",
256 (x == NR_TX) ? "Tx" : "Rx", qid);
260 ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid)
262 ena_netmap_reset_ring(adapter, qid, NR_RX);
266 ena_netmap_reset_tx_ring(struct ena_adapter *adapter, int qid)
268 ena_netmap_reset_ring(adapter, qid, NR_TX);
272 ena_netmap_reg(struct netmap_adapter *na, int onoff)
274 struct ifnet *ifp = na->ifp;
275 struct ena_adapter* adapter = ifp->if_softc;
276 struct netmap_kring *kring;
280 sx_xlock(&adapter->ioctl_sx);
281 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
285 ena_trace(ENA_NETMAP, "netmap on\n");
287 for (i = 0; i <= nma_get_nrings(na, t); i++) {
288 kring = NMR(na, t)[i];
289 if (nm_kring_pending_on(kring)) {
290 kring->nr_mode = NKR_NETMAP_ON;
294 nm_set_native_flags(na);
296 ena_trace(ENA_NETMAP, "netmap off\n");
297 nm_clear_native_flags(na);
299 for (i = 0; i <= nma_get_nrings(na, t); i++) {
300 kring = NMR(na, t)[i];
301 if (nm_kring_pending_off(kring)) {
302 kring->nr_mode = NKR_NETMAP_OFF;
308 rc = ena_up(adapter);
310 ena_trace(ENA_WARNING, "ena_up failed with rc=%d\n", rc);
311 adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE;
312 nm_clear_native_flags(na);
313 ena_destroy_device(adapter, false);
314 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
315 rc = ena_restore_device(adapter);
317 sx_unlock(&adapter->ioctl_sx);
323 ena_netmap_txsync(struct netmap_kring *kring, int flags)
325 struct ena_netmap_ctx ctx;
328 ena_netmap_fill_ctx(kring, &ctx, ENA_IO_TXQ_IDX(kring->ring_id));
329 ctx.ring = &ctx.adapter->tx_ring[kring->ring_id];
331 ENA_RING_MTX_LOCK(ctx.ring);
332 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, ctx.adapter)))
335 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
338 rc = ena_netmap_tx_frames(&ctx);
339 ena_netmap_tx_cleanup(&ctx);
342 ENA_RING_MTX_UNLOCK(ctx.ring);
347 ena_netmap_tx_frames(struct ena_netmap_ctx *ctx)
349 struct ena_ring *tx_ring = ctx->ring;
352 ctx->nm_i = ctx->kring->nr_hwcur;
353 ctx->nt = ctx->ring->next_to_use;
355 __builtin_prefetch(&ctx->slots[ctx->nm_i]);
357 while (ctx->nm_i != ctx->kring->rhead) {
358 if ((rc = ena_netmap_tx_frame(ctx)) != 0) {
360 * When there is no empty space in Tx ring, error is
361 * still being returned. It should not be passed to the
362 * netmap, as application knows current ring state from
363 * netmap ring pointers. Returning error there could
364 * cause application to exit, but the Tx ring is commonly
367 if (rc == ENA_COM_NO_MEM)
371 tx_ring->acum_pkts++;
374 /* If any packet was sent... */
375 if (likely(ctx->nm_i != ctx->kring->nr_hwcur)) {
377 /* ...send the doorbell to the device. */
378 ena_com_write_sq_doorbell(ctx->io_sq);
379 counter_u64_add(ctx->ring->tx_stats.doorbells, 1);
380 tx_ring->acum_pkts = 0;
382 ctx->ring->next_to_use = ctx->nt;
383 ctx->kring->nr_hwcur = ctx->nm_i;
390 ena_netmap_tx_frame(struct ena_netmap_ctx *ctx)
392 struct ena_com_tx_ctx ena_tx_ctx;
393 struct ena_adapter *adapter;
394 struct ena_ring *tx_ring;
395 struct ena_tx_buffer *tx_info;
403 adapter = ctx->adapter;
404 if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) {
405 ena_trace(ENA_WARNING, "Too many slots per packet\n");
411 req_id = tx_ring->free_tx_ids[ctx->nt];
412 tx_info = &tx_ring->tx_buffer_info[req_id];
413 tx_info->num_of_bufs = 0;
414 tx_info->nm_info.sockets_used = 0;
416 rc = ena_netmap_tx_map_slots(ctx, tx_info, &push_hdr, &header_len,
418 if (unlikely(rc != 0)) {
419 device_printf(adapter->pdev, "Failed to map Tx slot\n");
423 bzero(&ena_tx_ctx, sizeof(struct ena_com_tx_ctx));
424 ena_tx_ctx.ena_bufs = tx_info->bufs;
425 ena_tx_ctx.push_header = push_hdr;
426 ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
427 ena_tx_ctx.req_id = req_id;
428 ena_tx_ctx.header_len = header_len;
430 /* There are no any offloads, as the netmap doesn't support them */
432 if (tx_ring->acum_pkts == DB_THRESHOLD ||
433 ena_com_is_doorbell_needed(ctx->io_sq, &ena_tx_ctx)) {
435 ena_com_write_sq_doorbell(ctx->io_sq);
436 counter_u64_add(tx_ring->tx_stats.doorbells, 1);
437 tx_ring->acum_pkts = 0;
440 rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc);
441 if (unlikely(rc != 0)) {
442 if (likely(rc == ENA_COM_NO_MEM)) {
443 ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
444 "Tx ring[%d] is out of space\n", tx_ring->que->id);
446 device_printf(adapter->pdev,
447 "Failed to prepare Tx bufs\n");
449 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
451 ena_netmap_unmap_last_socket_chain(ctx, tx_info);
456 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
457 counter_u64_add_protected(tx_ring->tx_stats.bytes, packet_len);
458 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
459 counter_u64_add_protected(adapter->hw_stats.tx_bytes, packet_len);
462 tx_info->tx_descs = nb_hw_desc;
464 ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
466 for (unsigned int i = 0; i < tx_info->num_of_bufs; i++)
467 bus_dmamap_sync(adapter->tx_buf_tag,
468 tx_info->nm_info.map_seg[i], BUS_DMASYNC_PREWRITE);
473 static inline uint16_t
474 ena_netmap_count_slots(struct ena_netmap_ctx *ctx)
477 uint16_t nm = ctx->nm_i;
479 while ((ctx->slots[nm].flags & NS_MOREFRAG) != 0) {
481 nm = nm_next(nm, ctx->lim);
487 static inline uint16_t
488 ena_netmap_packet_len(struct netmap_slot *slots, u_int slot_index,
491 struct netmap_slot *nm_slot;
492 uint16_t packet_size = 0;
495 nm_slot = &slots[slot_index];
496 packet_size += nm_slot->len;
497 slot_index = nm_next(slot_index, limit);
498 } while ((nm_slot->flags & NS_MOREFRAG) != 0);
504 ena_netmap_copy_data(struct netmap_adapter *na, struct netmap_slot *slots,
505 u_int slot_index, uint16_t limit, uint16_t bytes_to_copy, void *destination)
507 struct netmap_slot *nm_slot;
509 uint16_t packet_size;
510 uint16_t data_amount;
514 nm_slot = &slots[slot_index];
515 slot_vaddr = NMB(na, nm_slot);
516 if (unlikely(slot_vaddr == NULL))
519 data_amount = min_t(uint16_t, bytes_to_copy, nm_slot->len);
520 memcpy(destination, slot_vaddr, data_amount);
521 bytes_to_copy -= data_amount;
523 slot_index = nm_next(slot_index, limit);
524 } while ((nm_slot->flags & NS_MOREFRAG) != 0 && bytes_to_copy > 0);
530 ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot,
531 bus_dma_tag_t dmatag, bus_dmamap_t dmamap, void **vaddr, uint64_t *paddr)
535 *vaddr = PNMB(na, slot, paddr);
536 if (unlikely(vaddr == NULL)) {
537 ena_trace(ENA_ALERT, "Slot address is NULL\n");
541 rc = netmap_load_map(na, dmatag, dmamap, *vaddr);
542 if (unlikely(rc != 0)) {
543 ena_trace(ENA_ALERT, "Failed to map slot %d for DMA\n",
552 ena_netmap_tx_map_slots(struct ena_netmap_ctx *ctx,
553 struct ena_tx_buffer *tx_info, void **push_hdr, uint16_t *header_len,
554 uint16_t *packet_len)
556 struct netmap_slot *slot;
557 struct ena_com_buf *ena_buf;
558 struct ena_adapter *adapter;
559 struct ena_ring *tx_ring;
560 struct ena_netmap_tx_info *nm_info;
561 bus_dmamap_t *nm_maps;
564 uint32_t *nm_buf_idx;
565 uint32_t slot_head_len;
567 uint32_t remaining_len;
572 adapter = ctx->adapter;
574 ena_buf = tx_info->bufs;
575 nm_info = &tx_info->nm_info;
576 nm_maps = nm_info->map_seg;
577 nm_buf_idx = nm_info->socket_buf_idx;
578 slot = &ctx->slots[ctx->nm_i];
580 slot_head_len = slot->len;
581 *packet_len = ena_netmap_packet_len(ctx->slots, ctx->nm_i, ctx->lim);
582 remaining_len = *packet_len;
585 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
586 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
588 * When the device is in LLQ mode, the driver will copy
589 * the header into the device memory space.
590 * The ena_com layer assumes that the header is in a linear
592 * This assumption might be wrong since part of the header
593 * can be in the fragmented buffers.
594 * First, check if header fits in the first slot. If not, copy
595 * it to separate buffer that will be holding linearized data.
597 push_len = min_t(uint32_t, *packet_len,
598 tx_ring->tx_max_header_size);
599 *header_len = push_len;
600 /* If header is in linear space, just point to socket's data. */
601 if (likely(push_len <= slot_head_len)) {
602 *push_hdr = NMB(ctx->na, slot);
603 if (unlikely(push_hdr == NULL)) {
604 device_printf(adapter->pdev,
605 "Slot vaddress is NULL\n");
609 * Otherwise, copy whole portion of header from multiple slots
610 * to intermediate buffer.
613 rc = ena_netmap_copy_data(ctx->na,
618 tx_ring->push_buf_intermediate_buf);
620 device_printf(adapter->pdev,
621 "Failed to copy data from slots to push_buf\n");
625 *push_hdr = tx_ring->push_buf_intermediate_buf;
626 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
628 delta = push_len - slot_head_len;
631 ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
632 "slot: %d header_buf->vaddr: %p push_len: %d\n",
633 slot->buf_idx, *push_hdr, push_len);
636 * If header was in linear memory space, map for the dma rest of the data
637 * in the first mbuf of the mbuf chain.
639 if (slot_head_len > push_len) {
640 rc = ena_netmap_map_single_slot(ctx->na,
646 if (unlikely(rc != 0)) {
647 device_printf(adapter->pdev,
648 "DMA mapping error\n");
653 ena_buf->paddr = paddr + push_len;
654 ena_buf->len = slot->len - push_len;
657 tx_info->num_of_bufs++;
660 remaining_len -= slot->len;
662 /* Save buf idx before advancing */
663 *nm_buf_idx = slot->buf_idx;
667 /* Advance to the next socket */
668 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
669 slot = &ctx->slots[ctx->nm_i];
670 nm_info->sockets_used++;
673 * If header is in non linear space (delta > 0), then skip mbufs
674 * containing header and map the last one containing both header
675 * and the packet data.
676 * The first segment is already counted in.
679 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
680 frag_len = slot->len;
683 * If whole segment contains header just move to the
684 * next one and reduce delta.
686 if (unlikely(delta >= frag_len)) {
690 * Map the data and then assign it with the
693 rc = ena_netmap_map_single_slot(ctx->na,
699 if (unlikely(rc != 0)) {
700 device_printf(adapter->pdev,
701 "DMA mapping error\n");
706 ena_buf->paddr = paddr + delta;
707 ena_buf->len = slot->len - delta;
710 tx_info->num_of_bufs++;
714 remaining_len -= slot->len;
716 /* Save buf idx before advancing */
717 *nm_buf_idx = slot->buf_idx;
721 /* Advance to the next socket */
722 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
723 slot = &ctx->slots[ctx->nm_i];
724 nm_info->sockets_used++;
729 * header_len is just a hint for the device. Because netmap is
730 * not giving us any information about packet header length and
731 * it is not guaranteed that all packet headers will be in the
732 * 1st slot, setting header_len to 0 is making the device ignore
733 * this value and resolve header on it's own.
738 /* Map all remaining data (regular routine for non-LLQ mode) */
739 while (remaining_len > 0) {
740 __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
742 rc = ena_netmap_map_single_slot(ctx->na,
748 if (unlikely(rc != 0)) {
749 device_printf(adapter->pdev,
750 "DMA mapping error\n");
755 ena_buf->paddr = paddr;
756 ena_buf->len = slot->len;
759 tx_info->num_of_bufs++;
761 remaining_len -= slot->len;
763 /* Save buf idx before advancing */
764 *nm_buf_idx = slot->buf_idx;
768 /* Advance to the next socket */
769 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
770 slot = &ctx->slots[ctx->nm_i];
771 nm_info->sockets_used++;
777 ena_netmap_unmap_last_socket_chain(ctx, tx_info);
783 ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *ctx,
784 struct ena_tx_buffer *tx_info)
786 struct ena_netmap_tx_info *nm_info;
789 nm_info = &tx_info->nm_info;
792 * As the used sockets must not be equal to the buffers used in the LLQ
793 * mode, they must be treated separately.
794 * First, unmap the DMA maps.
796 n = tx_info->num_of_bufs;
798 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
799 nm_info->map_seg[n]);
801 tx_info->num_of_bufs = 0;
803 /* Next, retain the sockets back to the userspace */
804 n = nm_info->sockets_used;
806 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
807 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
808 nm_info->socket_buf_idx[n] = 0;
809 ctx->nm_i = nm_prev(ctx->nm_i, ctx->lim);
811 nm_info->sockets_used = 0;
815 ena_netmap_tx_cleanup(struct ena_netmap_ctx *ctx)
818 uint16_t total_tx_descs = 0;
820 ctx->nm_i = ctx->kring->nr_hwtail;
821 ctx->nt = ctx->ring->next_to_clean;
823 /* Reclaim buffers for completed transmissions */
824 while (ena_com_tx_comp_req_id_get(ctx->io_cq, &req_id) >= 0) {
825 if (validate_tx_req_id(ctx->ring, req_id) != 0)
827 total_tx_descs += ena_netmap_tx_clean_one(ctx, req_id);
830 ctx->kring->nr_hwtail = ctx->nm_i;
832 if (total_tx_descs > 0) {
833 /* acknowledge completion of sent packets */
834 ctx->ring->next_to_clean = ctx->nt;
835 ena_com_comp_ack(ctx->ring->ena_com_io_sq, total_tx_descs);
836 ena_com_update_dev_comp_head(ctx->ring->ena_com_io_cq);
841 ena_netmap_tx_clean_one(struct ena_netmap_ctx *ctx, uint16_t req_id)
843 struct ena_tx_buffer *tx_info;
844 struct ena_netmap_tx_info *nm_info;
847 tx_info = &ctx->ring->tx_buffer_info[req_id];
848 nm_info = &tx_info->nm_info;
851 * As the used sockets must not be equal to the buffers used in the LLQ
852 * mode, they must be treated separately.
853 * First, unmap the DMA maps.
855 n = tx_info->num_of_bufs;
856 for (n = 0; n < tx_info->num_of_bufs; n++) {
857 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
858 nm_info->map_seg[n]);
860 tx_info->num_of_bufs = 0;
862 /* Next, retain the sockets back to the userspace */
863 for (n = 0; n < nm_info->sockets_used; n++) {
864 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
865 ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0,
866 "Tx idx is not 0.\n");
867 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
868 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
869 nm_info->socket_buf_idx[n] = 0;
871 nm_info->sockets_used = 0;
873 ctx->ring->free_tx_ids[ctx->nt] = req_id;
874 ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->lim);
876 return tx_info->tx_descs;
880 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
882 struct ena_adapter *adapter = tx_ring->adapter;
884 if (likely(req_id < tx_ring->ring_size))
887 ena_trace(ENA_WARNING, "Invalid req_id: %hu\n", req_id);
888 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
890 adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
891 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
897 ena_netmap_rxsync(struct netmap_kring *kring, int flags)
899 struct ena_netmap_ctx ctx;
902 ena_netmap_fill_ctx(kring, &ctx, ENA_IO_RXQ_IDX(kring->ring_id));
903 ctx.ring = &ctx.adapter->rx_ring[kring->ring_id];
905 if (ctx.kring->rhead > ctx.lim) {
906 /* Probably not needed to release slots from RX ring. */
907 return (netmap_ring_reinit(ctx.kring));
910 if (unlikely((if_getdrvflags(ctx.na->ifp) & IFF_DRV_RUNNING) == 0))
913 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
916 if ((rc = ena_netmap_rx_frames(&ctx)) != 0)
919 ena_netmap_rx_cleanup(&ctx);
925 ena_netmap_rx_frames(struct ena_netmap_ctx *ctx)
928 int frames_counter = 0;
930 ctx->nt = ctx->ring->next_to_clean;
931 ctx->nm_i = ctx->kring->nr_hwtail;
933 while((rc = ena_netmap_rx_frame(ctx)) == ENA_NETMAP_MORE_FRAMES) {
935 /* In case of multiple frames, it is not an error. */
937 if (frames_counter > ENA_MAX_FRAMES) {
938 device_printf(ctx->adapter->pdev,
939 "Driver is stuck in the Rx loop\n");
944 ctx->kring->nr_hwtail = ctx->nm_i;
945 ctx->kring->nr_kflags &= ~NKR_PENDINTR;
946 ctx->ring->next_to_clean = ctx->nt;
952 ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
954 struct ena_com_rx_ctx ena_rx_ctx;
958 ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs;
959 ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size;
960 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
961 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
963 rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx);
964 if (unlikely(rc != 0)) {
965 ena_trace(ENA_ALERT, "Too many desc from the device.\n");
966 counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
967 ctx->adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
968 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, ctx->adapter);
971 if (unlikely(ena_rx_ctx.descs == 0))
972 return (ENA_NETMAP_NO_MORE_FRAMES);
974 ena_trace(ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:"
975 " %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid,
976 ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto,
979 for (buf = 0; buf < ena_rx_ctx.descs; buf++)
980 if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0)
983 * ena_netmap_rx_load_desc doesn't know the number of descriptors.
984 * It just set flag NS_MOREFRAG to all slots, then here flag of
985 * last slot is cleared.
987 ctx->slots[nm_prev(ctx->nm_i, ctx->lim)].flags = NS_BUF_CHANGED;
993 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
994 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
997 counter_u64_add_protected(ctx->ring->rx_stats.bytes, len);
998 counter_u64_add_protected(ctx->adapter->hw_stats.rx_bytes, len);
999 counter_u64_add_protected(ctx->ring->rx_stats.cnt, 1);
1000 counter_u64_add_protected(ctx->adapter->hw_stats.rx_packets, 1);
1003 return (ENA_NETMAP_MORE_FRAMES);
1008 /* Remove failed packet from ring */
1010 ctx->slots[nm].flags = 0;
1011 ctx->slots[nm].len = 0;
1012 nm = nm_prev(nm, ctx->lim);
1019 ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len)
1021 struct ena_rx_buffer *rx_info;
1025 req_id = ctx->ring->ena_bufs[buf].req_id;
1026 rc = validate_rx_req_id(ctx->ring, req_id);
1027 if (unlikely(rc != 0))
1030 rx_info = &ctx->ring->rx_buffer_info[req_id];
1031 bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map,
1032 BUS_DMASYNC_POSTREAD);
1033 netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map);
1035 ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, "Rx idx is not 0.\n");
1037 ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx;
1038 rx_info->netmap_buf_idx = 0;
1040 * Set NS_MOREFRAG to all slots.
1041 * Then ena_netmap_rx_frame clears it from last one.
1043 ctx->slots[ctx->nm_i].flags |= NS_MOREFRAG | NS_BUF_CHANGED;
1044 ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len;
1045 *len += ctx->slots[ctx->nm_i].len;
1046 ctx->ring->free_rx_ids[ctx->nt] = req_id;
1047 ena_trace(ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n",
1048 rx_info, ctx->slots[ctx->nm_i].buf_idx,
1049 (uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i);
1051 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
1052 ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
1058 ena_netmap_rx_cleanup(struct ena_netmap_ctx *ctx)
1060 int refill_required;
1062 refill_required = ctx->kring->rhead - ctx->kring->nr_hwcur;
1063 if (ctx->kring->nr_hwcur != ctx->kring->nr_hwtail)
1064 refill_required -= 1;
1066 if (refill_required == 0)
1068 else if (refill_required < 0)
1069 refill_required += ctx->kring->nkr_num_slots;
1071 ena_refill_rx_bufs(ctx->ring, refill_required);
1075 ena_netmap_fill_ctx(struct netmap_kring *kring, struct ena_netmap_ctx *ctx,
1079 ctx->na = kring->na;
1080 ctx->adapter = ctx->na->ifp->if_softc;
1081 ctx->lim = kring->nkr_num_slots - 1;
1082 ctx->io_cq = &ctx->adapter->ena_dev->io_cq_queues[ena_qid];
1083 ctx->io_sq = &ctx->adapter->ena_dev->io_sq_queues[ena_qid];
1084 ctx->slots = kring->ring->slot;
1088 ena_netmap_unload(struct ena_adapter *adapter, bus_dmamap_t map)
1090 struct netmap_adapter *na = NA(adapter->ifp);
1092 netmap_unload_map(na, adapter->tx_buf_tag, map);
1095 #endif /* DEV_NETMAP */