1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright (c) 2021, Intel Corporation
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the Intel Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
34 * @file ice_iflib_txrx.c
35 * @brief iflib Tx/Rx hotpath
37 * Main location for the iflib Tx/Rx hotpath implementation.
39 * Contains the implementation for the iflib function callbacks and the
40 * if_txrx ops structure.
43 #include "ice_iflib.h"
45 /* Tx/Rx hotpath utility functions */
46 #include "ice_common_txrx.h"
49 * iflib txrx method declarations
51 static int ice_ift_txd_encap(void *arg, if_pkt_info_t pi);
52 static int ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri);
53 static void ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
54 static int ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear);
55 static int ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget);
56 static void ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx, qidx_t pidx);
57 static void ice_ift_rxd_refill(void *arg, if_rxd_update_t iru);
58 static qidx_t ice_ift_queue_select(void *arg, struct mbuf *m);
60 /* Macro to help extract the NIC mode flexible Rx descriptor fields from the
61 * advanced 32byte Rx descriptors.
63 #define RX_FLEX_NIC(desc, field) \
64 (((struct ice_32b_rx_flex_desc_nic *)desc)->field)
68 * @brief Tx/Rx operations for the iflib stack
70 * Structure defining the Tx and Rx related operations that iflib can request
71 * the driver to perform. These are the main entry points for the hot path of
72 * the transmit and receive paths in the iflib driver.
74 struct if_txrx ice_txrx = {
75 .ift_txd_encap = ice_ift_txd_encap,
76 .ift_txd_flush = ice_ift_txd_flush,
77 .ift_txd_credits_update = ice_ift_txd_credits_update,
78 .ift_rxd_available = ice_ift_rxd_available,
79 .ift_rxd_pkt_get = ice_ift_rxd_pkt_get,
80 .ift_rxd_refill = ice_ift_rxd_refill,
81 .ift_rxd_flush = ice_ift_rxd_flush,
82 .ift_txq_select = ice_ift_queue_select,
86 * ice_ift_txd_encap - prepare Tx descriptors for a packet
87 * @arg: the iflib softc structure pointer
90 * Prepares and encapsulates the given packet into into Tx descriptors, in
91 * preparation for sending to the transmit engine. Sets the necessary context
92 * descriptors for TSO and other offloads, and prepares the last descriptor
93 * for the writeback status.
95 * Return 0 on success, non-zero error code on failure.
98 ice_ift_txd_encap(void *arg, if_pkt_info_t pi)
100 struct ice_softc *sc = (struct ice_softc *)arg;
101 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx];
102 int nsegs = pi->ipi_nsegs;
103 bus_dma_segment_t *segs = pi->ipi_segs;
104 struct ice_tx_desc *txd = NULL;
105 int i, j, mask, pidx_last;
111 /* Set up the TSO/CSUM offload */
112 if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) {
113 /* Set up the TSO context descriptor if required */
114 if (pi->ipi_csum_flags & CSUM_TSO) {
115 if (ice_tso_detect_sparse(pi))
117 i = ice_tso_setup(txq, pi);
119 ice_tx_setup_offload(txq, pi, &cmd, &off);
121 if (pi->ipi_mflags & M_VLANTAG)
122 cmd |= ICE_TX_DESC_CMD_IL2TAG1;
124 mask = txq->desc_count - 1;
125 for (j = 0; j < nsegs; j++) {
128 txd = &txq->tx_base[i];
129 seglen = segs[j].ds_len;
131 txd->buf_addr = htole64(segs[j].ds_addr);
132 txd->cmd_type_offset_bsz =
133 htole64(ICE_TX_DESC_DTYPE_DATA
134 | ((u64)cmd << ICE_TXD_QW1_CMD_S)
135 | ((u64)off << ICE_TXD_QW1_OFFSET_S)
136 | ((u64)seglen << ICE_TXD_QW1_TX_BUF_SZ_S)
137 | ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S));
139 txq->stats.tx_bytes += seglen;
144 /* Set the last descriptor for report */
145 #define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS)
146 txd->cmd_type_offset_bsz |=
147 htole64(((u64)ICE_TXD_CMD << ICE_TXD_QW1_CMD_S));
149 /* Add to report status array */
150 txq->tx_rsq[txq->tx_rs_pidx] = pidx_last;
151 txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask;
152 MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx);
154 pi->ipi_new_pidx = i;
156 ++txq->stats.tx_packets;
161 * ice_ift_txd_flush - Flush Tx descriptors to hardware
162 * @arg: device specific softc pointer
163 * @txqid: the Tx queue to flush
164 * @pidx: descriptor index to advance tail to
166 * Advance the Transmit Descriptor Tail (TDT). This indicates to hardware that
167 * frames are available for transmit.
170 ice_ift_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
172 struct ice_softc *sc = (struct ice_softc *)arg;
173 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];
174 struct ice_hw *hw = &sc->hw;
176 wr32(hw, txq->tail, pidx);
180 * ice_ift_txd_credits_update - cleanup Tx descriptors
181 * @arg: device private softc
182 * @txqid: the Tx queue to update
183 * @clear: if false, only report, do not actually clean
185 * If clear is false, iflib is asking if we *could* clean up any Tx
188 * If clear is true, iflib is requesting to cleanup and reclaim used Tx
192 ice_ift_txd_credits_update(void *arg, uint16_t txqid, bool clear)
194 struct ice_softc *sc = (struct ice_softc *)arg;
195 struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];
197 qidx_t processed = 0;
198 qidx_t cur, prev, ntxd, rs_cidx;
202 rs_cidx = txq->tx_rs_cidx;
203 if (rs_cidx == txq->tx_rs_pidx)
205 cur = txq->tx_rsq[rs_cidx];
206 MPASS(cur != QIDX_INVALID);
207 is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);
211 else if (clear == false)
214 prev = txq->tx_cidx_processed;
215 ntxd = txq->desc_count;
218 delta = (int32_t)cur - (int32_t)prev;
224 rs_cidx = (rs_cidx + 1) & (ntxd-1);
225 if (rs_cidx == txq->tx_rs_pidx)
227 cur = txq->tx_rsq[rs_cidx];
228 MPASS(cur != QIDX_INVALID);
229 is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);
232 txq->tx_rs_cidx = rs_cidx;
233 txq->tx_cidx_processed = prev;
239 * ice_ift_rxd_available - Return number of available Rx packets
240 * @arg: device private softc
241 * @rxqid: the Rx queue id
242 * @pidx: descriptor start point
243 * @budget: maximum Rx budget
245 * Determines how many Rx packets are available on the queue, up to a maximum
246 * of the given budget.
249 ice_ift_rxd_available(void *arg, uint16_t rxqid, qidx_t pidx, qidx_t budget)
251 struct ice_softc *sc = (struct ice_softc *)arg;
252 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
253 union ice_32b_rx_flex_desc *rxd;
257 nrxd = rxq->desc_count;
259 for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) {
260 rxd = &rxq->rx_base[i];
261 status0 = le16toh(rxd->wb.status_error0);
263 if ((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) == 0)
267 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S))
275 * ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer
276 * @arg: device specific softc
277 * @ri: receive packet info
279 * This function is called by iflib, and executes in ithread context. It is
280 * called by iflib to obtain data which has been DMA'ed into host memory.
281 * Returns zero on success, and EBADMSG on failure.
284 ice_ift_rxd_pkt_get(void *arg, if_rxd_info_t ri)
286 struct ice_softc *sc = (struct ice_softc *)arg;
287 if_softc_ctx_t scctx = sc->scctx;
288 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx];
289 union ice_32b_rx_flex_desc *cur;
290 u16 status0, plen, ptype;
298 /* 5 descriptor receive limit */
299 MPASS(i < ICE_MAX_RX_SEGS);
301 cur = &rxq->rx_base[cidx];
302 status0 = le16toh(cur->wb.status_error0);
303 plen = le16toh(cur->wb.pkt_len) &
304 ICE_RX_FLX_DESC_PKT_LEN_M;
306 /* we should never be called without a valid descriptor */
307 MPASS((status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S)) != 0);
311 cur->wb.status_error0 = 0;
312 eop = (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_EOF_S));
314 ri->iri_frags[i].irf_flid = 0;
315 ri->iri_frags[i].irf_idx = cidx;
316 ri->iri_frags[i].irf_len = plen;
317 if (++cidx == rxq->desc_count)
322 /* End of Packet reached; cur is eop/last descriptor */
324 /* Make sure packets with bad L2 values are discarded.
325 * This bit is only valid in the last descriptor.
327 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S)) {
328 rxq->stats.desc_errs++;
332 /* Get VLAN tag information if one is in descriptor */
333 if (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
334 ri->iri_vtag = le16toh(cur->wb.l2tag1);
335 ri->iri_flags |= M_VLANTAG;
338 /* Capture soft statistics for this Rx queue */
339 rxq->stats.rx_packets++;
340 rxq->stats.rx_bytes += ri->iri_len;
342 /* Get packet type and set checksum flags */
343 ptype = le16toh(cur->wb.ptype_flex_flags0) &
344 ICE_RX_FLEX_DESC_PTYPE_M;
345 if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
346 ice_rx_checksum(rxq, &ri->iri_csum_flags,
347 &ri->iri_csum_data, status0, ptype);
349 /* Set remaining iflib RX descriptor info fields */
350 ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash));
351 ri->iri_rsstype = ice_ptype_to_hash(ptype);
357 * ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware
358 * @arg: device specific softc structure
359 * @iru: the Rx descriptor update structure
361 * Update the Rx descriptor indices for a given queue, assigning new physical
362 * addresses to the descriptors, preparing them for re-use by the hardware.
365 ice_ift_rxd_refill(void *arg, if_rxd_update_t iru)
367 struct ice_softc *sc = (struct ice_softc *)arg;
368 struct ice_rx_queue *rxq;
373 uint16_t qsidx, count;
375 paddrs = iru->iru_paddrs;
376 pidx = iru->iru_pidx;
377 qsidx = iru->iru_qsidx;
378 count = iru->iru_count;
380 rxq = &(sc->pf_vsi.rx_queues[qsidx]);
382 for (i = 0, next_pidx = pidx; i < count; i++) {
383 rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]);
384 if (++next_pidx == (uint32_t)rxq->desc_count)
390 * ice_ift_rxd_flush - Flush Rx descriptors to hardware
391 * @arg: device specific softc pointer
392 * @rxqid: the Rx queue to flush
393 * @flidx: unused parameter
394 * @pidx: descriptor index to advance tail to
396 * Advance the Receive Descriptor Tail (RDT). This indicates to hardware that
397 * software is done with the descriptor and it can be recycled.
400 ice_ift_rxd_flush(void *arg, uint16_t rxqid, uint8_t flidx __unused,
403 struct ice_softc *sc = (struct ice_softc *)arg;
404 struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];
405 struct ice_hw *hw = &sc->hw;
407 wr32(hw, rxq->tail, pidx);
411 ice_ift_queue_select(void *arg, struct mbuf *m)
413 struct ice_softc *sc = (struct ice_softc *)arg;
414 struct ice_vsi *vsi = &sc->pf_vsi;
415 u16 tc_base_queue, tc_qcount;
419 /* Included to match default iflib behavior */
420 /* Only go out on default queue if ALTQ is enabled */
421 struct ifnet *ifp = (struct ifnet *)iflib_get_ifp(sc->ctx);
422 if (ALTQ_IS_ENABLED(&ifp->if_snd))
426 if (!ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS)) {
427 if (M_HASHTYPE_GET(m)) {
428 /* Default iflib queue selection method */
429 return (m->m_pkthdr.flowid % sc->pf_vsi.num_tx_queues);
434 /* Use default TC unless overridden */
435 tc = 0; /* XXX: Get default TC for traffic if >1 TC? */
437 if (m->m_flags & M_VLANTAG) {
438 up = EVL_PRIOFTAG(m->m_pkthdr.ether_vtag);
439 tc = sc->hw.port_info->qos_cfg.local_dcbx_cfg.etscfg.prio_table[up];
442 tc_base_queue = vsi->tc_info[tc].qoffset;
443 tc_qcount = vsi->tc_info[tc].qcount_tx;
445 if (M_HASHTYPE_GET(m))
446 return ((m->m_pkthdr.flowid % tc_qcount) + tc_base_queue);
448 return (tc_base_queue);