2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org>
5 * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 #include <net/rss_config.h>
35 #include <netinet/in_rss.h>
39 #define DPRINTF device_printf
44 /*********************************************************************
45 * Local Function prototypes
46 *********************************************************************/
47 static int em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi,
48 uint32_t *txd_upper, uint32_t *txd_lower);
49 static int em_transmit_checksum_setup(struct e1000_softc *sc,
50 if_pkt_info_t pi, uint32_t *txd_upper, uint32_t *txd_lower);
51 static int em_isc_txd_encap(void *arg, if_pkt_info_t pi);
52 static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
53 static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
54 static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru);
55 static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
57 static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
59 static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
61 static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru);
63 static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
65 static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
67 static void em_receive_checksum(uint16_t, uint8_t, if_rxd_info_t);
68 static int em_determine_rsstype(uint32_t pkt_info);
69 extern int em_intr(void *arg);
71 struct if_txrx em_txrx = {
72 .ift_txd_encap = em_isc_txd_encap,
73 .ift_txd_flush = em_isc_txd_flush,
74 .ift_txd_credits_update = em_isc_txd_credits_update,
75 .ift_rxd_available = em_isc_rxd_available,
76 .ift_rxd_pkt_get = em_isc_rxd_pkt_get,
77 .ift_rxd_refill = em_isc_rxd_refill,
78 .ift_rxd_flush = em_isc_rxd_flush,
79 .ift_legacy_intr = em_intr
82 struct if_txrx lem_txrx = {
83 .ift_txd_encap = em_isc_txd_encap,
84 .ift_txd_flush = em_isc_txd_flush,
85 .ift_txd_credits_update = em_isc_txd_credits_update,
86 .ift_rxd_available = lem_isc_rxd_available,
87 .ift_rxd_pkt_get = lem_isc_rxd_pkt_get,
88 .ift_rxd_refill = lem_isc_rxd_refill,
89 .ift_rxd_flush = em_isc_rxd_flush,
90 .ift_legacy_intr = em_intr
93 extern if_shared_ctx_t em_sctx;
96 em_dump_rs(struct e1000_softc *sc)
98 if_softc_ctx_t scctx = sc->shared;
99 struct em_tx_queue *que;
101 qidx_t i, ntxd, qid, cur;
106 ntxd = scctx->isc_ntxd[0];
107 for (qid = 0; qid < sc->tx_num_queues; qid++) {
108 que = &sc->tx_queues[qid];
110 rs_cidx = txr->tx_rs_cidx;
111 if (rs_cidx != txr->tx_rs_pidx) {
112 cur = txr->tx_rsq[rs_cidx];
113 status = txr->tx_base[cur].upper.fields.status;
114 if (!(status & E1000_TXD_STAT_DD))
115 printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur);
117 rs_cidx = (rs_cidx-1)&(ntxd-1);
118 cur = txr->tx_rsq[rs_cidx];
119 printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur);
121 printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed,
123 for (i = 0; i < ntxd; i++) {
124 if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD)
125 printf("%d set ", i);
131 /**********************************************************************
133 * Setup work for hardware segmentation offload (TSO) on
134 * adapters using advanced tx descriptors
136 **********************************************************************/
138 em_tso_setup(struct e1000_softc *sc, if_pkt_info_t pi, uint32_t *txd_upper,
141 if_softc_ctx_t scctx = sc->shared;
142 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
143 struct tx_ring *txr = &que->txr;
144 struct e1000_hw *hw = &sc->hw;
145 struct e1000_context_desc *TXD;
147 uint32_t cmd_type_len;
149 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
150 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
151 E1000_TXD_DTYP_D | /* Data descr type */
152 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
154 /* IP and/or TCP header checksum calculation and insertion. */
155 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
158 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
161 * Start offset for header checksum calculation.
162 * End offset for header checksum calculation.
163 * Offset of place put the checksum.
165 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
166 TXD->lower_setup.ip_fields.ipcse =
167 htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
168 TXD->lower_setup.ip_fields.ipcso =
169 pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
172 * Start offset for payload checksum calculation.
173 * End offset for payload checksum calculation.
174 * Offset of place to put the checksum.
176 TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
177 TXD->upper_setup.tcp_fields.tucse = 0;
178 TXD->upper_setup.tcp_fields.tucso =
179 pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum);
182 * Payload size per packet w/o any headers.
183 * Length of all headers up to payload.
185 TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz);
186 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
189 * 8254x SDM4.0 page 45, and PCIe GbE SDM2.5 page 63
190 * - Set up basic TUCMDs
191 * - Enable IP bit on 82544
192 * - For others IP bit on indicates IPv4, while off indicates IPv6
194 cmd_type_len = sc->txd_cmd |
195 E1000_TXD_CMD_DEXT | /* Extended descr */
196 E1000_TXD_CMD_TSE | /* TSE context */
197 E1000_TXD_CMD_TCP; /* Do TCP checksum */
198 if (hw->mac.type == e1000_82544)
199 cmd_type_len |= E1000_TXD_CMD_IP;
200 else if (pi->ipi_etype == ETHERTYPE_IP)
201 cmd_type_len |= E1000_TXD_CMD_IP;
202 TXD->cmd_and_length = htole32(cmd_type_len |
203 (pi->ipi_len - hdr_len)); /* Total len */
207 if (++cur == scctx->isc_ntxd[0]) {
210 DPRINTF(iflib_get_dev(sc->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__,
215 #define TSO_WORKAROUND 4
216 #define DONT_FORCE_CTX 1
219 /*********************************************************************
220 * The offload context is protocol specific (TCP/UDP) and thus
221 * only needs to be set when the protocol changes. The occasion
222 * of a context change can be a performance detriment, and
223 * might be better just disabled. The reason arises in the way
224 * in which the controller supports pipelined requests from the
225 * Tx data DMA. Up to four requests can be pipelined, and they may
226 * belong to the same packet or to multiple packets. However all
227 * requests for one packet are issued before a request is issued
228 * for a subsequent packet and if a request for the next packet
229 * requires a context change, that request will be stalled
230 * until the previous request completes. This means setting up
231 * a new context effectively disables pipelined Tx data DMA which
232 * in turn greatly slow down performance to send small sized
234 **********************************************************************/
237 em_transmit_checksum_setup(struct e1000_softc *sc, if_pkt_info_t pi,
238 uint32_t *txd_upper, uint32_t *txd_lower)
240 struct e1000_context_desc *TXD = NULL;
241 if_softc_ctx_t scctx = sc->shared;
242 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
243 struct tx_ring *txr = &que->txr;
244 int csum_flags = pi->ipi_csum_flags;
249 hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
253 * The 82574L can only remember the *last* context used
254 * regardless of queue that it was use for. We cannot reuse
255 * contexts on this hardware platform and must generate a new
256 * context every time. 82574L hardware spec, section 7.2.6,
259 if (DONT_FORCE_CTX &&
260 sc->tx_num_queues == 1 &&
261 txr->csum_lhlen == pi->ipi_ehdrlen &&
262 txr->csum_iphlen == pi->ipi_ip_hlen &&
263 txr->csum_flags == csum_flags) {
265 * Same csum offload context as the previous packets;
268 *txd_upper = txr->csum_txd_upper;
269 *txd_lower = txr->csum_txd_lower;
273 TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
274 if (csum_flags & CSUM_IP) {
275 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
277 * Start offset for header checksum calculation.
278 * End offset for header checksum calculation.
279 * Offset of place to put the checksum.
281 TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen;
282 TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len);
283 TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen +
284 offsetof(struct ip, ip_sum);
285 cmd |= E1000_TXD_CMD_IP;
288 if (csum_flags & (CSUM_TCP|CSUM_UDP)) {
291 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
292 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
294 if (csum_flags & CSUM_TCP) {
295 tucso = hdr_len + offsetof(struct tcphdr, th_sum);
296 cmd |= E1000_TXD_CMD_TCP;
298 tucso = hdr_len + offsetof(struct udphdr, uh_sum);
299 TXD->upper_setup.tcp_fields.tucss = hdr_len;
300 TXD->upper_setup.tcp_fields.tucse = htole16(0);
301 TXD->upper_setup.tcp_fields.tucso = tucso;
304 txr->csum_lhlen = pi->ipi_ehdrlen;
305 txr->csum_iphlen = pi->ipi_ip_hlen;
306 txr->csum_flags = csum_flags;
307 txr->csum_txd_upper = *txd_upper;
308 txr->csum_txd_lower = *txd_lower;
310 TXD->tcp_seg_setup.data = htole32(0);
311 TXD->cmd_and_length =
312 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
314 if (++cur == scctx->isc_ntxd[0]) {
317 DPRINTF(iflib_get_dev(sc->ctx),
318 "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n",
319 csum_flags, *txd_upper, *txd_lower, hdr_len, cmd);
324 em_isc_txd_encap(void *arg, if_pkt_info_t pi)
326 struct e1000_softc *sc = arg;
327 if_softc_ctx_t scctx = sc->shared;
328 struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
329 struct tx_ring *txr = &que->txr;
330 bus_dma_segment_t *segs = pi->ipi_segs;
331 int nsegs = pi->ipi_nsegs;
332 int csum_flags = pi->ipi_csum_flags;
333 int i, j, first, pidx_last;
334 uint32_t txd_flags, txd_upper = 0, txd_lower = 0;
336 struct e1000_tx_desc *ctxd = NULL;
337 bool do_tso, tso_desc;
340 txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0;
341 i = first = pi->ipi_pidx;
342 do_tso = (csum_flags & CSUM_TSO);
344 ntxd = scctx->isc_ntxd[0];
346 * TSO Hardware workaround, if this packet is not
347 * TSO, and is only a single descriptor long, and
348 * it follows a TSO burst, then we need to add a
349 * sentinel descriptor to prevent premature writeback.
351 if ((!do_tso) && (txr->tx_tso == true)) {
357 /* Do hardware assists */
359 i = em_tso_setup(sc, pi, &txd_upper, &txd_lower);
361 } else if (csum_flags & EM_CSUM_OFFLOAD) {
362 i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower);
365 if (pi->ipi_mflags & M_VLANTAG) {
366 /* Set the vlan id. */
367 txd_upper |= htole16(pi->ipi_vtag) << 16;
368 /* Tell hardware to add tag */
369 txd_lower |= htole32(E1000_TXD_CMD_VLE);
372 DPRINTF(iflib_get_dev(sc->ctx),
373 "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
374 /* XXX sc->pcix_82544 -- lem_fill_descriptors */
376 /* Set up our transmit descriptors */
377 for (j = 0; j < nsegs; j++) {
382 ctxd = &txr->tx_base[i];
383 seg_addr = segs[j].ds_addr;
384 seg_len = segs[j].ds_len;
385 cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd;
389 * If this is the last descriptor, we want to
390 * split it so we have a small final sentinel
392 if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
393 seg_len -= TSO_WORKAROUND;
394 ctxd->buffer_addr = htole64(seg_addr);
395 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
396 ctxd->upper.data = htole32(txd_upper);
398 if (++i == scctx->isc_ntxd[0])
401 /* Now make the sentinel */
402 ctxd = &txr->tx_base[i];
403 ctxd->buffer_addr = htole64(seg_addr + seg_len);
404 ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND);
405 ctxd->upper.data = htole32(txd_upper);
407 if (++i == scctx->isc_ntxd[0])
409 DPRINTF(iflib_get_dev(sc->ctx),
410 "TSO path pidx_last=%d i=%d ntxd[0]=%d\n",
411 pidx_last, i, scctx->isc_ntxd[0]);
413 ctxd->buffer_addr = htole64(seg_addr);
414 ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
415 ctxd->upper.data = htole32(txd_upper);
417 if (++i == scctx->isc_ntxd[0])
419 DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n",
420 pidx_last, i, scctx->isc_ntxd[0]);
425 * Last Descriptor of Packet
426 * needs End Of Packet (EOP)
427 * and Report Status (RS)
429 if (txd_flags && nsegs) {
430 txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
431 DPRINTF(iflib_get_dev(sc->ctx),
432 "setting to RS on %d rs_pidx %d first: %d\n",
433 pidx_last, txr->tx_rs_pidx, first);
434 txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
435 MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
437 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags);
438 DPRINTF(iflib_get_dev(sc->ctx),
439 "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i);
440 pi->ipi_new_pidx = i;
446 em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
448 struct e1000_softc *sc = arg;
449 struct em_tx_queue *que = &sc->tx_queues[txqid];
450 struct tx_ring *txr = &que->txr;
452 E1000_WRITE_REG(&sc->hw, E1000_TDT(txr->me), pidx);
456 em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
458 struct e1000_softc *sc = arg;
459 if_softc_ctx_t scctx = sc->shared;
460 struct em_tx_queue *que = &sc->tx_queues[txqid];
461 struct tx_ring *txr = &que->txr;
463 qidx_t processed = 0;
465 qidx_t cur, prev, ntxd, rs_cidx;
469 rs_cidx = txr->tx_rs_cidx;
470 if (rs_cidx == txr->tx_rs_pidx)
472 cur = txr->tx_rsq[rs_cidx];
473 MPASS(cur != QIDX_INVALID);
474 status = txr->tx_base[cur].upper.fields.status;
475 updated = !!(status & E1000_TXD_STAT_DD);
480 /* If clear is false just let caller know that there
481 * are descriptors to reclaim */
485 prev = txr->tx_cidx_processed;
486 ntxd = scctx->isc_ntxd[0];
489 delta = (int32_t)cur - (int32_t)prev;
493 DPRINTF(iflib_get_dev(sc->ctx),
494 "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n",
495 __FUNCTION__, prev, cur, clear, delta);
499 rs_cidx = (rs_cidx + 1) & (ntxd-1);
500 if (rs_cidx == txr->tx_rs_pidx)
502 cur = txr->tx_rsq[rs_cidx];
503 MPASS(cur != QIDX_INVALID);
504 status = txr->tx_base[cur].upper.fields.status;
505 } while ((status & E1000_TXD_STAT_DD));
507 txr->tx_rs_cidx = rs_cidx;
508 txr->tx_cidx_processed = prev;
513 lem_isc_rxd_refill(void *arg, if_rxd_update_t iru)
515 struct e1000_softc *sc = arg;
516 if_softc_ctx_t scctx = sc->shared;
517 struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx];
518 struct rx_ring *rxr = &que->rxr;
519 struct e1000_rx_desc *rxd;
521 uint32_t next_pidx, pidx;
525 paddrs = iru->iru_paddrs;
526 pidx = iru->iru_pidx;
527 count = iru->iru_count;
529 for (i = 0, next_pidx = pidx; i < count; i++) {
530 rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
531 rxd->buffer_addr = htole64(paddrs[i]);
532 /* status bits must be cleared */
535 if (++next_pidx == scctx->isc_nrxd[0])
541 em_isc_rxd_refill(void *arg, if_rxd_update_t iru)
543 struct e1000_softc *sc = arg;
544 if_softc_ctx_t scctx = sc->shared;
545 uint16_t rxqid = iru->iru_qsidx;
546 struct em_rx_queue *que = &sc->rx_queues[rxqid];
547 struct rx_ring *rxr = &que->rxr;
548 union e1000_rx_desc_extended *rxd;
550 uint32_t next_pidx, pidx;
554 paddrs = iru->iru_paddrs;
555 pidx = iru->iru_pidx;
556 count = iru->iru_count;
558 for (i = 0, next_pidx = pidx; i < count; i++) {
559 rxd = &rxr->rx_base[next_pidx];
560 rxd->read.buffer_addr = htole64(paddrs[i]);
561 /* DD bits must be cleared */
562 rxd->wb.upper.status_error = 0;
564 if (++next_pidx == scctx->isc_nrxd[0])
570 em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
573 struct e1000_softc *sc = arg;
574 struct em_rx_queue *que = &sc->rx_queues[rxqid];
575 struct rx_ring *rxr = &que->rxr;
577 E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
581 lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
583 struct e1000_softc *sc = arg;
584 if_softc_ctx_t scctx = sc->shared;
585 struct em_rx_queue *que = &sc->rx_queues[rxqid];
586 struct rx_ring *rxr = &que->rxr;
587 struct e1000_rx_desc *rxd;
588 uint32_t staterr = 0;
591 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
592 rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
593 staterr = rxd->status;
595 if ((staterr & E1000_RXD_STAT_DD) == 0)
597 if (++i == scctx->isc_nrxd[0])
599 if (staterr & E1000_RXD_STAT_EOP)
606 em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
608 struct e1000_softc *sc = arg;
609 if_softc_ctx_t scctx = sc->shared;
610 struct em_rx_queue *que = &sc->rx_queues[rxqid];
611 struct rx_ring *rxr = &que->rxr;
612 union e1000_rx_desc_extended *rxd;
613 uint32_t staterr = 0;
616 for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
617 rxd = &rxr->rx_base[i];
618 staterr = le32toh(rxd->wb.upper.status_error);
620 if ((staterr & E1000_RXD_STAT_DD) == 0)
622 if (++i == scctx->isc_nrxd[0])
624 if (staterr & E1000_RXD_STAT_EOP)
631 lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
633 struct e1000_softc *sc = arg;
634 if_softc_ctx_t scctx = sc->shared;
635 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
636 struct rx_ring *rxr = &que->rxr;
637 struct e1000_rx_desc *rxd;
639 uint32_t status, errors;
643 status = errors = i = 0;
647 rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx];
648 status = rxd->status;
649 errors = rxd->errors;
651 /* Error Checking then decrement count */
652 MPASS ((status & E1000_RXD_STAT_DD) != 0);
654 len = le16toh(rxd->length);
657 eop = (status & E1000_RXD_STAT_EOP) != 0;
659 /* Make sure bad packets are discarded */
660 if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
662 /* XXX fixup if common */
666 ri->iri_frags[i].irf_flid = 0;
667 ri->iri_frags[i].irf_idx = cidx;
668 ri->iri_frags[i].irf_len = len;
669 /* Zero out the receive descriptors status. */
672 if (++cidx == scctx->isc_nrxd[0])
677 /* XXX add a faster way to look this up */
678 if (sc->hw.mac.type >= e1000_82543)
679 em_receive_checksum(status, errors, ri);
681 if (status & E1000_RXD_STAT_VP) {
682 ri->iri_vtag = le16toh(rxd->special);
683 ri->iri_flags |= M_VLANTAG;
692 em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
694 struct e1000_softc *sc = arg;
695 if_softc_ctx_t scctx = sc->shared;
696 struct em_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
697 struct rx_ring *rxr = &que->rxr;
698 union e1000_rx_desc_extended *rxd;
702 uint32_t staterr = 0;
710 rxd = &rxr->rx_base[cidx];
711 staterr = le32toh(rxd->wb.upper.status_error);
712 pkt_info = le32toh(rxd->wb.lower.mrq);
714 /* Error Checking then decrement count */
715 MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
717 len = le16toh(rxd->wb.upper.length);
720 eop = (staterr & E1000_RXD_STAT_EOP) != 0;
722 /* Make sure bad packets are discarded */
723 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
728 ri->iri_frags[i].irf_flid = 0;
729 ri->iri_frags[i].irf_idx = cidx;
730 ri->iri_frags[i].irf_len = len;
731 /* Zero out the receive descriptors status. */
732 rxd->wb.upper.status_error &= htole32(~0xFF);
734 if (++cidx == scctx->isc_nrxd[0])
739 if (scctx->isc_capenable & IFCAP_RXCSUM)
740 em_receive_checksum(staterr, staterr >> 24, ri);
742 if (staterr & E1000_RXD_STAT_VP) {
743 ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
744 ri->iri_flags |= M_VLANTAG;
747 ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss);
748 ri->iri_rsstype = em_determine_rsstype(pkt_info);
754 /*********************************************************************
756 * Verify that the hardware indicated that the checksum is valid.
757 * Inform the stack about the status of checksum so that stack
758 * doesn't spend time verifying the checksum.
760 *********************************************************************/
762 em_receive_checksum(uint16_t status, uint8_t errors, if_rxd_info_t ri)
764 if (__predict_false(status & E1000_RXD_STAT_IXSM))
767 /* If there is a layer 3 or 4 error we are done */
768 if (__predict_false(errors & (E1000_RXD_ERR_IPE | E1000_RXD_ERR_TCPE)))
771 /* IP Checksum Good */
772 if (status & E1000_RXD_STAT_IPCS)
773 ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
775 /* Valid L4E checksum */
776 if (__predict_true(status &
777 (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))) {
778 ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
779 ri->iri_csum_data = htons(0xffff);
783 /********************************************************************
785 * Parse the packet type to determine the appropriate hash
787 ******************************************************************/
789 em_determine_rsstype(uint32_t pkt_info)
791 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
792 case E1000_RXDADV_RSSTYPE_IPV4_TCP:
793 return M_HASHTYPE_RSS_TCP_IPV4;
794 case E1000_RXDADV_RSSTYPE_IPV4:
795 return M_HASHTYPE_RSS_IPV4;
796 case E1000_RXDADV_RSSTYPE_IPV6_TCP:
797 return M_HASHTYPE_RSS_TCP_IPV6;
798 case E1000_RXDADV_RSSTYPE_IPV6_EX:
799 return M_HASHTYPE_RSS_IPV6_EX;
800 case E1000_RXDADV_RSSTYPE_IPV6:
801 return M_HASHTYPE_RSS_IPV6;
802 case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
803 return M_HASHTYPE_RSS_TCP_IPV6_EX;
805 return M_HASHTYPE_OPAQUE;