sys/dev/bnxt/bnxt_txrx.c

   1 /*-
   2  * Broadcom NetXtreme-C/E network driver.
   3  *
   4  * Copyright (c) 2016 Broadcom, All Rights Reserved.
   5  * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS'
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
  20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  26  * THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/types.h>
  33 #include <sys/socket.h>
  34 #include <sys/endian.h>
  35 #include <net/if.h>
  36 #include <net/if_var.h>
  37 #include <net/ethernet.h>
  38 #include <net/iflib.h>
  39
  40 #include "opt_inet.h"
  41 #include "opt_inet6.h"
  42 #include "opt_rss.h"
  43
  44 #include "bnxt.h"
  45
  46 /*
  47  * Function prototypes
  48  */
  49
  50 static int bnxt_isc_txd_encap(void *sc, if_pkt_info_t pi);
  51 static void bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx);
  52 static int bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear);
  53
  54 static void bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru);
  55
  56 /*                              uint16_t rxqid, uint8_t flid,
  57     uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs, uint16_t count,
  58     uint16_t buf_size);
  59 */
  60 static void bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid,
  61     qidx_t pidx);
  62 static int bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx,
  63     qidx_t budget);
  64 static int bnxt_isc_rxd_pkt_get(void *sc, if_rxd_info_t ri);
  65
  66 static int bnxt_intr(void *sc);
  67
  68 struct if_txrx bnxt_txrx  = {
  69         .ift_txd_encap = bnxt_isc_txd_encap,
  70         .ift_txd_flush = bnxt_isc_txd_flush,
  71         .ift_txd_credits_update = bnxt_isc_txd_credits_update,
  72         .ift_rxd_available = bnxt_isc_rxd_available,
  73         .ift_rxd_pkt_get = bnxt_isc_rxd_pkt_get,
  74         .ift_rxd_refill = bnxt_isc_rxd_refill,
  75         .ift_rxd_flush = bnxt_isc_rxd_flush,
  76         .ift_legacy_intr = bnxt_intr
  77 };
  78
  79 /*
  80  * Device Dependent Packet Transmit and Receive Functions
  81  */
  82
  83 static const uint16_t bnxt_tx_lhint[] = {
  84         TX_BD_SHORT_FLAGS_LHINT_LT512,
  85         TX_BD_SHORT_FLAGS_LHINT_LT1K,
  86         TX_BD_SHORT_FLAGS_LHINT_LT2K,
  87         TX_BD_SHORT_FLAGS_LHINT_LT2K,
  88         TX_BD_SHORT_FLAGS_LHINT_GTE2K,
  89 };
  90
  91 static int
  92 bnxt_isc_txd_encap(void *sc, if_pkt_info_t pi)
  93 {
  94         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
  95         struct bnxt_ring *txr = &softc->tx_rings[pi->ipi_qsidx];
  96         struct tx_bd_long *tbd;
  97         struct tx_bd_long_hi *tbdh;
  98         bool need_hi = false;
  99         uint16_t flags_type;
 100         uint16_t lflags;
 101         uint32_t cfa_meta;
 102         int seg = 0;
 103
 104         /* If we have offloads enabled, we need to use two BDs. */
 105         if ((pi->ipi_csum_flags & (CSUM_OFFLOAD | CSUM_TSO | CSUM_IP)) ||
 106             pi->ipi_mflags & M_VLANTAG)
 107                 need_hi = true;
 108
 109         /* TODO: Devices before Cu+B1 need to not mix long and short BDs */
 110         need_hi = true;
 111
 112         pi->ipi_new_pidx = pi->ipi_pidx;
 113         tbd = &((struct tx_bd_long *)txr->vaddr)[pi->ipi_new_pidx];
 114         pi->ipi_ndescs = 0;
 115         /* No need to byte-swap the opaque value */
 116         tbd->opaque = ((pi->ipi_nsegs + need_hi) << 24) | pi->ipi_new_pidx;
 117         tbd->len = htole16(pi->ipi_segs[seg].ds_len);
 118         tbd->addr = htole64(pi->ipi_segs[seg++].ds_addr);
 119         flags_type = ((pi->ipi_nsegs + need_hi) <<
 120             TX_BD_SHORT_FLAGS_BD_CNT_SFT) & TX_BD_SHORT_FLAGS_BD_CNT_MASK;
 121         if (pi->ipi_len >= 2048)
 122                 flags_type |= TX_BD_SHORT_FLAGS_LHINT_GTE2K;
 123         else
 124                 flags_type |= bnxt_tx_lhint[pi->ipi_len >> 9];
 125
 126         if (need_hi) {
 127                 flags_type |= TX_BD_LONG_TYPE_TX_BD_LONG;
 128
 129                 pi->ipi_new_pidx = RING_NEXT(txr, pi->ipi_new_pidx);
 130                 tbdh = &((struct tx_bd_long_hi *)txr->vaddr)[pi->ipi_new_pidx];
 131                 tbdh->mss = htole16(pi->ipi_tso_segsz);
 132                 tbdh->hdr_size = htole16((pi->ipi_ehdrlen + pi->ipi_ip_hlen +
 133                     pi->ipi_tcp_hlen) >> 1);
 134                 tbdh->cfa_action = 0;
 135                 lflags = 0;
 136                 cfa_meta = 0;
 137                 if (pi->ipi_mflags & M_VLANTAG) {
 138                         /* TODO: Do we need to byte-swap the vtag here? */
 139                         cfa_meta = TX_BD_LONG_CFA_META_KEY_VLAN_TAG |
 140                             pi->ipi_vtag;
 141                         cfa_meta |= TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100;
 142                 }
 143                 tbdh->cfa_meta = htole32(cfa_meta);
 144                 if (pi->ipi_csum_flags & CSUM_TSO) {
 145                         lflags |= TX_BD_LONG_LFLAGS_LSO |
 146                             TX_BD_LONG_LFLAGS_T_IPID;
 147                 }
 148                 else if(pi->ipi_csum_flags & CSUM_OFFLOAD) {
 149                         lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM |
 150                             TX_BD_LONG_LFLAGS_IP_CHKSUM;
 151                 }
 152                 else if(pi->ipi_csum_flags & CSUM_IP) {
 153                         lflags |= TX_BD_LONG_LFLAGS_IP_CHKSUM;
 154                 }
 155                 tbdh->lflags = htole16(lflags);
 156         }
 157         else {
 158                 flags_type |= TX_BD_SHORT_TYPE_TX_BD_SHORT;
 159         }
 160
 161         for (; seg < pi->ipi_nsegs; seg++) {
 162                 tbd->flags_type = htole16(flags_type);
 163                 pi->ipi_new_pidx = RING_NEXT(txr, pi->ipi_new_pidx);
 164                 tbd = &((struct tx_bd_long *)txr->vaddr)[pi->ipi_new_pidx];
 165                 tbd->len = htole16(pi->ipi_segs[seg].ds_len);
 166                 tbd->addr = htole64(pi->ipi_segs[seg].ds_addr);
 167                 flags_type = TX_BD_SHORT_TYPE_TX_BD_SHORT;
 168         }
 169         flags_type |= TX_BD_SHORT_FLAGS_PACKET_END;
 170         tbd->flags_type = htole16(flags_type);
 171         pi->ipi_new_pidx = RING_NEXT(txr, pi->ipi_new_pidx);
 172
 173         return 0;
 174 }
 175
 176 static void
 177 bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx)
 178 {
 179         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
 180         struct bnxt_ring *tx_ring = &softc->tx_rings[txqid];
 181
 182         /* pidx is what we last set ipi_new_pidx to */
 183         BNXT_TX_DB(tx_ring, pidx);
 184         /* TODO: Cumulus+ doesn't need the double doorbell */
 185         BNXT_TX_DB(tx_ring, pidx);
 186         return;
 187 }
 188
 189 static int
 190 bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear)
 191 {
 192         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
 193         struct bnxt_cp_ring *cpr = &softc->tx_cp_rings[txqid];
 194         struct tx_cmpl *cmpl = (struct tx_cmpl *)cpr->ring.vaddr;
 195         int avail = 0;
 196         uint32_t cons = cpr->cons;
 197         bool v_bit = cpr->v_bit;
 198         bool last_v_bit;
 199         uint32_t last_cons;
 200         uint16_t type;
 201         uint16_t err;
 202
 203         for (;;) {
 204                 last_cons = cons;
 205                 last_v_bit = v_bit;
 206                 NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 207                 CMPL_PREFETCH_NEXT(cpr, cons);
 208
 209                 if (!CMP_VALID(&cmpl[cons], v_bit))
 210                         goto done;
 211
 212                 type = cmpl[cons].flags_type & TX_CMPL_TYPE_MASK;
 213                 switch (type) {
 214                 case TX_CMPL_TYPE_TX_L2:
 215                         err = (le16toh(cmpl[cons].errors_v) &
 216                             TX_CMPL_ERRORS_BUFFER_ERROR_MASK) >>
 217                             TX_CMPL_ERRORS_BUFFER_ERROR_SFT;
 218                         if (err)
 219                                 device_printf(softc->dev,
 220                                     "TX completion error %u\n", err);
 221                         /* No need to byte-swap the opaque value */
 222                         avail += cmpl[cons].opaque >> 24;
 223                         /*
 224                          * If we're not clearing, iflib only cares if there's
 225                          * at least one buffer.  Don't scan the whole ring in
 226                          * this case.
 227                          */
 228                         if (!clear)
 229                                 goto done;
 230                         break;
 231                 default:
 232                         if (type & 1) {
 233                                 NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 234                                 if (!CMP_VALID(&cmpl[cons], v_bit))
 235                                         goto done;
 236                         }
 237                         device_printf(softc->dev,
 238                             "Unhandled TX completion type %u\n", type);
 239                         break;
 240                 }
 241         }
 242 done:
 243
 244         if (clear && avail) {
 245                 cpr->cons = last_cons;
 246                 cpr->v_bit = last_v_bit;
 247                 BNXT_CP_IDX_DISABLE_DB(&cpr->ring, cpr->cons);
 248         }
 249
 250         return avail;
 251 }
 252
 253 static void
 254 bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru)
 255 {
 256         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
 257         struct bnxt_ring *rx_ring;
 258         struct rx_prod_pkt_bd *rxbd;
 259         uint16_t type;
 260         uint16_t i;
 261         uint16_t rxqid;
 262         uint16_t count, len;
 263         uint32_t pidx;
 264         uint8_t flid;
 265         uint64_t *paddrs;
 266         qidx_t  *frag_idxs;
 267
 268         rxqid = iru->iru_qsidx;
 269         count = iru->iru_count;
 270         len = iru->iru_buf_size;
 271         pidx = iru->iru_pidx;
 272         flid = iru->iru_flidx;
 273         paddrs = iru->iru_paddrs;
 274         frag_idxs = iru->iru_idxs;
 275
 276         if (flid == 0) {
 277                 rx_ring = &softc->rx_rings[rxqid];
 278                 type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT;
 279         }
 280         else {
 281                 rx_ring = &softc->ag_rings[rxqid];
 282                 type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG;
 283         }
 284         rxbd = (void *)rx_ring->vaddr;
 285
 286         for (i=0; i<count; i++) {
 287                 rxbd[pidx].flags_type = htole16(type);
 288                 rxbd[pidx].len = htole16(len);
 289                 /* No need to byte-swap the opaque value */
 290                 rxbd[pidx].opaque = (((rxqid & 0xff) << 24) | (flid << 16)
 291                     | (frag_idxs[i]));
 292                 rxbd[pidx].addr = htole64(paddrs[i]);
 293                 if (++pidx == rx_ring->ring_size)
 294                         pidx = 0;
 295         }
 296         return;
 297 }
 298
 299 static void
 300 bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid,
 301     qidx_t pidx)
 302 {
 303         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
 304         struct bnxt_ring *rx_ring;
 305
 306         if (flid == 0)
 307                 rx_ring = &softc->rx_rings[rxqid];
 308         else
 309                 rx_ring = &softc->ag_rings[rxqid];
 310
 311         /*
 312          * We *must* update the completion ring before updating the RX ring
 313          * or we will overrun the completion ring and the device will wedge for
 314          * RX.
 315          */
 316         if (softc->rx_cp_rings[rxqid].cons != UINT32_MAX)
 317                 BNXT_CP_IDX_DISABLE_DB(&softc->rx_cp_rings[rxqid].ring,
 318                     softc->rx_cp_rings[rxqid].cons);
 319         /* We're given the last filled RX buffer here, not the next empty one */
 320         BNXT_RX_DB(rx_ring, RING_NEXT(rx_ring, pidx));
 321         /* TODO: Cumulus+ doesn't need the double doorbell */
 322         BNXT_RX_DB(rx_ring, RING_NEXT(rx_ring, pidx));
 323         return;
 324 }
 325
 326 static int
 327 bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx, qidx_t budget)
 328 {
 329         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
 330         struct bnxt_cp_ring *cpr = &softc->rx_cp_rings[rxqid];
 331         struct rx_pkt_cmpl *rcp;
 332         struct rx_tpa_end_cmpl *rtpae;
 333         struct cmpl_base *cmp = (struct cmpl_base *)cpr->ring.vaddr;
 334         int avail = 0;
 335         uint32_t cons = cpr->cons;
 336         bool v_bit = cpr->v_bit;
 337         uint8_t ags;
 338         int i;
 339         uint16_t type;
 340
 341         for (;;) {
 342                 NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 343                 CMPL_PREFETCH_NEXT(cpr, cons);
 344
 345                 if (!CMP_VALID(&cmp[cons], v_bit))
 346                         goto cmpl_invalid;
 347
 348                 type = le16toh(cmp[cons].type) & CMPL_BASE_TYPE_MASK;
 349                 switch (type) {
 350                 case CMPL_BASE_TYPE_RX_L2:
 351                         rcp = (void *)&cmp[cons];
 352                         ags = (rcp->agg_bufs_v1 & RX_PKT_CMPL_AGG_BUFS_MASK) >>
 353                             RX_PKT_CMPL_AGG_BUFS_SFT;
 354                         NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 355                         CMPL_PREFETCH_NEXT(cpr, cons);
 356
 357                         if (!CMP_VALID(&cmp[cons], v_bit))
 358                                 goto cmpl_invalid;
 359
 360                         /* Now account for all the AG completions */
 361                         for (i=0; i<ags; i++) {
 362                                 NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 363                                 CMPL_PREFETCH_NEXT(cpr, cons);
 364                                 if (!CMP_VALID(&cmp[cons], v_bit))
 365                                         goto cmpl_invalid;
 366                         }
 367                         avail++;
 368                         break;
 369                 case CMPL_BASE_TYPE_RX_TPA_END:
 370                         rtpae = (void *)&cmp[cons];
 371                         ags = (rtpae->agg_bufs_v1 &
 372                             RX_TPA_END_CMPL_AGG_BUFS_MASK) >>
 373                             RX_TPA_END_CMPL_AGG_BUFS_SFT;
 374                         NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 375                         CMPL_PREFETCH_NEXT(cpr, cons);
 376
 377                         if (!CMP_VALID(&cmp[cons], v_bit))
 378                                 goto cmpl_invalid;
 379                         /* Now account for all the AG completions */
 380                         for (i=0; i<ags; i++) {
 381                                 NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 382                                 CMPL_PREFETCH_NEXT(cpr, cons);
 383                                 if (!CMP_VALID(&cmp[cons], v_bit))
 384                                         goto cmpl_invalid;
 385                         }
 386                         avail++;
 387                         break;
 388                 case CMPL_BASE_TYPE_RX_TPA_START:
 389                         NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 390                         CMPL_PREFETCH_NEXT(cpr, cons);
 391
 392                         if (!CMP_VALID(&cmp[cons], v_bit))
 393                                 goto cmpl_invalid;
 394                         break;
 395                 case CMPL_BASE_TYPE_RX_AGG:
 396                         break;
 397                 default:
 398                         device_printf(softc->dev,
 399                             "Unhandled completion type %d on RXQ %d\n",
 400                             type, rxqid);
 401
 402                         /* Odd completion types use two completions */
 403                         if (type & 1) {
 404                                 NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
 405                                 CMPL_PREFETCH_NEXT(cpr, cons);
 406
 407                                 if (!CMP_VALID(&cmp[cons], v_bit))
 408                                         goto cmpl_invalid;
 409                         }
 410                         break;
 411                 }
 412                 if (avail > budget)
 413                         break;
 414         }
 415 cmpl_invalid:
 416
 417         return avail;
 418 }
 419
 420 static void
 421 bnxt_set_rsstype(if_rxd_info_t ri, uint8_t rss_hash_type)
 422 {
 423         uint8_t rss_profile_id;
 424
 425         rss_profile_id = BNXT_GET_RSS_PROFILE_ID(rss_hash_type);
 426         switch (rss_profile_id) {
 427         case BNXT_RSS_HASH_TYPE_TCPV4:
 428                 ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV4;
 429                 break;
 430         case BNXT_RSS_HASH_TYPE_UDPV4:
 431                 ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV4;
 432                 break;
 433         case BNXT_RSS_HASH_TYPE_IPV4:
 434                 ri->iri_rsstype = M_HASHTYPE_RSS_IPV4;
 435                 break;
 436         case BNXT_RSS_HASH_TYPE_TCPV6:
 437                 ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV6;
 438                 break;
 439         case BNXT_RSS_HASH_TYPE_UDPV6:
 440                 ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV6;
 441                 break;
 442         case BNXT_RSS_HASH_TYPE_IPV6:
 443                 ri->iri_rsstype = M_HASHTYPE_RSS_IPV6;
 444                 break;
 445         default:
 446                 ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
 447                 break;
 448         }
 449 }
 450
 451 static int
 452 bnxt_pkt_get_l2(struct bnxt_softc *softc, if_rxd_info_t ri,
 453     struct bnxt_cp_ring *cpr, uint16_t flags_type)
 454 {
 455         struct rx_pkt_cmpl *rcp;
 456         struct rx_pkt_cmpl_hi *rcph;
 457         struct rx_abuf_cmpl *acp;
 458         uint32_t flags2;
 459         uint32_t errors;
 460         uint8_t ags;
 461         int i;
 462
 463         rcp = &((struct rx_pkt_cmpl *)cpr->ring.vaddr)[cpr->cons];
 464
 465         /* Extract from the first 16-byte BD */
 466         if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) {
 467                 ri->iri_flowid = le32toh(rcp->rss_hash);
 468                 bnxt_set_rsstype(ri, rcp->rss_hash_type);
 469         }
 470         else {
 471                 ri->iri_rsstype = M_HASHTYPE_NONE;
 472         }
 473         ags = (rcp->agg_bufs_v1 & RX_PKT_CMPL_AGG_BUFS_MASK) >>
 474             RX_PKT_CMPL_AGG_BUFS_SFT;
 475         ri->iri_nfrags = ags + 1;
 476         /* No need to byte-swap the opaque value */
 477         ri->iri_frags[0].irf_flid = (rcp->opaque >> 16) & 0xff;
 478         ri->iri_frags[0].irf_idx = rcp->opaque & 0xffff;
 479         ri->iri_frags[0].irf_len = le16toh(rcp->len);
 480         ri->iri_len = le16toh(rcp->len);
 481
 482         /* Now the second 16-byte BD */
 483         NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
 484         ri->iri_cidx = RING_NEXT(&cpr->ring, ri->iri_cidx);
 485         rcph = &((struct rx_pkt_cmpl_hi *)cpr->ring.vaddr)[cpr->cons];
 486
 487         flags2 = le32toh(rcph->flags2);
 488         errors = le16toh(rcph->errors_v2);
 489         if ((flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_MASK) ==
 490             RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) {
 491                 ri->iri_flags |= M_VLANTAG;
 492                 /* TODO: Should this be the entire 16-bits? */
 493                 ri->iri_vtag = le32toh(rcph->metadata) &
 494                     (RX_PKT_CMPL_METADATA_VID_MASK | RX_PKT_CMPL_METADATA_DE |
 495                     RX_PKT_CMPL_METADATA_PRI_MASK);
 496         }
 497         if (flags2 & RX_PKT_CMPL_FLAGS2_IP_CS_CALC) {
 498                 ri->iri_csum_flags |= CSUM_IP_CHECKED;
 499                 if (!(errors & RX_PKT_CMPL_ERRORS_IP_CS_ERROR))
 500                         ri->iri_csum_flags |= CSUM_IP_VALID;
 501         }
 502         if (flags2 & (RX_PKT_CMPL_FLAGS2_L4_CS_CALC |
 503                       RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC)) {
 504                 ri->iri_csum_flags |= CSUM_L4_CALC;
 505                 if (!(errors & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR |
 506                                 RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR))) {
 507                         ri->iri_csum_flags |= CSUM_L4_VALID;
 508                         ri->iri_csum_data = 0xffff;
 509                 }
 510         }
 511
 512         /* And finally the ag ring stuff. */
 513         for (i=1; i < ri->iri_nfrags; i++) {
 514                 NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
 515                 ri->iri_cidx = RING_NEXT(&cpr->ring, ri->iri_cidx);
 516                 acp = &((struct rx_abuf_cmpl *)cpr->ring.vaddr)[cpr->cons];
 517
 518                 /* No need to byte-swap the opaque value */
 519                 ri->iri_frags[i].irf_flid = (acp->opaque >> 16 & 0xff);
 520                 ri->iri_frags[i].irf_idx = acp->opaque & 0xffff;
 521                 ri->iri_frags[i].irf_len = le16toh(acp->len);
 522                 ri->iri_len += le16toh(acp->len);
 523         }
 524
 525         return 0;
 526 }
 527
 528 static int
 529 bnxt_pkt_get_tpa(struct bnxt_softc *softc, if_rxd_info_t ri,
 530     struct bnxt_cp_ring *cpr, uint16_t flags_type)
 531 {
 532         struct rx_tpa_end_cmpl *agend =
 533             &((struct rx_tpa_end_cmpl *)cpr->ring.vaddr)[cpr->cons];
 534         struct rx_abuf_cmpl *acp;
 535         struct bnxt_full_tpa_start *tpas;
 536         uint32_t flags2;
 537         uint8_t ags;
 538         uint8_t agg_id;
 539         int i;
 540
 541         /* Get the agg_id */
 542         agg_id = (agend->agg_id & RX_TPA_END_CMPL_AGG_ID_MASK) >>
 543             RX_TPA_END_CMPL_AGG_ID_SFT;
 544         tpas = &(softc->rx_rings[ri->iri_qsidx].tpa_start[agg_id]);
 545
 546         /* Extract from the first 16-byte BD */
 547         if (le16toh(tpas->low.flags_type) & RX_TPA_START_CMPL_FLAGS_RSS_VALID) {
 548                 ri->iri_flowid = le32toh(tpas->low.rss_hash);
 549                 bnxt_set_rsstype(ri, tpas->low.rss_hash_type);
 550         }
 551         else {
 552                 ri->iri_rsstype = M_HASHTYPE_NONE;
 553         }
 554         ags = (agend->agg_bufs_v1 & RX_TPA_END_CMPL_AGG_BUFS_MASK) >>
 555             RX_TPA_END_CMPL_AGG_BUFS_SFT;
 556         ri->iri_nfrags = ags + 1;
 557         /* No need to byte-swap the opaque value */
 558         ri->iri_frags[0].irf_flid = ((tpas->low.opaque >> 16) & 0xff);
 559         ri->iri_frags[0].irf_idx = (tpas->low.opaque & 0xffff);
 560         ri->iri_frags[0].irf_len = le16toh(tpas->low.len);
 561         ri->iri_len = le16toh(tpas->low.len);
 562
 563         /* Now the second 16-byte BD */
 564         NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
 565         ri->iri_cidx = RING_NEXT(&cpr->ring, ri->iri_cidx);
 566
 567         flags2 = le32toh(tpas->high.flags2);
 568         if ((flags2 & RX_TPA_START_CMPL_FLAGS2_META_FORMAT_MASK) ==
 569             RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN) {
 570                 ri->iri_flags |= M_VLANTAG;
 571                 /* TODO: Should this be the entire 16-bits? */
 572                 ri->iri_vtag = le32toh(tpas->high.metadata) &
 573                     (RX_TPA_START_CMPL_METADATA_VID_MASK |
 574                     RX_TPA_START_CMPL_METADATA_DE |
 575                     RX_TPA_START_CMPL_METADATA_PRI_MASK);
 576         }
 577         if (flags2 & RX_TPA_START_CMPL_FLAGS2_IP_CS_CALC) {
 578                 ri->iri_csum_flags |= CSUM_IP_CHECKED;
 579                 ri->iri_csum_flags |= CSUM_IP_VALID;
 580         }
 581         if (flags2 & RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC) {
 582                 ri->iri_csum_flags |= CSUM_L4_CALC;
 583                 ri->iri_csum_flags |= CSUM_L4_VALID;
 584                 ri->iri_csum_data = 0xffff;
 585         }
 586
 587         /* Now the ag ring stuff. */
 588         for (i=1; i < ri->iri_nfrags; i++) {
 589                 NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
 590                 ri->iri_cidx = RING_NEXT(&cpr->ring, ri->iri_cidx);
 591                 acp = &((struct rx_abuf_cmpl *)cpr->ring.vaddr)[cpr->cons];
 592
 593                 /* No need to byte-swap the opaque value */
 594                 ri->iri_frags[i].irf_flid = ((acp->opaque >> 16) & 0xff);
 595                 ri->iri_frags[i].irf_idx = (acp->opaque & 0xffff);
 596                 ri->iri_frags[i].irf_len = le16toh(acp->len);
 597                 ri->iri_len += le16toh(acp->len);
 598         }
 599
 600         /* And finally, the empty BD at the end... */
 601         ri->iri_nfrags++;
 602         /* No need to byte-swap the opaque value */
 603         ri->iri_frags[i].irf_flid = ((agend->opaque >> 16) & 0xff);
 604         ri->iri_frags[i].irf_idx = (agend->opaque & 0xffff);
 605         ri->iri_frags[i].irf_len = le16toh(agend->len);
 606         ri->iri_len += le16toh(agend->len);
 607
 608         return 0;
 609 }
 610
 611 /* If we return anything but zero, iflib will assert... */
 612 static int
 613 bnxt_isc_rxd_pkt_get(void *sc, if_rxd_info_t ri)
 614 {
 615         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
 616         struct bnxt_cp_ring *cpr = &softc->rx_cp_rings[ri->iri_qsidx];
 617         struct cmpl_base *cmp_q = (struct cmpl_base *)cpr->ring.vaddr;
 618         struct cmpl_base *cmp;
 619         struct rx_tpa_start_cmpl *rtpa;
 620         uint16_t flags_type;
 621         uint16_t type;
 622         uint8_t agg_id;
 623
 624         for (;;) {
 625                 NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
 626                 ri->iri_cidx = RING_NEXT(&cpr->ring, ri->iri_cidx);
 627                 CMPL_PREFETCH_NEXT(cpr, cpr->cons);
 628                 cmp = &((struct cmpl_base *)cpr->ring.vaddr)[cpr->cons];
 629
 630                 flags_type = le16toh(cmp->type);
 631                 type = flags_type & CMPL_BASE_TYPE_MASK;
 632
 633                 switch (type) {
 634                 case CMPL_BASE_TYPE_RX_L2:
 635                         return bnxt_pkt_get_l2(softc, ri, cpr, flags_type);
 636                 case CMPL_BASE_TYPE_RX_TPA_END:
 637                         return bnxt_pkt_get_tpa(softc, ri, cpr, flags_type);
 638                 case CMPL_BASE_TYPE_RX_TPA_START:
 639                         rtpa = (void *)&cmp_q[cpr->cons];
 640                         agg_id = (rtpa->agg_id &
 641                             RX_TPA_START_CMPL_AGG_ID_MASK) >>
 642                             RX_TPA_START_CMPL_AGG_ID_SFT;
 643                         softc->rx_rings[ri->iri_qsidx].tpa_start[agg_id].low = *rtpa;
 644
 645                         NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
 646                         ri->iri_cidx = RING_NEXT(&cpr->ring, ri->iri_cidx);
 647                         CMPL_PREFETCH_NEXT(cpr, cpr->cons);
 648
 649                         softc->rx_rings[ri->iri_qsidx].tpa_start[agg_id].high =
 650                             ((struct rx_tpa_start_cmpl_hi *)cmp_q)[cpr->cons];
 651                         break;
 652                 default:
 653                         device_printf(softc->dev,
 654                             "Unhandled completion type %d on RXQ %d get\n",
 655                             type, ri->iri_qsidx);
 656                         if (type & 1) {
 657                                 NEXT_CP_CONS_V(&cpr->ring, cpr->cons,
 658                                     cpr->v_bit);
 659                                 ri->iri_cidx = RING_NEXT(&cpr->ring,
 660                                     ri->iri_cidx);
 661                                 CMPL_PREFETCH_NEXT(cpr, cpr->cons);
 662                         }
 663                         break;
 664                 }
 665         }
 666
 667         return 0;
 668 }
 669
 670 static int
 671 bnxt_intr(void *sc)
 672 {
 673         struct bnxt_softc *softc = (struct bnxt_softc *)sc;
 674
 675         device_printf(softc->dev, "STUB: %s @ %s:%d\n", __func__, __FILE__, __LINE__);
 676         return ENOSYS;
 677 }