sys/dev/ixl/ixl_txrx.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2015, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 /*
  36 **      IXL driver TX/RX Routines:
  37 **          This was seperated to allow usage by
  38 **          both the PF and VF drivers.
  39 */
  40
  41 #ifndef IXL_STANDALONE_BUILD
  42 #include "opt_inet.h"
  43 #include "opt_inet6.h"
  44 #include "opt_rss.h"
  45 #endif
  46
  47 #include "ixl.h"
  48
  49 #ifdef RSS
  50 #include <net/rss_config.h>
  51 #endif
  52
  53 /* Local Prototypes */
  54 static void     ixl_rx_checksum(struct mbuf *, u32, u32, u8);
  55 static void     ixl_refresh_mbufs(struct ixl_queue *, int);
  56 static int      ixl_xmit(struct ixl_queue *, struct mbuf **);
  57 static int      ixl_tx_setup_offload(struct ixl_queue *,
  58                     struct mbuf *, u32 *, u32 *);
  59 static bool     ixl_tso_setup(struct ixl_queue *, struct mbuf *);
  60
  61 static inline void ixl_rx_discard(struct rx_ring *, int);
  62 static inline void ixl_rx_input(struct rx_ring *, struct ifnet *,
  63                     struct mbuf *, u8);
  64
  65 static inline bool ixl_tso_detect_sparse(struct mbuf *mp);
  66 static int      ixl_tx_setup_offload(struct ixl_queue *que,
  67     struct mbuf *mp, u32 *cmd, u32 *off);
  68 static inline u32 ixl_get_tx_head(struct ixl_queue *que);
  69
  70 #ifdef DEV_NETMAP
  71 #include <dev/netmap/if_ixl_netmap.h>
  72 int ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip = 1;
  73 #endif /* DEV_NETMAP */
  74
  75 /*
  76  * @key key is saved into this parameter
  77  */
  78 void
  79 ixl_get_default_rss_key(u32 *key)
  80 {
  81         MPASS(key != NULL);
  82
  83         u32 rss_seed[IXL_RSS_KEY_SIZE_REG] = {0x41b01687,
  84             0x183cfd8c, 0xce880440, 0x580cbc3c,
  85             0x35897377, 0x328b25e1, 0x4fa98922,
  86             0xb7d90c14, 0xd5bad70d, 0xcd15a2c1,
  87             0x0, 0x0, 0x0};
  88
  89         bcopy(rss_seed, key, IXL_RSS_KEY_SIZE);
  90 }
  91
  92 /*
  93 ** Multiqueue Transmit driver
  94 */
  95 int
  96 ixl_mq_start(struct ifnet *ifp, struct mbuf *m)
  97 {
  98         struct ixl_vsi          *vsi = ifp->if_softc;
  99         struct ixl_queue        *que;
 100         struct tx_ring          *txr;
 101         int                     err, i;
 102 #ifdef RSS
 103         u32                     bucket_id;
 104 #endif
 105
 106         /*
 107         ** Which queue to use:
 108         **
 109         ** When doing RSS, map it to the same outbound
 110         ** queue as the incoming flow would be mapped to.
 111         ** If everything is setup correctly, it should be
 112         ** the same bucket that the current CPU we're on is.
 113         */
 114         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
 115 #ifdef  RSS
 116                 if (rss_hash2bucket(m->m_pkthdr.flowid,
 117                     M_HASHTYPE_GET(m), &bucket_id) == 0) {
 118                         i = bucket_id % vsi->num_queues;
 119                 } else
 120 #endif
 121                         i = m->m_pkthdr.flowid % vsi->num_queues;
 122         } else
 123                 i = curcpu % vsi->num_queues;
 124
 125         que = &vsi->queues[i];
 126         txr = &que->txr;
 127
 128         err = drbr_enqueue(ifp, txr->br, m);
 129         if (err)
 130                 return (err);
 131         if (IXL_TX_TRYLOCK(txr)) {
 132                 ixl_mq_start_locked(ifp, txr);
 133                 IXL_TX_UNLOCK(txr);
 134         } else
 135                 taskqueue_enqueue(que->tq, &que->tx_task);
 136
 137         return (0);
 138 }
 139
 140 int
 141 ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
 142 {
 143         struct ixl_queue        *que = txr->que;
 144         struct ixl_vsi          *vsi = que->vsi;
 145         struct mbuf             *next;
 146         int                     err = 0;
 147
 148
 149         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 150             vsi->link_active == 0)
 151                 return (ENETDOWN);
 152
 153         /* Process the transmit queue */
 154         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 155                 if ((err = ixl_xmit(que, &next)) != 0) {
 156                         if (next == NULL)
 157                                 drbr_advance(ifp, txr->br);
 158                         else
 159                                 drbr_putback(ifp, txr->br, next);
 160                         break;
 161                 }
 162                 drbr_advance(ifp, txr->br);
 163                 /* Send a copy of the frame to the BPF listener */
 164                 ETHER_BPF_MTAP(ifp, next);
 165                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 166                         break;
 167         }
 168
 169         if (txr->avail < IXL_TX_CLEANUP_THRESHOLD)
 170                 ixl_txeof(que);
 171
 172         return (err);
 173 }
 174
 175 /*
 176  * Called from a taskqueue to drain queued transmit packets.
 177  */
 178 void
 179 ixl_deferred_mq_start(void *arg, int pending)
 180 {
 181         struct ixl_queue        *que = arg;
 182         struct tx_ring          *txr = &que->txr;
 183         struct ixl_vsi          *vsi = que->vsi;
 184         struct ifnet            *ifp = vsi->ifp;
 185
 186         IXL_TX_LOCK(txr);
 187         if (!drbr_empty(ifp, txr->br))
 188                 ixl_mq_start_locked(ifp, txr);
 189         IXL_TX_UNLOCK(txr);
 190 }
 191
 192 /*
 193 ** Flush all queue ring buffers
 194 */
 195 void
 196 ixl_qflush(struct ifnet *ifp)
 197 {
 198         struct ixl_vsi  *vsi = ifp->if_softc;
 199
 200         for (int i = 0; i < vsi->num_queues; i++) {
 201                 struct ixl_queue *que = &vsi->queues[i];
 202                 struct tx_ring  *txr = &que->txr;
 203                 struct mbuf     *m;
 204                 IXL_TX_LOCK(txr);
 205                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 206                         m_freem(m);
 207                 IXL_TX_UNLOCK(txr);
 208         }
 209         if_qflush(ifp);
 210 }
 211
 212 /*
 213 ** Find mbuf chains passed to the driver
 214 ** that are 'sparse', using more than 8
 215 ** mbufs to deliver an mss-size chunk of data
 216 */
 217 static inline bool
 218 ixl_tso_detect_sparse(struct mbuf *mp)
 219 {
 220         struct mbuf     *m;
 221         int             num, mss;
 222
 223         num = 0;
 224         mss = mp->m_pkthdr.tso_segsz;
 225
 226         /* Exclude first mbuf; assume it contains all headers */
 227         for (m = mp->m_next; m != NULL; m = m->m_next) {
 228                 if (m == NULL)
 229                         break;
 230                 num++;
 231                 mss -= m->m_len % mp->m_pkthdr.tso_segsz;
 232
 233                 if (mss < 1) {
 234                         if (num > IXL_SPARSE_CHAIN)
 235                                 return (true);
 236                         num = (mss == 0) ? 0 : 1;
 237                         mss += mp->m_pkthdr.tso_segsz;
 238                 }
 239         }
 240
 241         return (false);
 242 }
 243
 244
 245 /*********************************************************************
 246  *
 247  *  This routine maps the mbufs to tx descriptors, allowing the
 248  *  TX engine to transmit the packets.
 249  *      - return 0 on success, positive on failure
 250  *
 251  **********************************************************************/
 252 #define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
 253
 254 static int
 255 ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp)
 256 {
 257         struct ixl_vsi          *vsi = que->vsi;
 258         struct i40e_hw          *hw = vsi->hw;
 259         struct tx_ring          *txr = &que->txr;
 260         struct ixl_tx_buf       *buf;
 261         struct i40e_tx_desc     *txd = NULL;
 262         struct mbuf             *m_head, *m;
 263         int                     i, j, error, nsegs;
 264         int                     first, last = 0;
 265         u16                     vtag = 0;
 266         u32                     cmd, off;
 267         bus_dmamap_t            map;
 268         bus_dma_tag_t           tag;
 269         bus_dma_segment_t       segs[IXL_MAX_TSO_SEGS];
 270
 271         cmd = off = 0;
 272         m_head = *m_headp;
 273
 274         /*
 275          * Important to capture the first descriptor
 276          * used because it will contain the index of
 277          * the one we tell the hardware to report back
 278          */
 279         first = txr->next_avail;
 280         buf = &txr->buffers[first];
 281         map = buf->map;
 282         tag = txr->tx_tag;
 283
 284         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
 285                 /* Use larger mapping for TSO */
 286                 tag = txr->tso_tag;
 287                 if (ixl_tso_detect_sparse(m_head)) {
 288                         m = m_defrag(m_head, M_NOWAIT);
 289                         if (m == NULL) {
 290                                 m_freem(*m_headp);
 291                                 *m_headp = NULL;
 292                                 return (ENOBUFS);
 293                         }
 294                         *m_headp = m;
 295                 }
 296         }
 297
 298         /*
 299          * Map the packet for DMA.
 300          */
 301         error = bus_dmamap_load_mbuf_sg(tag, map,
 302             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 303
 304         if (error == EFBIG) {
 305                 struct mbuf *m;
 306
 307                 m = m_defrag(*m_headp, M_NOWAIT);
 308                 if (m == NULL) {
 309                         que->mbuf_defrag_failed++;
 310                         m_freem(*m_headp);
 311                         *m_headp = NULL;
 312                         return (ENOBUFS);
 313                 }
 314                 *m_headp = m;
 315
 316                 /* Try it again */
 317                 error = bus_dmamap_load_mbuf_sg(tag, map,
 318                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 319
 320                 if (error != 0) {
 321                         que->tx_dmamap_failed++;
 322                         m_freem(*m_headp);
 323                         *m_headp = NULL;
 324                         return (error);
 325                 }
 326         } else if (error != 0) {
 327                 que->tx_dmamap_failed++;
 328                 m_freem(*m_headp);
 329                 *m_headp = NULL;
 330                 return (error);
 331         }
 332
 333         /* Make certain there are enough descriptors */
 334         if (nsegs > txr->avail - 2) {
 335                 txr->no_desc++;
 336                 error = ENOBUFS;
 337                 goto xmit_fail;
 338         }
 339         m_head = *m_headp;
 340
 341         /* Set up the TSO/CSUM offload */
 342         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
 343                 error = ixl_tx_setup_offload(que, m_head, &cmd, &off);
 344                 if (error)
 345                         goto xmit_fail;
 346         }
 347
 348         cmd |= I40E_TX_DESC_CMD_ICRC;
 349         /* Grab the VLAN tag */
 350         if (m_head->m_flags & M_VLANTAG) {
 351                 cmd |= I40E_TX_DESC_CMD_IL2TAG1;
 352                 vtag = htole16(m_head->m_pkthdr.ether_vtag);
 353         }
 354
 355         i = txr->next_avail;
 356         for (j = 0; j < nsegs; j++) {
 357                 bus_size_t seglen;
 358
 359                 buf = &txr->buffers[i];
 360                 buf->tag = tag; /* Keep track of the type tag */
 361                 txd = &txr->base[i];
 362                 seglen = segs[j].ds_len;
 363
 364                 txd->buffer_addr = htole64(segs[j].ds_addr);
 365                 txd->cmd_type_offset_bsz =
 366                     htole64(I40E_TX_DESC_DTYPE_DATA
 367                     | ((u64)cmd  << I40E_TXD_QW1_CMD_SHIFT)
 368                     | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT)
 369                     | ((u64)seglen  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)
 370                     | ((u64)vtag  << I40E_TXD_QW1_L2TAG1_SHIFT));
 371
 372                 last = i; /* descriptor that will get completion IRQ */
 373
 374                 if (++i == que->num_desc)
 375                         i = 0;
 376
 377                 buf->m_head = NULL;
 378                 buf->eop_index = -1;
 379         }
 380         /* Set the last descriptor for report */
 381         txd->cmd_type_offset_bsz |=
 382             htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT));
 383         txr->avail -= nsegs;
 384         txr->next_avail = i;
 385
 386         buf->m_head = m_head;
 387         /* Swap the dma map between the first and last descriptor */
 388         txr->buffers[first].map = buf->map;
 389         buf->map = map;
 390         bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE);
 391
 392         /* Set the index of the descriptor that will be marked done */
 393         buf = &txr->buffers[first];
 394         buf->eop_index = last;
 395
 396         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 397             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 398         /*
 399          * Advance the Transmit Descriptor Tail (Tdt), this tells the
 400          * hardware that this frame is available to transmit.
 401          */
 402         ++txr->total_packets;
 403         wr32(hw, txr->tail, i);
 404
 405         /* Mark outstanding work */
 406         atomic_store_rel_32(&txr->watchdog_timer, IXL_WATCHDOG);
 407         return (0);
 408
 409 xmit_fail:
 410         bus_dmamap_unload(tag, buf->map);
 411         return (error);
 412 }
 413
 414
 415 /*********************************************************************
 416  *
 417  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
 418  *  the information needed to transmit a packet on the wire. This is
 419  *  called only once at attach, setup is done every reset.
 420  *
 421  **********************************************************************/
 422 int
 423 ixl_allocate_tx_data(struct ixl_queue *que)
 424 {
 425         struct tx_ring          *txr = &que->txr;
 426         struct ixl_vsi          *vsi = que->vsi;
 427         device_t                dev = vsi->dev;
 428         struct ixl_tx_buf       *buf;
 429         int                     error = 0;
 430
 431         /*
 432          * Setup DMA descriptor areas.
 433          */
 434         if ((error = bus_dma_tag_create(NULL,           /* parent */
 435                                1, 0,                    /* alignment, bounds */
 436                                BUS_SPACE_MAXADDR,       /* lowaddr */
 437                                BUS_SPACE_MAXADDR,       /* highaddr */
 438                                NULL, NULL,              /* filter, filterarg */
 439                                IXL_TSO_SIZE,            /* maxsize */
 440                                IXL_MAX_TX_SEGS,         /* nsegments */
 441                                PAGE_SIZE,               /* maxsegsize */
 442                                0,                       /* flags */
 443                                NULL,                    /* lockfunc */
 444                                NULL,                    /* lockfuncarg */
 445                                &txr->tx_tag))) {
 446                 device_printf(dev,"Unable to allocate TX DMA tag\n");
 447                 goto fail;
 448         }
 449
 450         /* Make a special tag for TSO */
 451         if ((error = bus_dma_tag_create(NULL,           /* parent */
 452                                1, 0,                    /* alignment, bounds */
 453                                BUS_SPACE_MAXADDR,       /* lowaddr */
 454                                BUS_SPACE_MAXADDR,       /* highaddr */
 455                                NULL, NULL,              /* filter, filterarg */
 456                                IXL_TSO_SIZE,            /* maxsize */
 457                                IXL_MAX_TSO_SEGS,        /* nsegments */
 458                                PAGE_SIZE,               /* maxsegsize */
 459                                0,                       /* flags */
 460                                NULL,                    /* lockfunc */
 461                                NULL,                    /* lockfuncarg */
 462                                &txr->tso_tag))) {
 463                 device_printf(dev,"Unable to allocate TX TSO DMA tag\n");
 464                 goto fail;
 465         }
 466
 467         if (!(txr->buffers =
 468             (struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) *
 469             que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 470                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 471                 error = ENOMEM;
 472                 goto fail;
 473         }
 474
 475         /* Create the descriptor buffer default dma maps */
 476         buf = txr->buffers;
 477         for (int i = 0; i < que->num_desc; i++, buf++) {
 478                 buf->tag = txr->tx_tag;
 479                 error = bus_dmamap_create(buf->tag, 0, &buf->map);
 480                 if (error != 0) {
 481                         device_printf(dev, "Unable to create TX DMA map\n");
 482                         goto fail;
 483                 }
 484         }
 485 fail:
 486         return (error);
 487 }
 488
 489
 490 /*********************************************************************
 491  *
 492  *  (Re)Initialize a queue transmit ring.
 493  *      - called by init, it clears the descriptor ring,
 494  *        and frees any stale mbufs
 495  *
 496  **********************************************************************/
 497 void
 498 ixl_init_tx_ring(struct ixl_queue *que)
 499 {
 500 #ifdef DEV_NETMAP
 501         struct netmap_adapter *na = NA(que->vsi->ifp);
 502         struct netmap_slot *slot;
 503 #endif /* DEV_NETMAP */
 504         struct tx_ring          *txr = &que->txr;
 505         struct ixl_tx_buf       *buf;
 506
 507         /* Clear the old ring contents */
 508         IXL_TX_LOCK(txr);
 509
 510 #ifdef DEV_NETMAP
 511         /*
 512          * (under lock): if in netmap mode, do some consistency
 513          * checks and set slot to entry 0 of the netmap ring.
 514          */
 515         slot = netmap_reset(na, NR_TX, que->me, 0);
 516 #endif /* DEV_NETMAP */
 517
 518         bzero((void *)txr->base,
 519               (sizeof(struct i40e_tx_desc)) * que->num_desc);
 520
 521         /* Reset indices */
 522         txr->next_avail = 0;
 523         txr->next_to_clean = 0;
 524
 525         /* Reset watchdog status */
 526         txr->watchdog_timer = 0;
 527
 528 #ifdef IXL_FDIR
 529         /* Initialize flow director */
 530         txr->atr_rate = ixl_atr_rate;
 531         txr->atr_count = 0;
 532 #endif
 533         /* Free any existing tx mbufs. */
 534         buf = txr->buffers;
 535         for (int i = 0; i < que->num_desc; i++, buf++) {
 536                 if (buf->m_head != NULL) {
 537                         bus_dmamap_sync(buf->tag, buf->map,
 538                             BUS_DMASYNC_POSTWRITE);
 539                         bus_dmamap_unload(buf->tag, buf->map);
 540                         m_freem(buf->m_head);
 541                         buf->m_head = NULL;
 542                 }
 543 #ifdef DEV_NETMAP
 544                 /*
 545                  * In netmap mode, set the map for the packet buffer.
 546                  * NOTE: Some drivers (not this one) also need to set
 547                  * the physical buffer address in the NIC ring.
 548                  * netmap_idx_n2k() maps a nic index, i, into the corresponding
 549                  * netmap slot index, si
 550                  */
 551                 if (slot) {
 552                         int si = netmap_idx_n2k(&na->tx_rings[que->me], i);
 553                         netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si));
 554                 }
 555 #endif /* DEV_NETMAP */
 556                 /* Clear the EOP index */
 557                 buf->eop_index = -1;
 558         }
 559
 560         /* Set number of descriptors available */
 561         txr->avail = que->num_desc;
 562
 563         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 564             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 565         IXL_TX_UNLOCK(txr);
 566 }
 567
 568
 569 /*********************************************************************
 570  *
 571  *  Free transmit ring related data structures.
 572  *
 573  **********************************************************************/
 574 void
 575 ixl_free_que_tx(struct ixl_queue *que)
 576 {
 577         struct tx_ring *txr = &que->txr;
 578         struct ixl_tx_buf *buf;
 579
 580         INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
 581
 582         for (int i = 0; i < que->num_desc; i++) {
 583                 buf = &txr->buffers[i];
 584                 if (buf->m_head != NULL) {
 585                         bus_dmamap_sync(buf->tag, buf->map,
 586                             BUS_DMASYNC_POSTWRITE);
 587                         bus_dmamap_unload(buf->tag,
 588                             buf->map);
 589                         m_freem(buf->m_head);
 590                         buf->m_head = NULL;
 591                         if (buf->map != NULL) {
 592                                 bus_dmamap_destroy(buf->tag,
 593                                     buf->map);
 594                                 buf->map = NULL;
 595                         }
 596                 } else if (buf->map != NULL) {
 597                         bus_dmamap_unload(buf->tag,
 598                             buf->map);
 599                         bus_dmamap_destroy(buf->tag,
 600                             buf->map);
 601                         buf->map = NULL;
 602                 }
 603         }
 604         if (txr->br != NULL)
 605                 buf_ring_free(txr->br, M_DEVBUF);
 606         if (txr->buffers != NULL) {
 607                 free(txr->buffers, M_DEVBUF);
 608                 txr->buffers = NULL;
 609         }
 610         if (txr->tx_tag != NULL) {
 611                 bus_dma_tag_destroy(txr->tx_tag);
 612                 txr->tx_tag = NULL;
 613         }
 614         if (txr->tso_tag != NULL) {
 615                 bus_dma_tag_destroy(txr->tso_tag);
 616                 txr->tso_tag = NULL;
 617         }
 618
 619         INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
 620         return;
 621 }
 622
 623 /*********************************************************************
 624  *
 625  *  Setup descriptor for hw offloads
 626  *
 627  **********************************************************************/
 628
 629 static int
 630 ixl_tx_setup_offload(struct ixl_queue *que,
 631     struct mbuf *mp, u32 *cmd, u32 *off)
 632 {
 633         struct ether_vlan_header        *eh;
 634 #ifdef INET
 635         struct ip                       *ip = NULL;
 636 #endif
 637         struct tcphdr                   *th = NULL;
 638 #ifdef INET6
 639         struct ip6_hdr                  *ip6;
 640 #endif
 641         int                             elen, ip_hlen = 0, tcp_hlen;
 642         u16                             etype;
 643         u8                              ipproto = 0;
 644         bool                            tso = FALSE;
 645
 646         /* Set up the TSO context descriptor if required */
 647         if (mp->m_pkthdr.csum_flags & CSUM_TSO) {
 648                 tso = ixl_tso_setup(que, mp);
 649                 if (tso)
 650                         ++que->tso;
 651                 else
 652                         return (ENXIO);
 653         }
 654
 655         /*
 656          * Determine where frame payload starts.
 657          * Jump over vlan headers if already present,
 658          * helpful for QinQ too.
 659          */
 660         eh = mtod(mp, struct ether_vlan_header *);
 661         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 662                 etype = ntohs(eh->evl_proto);
 663                 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 664         } else {
 665                 etype = ntohs(eh->evl_encap_proto);
 666                 elen = ETHER_HDR_LEN;
 667         }
 668
 669         switch (etype) {
 670 #ifdef INET
 671                 case ETHERTYPE_IP:
 672                         ip = (struct ip *)(mp->m_data + elen);
 673                         ip_hlen = ip->ip_hl << 2;
 674                         ipproto = ip->ip_p;
 675                         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 676                         /* The IP checksum must be recalculated with TSO */
 677                         if (tso)
 678                                 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
 679                         else
 680                                 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
 681                         break;
 682 #endif
 683 #ifdef INET6
 684                 case ETHERTYPE_IPV6:
 685                         ip6 = (struct ip6_hdr *)(mp->m_data + elen);
 686                         ip_hlen = sizeof(struct ip6_hdr);
 687                         ipproto = ip6->ip6_nxt;
 688                         th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
 689                         *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
 690                         break;
 691 #endif
 692                 default:
 693                         break;
 694         }
 695
 696         *off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
 697         *off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
 698
 699         switch (ipproto) {
 700                 case IPPROTO_TCP:
 701                         tcp_hlen = th->th_off << 2;
 702                         if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) {
 703                                 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
 704                                 *off |= (tcp_hlen >> 2) <<
 705                                     I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 706                         }
 707 #ifdef IXL_FDIR
 708                         ixl_atr(que, th, etype);
 709 #endif
 710                         break;
 711                 case IPPROTO_UDP:
 712                         if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) {
 713                                 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
 714                                 *off |= (sizeof(struct udphdr) >> 2) <<
 715                                     I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 716                         }
 717                         break;
 718
 719                 case IPPROTO_SCTP:
 720                         if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) {
 721                                 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
 722                                 *off |= (sizeof(struct sctphdr) >> 2) <<
 723                                     I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 724                         }
 725                         /* Fall Thru */
 726                 default:
 727                         break;
 728         }
 729
 730         return (0);
 731 }
 732
 733
 734 /**********************************************************************
 735  *
 736  *  Setup context for hardware segmentation offload (TSO)
 737  *
 738  **********************************************************************/
 739 static bool
 740 ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp)
 741 {
 742         struct tx_ring                  *txr = &que->txr;
 743         struct i40e_tx_context_desc     *TXD;
 744         struct ixl_tx_buf               *buf;
 745         u32                             cmd, mss, type, tsolen;
 746         u16                             etype;
 747         int                             idx, elen, ip_hlen, tcp_hlen;
 748         struct ether_vlan_header        *eh;
 749 #ifdef INET
 750         struct ip                       *ip;
 751 #endif
 752 #ifdef INET6
 753         struct ip6_hdr                  *ip6;
 754 #endif
 755 #if defined(INET6) || defined(INET)
 756         struct tcphdr                   *th;
 757 #endif
 758         u64                             type_cmd_tso_mss;
 759
 760         /*
 761          * Determine where frame payload starts.
 762          * Jump over vlan headers if already present
 763          */
 764         eh = mtod(mp, struct ether_vlan_header *);
 765         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 766                 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 767                 etype = eh->evl_proto;
 768         } else {
 769                 elen = ETHER_HDR_LEN;
 770                 etype = eh->evl_encap_proto;
 771         }
 772
 773         switch (ntohs(etype)) {
 774 #ifdef INET6
 775         case ETHERTYPE_IPV6:
 776                 ip6 = (struct ip6_hdr *)(mp->m_data + elen);
 777                 if (ip6->ip6_nxt != IPPROTO_TCP)
 778                         return (ENXIO);
 779                 ip_hlen = sizeof(struct ip6_hdr);
 780                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
 781                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
 782                 tcp_hlen = th->th_off << 2;
 783                 /*
 784                  * The corresponding flag is set by the stack in the IPv4
 785                  * TSO case, but not in IPv6 (at least in FreeBSD 10.2).
 786                  * So, set it here because the rest of the flow requires it.
 787                  */
 788                 mp->m_pkthdr.csum_flags |= CSUM_TCP_IPV6;
 789                 break;
 790 #endif
 791 #ifdef INET
 792         case ETHERTYPE_IP:
 793                 ip = (struct ip *)(mp->m_data + elen);
 794                 if (ip->ip_p != IPPROTO_TCP)
 795                         return (ENXIO);
 796                 ip->ip_sum = 0;
 797                 ip_hlen = ip->ip_hl << 2;
 798                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 799                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
 800                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 801                 tcp_hlen = th->th_off << 2;
 802                 break;
 803 #endif
 804         default:
 805                 printf("%s: CSUM_TSO but no supported IP version (0x%04x)",
 806                     __func__, ntohs(etype));
 807                 return FALSE;
 808         }
 809
 810         /* Ensure we have at least the IP+TCP header in the first mbuf. */
 811         if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr))
 812                 return FALSE;
 813
 814         idx = txr->next_avail;
 815         buf = &txr->buffers[idx];
 816         TXD = (struct i40e_tx_context_desc *) &txr->base[idx];
 817         tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen);
 818
 819         type = I40E_TX_DESC_DTYPE_CONTEXT;
 820         cmd = I40E_TX_CTX_DESC_TSO;
 821         /* TSO MSS must not be less than 64 */
 822         if (mp->m_pkthdr.tso_segsz < IXL_MIN_TSO_MSS) {
 823                 que->mss_too_small++;
 824                 mp->m_pkthdr.tso_segsz = IXL_MIN_TSO_MSS;
 825         }
 826         mss = mp->m_pkthdr.tso_segsz;
 827
 828         type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) |
 829             ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
 830             ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
 831             ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
 832         TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss);
 833
 834         TXD->tunneling_params = htole32(0);
 835         buf->m_head = NULL;
 836         buf->eop_index = -1;
 837
 838         if (++idx == que->num_desc)
 839                 idx = 0;
 840
 841         txr->avail--;
 842         txr->next_avail = idx;
 843
 844         return TRUE;
 845 }
 846
 847 /*
 848 ** ixl_get_tx_head - Retrieve the value from the
 849 **    location the HW records its HEAD index
 850 */
 851 static inline u32
 852 ixl_get_tx_head(struct ixl_queue *que)
 853 {
 854         struct tx_ring  *txr = &que->txr;
 855         void *head = &txr->base[que->num_desc];
 856         return LE32_TO_CPU(*(volatile __le32 *)head);
 857 }
 858
 859 /**********************************************************************
 860  *
 861  *  Examine each tx_buffer in the used queue. If the hardware is done
 862  *  processing the packet then free associated resources. The
 863  *  tx_buffer is put back on the free queue.
 864  *
 865  **********************************************************************/
 866 bool
 867 ixl_txeof(struct ixl_queue *que)
 868 {
 869         struct tx_ring          *txr = &que->txr;
 870         u32                     first, last, head, done, processed;
 871         struct ixl_tx_buf       *buf;
 872         struct i40e_tx_desc     *tx_desc, *eop_desc;
 873
 874
 875         mtx_assert(&txr->mtx, MA_OWNED);
 876
 877 #ifdef DEV_NETMAP
 878         // XXX todo: implement moderation
 879         if (netmap_tx_irq(que->vsi->ifp, que->me))
 880                 return FALSE;
 881 #endif /* DEF_NETMAP */
 882
 883         /* These are not the descriptors you seek, move along :) */
 884         if (txr->avail == que->num_desc) {
 885                 atomic_store_rel_32(&txr->watchdog_timer, 0);
 886                 return FALSE;
 887         }
 888
 889         processed = 0;
 890         first = txr->next_to_clean;
 891         buf = &txr->buffers[first];
 892         tx_desc = (struct i40e_tx_desc *)&txr->base[first];
 893         last = buf->eop_index;
 894         if (last == -1)
 895                 return FALSE;
 896         eop_desc = (struct i40e_tx_desc *)&txr->base[last];
 897
 898         /* Get the Head WB value */
 899         head = ixl_get_tx_head(que);
 900
 901         /*
 902         ** Get the index of the first descriptor
 903         ** BEYOND the EOP and call that 'done'.
 904         ** I do this so the comparison in the
 905         ** inner while loop below can be simple
 906         */
 907         if (++last == que->num_desc) last = 0;
 908         done = last;
 909
 910         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 911             BUS_DMASYNC_POSTREAD);
 912         /*
 913         ** The HEAD index of the ring is written in a
 914         ** defined location, this rather than a done bit
 915         ** is what is used to keep track of what must be
 916         ** 'cleaned'.
 917         */
 918         while (first != head) {
 919                 /* We clean the range of the packet */
 920                 while (first != done) {
 921                         ++txr->avail;
 922                         ++processed;
 923
 924                         if (buf->m_head) {
 925                                 txr->bytes += /* for ITR adjustment */
 926                                     buf->m_head->m_pkthdr.len;
 927                                 txr->tx_bytes += /* for TX stats */
 928                                     buf->m_head->m_pkthdr.len;
 929                                 bus_dmamap_sync(buf->tag,
 930                                     buf->map,
 931                                     BUS_DMASYNC_POSTWRITE);
 932                                 bus_dmamap_unload(buf->tag,
 933                                     buf->map);
 934                                 m_freem(buf->m_head);
 935                                 buf->m_head = NULL;
 936                                 buf->map = NULL;
 937                         }
 938                         buf->eop_index = -1;
 939
 940                         if (++first == que->num_desc)
 941                                 first = 0;
 942
 943                         buf = &txr->buffers[first];
 944                         tx_desc = &txr->base[first];
 945                 }
 946                 ++txr->packets;
 947                 /* See if there is more work now */
 948                 last = buf->eop_index;
 949                 if (last != -1) {
 950                         eop_desc = &txr->base[last];
 951                         /* Get next done point */
 952                         if (++last == que->num_desc) last = 0;
 953                         done = last;
 954                 } else
 955                         break;
 956         }
 957         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 958             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 959
 960         txr->next_to_clean = first;
 961
 962
 963         /*
 964          * If there are no pending descriptors, clear the timeout.
 965          */
 966         if (txr->avail == que->num_desc) {
 967                 atomic_store_rel_32(&txr->watchdog_timer, 0);
 968                 return FALSE;
 969         }
 970
 971         return TRUE;
 972 }
 973
 974 /*********************************************************************
 975  *
 976  *  Refresh mbuf buffers for RX descriptor rings
 977  *   - now keeps its own state so discards due to resource
 978  *     exhaustion are unnecessary, if an mbuf cannot be obtained
 979  *     it just returns, keeping its placeholder, thus it can simply
 980  *     be recalled to try again.
 981  *
 982  **********************************************************************/
 983 static void
 984 ixl_refresh_mbufs(struct ixl_queue *que, int limit)
 985 {
 986         struct ixl_vsi          *vsi = que->vsi;
 987         struct rx_ring          *rxr = &que->rxr;
 988         bus_dma_segment_t       hseg[1];
 989         bus_dma_segment_t       pseg[1];
 990         struct ixl_rx_buf       *buf;
 991         struct mbuf             *mh, *mp;
 992         int                     i, j, nsegs, error;
 993         bool                    refreshed = FALSE;
 994
 995         i = j = rxr->next_refresh;
 996         /* Control the loop with one beyond */
 997         if (++j == que->num_desc)
 998                 j = 0;
 999
1000         while (j != limit) {
1001                 buf = &rxr->buffers[i];
1002                 if (rxr->hdr_split == FALSE)
1003                         goto no_split;
1004
1005                 if (buf->m_head == NULL) {
1006                         mh = m_gethdr(M_NOWAIT, MT_DATA);
1007                         if (mh == NULL)
1008                                 goto update;
1009                 } else
1010                         mh = buf->m_head;
1011
1012                 mh->m_pkthdr.len = mh->m_len = MHLEN;
1013                 mh->m_len = MHLEN;
1014                 mh->m_flags |= M_PKTHDR;
1015                 /* Get the memory mapping */
1016                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
1017                     buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
1018                 if (error != 0) {
1019                         printf("Refresh mbufs: hdr dmamap load"
1020                             " failure - %d\n", error);
1021                         m_free(mh);
1022                         buf->m_head = NULL;
1023                         goto update;
1024                 }
1025                 buf->m_head = mh;
1026                 bus_dmamap_sync(rxr->htag, buf->hmap,
1027                     BUS_DMASYNC_PREREAD);
1028                 rxr->base[i].read.hdr_addr =
1029                    htole64(hseg[0].ds_addr);
1030
1031 no_split:
1032                 if (buf->m_pack == NULL) {
1033                         mp = m_getjcl(M_NOWAIT, MT_DATA,
1034                             M_PKTHDR, rxr->mbuf_sz);
1035                         if (mp == NULL)
1036                                 goto update;
1037                 } else
1038                         mp = buf->m_pack;
1039
1040                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1041                 /* Get the memory mapping */
1042                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1043                     buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
1044                 if (error != 0) {
1045                         printf("Refresh mbufs: payload dmamap load"
1046                             " failure - %d\n", error);
1047                         m_free(mp);
1048                         buf->m_pack = NULL;
1049                         goto update;
1050                 }
1051                 buf->m_pack = mp;
1052                 bus_dmamap_sync(rxr->ptag, buf->pmap,
1053                     BUS_DMASYNC_PREREAD);
1054                 rxr->base[i].read.pkt_addr =
1055                    htole64(pseg[0].ds_addr);
1056                 /* Used only when doing header split */
1057                 rxr->base[i].read.hdr_addr = 0;
1058
1059                 refreshed = TRUE;
1060                 /* Next is precalculated */
1061                 i = j;
1062                 rxr->next_refresh = i;
1063                 if (++j == que->num_desc)
1064                         j = 0;
1065         }
1066 update:
1067         if (refreshed) /* Update hardware tail index */
1068                 wr32(vsi->hw, rxr->tail, rxr->next_refresh);
1069         return;
1070 }
1071
1072
1073 /*********************************************************************
1074  *
1075  *  Allocate memory for rx_buffer structures. Since we use one
1076  *  rx_buffer per descriptor, the maximum number of rx_buffer's
1077  *  that we'll need is equal to the number of receive descriptors
1078  *  that we've defined.
1079  *
1080  **********************************************************************/
1081 int
1082 ixl_allocate_rx_data(struct ixl_queue *que)
1083 {
1084         struct rx_ring          *rxr = &que->rxr;
1085         struct ixl_vsi          *vsi = que->vsi;
1086         device_t                dev = vsi->dev;
1087         struct ixl_rx_buf       *buf;
1088         int                     i, bsize, error;
1089
1090         bsize = sizeof(struct ixl_rx_buf) * que->num_desc;
1091         if (!(rxr->buffers =
1092             (struct ixl_rx_buf *) malloc(bsize,
1093             M_DEVBUF, M_NOWAIT | M_ZERO))) {
1094                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
1095                 error = ENOMEM;
1096                 return (error);
1097         }
1098
1099         if ((error = bus_dma_tag_create(NULL,   /* parent */
1100                                    1, 0,        /* alignment, bounds */
1101                                    BUS_SPACE_MAXADDR,   /* lowaddr */
1102                                    BUS_SPACE_MAXADDR,   /* highaddr */
1103                                    NULL, NULL,          /* filter, filterarg */
1104                                    MSIZE,               /* maxsize */
1105                                    1,                   /* nsegments */
1106                                    MSIZE,               /* maxsegsize */
1107                                    0,                   /* flags */
1108                                    NULL,                /* lockfunc */
1109                                    NULL,                /* lockfuncarg */
1110                                    &rxr->htag))) {
1111                 device_printf(dev, "Unable to create RX DMA htag\n");
1112                 return (error);
1113         }
1114
1115         if ((error = bus_dma_tag_create(NULL,   /* parent */
1116                                    1, 0,        /* alignment, bounds */
1117                                    BUS_SPACE_MAXADDR,   /* lowaddr */
1118                                    BUS_SPACE_MAXADDR,   /* highaddr */
1119                                    NULL, NULL,          /* filter, filterarg */
1120                                    MJUM16BYTES,         /* maxsize */
1121                                    1,                   /* nsegments */
1122                                    MJUM16BYTES,         /* maxsegsize */
1123                                    0,                   /* flags */
1124                                    NULL,                /* lockfunc */
1125                                    NULL,                /* lockfuncarg */
1126                                    &rxr->ptag))) {
1127                 device_printf(dev, "Unable to create RX DMA ptag\n");
1128                 return (error);
1129         }
1130
1131         for (i = 0; i < que->num_desc; i++) {
1132                 buf = &rxr->buffers[i];
1133                 error = bus_dmamap_create(rxr->htag,
1134                     BUS_DMA_NOWAIT, &buf->hmap);
1135                 if (error) {
1136                         device_printf(dev, "Unable to create RX head map\n");
1137                         break;
1138                 }
1139                 error = bus_dmamap_create(rxr->ptag,
1140                     BUS_DMA_NOWAIT, &buf->pmap);
1141                 if (error) {
1142                         device_printf(dev, "Unable to create RX pkt map\n");
1143                         break;
1144                 }
1145         }
1146
1147         return (error);
1148 }
1149
1150
1151 /*********************************************************************
1152  *
1153  *  (Re)Initialize the queue receive ring and its buffers.
1154  *
1155  **********************************************************************/
1156 int
1157 ixl_init_rx_ring(struct ixl_queue *que)
1158 {
1159         struct  rx_ring         *rxr = &que->rxr;
1160         struct ixl_vsi          *vsi = que->vsi;
1161 #if defined(INET6) || defined(INET)
1162         struct ifnet            *ifp = vsi->ifp;
1163         struct lro_ctrl         *lro = &rxr->lro;
1164 #endif
1165         struct ixl_rx_buf       *buf;
1166         bus_dma_segment_t       pseg[1], hseg[1];
1167         int                     rsize, nsegs, error = 0;
1168 #ifdef DEV_NETMAP
1169         struct netmap_adapter *na = NA(que->vsi->ifp);
1170         struct netmap_slot *slot;
1171 #endif /* DEV_NETMAP */
1172
1173         IXL_RX_LOCK(rxr);
1174 #ifdef DEV_NETMAP
1175         /* same as in ixl_init_tx_ring() */
1176         slot = netmap_reset(na, NR_RX, que->me, 0);
1177 #endif /* DEV_NETMAP */
1178         /* Clear the ring contents */
1179         rsize = roundup2(que->num_desc *
1180             sizeof(union i40e_rx_desc), DBA_ALIGN);
1181         bzero((void *)rxr->base, rsize);
1182         /* Cleanup any existing buffers */
1183         for (int i = 0; i < que->num_desc; i++) {
1184                 buf = &rxr->buffers[i];
1185                 if (buf->m_head != NULL) {
1186                         bus_dmamap_sync(rxr->htag, buf->hmap,
1187                             BUS_DMASYNC_POSTREAD);
1188                         bus_dmamap_unload(rxr->htag, buf->hmap);
1189                         buf->m_head->m_flags |= M_PKTHDR;
1190                         m_freem(buf->m_head);
1191                 }
1192                 if (buf->m_pack != NULL) {
1193                         bus_dmamap_sync(rxr->ptag, buf->pmap,
1194                             BUS_DMASYNC_POSTREAD);
1195                         bus_dmamap_unload(rxr->ptag, buf->pmap);
1196                         buf->m_pack->m_flags |= M_PKTHDR;
1197                         m_freem(buf->m_pack);
1198                 }
1199                 buf->m_head = NULL;
1200                 buf->m_pack = NULL;
1201         }
1202
1203         /* header split is off */
1204         rxr->hdr_split = FALSE;
1205
1206         /* Now replenish the mbufs */
1207         for (int j = 0; j != que->num_desc; ++j) {
1208                 struct mbuf     *mh, *mp;
1209
1210                 buf = &rxr->buffers[j];
1211 #ifdef DEV_NETMAP
1212                 /*
1213                  * In netmap mode, fill the map and set the buffer
1214                  * address in the NIC ring, considering the offset
1215                  * between the netmap and NIC rings (see comment in
1216                  * ixgbe_setup_transmit_ring() ). No need to allocate
1217                  * an mbuf, so end the block with a continue;
1218                  */
1219                 if (slot) {
1220                         int sj = netmap_idx_n2k(&na->rx_rings[que->me], j);
1221                         uint64_t paddr;
1222                         void *addr;
1223
1224                         addr = PNMB(na, slot + sj, &paddr);
1225                         netmap_load_map(na, rxr->dma.tag, buf->pmap, addr);
1226                         /* Update descriptor and the cached value */
1227                         rxr->base[j].read.pkt_addr = htole64(paddr);
1228                         rxr->base[j].read.hdr_addr = 0;
1229                         continue;
1230                 }
1231 #endif /* DEV_NETMAP */
1232                 /*
1233                 ** Don't allocate mbufs if not
1234                 ** doing header split, its wasteful
1235                 */
1236                 if (rxr->hdr_split == FALSE)
1237                         goto skip_head;
1238
1239                 /* First the header */
1240                 buf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
1241                 if (buf->m_head == NULL) {
1242                         error = ENOBUFS;
1243                         goto fail;
1244                 }
1245                 m_adj(buf->m_head, ETHER_ALIGN);
1246                 mh = buf->m_head;
1247                 mh->m_len = mh->m_pkthdr.len = MHLEN;
1248                 mh->m_flags |= M_PKTHDR;
1249                 /* Get the memory mapping */
1250                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
1251                     buf->hmap, buf->m_head, hseg,
1252                     &nsegs, BUS_DMA_NOWAIT);
1253                 if (error != 0) /* Nothing elegant to do here */
1254                         goto fail;
1255                 bus_dmamap_sync(rxr->htag,
1256                     buf->hmap, BUS_DMASYNC_PREREAD);
1257                 /* Update descriptor */
1258                 rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
1259
1260 skip_head:
1261                 /* Now the payload cluster */
1262                 buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
1263                     M_PKTHDR, rxr->mbuf_sz);
1264                 if (buf->m_pack == NULL) {
1265                         error = ENOBUFS;
1266                         goto fail;
1267                 }
1268                 mp = buf->m_pack;
1269                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1270                 /* Get the memory mapping */
1271                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1272                     buf->pmap, mp, pseg,
1273                     &nsegs, BUS_DMA_NOWAIT);
1274                 if (error != 0)
1275                         goto fail;
1276                 bus_dmamap_sync(rxr->ptag,
1277                     buf->pmap, BUS_DMASYNC_PREREAD);
1278                 /* Update descriptor */
1279                 rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
1280                 rxr->base[j].read.hdr_addr = 0;
1281         }
1282
1283
1284         /* Setup our descriptor indices */
1285         rxr->next_check = 0;
1286         rxr->next_refresh = 0;
1287         rxr->lro_enabled = FALSE;
1288         rxr->split = 0;
1289         rxr->bytes = 0;
1290         rxr->discard = FALSE;
1291
1292         wr32(vsi->hw, rxr->tail, que->num_desc - 1);
1293         ixl_flush(vsi->hw);
1294
1295 #if defined(INET6) || defined(INET)
1296         /*
1297         ** Now set up the LRO interface:
1298         */
1299         if (ifp->if_capenable & IFCAP_LRO) {
1300                 int err = tcp_lro_init(lro);
1301                 if (err) {
1302                         if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me);
1303                         goto fail;
1304                 }
1305                 INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me);
1306                 rxr->lro_enabled = TRUE;
1307                 lro->ifp = vsi->ifp;
1308         }
1309 #endif
1310
1311         bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1312             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1313
1314 fail:
1315         IXL_RX_UNLOCK(rxr);
1316         return (error);
1317 }
1318
1319
1320 /*********************************************************************
1321  *
1322  *  Free station receive ring data structures
1323  *
1324  **********************************************************************/
1325 void
1326 ixl_free_que_rx(struct ixl_queue *que)
1327 {
1328         struct rx_ring          *rxr = &que->rxr;
1329         struct ixl_rx_buf       *buf;
1330
1331         INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
1332
1333         /* Cleanup any existing buffers */
1334         if (rxr->buffers != NULL) {
1335                 for (int i = 0; i < que->num_desc; i++) {
1336                         buf = &rxr->buffers[i];
1337                         if (buf->m_head != NULL) {
1338                                 bus_dmamap_sync(rxr->htag, buf->hmap,
1339                                     BUS_DMASYNC_POSTREAD);
1340                                 bus_dmamap_unload(rxr->htag, buf->hmap);
1341                                 buf->m_head->m_flags |= M_PKTHDR;
1342                                 m_freem(buf->m_head);
1343                         }
1344                         if (buf->m_pack != NULL) {
1345                                 bus_dmamap_sync(rxr->ptag, buf->pmap,
1346                                     BUS_DMASYNC_POSTREAD);
1347                                 bus_dmamap_unload(rxr->ptag, buf->pmap);
1348                                 buf->m_pack->m_flags |= M_PKTHDR;
1349                                 m_freem(buf->m_pack);
1350                         }
1351                         buf->m_head = NULL;
1352                         buf->m_pack = NULL;
1353                         if (buf->hmap != NULL) {
1354                                 bus_dmamap_destroy(rxr->htag, buf->hmap);
1355                                 buf->hmap = NULL;
1356                         }
1357                         if (buf->pmap != NULL) {
1358                                 bus_dmamap_destroy(rxr->ptag, buf->pmap);
1359                                 buf->pmap = NULL;
1360                         }
1361                 }
1362                 if (rxr->buffers != NULL) {
1363                         free(rxr->buffers, M_DEVBUF);
1364                         rxr->buffers = NULL;
1365                 }
1366         }
1367
1368         if (rxr->htag != NULL) {
1369                 bus_dma_tag_destroy(rxr->htag);
1370                 rxr->htag = NULL;
1371         }
1372         if (rxr->ptag != NULL) {
1373                 bus_dma_tag_destroy(rxr->ptag);
1374                 rxr->ptag = NULL;
1375         }
1376
1377         INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
1378         return;
1379 }
1380
1381 static inline void
1382 ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype)
1383 {
1384
1385 #if defined(INET6) || defined(INET)
1386         /*
1387          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
1388          * should be computed by hardware. Also it should not have VLAN tag in
1389          * ethernet header.
1390          */
1391         if (rxr->lro_enabled &&
1392             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
1393             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
1394             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
1395                 /*
1396                  * Send to the stack if:
1397                  **  - LRO not enabled, or
1398                  **  - no LRO resources, or
1399                  **  - lro enqueue fails
1400                  */
1401                 if (rxr->lro.lro_cnt != 0)
1402                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
1403                                 return;
1404         }
1405 #endif
1406         IXL_RX_UNLOCK(rxr);
1407         (*ifp->if_input)(ifp, m);
1408         IXL_RX_LOCK(rxr);
1409 }
1410
1411
1412 static inline void
1413 ixl_rx_discard(struct rx_ring *rxr, int i)
1414 {
1415         struct ixl_rx_buf       *rbuf;
1416
1417         rbuf = &rxr->buffers[i];
1418
1419         if (rbuf->fmp != NULL) {/* Partial chain ? */
1420                 rbuf->fmp->m_flags |= M_PKTHDR;
1421                 m_freem(rbuf->fmp);
1422                 rbuf->fmp = NULL;
1423         }
1424
1425         /*
1426         ** With advanced descriptors the writeback
1427         ** clobbers the buffer addrs, so its easier
1428         ** to just free the existing mbufs and take
1429         ** the normal refresh path to get new buffers
1430         ** and mapping.
1431         */
1432         if (rbuf->m_head) {
1433                 m_free(rbuf->m_head);
1434                 rbuf->m_head = NULL;
1435         }
1436
1437         if (rbuf->m_pack) {
1438                 m_free(rbuf->m_pack);
1439                 rbuf->m_pack = NULL;
1440         }
1441
1442         return;
1443 }
1444
1445 #ifdef RSS
1446 /*
1447 ** i40e_ptype_to_hash: parse the packet type
1448 ** to determine the appropriate hash.
1449 */
1450 static inline int
1451 ixl_ptype_to_hash(u8 ptype)
1452 {
1453         struct i40e_rx_ptype_decoded    decoded;
1454         u8                              ex = 0;
1455
1456         decoded = decode_rx_desc_ptype(ptype);
1457         ex = decoded.outer_frag;
1458
1459         if (!decoded.known)
1460                 return M_HASHTYPE_OPAQUE_HASH;
1461
1462         if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2)
1463                 return M_HASHTYPE_OPAQUE_HASH;
1464
1465         /* Note: anything that gets to this point is IP */
1466         if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) {
1467                 switch (decoded.inner_prot) {
1468                         case I40E_RX_PTYPE_INNER_PROT_TCP:
1469                                 if (ex)
1470                                         return M_HASHTYPE_RSS_TCP_IPV6_EX;
1471                                 else
1472                                         return M_HASHTYPE_RSS_TCP_IPV6;
1473                         case I40E_RX_PTYPE_INNER_PROT_UDP:
1474                                 if (ex)
1475                                         return M_HASHTYPE_RSS_UDP_IPV6_EX;
1476                                 else
1477                                         return M_HASHTYPE_RSS_UDP_IPV6;
1478                         default:
1479                                 if (ex)
1480                                         return M_HASHTYPE_RSS_IPV6_EX;
1481                                 else
1482                                         return M_HASHTYPE_RSS_IPV6;
1483                 }
1484         }
1485         if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) {
1486                 switch (decoded.inner_prot) {
1487                         case I40E_RX_PTYPE_INNER_PROT_TCP:
1488                                         return M_HASHTYPE_RSS_TCP_IPV4;
1489                         case I40E_RX_PTYPE_INNER_PROT_UDP:
1490                                 if (ex)
1491                                         return M_HASHTYPE_RSS_UDP_IPV4_EX;
1492                                 else
1493                                         return M_HASHTYPE_RSS_UDP_IPV4;
1494                         default:
1495                                         return M_HASHTYPE_RSS_IPV4;
1496                 }
1497         }
1498         /* We should never get here!! */
1499         return M_HASHTYPE_OPAQUE_HASH;
1500 }
1501 #endif /* RSS */
1502
1503 /*********************************************************************
1504  *
1505  *  This routine executes in interrupt context. It replenishes
1506  *  the mbufs in the descriptor and sends data which has been
1507  *  dma'ed into host memory to upper layer.
1508  *
1509  *  We loop at most count times if count is > 0, or until done if
1510  *  count < 0.
1511  *
1512  *  Return TRUE for more work, FALSE for all clean.
1513  *********************************************************************/
1514 bool
1515 ixl_rxeof(struct ixl_queue *que, int count)
1516 {
1517         struct ixl_vsi          *vsi = que->vsi;
1518         struct rx_ring          *rxr = &que->rxr;
1519         struct ifnet            *ifp = vsi->ifp;
1520 #if defined(INET6) || defined(INET)
1521         struct lro_ctrl         *lro = &rxr->lro;
1522 #endif
1523         int                     i, nextp, processed = 0;
1524         union i40e_rx_desc      *cur;
1525         struct ixl_rx_buf       *rbuf, *nbuf;
1526
1527
1528         IXL_RX_LOCK(rxr);
1529
1530 #ifdef DEV_NETMAP
1531         if (netmap_rx_irq(ifp, que->me, &count)) {
1532                 IXL_RX_UNLOCK(rxr);
1533                 return (FALSE);
1534         }
1535 #endif /* DEV_NETMAP */
1536
1537         for (i = rxr->next_check; count != 0;) {
1538                 struct mbuf     *sendmp, *mh, *mp;
1539                 u32             status, error;
1540                 u16             hlen, plen, vtag;
1541                 u64             qword;
1542                 u8              ptype;
1543                 bool            eop;
1544
1545                 /* Sync the ring. */
1546                 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1547                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1548
1549                 cur = &rxr->base[i];
1550                 qword = le64toh(cur->wb.qword1.status_error_len);
1551                 status = (qword & I40E_RXD_QW1_STATUS_MASK)
1552                     >> I40E_RXD_QW1_STATUS_SHIFT;
1553                 error = (qword & I40E_RXD_QW1_ERROR_MASK)
1554                     >> I40E_RXD_QW1_ERROR_SHIFT;
1555                 plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK)
1556                     >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1557                 hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK)
1558                     >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1559                 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK)
1560                     >> I40E_RXD_QW1_PTYPE_SHIFT;
1561
1562                 if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) {
1563                         ++rxr->not_done;
1564                         break;
1565                 }
1566                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1567                         break;
1568
1569                 count--;
1570                 sendmp = NULL;
1571                 nbuf = NULL;
1572                 cur->wb.qword1.status_error_len = 0;
1573                 rbuf = &rxr->buffers[i];
1574                 mh = rbuf->m_head;
1575                 mp = rbuf->m_pack;
1576                 eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT));
1577                 if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT))
1578                         vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1);
1579                 else
1580                         vtag = 0;
1581
1582                 /*
1583                 ** Make sure bad packets are discarded,
1584                 ** note that only EOP descriptor has valid
1585                 ** error results.
1586                 */
1587                 if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1588                         rxr->desc_errs++;
1589                         ixl_rx_discard(rxr, i);
1590                         goto next_desc;
1591                 }
1592
1593                 /* Prefetch the next buffer */
1594                 if (!eop) {
1595                         nextp = i + 1;
1596                         if (nextp == que->num_desc)
1597                                 nextp = 0;
1598                         nbuf = &rxr->buffers[nextp];
1599                         prefetch(nbuf);
1600                 }
1601
1602                 /*
1603                 ** The header mbuf is ONLY used when header
1604                 ** split is enabled, otherwise we get normal
1605                 ** behavior, ie, both header and payload
1606                 ** are DMA'd into the payload buffer.
1607                 **
1608                 ** Rather than using the fmp/lmp global pointers
1609                 ** we now keep the head of a packet chain in the
1610                 ** buffer struct and pass this along from one
1611                 ** descriptor to the next, until we get EOP.
1612                 */
1613                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
1614                         if (hlen > IXL_RX_HDR)
1615                                 hlen = IXL_RX_HDR;
1616                         mh->m_len = hlen;
1617                         mh->m_flags |= M_PKTHDR;
1618                         mh->m_next = NULL;
1619                         mh->m_pkthdr.len = mh->m_len;
1620                         /* Null buf pointer so it is refreshed */
1621                         rbuf->m_head = NULL;
1622                         /*
1623                         ** Check the payload length, this
1624                         ** could be zero if its a small
1625                         ** packet.
1626                         */
1627                         if (plen > 0) {
1628                                 mp->m_len = plen;
1629                                 mp->m_next = NULL;
1630                                 mp->m_flags &= ~M_PKTHDR;
1631                                 mh->m_next = mp;
1632                                 mh->m_pkthdr.len += mp->m_len;
1633                                 /* Null buf pointer so it is refreshed */
1634                                 rbuf->m_pack = NULL;
1635                                 rxr->split++;
1636                         }
1637                         /*
1638                         ** Now create the forward
1639                         ** chain so when complete
1640                         ** we wont have to.
1641                         */
1642                         if (eop == 0) {
1643                                 /* stash the chain head */
1644                                 nbuf->fmp = mh;
1645                                 /* Make forward chain */
1646                                 if (plen)
1647                                         mp->m_next = nbuf->m_pack;
1648                                 else
1649                                         mh->m_next = nbuf->m_pack;
1650                         } else {
1651                                 /* Singlet, prepare to send */
1652                                 sendmp = mh;
1653                                 if (vtag) {
1654                                         sendmp->m_pkthdr.ether_vtag = vtag;
1655                                         sendmp->m_flags |= M_VLANTAG;
1656                                 }
1657                         }
1658                 } else {
1659                         /*
1660                         ** Either no header split, or a
1661                         ** secondary piece of a fragmented
1662                         ** split packet.
1663                         */
1664                         mp->m_len = plen;
1665                         /*
1666                         ** See if there is a stored head
1667                         ** that determines what we are
1668                         */
1669                         sendmp = rbuf->fmp;
1670                         rbuf->m_pack = rbuf->fmp = NULL;
1671
1672                         if (sendmp != NULL) /* secondary frag */
1673                                 sendmp->m_pkthdr.len += mp->m_len;
1674                         else {
1675                                 /* first desc of a non-ps chain */
1676                                 sendmp = mp;
1677                                 sendmp->m_flags |= M_PKTHDR;
1678                                 sendmp->m_pkthdr.len = mp->m_len;
1679                         }
1680                         /* Pass the head pointer on */
1681                         if (eop == 0) {
1682                                 nbuf->fmp = sendmp;
1683                                 sendmp = NULL;
1684                                 mp->m_next = nbuf->m_pack;
1685                         }
1686                 }
1687                 ++processed;
1688                 /* Sending this frame? */
1689                 if (eop) {
1690                         sendmp->m_pkthdr.rcvif = ifp;
1691                         /* gather stats */
1692                         rxr->rx_packets++;
1693                         rxr->rx_bytes += sendmp->m_pkthdr.len;
1694                         /* capture data for dynamic ITR adjustment */
1695                         rxr->packets++;
1696                         rxr->bytes += sendmp->m_pkthdr.len;
1697                         /* Set VLAN tag (field only valid in eop desc) */
1698                         if (vtag) {
1699                                 sendmp->m_pkthdr.ether_vtag = vtag;
1700                                 sendmp->m_flags |= M_VLANTAG;
1701                         }
1702                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
1703                                 ixl_rx_checksum(sendmp, status, error, ptype);
1704 #ifdef RSS
1705                         sendmp->m_pkthdr.flowid =
1706                             le32toh(cur->wb.qword0.hi_dword.rss);
1707                         M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype));
1708 #else
1709                         sendmp->m_pkthdr.flowid = que->msix;
1710                         M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
1711 #endif
1712                 }
1713 next_desc:
1714                 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1715                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1716
1717                 /* Advance our pointers to the next descriptor. */
1718                 if (++i == que->num_desc)
1719                         i = 0;
1720
1721                 /* Now send to the stack or do LRO */
1722                 if (sendmp != NULL) {
1723                         rxr->next_check = i;
1724                         ixl_rx_input(rxr, ifp, sendmp, ptype);
1725                         i = rxr->next_check;
1726                 }
1727
1728                /* Every 8 descriptors we go to refresh mbufs */
1729                 if (processed == 8) {
1730                         ixl_refresh_mbufs(que, i);
1731                         processed = 0;
1732                 }
1733         }
1734
1735         /* Refresh any remaining buf structs */
1736         if (ixl_rx_unrefreshed(que))
1737                 ixl_refresh_mbufs(que, i);
1738
1739         rxr->next_check = i;
1740
1741 #if defined(INET6) || defined(INET)
1742         /*
1743          * Flush any outstanding LRO work
1744          */
1745 #if __FreeBSD_version >= 1100105
1746         tcp_lro_flush_all(lro);
1747 #else
1748         struct lro_entry *queued;
1749         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
1750                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
1751                 tcp_lro_flush(lro, queued);
1752         }
1753 #endif
1754 #endif /* defined(INET6) || defined(INET) */
1755
1756         IXL_RX_UNLOCK(rxr);
1757         return (FALSE);
1758 }
1759
1760
1761 /*********************************************************************
1762  *
1763  *  Verify that the hardware indicated that the checksum is valid.
1764  *  Inform the stack about the status of checksum so that stack
1765  *  doesn't spend time verifying the checksum.
1766  *
1767  *********************************************************************/
1768 static void
1769 ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype)
1770 {
1771         struct i40e_rx_ptype_decoded decoded;
1772
1773         decoded = decode_rx_desc_ptype(ptype);
1774
1775         /* Errors? */
1776         if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1777             (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) {
1778                 mp->m_pkthdr.csum_flags = 0;
1779                 return;
1780         }
1781
1782         /* IPv6 with extension headers likely have bad csum */
1783         if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1784             decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1785                 if (status &
1786                     (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) {
1787                         mp->m_pkthdr.csum_flags = 0;
1788                         return;
1789                 }
1790
1791
1792         /* IP Checksum Good */
1793         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
1794         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1795
1796         if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) {
1797                 mp->m_pkthdr.csum_flags |=
1798                     (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1799                 mp->m_pkthdr.csum_data |= htons(0xffff);
1800         }
1801         return;
1802 }
1803
1804 #if __FreeBSD_version >= 1100000
1805 uint64_t
1806 ixl_get_counter(if_t ifp, ift_counter cnt)
1807 {
1808         struct ixl_vsi *vsi;
1809
1810         vsi = if_getsoftc(ifp);
1811
1812         switch (cnt) {
1813         case IFCOUNTER_IPACKETS:
1814                 return (vsi->ipackets);
1815         case IFCOUNTER_IERRORS:
1816                 return (vsi->ierrors);
1817         case IFCOUNTER_OPACKETS:
1818                 return (vsi->opackets);
1819         case IFCOUNTER_OERRORS:
1820                 return (vsi->oerrors);
1821         case IFCOUNTER_COLLISIONS:
1822                 /* Collisions are by standard impossible in 40G/10G Ethernet */
1823                 return (0);
1824         case IFCOUNTER_IBYTES:
1825                 return (vsi->ibytes);
1826         case IFCOUNTER_OBYTES:
1827                 return (vsi->obytes);
1828         case IFCOUNTER_IMCASTS:
1829                 return (vsi->imcasts);
1830         case IFCOUNTER_OMCASTS:
1831                 return (vsi->omcasts);
1832         case IFCOUNTER_IQDROPS:
1833                 return (vsi->iqdrops);
1834         case IFCOUNTER_OQDROPS:
1835                 return (vsi->oqdrops);
1836         case IFCOUNTER_NOPROTO:
1837                 return (vsi->noproto);
1838         default:
1839                 return (if_get_counter_default(ifp, cnt));
1840         }
1841 }
1842 #endif
1843