sys/dev/ixl/ixl_txrx.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2013-2014, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 /*
  36 **      IXL driver TX/RX Routines:
  37 **          This was seperated to allow usage by
  38 **          both the BASE and the VF drivers.
  39 */
  40
  41 #include "opt_inet.h"
  42 #include "opt_inet6.h"
  43 #include "ixl.h"
  44
  45 /* Local Prototypes */
  46 static void     ixl_rx_checksum(struct mbuf *, u32, u32, u8);
  47 static void     ixl_refresh_mbufs(struct ixl_queue *, int);
  48 static int      ixl_xmit(struct ixl_queue *, struct mbuf **);
  49 static int      ixl_tx_setup_offload(struct ixl_queue *,
  50                     struct mbuf *, u32 *, u32 *);
  51 static bool     ixl_tso_setup(struct ixl_queue *, struct mbuf *);
  52
  53 static __inline void ixl_rx_discard(struct rx_ring *, int);
  54 static __inline void ixl_rx_input(struct rx_ring *, struct ifnet *,
  55                     struct mbuf *, u8);
  56
  57 /*
  58 ** Multiqueue Transmit driver
  59 **
  60 */
  61 int
  62 ixl_mq_start(struct ifnet *ifp, struct mbuf *m)
  63 {
  64         struct ixl_vsi          *vsi = ifp->if_softc;
  65         struct ixl_queue        *que;
  66         struct tx_ring          *txr;
  67         int                     err, i;
  68
  69         /* Which queue to use */
  70         if ((m->m_flags & M_FLOWID) != 0)
  71                 i = m->m_pkthdr.flowid % vsi->num_queues;
  72         else
  73                 i = curcpu % vsi->num_queues;
  74
  75         /* Check for a hung queue and pick alternative */
  76         if (((1 << i) & vsi->active_queues) == 0)
  77                 i = ffsl(vsi->active_queues);
  78
  79         que = &vsi->queues[i];
  80         txr = &que->txr;
  81
  82         err = drbr_enqueue(ifp, txr->br, m);
  83         if (err)
  84                 return(err);
  85         if (IXL_TX_TRYLOCK(txr)) {
  86                 ixl_mq_start_locked(ifp, txr);
  87                 IXL_TX_UNLOCK(txr);
  88         } else
  89                 taskqueue_enqueue(que->tq, &que->tx_task);
  90
  91         return (0);
  92 }
  93
  94 int
  95 ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
  96 {
  97         struct ixl_queue        *que = txr->que;
  98         struct ixl_vsi          *vsi = que->vsi;
  99         struct mbuf             *next;
 100         int                     err = 0;
 101
 102
 103         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 104             vsi->link_active == 0)
 105                 return (ENETDOWN);
 106
 107         /* Process the transmit queue */
 108         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 109                 if ((err = ixl_xmit(que, &next)) != 0) {
 110                         if (next == NULL)
 111                                 drbr_advance(ifp, txr->br);
 112                         else
 113                                 drbr_putback(ifp, txr->br, next);
 114                         break;
 115                 }
 116                 drbr_advance(ifp, txr->br);
 117                 /* Send a copy of the frame to the BPF listener */
 118                 ETHER_BPF_MTAP(ifp, next);
 119                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 120                         break;
 121         }
 122
 123         if (txr->avail < IXL_TX_CLEANUP_THRESHOLD)
 124                 ixl_txeof(que);
 125
 126         return (err);
 127 }
 128
 129 /*
 130  * Called from a taskqueue to drain queued transmit packets.
 131  */
 132 void
 133 ixl_deferred_mq_start(void *arg, int pending)
 134 {
 135         struct ixl_queue        *que = arg;
 136         struct tx_ring          *txr = &que->txr;
 137         struct ixl_vsi          *vsi = que->vsi;
 138         struct ifnet            *ifp = vsi->ifp;
 139
 140         IXL_TX_LOCK(txr);
 141         if (!drbr_empty(ifp, txr->br))
 142                 ixl_mq_start_locked(ifp, txr);
 143         IXL_TX_UNLOCK(txr);
 144 }
 145
 146 /*
 147 ** Flush all queue ring buffers
 148 */
 149 void
 150 ixl_qflush(struct ifnet *ifp)
 151 {
 152         struct ixl_vsi  *vsi = ifp->if_softc;
 153
 154         for (int i = 0; i < vsi->num_queues; i++) {
 155                 struct ixl_queue *que = &vsi->queues[i];
 156                 struct tx_ring  *txr = &que->txr;
 157                 struct mbuf     *m;
 158                 IXL_TX_LOCK(txr);
 159                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 160                         m_freem(m);
 161                 IXL_TX_UNLOCK(txr);
 162         }
 163         if_qflush(ifp);
 164 }
 165
 166 /*
 167 ** Find mbuf chains passed to the driver
 168 ** that are 'sparse', using more than 8
 169 ** mbufs to deliver an mss-size chunk of data
 170 */
 171 static inline bool
 172 ixl_tso_detect_sparse(struct mbuf *mp)
 173 {
 174         struct mbuf     *m;
 175         int             num = 0, mss;
 176         bool            ret = FALSE;
 177
 178         mss = mp->m_pkthdr.tso_segsz;
 179         for (m = mp->m_next; m != NULL; m = m->m_next) {
 180                 num++;
 181                 mss -= m->m_len;
 182                 if (mss < 1)
 183                         break;
 184                 if (m->m_next == NULL)
 185                         break;
 186         }
 187         if (num > IXL_SPARSE_CHAIN)
 188                 ret = TRUE;
 189
 190         return (ret);
 191 }
 192
 193
 194 /*********************************************************************
 195  *
 196  *  This routine maps the mbufs to tx descriptors, allowing the
 197  *  TX engine to transmit the packets.
 198  *      - return 0 on success, positive on failure
 199  *
 200  **********************************************************************/
 201 #define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
 202
 203 static int
 204 ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp)
 205 {
 206         struct ixl_vsi          *vsi = que->vsi;
 207         struct i40e_hw          *hw = vsi->hw;
 208         struct tx_ring          *txr = &que->txr;
 209         struct ixl_tx_buf       *buf;
 210         struct i40e_tx_desc     *txd = NULL;
 211         struct mbuf             *m_head, *m;
 212         int                     i, j, error, nsegs, maxsegs;
 213         int                     first, last = 0;
 214         u16                     vtag = 0;
 215         u32                     cmd, off;
 216         bus_dmamap_t            map;
 217         bus_dma_tag_t           tag;
 218         bus_dma_segment_t       segs[IXL_MAX_TSO_SEGS];
 219
 220
 221         cmd = off = 0;
 222         m_head = *m_headp;
 223
 224         /*
 225          * Important to capture the first descriptor
 226          * used because it will contain the index of
 227          * the one we tell the hardware to report back
 228          */
 229         first = txr->next_avail;
 230         buf = &txr->buffers[first];
 231         map = buf->map;
 232         tag = txr->tx_tag;
 233         maxsegs = IXL_MAX_TX_SEGS;
 234
 235         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
 236                 /* Use larger mapping for TSO */
 237                 tag = txr->tso_tag;
 238                 maxsegs = IXL_MAX_TSO_SEGS;
 239                 if (ixl_tso_detect_sparse(m_head)) {
 240                         m = m_defrag(m_head, M_NOWAIT);
 241                         if (m == NULL) {
 242                                 m_freem(*m_headp);
 243                                 *m_headp = NULL;
 244                                 return (ENOBUFS);
 245                         }
 246                         *m_headp = m;
 247                 }
 248         }
 249
 250         /*
 251          * Map the packet for DMA.
 252          */
 253         error = bus_dmamap_load_mbuf_sg(tag, map,
 254             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 255
 256         if (error == EFBIG) {
 257                 struct mbuf *m;
 258
 259                 m = m_collapse(*m_headp, M_NOWAIT, maxsegs);
 260                 if (m == NULL) {
 261                         que->mbuf_defrag_failed++;
 262                         m_freem(*m_headp);
 263                         *m_headp = NULL;
 264                         return (ENOBUFS);
 265                 }
 266                 *m_headp = m;
 267
 268                 /* Try it again */
 269                 error = bus_dmamap_load_mbuf_sg(tag, map,
 270                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 271
 272                 if (error == ENOMEM) {
 273                         que->tx_dma_setup++;
 274                         return (error);
 275                 } else if (error != 0) {
 276                         que->tx_dma_setup++;
 277                         m_freem(*m_headp);
 278                         *m_headp = NULL;
 279                         return (error);
 280                 }
 281         } else if (error == ENOMEM) {
 282                 que->tx_dma_setup++;
 283                 return (error);
 284         } else if (error != 0) {
 285                 que->tx_dma_setup++;
 286                 m_freem(*m_headp);
 287                 *m_headp = NULL;
 288                 return (error);
 289         }
 290
 291         /* Make certain there are enough descriptors */
 292         if (nsegs > txr->avail - 2) {
 293                 txr->no_desc++;
 294                 error = ENOBUFS;
 295                 goto xmit_fail;
 296         }
 297         m_head = *m_headp;
 298
 299         /* Set up the TSO/CSUM offload */
 300         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) {
 301                 error = ixl_tx_setup_offload(que, m_head, &cmd, &off);
 302                 if (error)
 303                         goto xmit_fail;
 304         }
 305
 306         cmd |= I40E_TX_DESC_CMD_ICRC;
 307         /* Grab the VLAN tag */
 308         if (m_head->m_flags & M_VLANTAG) {
 309                 cmd |= I40E_TX_DESC_CMD_IL2TAG1;
 310                 vtag = htole16(m_head->m_pkthdr.ether_vtag);
 311         }
 312
 313         i = txr->next_avail;
 314         for (j = 0; j < nsegs; j++) {
 315                 bus_size_t seglen;
 316
 317                 buf = &txr->buffers[i];
 318                 buf->tag = tag; /* Keep track of the type tag */
 319                 txd = &txr->base[i];
 320                 seglen = segs[j].ds_len;
 321
 322                 txd->buffer_addr = htole64(segs[j].ds_addr);
 323                 txd->cmd_type_offset_bsz =
 324                     htole64(I40E_TX_DESC_DTYPE_DATA
 325                     | ((u64)cmd  << I40E_TXD_QW1_CMD_SHIFT)
 326                     | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT)
 327                     | ((u64)seglen  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT)
 328                     | ((u64)vtag  << I40E_TXD_QW1_L2TAG1_SHIFT));
 329
 330                 last = i; /* descriptor that will get completion IRQ */
 331
 332                 if (++i == que->num_desc)
 333                         i = 0;
 334
 335                 buf->m_head = NULL;
 336                 buf->eop_index = -1;
 337         }
 338         /* Set the last descriptor for report */
 339         txd->cmd_type_offset_bsz |=
 340             htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT));
 341         txr->avail -= nsegs;
 342         txr->next_avail = i;
 343
 344         buf->m_head = m_head;
 345         /* Swap the dma map between the first and last descriptor */
 346         txr->buffers[first].map = buf->map;
 347         buf->map = map;
 348         bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE);
 349
 350         /* Set the index of the descriptor that will be marked done */
 351         buf = &txr->buffers[first];
 352         buf->eop_index = last;
 353
 354         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 355             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 356         /*
 357          * Advance the Transmit Descriptor Tail (Tdt), this tells the
 358          * hardware that this frame is available to transmit.
 359          */
 360         ++txr->total_packets;
 361         wr32(hw, txr->tail, i);
 362
 363         ixl_flush(hw);
 364         /* Mark outstanding work */
 365         if (que->busy == 0)
 366                 que->busy = 1;
 367         return (0);
 368
 369 xmit_fail:
 370         bus_dmamap_unload(tag, buf->map);
 371         return (error);
 372 }
 373
 374
 375 /*********************************************************************
 376  *
 377  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
 378  *  the information needed to transmit a packet on the wire. This is
 379  *  called only once at attach, setup is done every reset.
 380  *
 381  **********************************************************************/
 382 int
 383 ixl_allocate_tx_data(struct ixl_queue *que)
 384 {
 385         struct tx_ring          *txr = &que->txr;
 386         struct ixl_vsi          *vsi = que->vsi;
 387         device_t                dev = vsi->dev;
 388         struct ixl_tx_buf       *buf;
 389         int                     error = 0;
 390
 391         /*
 392          * Setup DMA descriptor areas.
 393          */
 394         if ((error = bus_dma_tag_create(NULL,           /* parent */
 395                                1, 0,                    /* alignment, bounds */
 396                                BUS_SPACE_MAXADDR,       /* lowaddr */
 397                                BUS_SPACE_MAXADDR,       /* highaddr */
 398                                NULL, NULL,              /* filter, filterarg */
 399                                IXL_TSO_SIZE,            /* maxsize */
 400                                IXL_MAX_TX_SEGS,         /* nsegments */
 401                                PAGE_SIZE,               /* maxsegsize */
 402                                0,                       /* flags */
 403                                NULL,                    /* lockfunc */
 404                                NULL,                    /* lockfuncarg */
 405                                &txr->tx_tag))) {
 406                 device_printf(dev,"Unable to allocate TX DMA tag\n");
 407                 goto fail;
 408         }
 409
 410         /* Make a special tag for TSO */
 411         if ((error = bus_dma_tag_create(NULL,           /* parent */
 412                                1, 0,                    /* alignment, bounds */
 413                                BUS_SPACE_MAXADDR,       /* lowaddr */
 414                                BUS_SPACE_MAXADDR,       /* highaddr */
 415                                NULL, NULL,              /* filter, filterarg */
 416                                IXL_TSO_SIZE,            /* maxsize */
 417                                IXL_MAX_TSO_SEGS,        /* nsegments */
 418                                PAGE_SIZE,               /* maxsegsize */
 419                                0,                       /* flags */
 420                                NULL,                    /* lockfunc */
 421                                NULL,                    /* lockfuncarg */
 422                                &txr->tso_tag))) {
 423                 device_printf(dev,"Unable to allocate TX TSO DMA tag\n");
 424                 goto fail;
 425         }
 426
 427         if (!(txr->buffers =
 428             (struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) *
 429             que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 430                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 431                 error = ENOMEM;
 432                 goto fail;
 433         }
 434
 435         /* Create the descriptor buffer default dma maps */
 436         buf = txr->buffers;
 437         for (int i = 0; i < que->num_desc; i++, buf++) {
 438                 buf->tag = txr->tx_tag;
 439                 error = bus_dmamap_create(buf->tag, 0, &buf->map);
 440                 if (error != 0) {
 441                         device_printf(dev, "Unable to create TX DMA map\n");
 442                         goto fail;
 443                 }
 444         }
 445 fail:
 446         return (error);
 447 }
 448
 449
 450 /*********************************************************************
 451  *
 452  *  (Re)Initialize a queue transmit ring.
 453  *      - called by init, it clears the descriptor ring,
 454  *        and frees any stale mbufs
 455  *
 456  **********************************************************************/
 457 void
 458 ixl_init_tx_ring(struct ixl_queue *que)
 459 {
 460         struct tx_ring *txr = &que->txr;
 461         struct ixl_tx_buf *buf;
 462
 463         /* Clear the old ring contents */
 464         IXL_TX_LOCK(txr);
 465         bzero((void *)txr->base,
 466               (sizeof(struct i40e_tx_desc)) * que->num_desc);
 467
 468         /* Reset indices */
 469         txr->next_avail = 0;
 470         txr->next_to_clean = 0;
 471
 472 #ifdef IXL_FDIR
 473         /* Initialize flow director */
 474         txr->atr_rate = ixl_atr_rate;
 475         txr->atr_count = 0;
 476 #endif
 477
 478         /* Free any existing tx mbufs. */
 479         buf = txr->buffers;
 480         for (int i = 0; i < que->num_desc; i++, buf++) {
 481                 if (buf->m_head != NULL) {
 482                         bus_dmamap_sync(buf->tag, buf->map,
 483                             BUS_DMASYNC_POSTWRITE);
 484                         bus_dmamap_unload(buf->tag, buf->map);
 485                         m_freem(buf->m_head);
 486                         buf->m_head = NULL;
 487                 }
 488                 /* Clear the EOP index */
 489                 buf->eop_index = -1;
 490         }
 491
 492         /* Set number of descriptors available */
 493         txr->avail = que->num_desc;
 494
 495         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 496             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 497         IXL_TX_UNLOCK(txr);
 498 }
 499
 500
 501 /*********************************************************************
 502  *
 503  *  Free transmit ring related data structures.
 504  *
 505  **********************************************************************/
 506 void
 507 ixl_free_que_tx(struct ixl_queue *que)
 508 {
 509         struct tx_ring *txr = &que->txr;
 510         struct ixl_tx_buf *buf;
 511
 512         INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
 513
 514         for (int i = 0; i < que->num_desc; i++) {
 515                 buf = &txr->buffers[i];
 516                 if (buf->m_head != NULL) {
 517                         bus_dmamap_sync(buf->tag, buf->map,
 518                             BUS_DMASYNC_POSTWRITE);
 519                         bus_dmamap_unload(buf->tag,
 520                             buf->map);
 521                         m_freem(buf->m_head);
 522                         buf->m_head = NULL;
 523                         if (buf->map != NULL) {
 524                                 bus_dmamap_destroy(buf->tag,
 525                                     buf->map);
 526                                 buf->map = NULL;
 527                         }
 528                 } else if (buf->map != NULL) {
 529                         bus_dmamap_unload(buf->tag,
 530                             buf->map);
 531                         bus_dmamap_destroy(buf->tag,
 532                             buf->map);
 533                         buf->map = NULL;
 534                 }
 535         }
 536         if (txr->br != NULL)
 537                 buf_ring_free(txr->br, M_DEVBUF);
 538         if (txr->buffers != NULL) {
 539                 free(txr->buffers, M_DEVBUF);
 540                 txr->buffers = NULL;
 541         }
 542         if (txr->tx_tag != NULL) {
 543                 bus_dma_tag_destroy(txr->tx_tag);
 544                 txr->tx_tag = NULL;
 545         }
 546         if (txr->tso_tag != NULL) {
 547                 bus_dma_tag_destroy(txr->tso_tag);
 548                 txr->tso_tag = NULL;
 549         }
 550
 551         INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
 552         return;
 553 }
 554
 555 /*********************************************************************
 556  *
 557  *  Setup descriptor for hw offloads
 558  *
 559  **********************************************************************/
 560
 561 static int
 562 ixl_tx_setup_offload(struct ixl_queue *que,
 563     struct mbuf *mp, u32 *cmd, u32 *off)
 564 {
 565         struct ether_vlan_header        *eh;
 566 #ifdef INET
 567         struct ip                       *ip = NULL;
 568 #endif
 569         struct tcphdr                   *th = NULL;
 570 #ifdef INET6
 571         struct ip6_hdr                  *ip6;
 572 #endif
 573         int                             elen, ip_hlen = 0, tcp_hlen;
 574         u16                             etype;
 575         u8                              ipproto = 0;
 576         bool                            tso = FALSE;
 577
 578
 579         /* Set up the TSO context descriptor if required */
 580         if (mp->m_pkthdr.csum_flags & CSUM_TSO) {
 581                 tso = ixl_tso_setup(que, mp);
 582                 if (tso)
 583                         ++que->tso;
 584                 else
 585                         return (ENXIO);
 586         }
 587
 588         /*
 589          * Determine where frame payload starts.
 590          * Jump over vlan headers if already present,
 591          * helpful for QinQ too.
 592          */
 593         eh = mtod(mp, struct ether_vlan_header *);
 594         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 595                 etype = ntohs(eh->evl_proto);
 596                 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 597         } else {
 598                 etype = ntohs(eh->evl_encap_proto);
 599                 elen = ETHER_HDR_LEN;
 600         }
 601
 602         switch (etype) {
 603 #ifdef INET
 604                 case ETHERTYPE_IP:
 605                         ip = (struct ip *)(mp->m_data + elen);
 606                         ip_hlen = ip->ip_hl << 2;
 607                         ipproto = ip->ip_p;
 608                         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 609                         /* The IP checksum must be recalculated with TSO */
 610                         if (tso)
 611                                 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
 612                         else
 613                                 *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
 614                         break;
 615 #endif
 616 #ifdef INET6
 617                 case ETHERTYPE_IPV6:
 618                         ip6 = (struct ip6_hdr *)(mp->m_data + elen);
 619                         ip_hlen = sizeof(struct ip6_hdr);
 620                         ipproto = ip6->ip6_nxt;
 621                         th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
 622                         *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
 623                         break;
 624 #endif
 625                 default:
 626                         break;
 627         }
 628
 629         *off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
 630         *off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
 631
 632         switch (ipproto) {
 633                 case IPPROTO_TCP:
 634                         tcp_hlen = th->th_off << 2;
 635                         if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) {
 636                                 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
 637                                 *off |= (tcp_hlen >> 2) <<
 638                                     I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 639                         }
 640 #ifdef IXL_FDIR
 641                         ixl_atr(que, th, etype);
 642 #endif
 643                         break;
 644                 case IPPROTO_UDP:
 645                         if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) {
 646                                 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
 647                                 *off |= (sizeof(struct udphdr) >> 2) <<
 648                                     I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 649                         }
 650                         break;
 651
 652                 case IPPROTO_SCTP:
 653                         if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) {
 654                                 *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
 655                                 *off |= (sizeof(struct sctphdr) >> 2) <<
 656                                     I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 657                         }
 658                         /* Fall Thru */
 659                 default:
 660                         break;
 661         }
 662
 663         return (0);
 664 }
 665
 666
 667 /**********************************************************************
 668  *
 669  *  Setup context for hardware segmentation offload (TSO)
 670  *
 671  **********************************************************************/
 672 static bool
 673 ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp)
 674 {
 675         struct tx_ring                  *txr = &que->txr;
 676         struct i40e_tx_context_desc     *TXD;
 677         struct ixl_tx_buf               *buf;
 678         u32                             cmd, mss, type, tsolen;
 679         u16                             etype;
 680         int                             idx, elen, ip_hlen, tcp_hlen;
 681         struct ether_vlan_header        *eh;
 682 #ifdef INET
 683         struct ip                       *ip;
 684 #endif
 685 #ifdef INET6
 686         struct ip6_hdr                  *ip6;
 687 #endif
 688 #if defined(INET6) || defined(INET)
 689         struct tcphdr                   *th;
 690 #endif
 691         u64                             type_cmd_tso_mss;
 692
 693         /*
 694          * Determine where frame payload starts.
 695          * Jump over vlan headers if already present
 696          */
 697         eh = mtod(mp, struct ether_vlan_header *);
 698         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 699                 elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 700                 etype = eh->evl_proto;
 701         } else {
 702                 elen = ETHER_HDR_LEN;
 703                 etype = eh->evl_encap_proto;
 704         }
 705
 706         switch (ntohs(etype)) {
 707 #ifdef INET6
 708         case ETHERTYPE_IPV6:
 709                 ip6 = (struct ip6_hdr *)(mp->m_data + elen);
 710                 if (ip6->ip6_nxt != IPPROTO_TCP)
 711                         return (ENXIO);
 712                 ip_hlen = sizeof(struct ip6_hdr);
 713                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
 714                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
 715                 tcp_hlen = th->th_off << 2;
 716                 break;
 717 #endif
 718 #ifdef INET
 719         case ETHERTYPE_IP:
 720                 ip = (struct ip *)(mp->m_data + elen);
 721                 if (ip->ip_p != IPPROTO_TCP)
 722                         return (ENXIO);
 723                 ip->ip_sum = 0;
 724                 ip_hlen = ip->ip_hl << 2;
 725                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 726                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
 727                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 728                 tcp_hlen = th->th_off << 2;
 729                 break;
 730 #endif
 731         default:
 732                 printf("%s: CSUM_TSO but no supported IP version (0x%04x)",
 733                     __func__, ntohs(etype));
 734                 return FALSE;
 735         }
 736
 737         /* Ensure we have at least the IP+TCP header in the first mbuf. */
 738         if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr))
 739                 return FALSE;
 740
 741         idx = txr->next_avail;
 742         buf = &txr->buffers[idx];
 743         TXD = (struct i40e_tx_context_desc *) &txr->base[idx];
 744         tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen);
 745
 746         type = I40E_TX_DESC_DTYPE_CONTEXT;
 747         cmd = I40E_TX_CTX_DESC_TSO;
 748         mss = mp->m_pkthdr.tso_segsz;
 749
 750         type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) |
 751             ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
 752             ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
 753             ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
 754         TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss);
 755
 756         TXD->tunneling_params = htole32(0);
 757         buf->m_head = NULL;
 758         buf->eop_index = -1;
 759
 760         if (++idx == que->num_desc)
 761                 idx = 0;
 762
 763         txr->avail--;
 764         txr->next_avail = idx;
 765
 766         return TRUE;
 767 }
 768
 769 /*
 770 ** ixl_get_tx_head - Retrieve the value from the
 771 **    location the HW records its HEAD index
 772 */
 773 static inline u32
 774 ixl_get_tx_head(struct ixl_queue *que)
 775 {
 776         struct tx_ring  *txr = &que->txr;
 777         void *head = &txr->base[que->num_desc];
 778         return LE32_TO_CPU(*(volatile __le32 *)head);
 779 }
 780
 781 /**********************************************************************
 782  *
 783  *  Examine each tx_buffer in the used queue. If the hardware is done
 784  *  processing the packet then free associated resources. The
 785  *  tx_buffer is put back on the free queue.
 786  *
 787  **********************************************************************/
 788 bool
 789 ixl_txeof(struct ixl_queue *que)
 790 {
 791         struct tx_ring          *txr = &que->txr;
 792         u32                     first, last, head, done, processed;
 793         struct ixl_tx_buf       *buf;
 794         struct i40e_tx_desc     *tx_desc, *eop_desc;
 795
 796
 797         mtx_assert(&txr->mtx, MA_OWNED);
 798
 799
 800         /* These are not the descriptors you seek, move along :) */
 801         if (txr->avail == que->num_desc) {
 802                 que->busy = 0;
 803                 return FALSE;
 804         }
 805
 806         processed = 0;
 807         first = txr->next_to_clean;
 808         buf = &txr->buffers[first];
 809         tx_desc = (struct i40e_tx_desc *)&txr->base[first];
 810         last = buf->eop_index;
 811         if (last == -1)
 812                 return FALSE;
 813         eop_desc = (struct i40e_tx_desc *)&txr->base[last];
 814
 815         /* Get the Head WB value */
 816         head = ixl_get_tx_head(que);
 817
 818         /*
 819         ** Get the index of the first descriptor
 820         ** BEYOND the EOP and call that 'done'.
 821         ** I do this so the comparison in the
 822         ** inner while loop below can be simple
 823         */
 824         if (++last == que->num_desc) last = 0;
 825         done = last;
 826
 827         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 828             BUS_DMASYNC_POSTREAD);
 829         /*
 830         ** The HEAD index of the ring is written in a
 831         ** defined location, this rather than a done bit
 832         ** is what is used to keep track of what must be
 833         ** 'cleaned'.
 834         */
 835         while (first != head) {
 836                 /* We clean the range of the packet */
 837                 while (first != done) {
 838                         ++txr->avail;
 839                         ++processed;
 840
 841                         if (buf->m_head) {
 842                                 txr->bytes += /* for ITR adjustment */
 843                                     buf->m_head->m_pkthdr.len;
 844                                 txr->tx_bytes += /* for TX stats */
 845                                     buf->m_head->m_pkthdr.len;
 846                                 bus_dmamap_sync(buf->tag,
 847                                     buf->map,
 848                                     BUS_DMASYNC_POSTWRITE);
 849                                 bus_dmamap_unload(buf->tag,
 850                                     buf->map);
 851                                 m_freem(buf->m_head);
 852                                 buf->m_head = NULL;
 853                                 buf->map = NULL;
 854                         }
 855                         buf->eop_index = -1;
 856
 857                         if (++first == que->num_desc)
 858                                 first = 0;
 859
 860                         buf = &txr->buffers[first];
 861                         tx_desc = &txr->base[first];
 862                 }
 863                 ++txr->packets;
 864                 /* See if there is more work now */
 865                 last = buf->eop_index;
 866                 if (last != -1) {
 867                         eop_desc = &txr->base[last];
 868                         /* Get next done point */
 869                         if (++last == que->num_desc) last = 0;
 870                         done = last;
 871                 } else
 872                         break;
 873         }
 874         bus_dmamap_sync(txr->dma.tag, txr->dma.map,
 875             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 876
 877         txr->next_to_clean = first;
 878
 879
 880         /*
 881         ** Hang detection, we know there's
 882         ** work outstanding or the first return
 883         ** would have been taken, so indicate an
 884         ** unsuccessful pass, in local_timer if
 885         ** the value is too great the queue will
 886         ** be considered hung. If anything has been
 887         ** cleaned then reset the state.
 888         */
 889         if ((processed == 0) && (que->busy != IXL_QUEUE_HUNG))
 890                 ++que->busy;
 891
 892         if (processed)
 893                 que->busy = 1; /* Note this turns off HUNG */
 894
 895         /*
 896          * If there are no pending descriptors, clear the timeout.
 897          */
 898         if (txr->avail == que->num_desc) {
 899                 que->busy = 0;
 900                 return FALSE;
 901         }
 902
 903         return TRUE;
 904 }
 905
 906 /*********************************************************************
 907  *
 908  *  Refresh mbuf buffers for RX descriptor rings
 909  *   - now keeps its own state so discards due to resource
 910  *     exhaustion are unnecessary, if an mbuf cannot be obtained
 911  *     it just returns, keeping its placeholder, thus it can simply
 912  *     be recalled to try again.
 913  *
 914  **********************************************************************/
 915 static void
 916 ixl_refresh_mbufs(struct ixl_queue *que, int limit)
 917 {
 918         struct ixl_vsi          *vsi = que->vsi;
 919         struct rx_ring          *rxr = &que->rxr;
 920         bus_dma_segment_t       hseg[1];
 921         bus_dma_segment_t       pseg[1];
 922         struct ixl_rx_buf       *buf;
 923         struct mbuf             *mh, *mp;
 924         int                     i, j, nsegs, error;
 925         bool                    refreshed = FALSE;
 926
 927         i = j = rxr->next_refresh;
 928         /* Control the loop with one beyond */
 929         if (++j == que->num_desc)
 930                 j = 0;
 931
 932         while (j != limit) {
 933                 buf = &rxr->buffers[i];
 934                 if (rxr->hdr_split == FALSE)
 935                         goto no_split;
 936
 937                 if (buf->m_head == NULL) {
 938                         mh = m_gethdr(M_NOWAIT, MT_DATA);
 939                         if (mh == NULL)
 940                                 goto update;
 941                 } else
 942                         mh = buf->m_head;
 943
 944                 mh->m_pkthdr.len = mh->m_len = MHLEN;
 945                 mh->m_len = MHLEN;
 946                 mh->m_flags |= M_PKTHDR;
 947                 /* Get the memory mapping */
 948                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
 949                     buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
 950                 if (error != 0) {
 951                         printf("Refresh mbufs: hdr dmamap load"
 952                             " failure - %d\n", error);
 953                         m_free(mh);
 954                         buf->m_head = NULL;
 955                         goto update;
 956                 }
 957                 buf->m_head = mh;
 958                 bus_dmamap_sync(rxr->htag, buf->hmap,
 959                     BUS_DMASYNC_PREREAD);
 960                 rxr->base[i].read.hdr_addr =
 961                    htole64(hseg[0].ds_addr);
 962
 963 no_split:
 964                 if (buf->m_pack == NULL) {
 965                         mp = m_getjcl(M_NOWAIT, MT_DATA,
 966                             M_PKTHDR, rxr->mbuf_sz);
 967                         if (mp == NULL)
 968                                 goto update;
 969                 } else
 970                         mp = buf->m_pack;
 971
 972                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
 973                 /* Get the memory mapping */
 974                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
 975                     buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
 976                 if (error != 0) {
 977                         printf("Refresh mbufs: payload dmamap load"
 978                             " failure - %d\n", error);
 979                         m_free(mp);
 980                         buf->m_pack = NULL;
 981                         goto update;
 982                 }
 983                 buf->m_pack = mp;
 984                 bus_dmamap_sync(rxr->ptag, buf->pmap,
 985                     BUS_DMASYNC_PREREAD);
 986                 rxr->base[i].read.pkt_addr =
 987                    htole64(pseg[0].ds_addr);
 988                 /* Used only when doing header split */
 989                 rxr->base[i].read.hdr_addr = 0;
 990
 991                 refreshed = TRUE;
 992                 /* Next is precalculated */
 993                 i = j;
 994                 rxr->next_refresh = i;
 995                 if (++j == que->num_desc)
 996                         j = 0;
 997         }
 998 update:
 999         if (refreshed) /* Update hardware tail index */
1000                 wr32(vsi->hw, rxr->tail, rxr->next_refresh);
1001         return;
1002 }
1003
1004
1005 /*********************************************************************
1006  *
1007  *  Allocate memory for rx_buffer structures. Since we use one
1008  *  rx_buffer per descriptor, the maximum number of rx_buffer's
1009  *  that we'll need is equal to the number of receive descriptors
1010  *  that we've defined.
1011  *
1012  **********************************************************************/
1013 int
1014 ixl_allocate_rx_data(struct ixl_queue *que)
1015 {
1016         struct rx_ring          *rxr = &que->rxr;
1017         struct ixl_vsi          *vsi = que->vsi;
1018         device_t                dev = vsi->dev;
1019         struct ixl_rx_buf       *buf;
1020         int                     i, bsize, error;
1021
1022         bsize = sizeof(struct ixl_rx_buf) * que->num_desc;
1023         if (!(rxr->buffers =
1024             (struct ixl_rx_buf *) malloc(bsize,
1025             M_DEVBUF, M_NOWAIT | M_ZERO))) {
1026                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
1027                 error = ENOMEM;
1028                 return (error);
1029         }
1030
1031         if ((error = bus_dma_tag_create(NULL,   /* parent */
1032                                    1, 0,        /* alignment, bounds */
1033                                    BUS_SPACE_MAXADDR,   /* lowaddr */
1034                                    BUS_SPACE_MAXADDR,   /* highaddr */
1035                                    NULL, NULL,          /* filter, filterarg */
1036                                    MSIZE,               /* maxsize */
1037                                    1,                   /* nsegments */
1038                                    MSIZE,               /* maxsegsize */
1039                                    0,                   /* flags */
1040                                    NULL,                /* lockfunc */
1041                                    NULL,                /* lockfuncarg */
1042                                    &rxr->htag))) {
1043                 device_printf(dev, "Unable to create RX DMA htag\n");
1044                 return (error);
1045         }
1046
1047         if ((error = bus_dma_tag_create(NULL,   /* parent */
1048                                    1, 0,        /* alignment, bounds */
1049                                    BUS_SPACE_MAXADDR,   /* lowaddr */
1050                                    BUS_SPACE_MAXADDR,   /* highaddr */
1051                                    NULL, NULL,          /* filter, filterarg */
1052                                    MJUM16BYTES,         /* maxsize */
1053                                    1,                   /* nsegments */
1054                                    MJUM16BYTES,         /* maxsegsize */
1055                                    0,                   /* flags */
1056                                    NULL,                /* lockfunc */
1057                                    NULL,                /* lockfuncarg */
1058                                    &rxr->ptag))) {
1059                 device_printf(dev, "Unable to create RX DMA ptag\n");
1060                 return (error);
1061         }
1062
1063         for (i = 0; i < que->num_desc; i++) {
1064                 buf = &rxr->buffers[i];
1065                 error = bus_dmamap_create(rxr->htag,
1066                     BUS_DMA_NOWAIT, &buf->hmap);
1067                 if (error) {
1068                         device_printf(dev, "Unable to create RX head map\n");
1069                         break;
1070                 }
1071                 error = bus_dmamap_create(rxr->ptag,
1072                     BUS_DMA_NOWAIT, &buf->pmap);
1073                 if (error) {
1074                         device_printf(dev, "Unable to create RX pkt map\n");
1075                         break;
1076                 }
1077         }
1078
1079         return (error);
1080 }
1081
1082
1083 /*********************************************************************
1084  *
1085  *  (Re)Initialize the queue receive ring and its buffers.
1086  *
1087  **********************************************************************/
1088 int
1089 ixl_init_rx_ring(struct ixl_queue *que)
1090 {
1091         struct  rx_ring         *rxr = &que->rxr;
1092 #if defined(INET6) || defined(INET)
1093         struct ixl_vsi          *vsi = que->vsi;
1094         struct ifnet            *ifp = vsi->ifp;
1095         struct lro_ctrl         *lro = &rxr->lro;
1096 #endif
1097         struct ixl_rx_buf       *buf;
1098         bus_dma_segment_t       pseg[1], hseg[1];
1099         int                     rsize, nsegs, error = 0;
1100
1101         IXL_RX_LOCK(rxr);
1102         /* Clear the ring contents */
1103         rsize = roundup2(que->num_desc *
1104             sizeof(union i40e_rx_desc), DBA_ALIGN);
1105         bzero((void *)rxr->base, rsize);
1106         /* Cleanup any existing buffers */
1107         for (int i = 0; i < que->num_desc; i++) {
1108                 buf = &rxr->buffers[i];
1109                 if (buf->m_head != NULL) {
1110                         bus_dmamap_sync(rxr->htag, buf->hmap,
1111                             BUS_DMASYNC_POSTREAD);
1112                         bus_dmamap_unload(rxr->htag, buf->hmap);
1113                         buf->m_head->m_flags |= M_PKTHDR;
1114                         m_freem(buf->m_head);
1115                 }
1116                 if (buf->m_pack != NULL) {
1117                         bus_dmamap_sync(rxr->ptag, buf->pmap,
1118                             BUS_DMASYNC_POSTREAD);
1119                         bus_dmamap_unload(rxr->ptag, buf->pmap);
1120                         buf->m_pack->m_flags |= M_PKTHDR;
1121                         m_freem(buf->m_pack);
1122                 }
1123                 buf->m_head = NULL;
1124                 buf->m_pack = NULL;
1125         }
1126
1127         /* header split is off */
1128         rxr->hdr_split = FALSE;
1129
1130         /* Now replenish the mbufs */
1131         for (int j = 0; j != que->num_desc; ++j) {
1132                 struct mbuf     *mh, *mp;
1133
1134                 buf = &rxr->buffers[j];
1135                 /*
1136                 ** Don't allocate mbufs if not
1137                 ** doing header split, its wasteful
1138                 */
1139                 if (rxr->hdr_split == FALSE)
1140                         goto skip_head;
1141
1142                 /* First the header */
1143                 buf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
1144                 if (buf->m_head == NULL) {
1145                         error = ENOBUFS;
1146                         goto fail;
1147                 }
1148                 m_adj(buf->m_head, ETHER_ALIGN);
1149                 mh = buf->m_head;
1150                 mh->m_len = mh->m_pkthdr.len = MHLEN;
1151                 mh->m_flags |= M_PKTHDR;
1152                 /* Get the memory mapping */
1153                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
1154                     buf->hmap, buf->m_head, hseg,
1155                     &nsegs, BUS_DMA_NOWAIT);
1156                 if (error != 0) /* Nothing elegant to do here */
1157                         goto fail;
1158                 bus_dmamap_sync(rxr->htag,
1159                     buf->hmap, BUS_DMASYNC_PREREAD);
1160                 /* Update descriptor */
1161                 rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
1162
1163 skip_head:
1164                 /* Now the payload cluster */
1165                 buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
1166                     M_PKTHDR, rxr->mbuf_sz);
1167                 if (buf->m_pack == NULL) {
1168                         error = ENOBUFS;
1169                         goto fail;
1170                 }
1171                 mp = buf->m_pack;
1172                 mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz;
1173                 /* Get the memory mapping */
1174                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
1175                     buf->pmap, mp, pseg,
1176                     &nsegs, BUS_DMA_NOWAIT);
1177                 if (error != 0)
1178                         goto fail;
1179                 bus_dmamap_sync(rxr->ptag,
1180                     buf->pmap, BUS_DMASYNC_PREREAD);
1181                 /* Update descriptor */
1182                 rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
1183                 rxr->base[j].read.hdr_addr = 0;
1184         }
1185
1186
1187         /* Setup our descriptor indices */
1188         rxr->next_check = 0;
1189         rxr->next_refresh = 0;
1190         rxr->lro_enabled = FALSE;
1191         rxr->split = 0;
1192         rxr->bytes = 0;
1193         rxr->discard = FALSE;
1194
1195         wr32(vsi->hw, rxr->tail, que->num_desc - 1);
1196         ixl_flush(vsi->hw);
1197
1198 #if defined(INET6) || defined(INET)
1199         /*
1200         ** Now set up the LRO interface:
1201         */
1202         if (ifp->if_capenable & IFCAP_LRO) {
1203                 int err = tcp_lro_init(lro);
1204                 if (err) {
1205                         if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me);
1206                         goto fail;
1207                 }
1208                 INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me);
1209                 rxr->lro_enabled = TRUE;
1210                 lro->ifp = vsi->ifp;
1211         }
1212 #endif
1213
1214         bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1215             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1216
1217 fail:
1218         IXL_RX_UNLOCK(rxr);
1219         return (error);
1220 }
1221
1222
1223 /*********************************************************************
1224  *
1225  *  Free station receive ring data structures
1226  *
1227  **********************************************************************/
1228 void
1229 ixl_free_que_rx(struct ixl_queue *que)
1230 {
1231         struct rx_ring          *rxr = &que->rxr;
1232         struct ixl_rx_buf       *buf;
1233
1234         INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
1235
1236         /* Cleanup any existing buffers */
1237         if (rxr->buffers != NULL) {
1238                 for (int i = 0; i < que->num_desc; i++) {
1239                         buf = &rxr->buffers[i];
1240                         if (buf->m_head != NULL) {
1241                                 bus_dmamap_sync(rxr->htag, buf->hmap,
1242                                     BUS_DMASYNC_POSTREAD);
1243                                 bus_dmamap_unload(rxr->htag, buf->hmap);
1244                                 buf->m_head->m_flags |= M_PKTHDR;
1245                                 m_freem(buf->m_head);
1246                         }
1247                         if (buf->m_pack != NULL) {
1248                                 bus_dmamap_sync(rxr->ptag, buf->pmap,
1249                                     BUS_DMASYNC_POSTREAD);
1250                                 bus_dmamap_unload(rxr->ptag, buf->pmap);
1251                                 buf->m_pack->m_flags |= M_PKTHDR;
1252                                 m_freem(buf->m_pack);
1253                         }
1254                         buf->m_head = NULL;
1255                         buf->m_pack = NULL;
1256                         if (buf->hmap != NULL) {
1257                                 bus_dmamap_destroy(rxr->htag, buf->hmap);
1258                                 buf->hmap = NULL;
1259                         }
1260                         if (buf->pmap != NULL) {
1261                                 bus_dmamap_destroy(rxr->ptag, buf->pmap);
1262                                 buf->pmap = NULL;
1263                         }
1264                 }
1265                 if (rxr->buffers != NULL) {
1266                         free(rxr->buffers, M_DEVBUF);
1267                         rxr->buffers = NULL;
1268                 }
1269         }
1270
1271         if (rxr->htag != NULL) {
1272                 bus_dma_tag_destroy(rxr->htag);
1273                 rxr->htag = NULL;
1274         }
1275         if (rxr->ptag != NULL) {
1276                 bus_dma_tag_destroy(rxr->ptag);
1277                 rxr->ptag = NULL;
1278         }
1279
1280         INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
1281         return;
1282 }
1283
1284 static __inline void
1285 ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype)
1286 {
1287
1288 #if defined(INET6) || defined(INET)
1289         /*
1290          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
1291          * should be computed by hardware. Also it should not have VLAN tag in
1292          * ethernet header.
1293          */
1294         if (rxr->lro_enabled &&
1295             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
1296             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
1297             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
1298                 /*
1299                  * Send to the stack if:
1300                  **  - LRO not enabled, or
1301                  **  - no LRO resources, or
1302                  **  - lro enqueue fails
1303                  */
1304                 if (rxr->lro.lro_cnt != 0)
1305                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
1306                                 return;
1307         }
1308 #endif
1309         IXL_RX_UNLOCK(rxr);
1310         (*ifp->if_input)(ifp, m);
1311         IXL_RX_LOCK(rxr);
1312 }
1313
1314
1315 static __inline void
1316 ixl_rx_discard(struct rx_ring *rxr, int i)
1317 {
1318         struct ixl_rx_buf       *rbuf;
1319
1320         rbuf = &rxr->buffers[i];
1321
1322         if (rbuf->fmp != NULL) {/* Partial chain ? */
1323                 rbuf->fmp->m_flags |= M_PKTHDR;
1324                 m_freem(rbuf->fmp);
1325                 rbuf->fmp = NULL;
1326         }
1327
1328         /*
1329         ** With advanced descriptors the writeback
1330         ** clobbers the buffer addrs, so its easier
1331         ** to just free the existing mbufs and take
1332         ** the normal refresh path to get new buffers
1333         ** and mapping.
1334         */
1335         if (rbuf->m_head) {
1336                 m_free(rbuf->m_head);
1337                 rbuf->m_head = NULL;
1338         }
1339
1340         if (rbuf->m_pack) {
1341                 m_free(rbuf->m_pack);
1342                 rbuf->m_pack = NULL;
1343         }
1344
1345         return;
1346 }
1347
1348
1349 /*********************************************************************
1350  *
1351  *  This routine executes in interrupt context. It replenishes
1352  *  the mbufs in the descriptor and sends data which has been
1353  *  dma'ed into host memory to upper layer.
1354  *
1355  *  We loop at most count times if count is > 0, or until done if
1356  *  count < 0.
1357  *
1358  *  Return TRUE for more work, FALSE for all clean.
1359  *********************************************************************/
1360 bool
1361 ixl_rxeof(struct ixl_queue *que, int count)
1362 {
1363         struct ixl_vsi          *vsi = que->vsi;
1364         struct rx_ring          *rxr = &que->rxr;
1365         struct ifnet            *ifp = vsi->ifp;
1366 #if defined(INET6) || defined(INET)
1367         struct lro_ctrl         *lro = &rxr->lro;
1368         struct lro_entry        *queued;
1369 #endif
1370         int                     i, nextp, processed = 0;
1371         union i40e_rx_desc      *cur;
1372         struct ixl_rx_buf       *rbuf, *nbuf;
1373
1374
1375         IXL_RX_LOCK(rxr);
1376
1377
1378         for (i = rxr->next_check; count != 0;) {
1379                 struct mbuf     *sendmp, *mh, *mp;
1380                 u32             rsc, status, error;
1381                 u16             hlen, plen, vtag;
1382                 u64             qword;
1383                 u8              ptype;
1384                 bool            eop;
1385
1386                 /* Sync the ring. */
1387                 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1388                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1389
1390                 cur = &rxr->base[i];
1391                 qword = le64toh(cur->wb.qword1.status_error_len);
1392                 status = (qword & I40E_RXD_QW1_STATUS_MASK)
1393                     >> I40E_RXD_QW1_STATUS_SHIFT;
1394                 error = (qword & I40E_RXD_QW1_ERROR_MASK)
1395                     >> I40E_RXD_QW1_ERROR_SHIFT;
1396                 plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK)
1397                     >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1398                 hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK)
1399                     >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1400                 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK)
1401                     >> I40E_RXD_QW1_PTYPE_SHIFT;
1402
1403                 if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) {
1404                         ++rxr->not_done;
1405                         break;
1406                 }
1407                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1408                         break;
1409
1410                 count--;
1411                 sendmp = NULL;
1412                 nbuf = NULL;
1413                 rsc = 0;
1414                 cur->wb.qword1.status_error_len = 0;
1415                 rbuf = &rxr->buffers[i];
1416                 mh = rbuf->m_head;
1417                 mp = rbuf->m_pack;
1418                 eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT));
1419                 if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT))
1420                         vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1);
1421                 else
1422                         vtag = 0;
1423
1424                 /*
1425                 ** Make sure bad packets are discarded,
1426                 ** note that only EOP descriptor has valid
1427                 ** error results.
1428                 */
1429                 if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1430                         rxr->discarded++;
1431                         ixl_rx_discard(rxr, i);
1432                         goto next_desc;
1433                 }
1434
1435                 /* Prefetch the next buffer */
1436                 if (!eop) {
1437                         nextp = i + 1;
1438                         if (nextp == que->num_desc)
1439                                 nextp = 0;
1440                         nbuf = &rxr->buffers[nextp];
1441                         prefetch(nbuf);
1442                 }
1443
1444                 /*
1445                 ** The header mbuf is ONLY used when header
1446                 ** split is enabled, otherwise we get normal
1447                 ** behavior, ie, both header and payload
1448                 ** are DMA'd into the payload buffer.
1449                 **
1450                 ** Rather than using the fmp/lmp global pointers
1451                 ** we now keep the head of a packet chain in the
1452                 ** buffer struct and pass this along from one
1453                 ** descriptor to the next, until we get EOP.
1454                 */
1455                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
1456                         if (hlen > IXL_RX_HDR)
1457                                 hlen = IXL_RX_HDR;
1458                         mh->m_len = hlen;
1459                         mh->m_flags |= M_PKTHDR;
1460                         mh->m_next = NULL;
1461                         mh->m_pkthdr.len = mh->m_len;
1462                         /* Null buf pointer so it is refreshed */
1463                         rbuf->m_head = NULL;
1464                         /*
1465                         ** Check the payload length, this
1466                         ** could be zero if its a small
1467                         ** packet.
1468                         */
1469                         if (plen > 0) {
1470                                 mp->m_len = plen;
1471                                 mp->m_next = NULL;
1472                                 mp->m_flags &= ~M_PKTHDR;
1473                                 mh->m_next = mp;
1474                                 mh->m_pkthdr.len += mp->m_len;
1475                                 /* Null buf pointer so it is refreshed */
1476                                 rbuf->m_pack = NULL;
1477                                 rxr->split++;
1478                         }
1479                         /*
1480                         ** Now create the forward
1481                         ** chain so when complete
1482                         ** we wont have to.
1483                         */
1484                         if (eop == 0) {
1485                                 /* stash the chain head */
1486                                 nbuf->fmp = mh;
1487                                 /* Make forward chain */
1488                                 if (plen)
1489                                         mp->m_next = nbuf->m_pack;
1490                                 else
1491                                         mh->m_next = nbuf->m_pack;
1492                         } else {
1493                                 /* Singlet, prepare to send */
1494                                 sendmp = mh;
1495                                 if (vtag) {
1496                                         sendmp->m_pkthdr.ether_vtag = vtag;
1497                                         sendmp->m_flags |= M_VLANTAG;
1498                                 }
1499                         }
1500                 } else {
1501                         /*
1502                         ** Either no header split, or a
1503                         ** secondary piece of a fragmented
1504                         ** split packet.
1505                         */
1506                         mp->m_len = plen;
1507                         /*
1508                         ** See if there is a stored head
1509                         ** that determines what we are
1510                         */
1511                         sendmp = rbuf->fmp;
1512                         rbuf->m_pack = rbuf->fmp = NULL;
1513
1514                         if (sendmp != NULL) /* secondary frag */
1515                                 sendmp->m_pkthdr.len += mp->m_len;
1516                         else {
1517                                 /* first desc of a non-ps chain */
1518                                 sendmp = mp;
1519                                 sendmp->m_flags |= M_PKTHDR;
1520                                 sendmp->m_pkthdr.len = mp->m_len;
1521                                 if (vtag) {
1522                                         sendmp->m_pkthdr.ether_vtag = vtag;
1523                                         sendmp->m_flags |= M_VLANTAG;
1524                                 }
1525                         }
1526                         /* Pass the head pointer on */
1527                         if (eop == 0) {
1528                                 nbuf->fmp = sendmp;
1529                                 sendmp = NULL;
1530                                 mp->m_next = nbuf->m_pack;
1531                         }
1532                 }
1533                 ++processed;
1534                 /* Sending this frame? */
1535                 if (eop) {
1536                         sendmp->m_pkthdr.rcvif = ifp;
1537                         /* gather stats */
1538                         rxr->rx_packets++;
1539                         rxr->rx_bytes += sendmp->m_pkthdr.len;
1540                         /* capture data for dynamic ITR adjustment */
1541                         rxr->packets++;
1542                         rxr->bytes += sendmp->m_pkthdr.len;
1543                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
1544                                 ixl_rx_checksum(sendmp, status, error, ptype);
1545                         sendmp->m_pkthdr.flowid = que->msix;
1546                         sendmp->m_flags |= M_FLOWID;
1547                 }
1548 next_desc:
1549                 bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
1550                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1551
1552                 /* Advance our pointers to the next descriptor. */
1553                 if (++i == que->num_desc)
1554                         i = 0;
1555
1556                 /* Now send to the stack or do LRO */
1557                 if (sendmp != NULL) {
1558                         rxr->next_check = i;
1559                         ixl_rx_input(rxr, ifp, sendmp, ptype);
1560                         i = rxr->next_check;
1561                 }
1562
1563                /* Every 8 descriptors we go to refresh mbufs */
1564                 if (processed == 8) {
1565                         ixl_refresh_mbufs(que, i);
1566                         processed = 0;
1567                 }
1568         }
1569
1570         /* Refresh any remaining buf structs */
1571         if (ixl_rx_unrefreshed(que))
1572                 ixl_refresh_mbufs(que, i);
1573
1574         rxr->next_check = i;
1575
1576 #if defined(INET6) || defined(INET)
1577         /*
1578          * Flush any outstanding LRO work
1579          */
1580         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
1581                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
1582                 tcp_lro_flush(lro, queued);
1583         }
1584 #endif
1585
1586         IXL_RX_UNLOCK(rxr);
1587         return (FALSE);
1588 }
1589
1590
1591 /*********************************************************************
1592  *
1593  *  Verify that the hardware indicated that the checksum is valid.
1594  *  Inform the stack about the status of checksum so that stack
1595  *  doesn't spend time verifying the checksum.
1596  *
1597  *********************************************************************/
1598 static void
1599 ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype)
1600 {
1601         struct i40e_rx_ptype_decoded decoded;
1602
1603         decoded = decode_rx_desc_ptype(ptype);
1604
1605         /* Errors? */
1606         if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1607             (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) {
1608                 mp->m_pkthdr.csum_flags = 0;
1609                 return;
1610         }
1611
1612         /* IPv6 with extension headers likely have bad csum */
1613         if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1614             decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6)
1615                 if (status &
1616                     (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) {
1617                         mp->m_pkthdr.csum_flags = 0;
1618                         return;
1619                 }
1620
1621
1622         /* IP Checksum Good */
1623         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
1624         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1625
1626         if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) {
1627                 mp->m_pkthdr.csum_flags |=
1628                     (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1629                 mp->m_pkthdr.csum_data |= htons(0xffff);
1630         }
1631         return;
1632 }
1633
1634 #if __FreeBSD_version >= 1100000
1635 uint64_t
1636 ixl_get_counter(if_t ifp, ift_counter cnt)
1637 {
1638         struct ixl_vsi *vsi;
1639
1640         vsi = if_getsoftc(ifp);
1641
1642         switch (cnt) {
1643         case IFCOUNTER_IPACKETS:
1644                 return (vsi->ipackets);
1645         case IFCOUNTER_IERRORS:
1646                 return (vsi->ierrors);
1647         case IFCOUNTER_OPACKETS:
1648                 return (vsi->opackets);
1649         case IFCOUNTER_OERRORS:
1650                 return (vsi->oerrors);
1651         case IFCOUNTER_COLLISIONS:
1652                 /* Collisions are by standard impossible in 40G/10G Ethernet */
1653                 return (0);
1654         case IFCOUNTER_IBYTES:
1655                 return (vsi->ibytes);
1656         case IFCOUNTER_OBYTES:
1657                 return (vsi->obytes);
1658         case IFCOUNTER_IMCASTS:
1659                 return (vsi->imcasts);
1660         case IFCOUNTER_OMCASTS:
1661                 return (vsi->omcasts);
1662         case IFCOUNTER_IQDROPS:
1663                 return (vsi->iqdrops);
1664         case IFCOUNTER_OQDROPS:
1665                 return (vsi->oqdrops);
1666         case IFCOUNTER_NOPROTO:
1667                 return (vsi->noproto);
1668         default:
1669                 return (if_get_counter_default(ifp, cnt));
1670         }
1671 }
1672 #endif
1673