sys/dev/ixgbe/ixv.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2013, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37 #include "ixv.h"
  38
  39 /*********************************************************************
  40  *  Driver version
  41  *********************************************************************/
  42 char ixv_driver_version[] = "1.1.4";
  43
  44 /*********************************************************************
  45  *  PCI Device ID Table
  46  *
  47  *  Used by probe to select devices to load on
  48  *  Last field stores an index into ixv_strings
  49  *  Last entry must be all 0s
  50  *
  51  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  52  *********************************************************************/
  53
  54 static ixv_vendor_info_t ixv_vendor_info_array[] =
  55 {
  56         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0},
  57         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0},
  58         /* required last entry */
  59         {0, 0, 0, 0, 0}
  60 };
  61
  62 /*********************************************************************
  63  *  Table of branding strings
  64  *********************************************************************/
  65
  66 static char    *ixv_strings[] = {
  67         "Intel(R) PRO/10GbE Virtual Function Network Driver"
  68 };
  69
  70 /*********************************************************************
  71  *  Function prototypes
  72  *********************************************************************/
  73 static int      ixv_probe(device_t);
  74 static int      ixv_attach(device_t);
  75 static int      ixv_detach(device_t);
  76 static int      ixv_shutdown(device_t);
  77 #if __FreeBSD_version < 800000
  78 static void     ixv_start(struct ifnet *);
  79 static void     ixv_start_locked(struct tx_ring *, struct ifnet *);
  80 #else
  81 static int      ixv_mq_start(struct ifnet *, struct mbuf *);
  82 static int      ixv_mq_start_locked(struct ifnet *,
  83                     struct tx_ring *, struct mbuf *);
  84 static void     ixv_qflush(struct ifnet *);
  85 #endif
  86 static int      ixv_ioctl(struct ifnet *, u_long, caddr_t);
  87 static void     ixv_init(void *);
  88 static void     ixv_init_locked(struct adapter *);
  89 static void     ixv_stop(void *);
  90 static void     ixv_media_status(struct ifnet *, struct ifmediareq *);
  91 static int      ixv_media_change(struct ifnet *);
  92 static void     ixv_identify_hardware(struct adapter *);
  93 static int      ixv_allocate_pci_resources(struct adapter *);
  94 static int      ixv_allocate_msix(struct adapter *);
  95 static int      ixv_allocate_queues(struct adapter *);
  96 static int      ixv_setup_msix(struct adapter *);
  97 static void     ixv_free_pci_resources(struct adapter *);
  98 static void     ixv_local_timer(void *);
  99 static void     ixv_setup_interface(device_t, struct adapter *);
 100 static void     ixv_config_link(struct adapter *);
 101
 102 static int      ixv_allocate_transmit_buffers(struct tx_ring *);
 103 static int      ixv_setup_transmit_structures(struct adapter *);
 104 static void     ixv_setup_transmit_ring(struct tx_ring *);
 105 static void     ixv_initialize_transmit_units(struct adapter *);
 106 static void     ixv_free_transmit_structures(struct adapter *);
 107 static void     ixv_free_transmit_buffers(struct tx_ring *);
 108
 109 static int      ixv_allocate_receive_buffers(struct rx_ring *);
 110 static int      ixv_setup_receive_structures(struct adapter *);
 111 static int      ixv_setup_receive_ring(struct rx_ring *);
 112 static void     ixv_initialize_receive_units(struct adapter *);
 113 static void     ixv_free_receive_structures(struct adapter *);
 114 static void     ixv_free_receive_buffers(struct rx_ring *);
 115
 116 static void     ixv_enable_intr(struct adapter *);
 117 static void     ixv_disable_intr(struct adapter *);
 118 static bool     ixv_txeof(struct tx_ring *);
 119 static bool     ixv_rxeof(struct ix_queue *, int);
 120 static void     ixv_rx_checksum(u32, struct mbuf *, u32);
 121 static void     ixv_set_multi(struct adapter *);
 122 static void     ixv_update_link_status(struct adapter *);
 123 static void     ixv_refresh_mbufs(struct rx_ring *, int);
 124 static int      ixv_xmit(struct tx_ring *, struct mbuf **);
 125 static int      ixv_sysctl_stats(SYSCTL_HANDLER_ARGS);
 126 static int      ixv_sysctl_debug(SYSCTL_HANDLER_ARGS);
 127 static int      ixv_set_flowcntl(SYSCTL_HANDLER_ARGS);
 128 static int      ixv_dma_malloc(struct adapter *, bus_size_t,
 129                     struct ixv_dma_alloc *, int);
 130 static void     ixv_dma_free(struct adapter *, struct ixv_dma_alloc *);
 131 static void     ixv_add_rx_process_limit(struct adapter *, const char *,
 132                     const char *, int *, int);
 133 static bool     ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 134 static bool     ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 135 static void     ixv_set_ivar(struct adapter *, u8, u8, s8);
 136 static void     ixv_configure_ivars(struct adapter *);
 137 static u8 *     ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 138
 139 static void     ixv_setup_vlan_support(struct adapter *);
 140 static void     ixv_register_vlan(void *, struct ifnet *, u16);
 141 static void     ixv_unregister_vlan(void *, struct ifnet *, u16);
 142
 143 static void     ixv_save_stats(struct adapter *);
 144 static void     ixv_init_stats(struct adapter *);
 145 static void     ixv_update_stats(struct adapter *);
 146
 147 static __inline void ixv_rx_discard(struct rx_ring *, int);
 148 static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *,
 149                     struct mbuf *, u32);
 150
 151 /* The MSI/X Interrupt handlers */
 152 static void     ixv_msix_que(void *);
 153 static void     ixv_msix_mbx(void *);
 154
 155 /* Deferred interrupt tasklets */
 156 static void     ixv_handle_que(void *, int);
 157 static void     ixv_handle_mbx(void *, int);
 158
 159 /*********************************************************************
 160  *  FreeBSD Device Interface Entry Points
 161  *********************************************************************/
 162
 163 static device_method_t ixv_methods[] = {
 164         /* Device interface */
 165         DEVMETHOD(device_probe, ixv_probe),
 166         DEVMETHOD(device_attach, ixv_attach),
 167         DEVMETHOD(device_detach, ixv_detach),
 168         DEVMETHOD(device_shutdown, ixv_shutdown),
 169         DEVMETHOD_END
 170 };
 171
 172 static driver_t ixv_driver = {
 173         "ix", ixv_methods, sizeof(struct adapter),
 174 };
 175
 176 extern devclass_t ixgbe_devclass;
 177 DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0);
 178 MODULE_DEPEND(ixv, pci, 1, 1, 1);
 179 MODULE_DEPEND(ixv, ether, 1, 1, 1);
 180
 181 /*
 182 ** TUNEABLE PARAMETERS:
 183 */
 184
 185 /*
 186 ** AIM: Adaptive Interrupt Moderation
 187 ** which means that the interrupt rate
 188 ** is varied over time based on the
 189 ** traffic for that interrupt vector
 190 */
 191 static int ixv_enable_aim = FALSE;
 192 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim);
 193
 194 /* How many packets rxeof tries to clean at a time */
 195 static int ixv_rx_process_limit = 128;
 196 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit);
 197
 198 /* Flow control setting, default to full */
 199 static int ixv_flow_control = ixgbe_fc_full;
 200 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control);
 201
 202 /*
 203  * Header split: this causes the hardware to DMA
 204  * the header into a seperate mbuf from the payload,
 205  * it can be a performance win in some workloads, but
 206  * in others it actually hurts, its off by default.
 207  */
 208 static int ixv_header_split = FALSE;
 209 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split);
 210
 211 /*
 212 ** Number of TX descriptors per ring,
 213 ** setting higher than RX as this seems
 214 ** the better performing choice.
 215 */
 216 static int ixv_txd = DEFAULT_TXD;
 217 TUNABLE_INT("hw.ixv.txd", &ixv_txd);
 218
 219 /* Number of RX descriptors per ring */
 220 static int ixv_rxd = DEFAULT_RXD;
 221 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd);
 222
 223 /*
 224 ** Shadow VFTA table, this is needed because
 225 ** the real filter table gets cleared during
 226 ** a soft reset and we need to repopulate it.
 227 */
 228 static u32 ixv_shadow_vfta[VFTA_SIZE];
 229
 230 /*********************************************************************
 231  *  Device identification routine
 232  *
 233  *  ixv_probe determines if the driver should be loaded on
 234  *  adapter based on PCI vendor/device id of the adapter.
 235  *
 236  *  return BUS_PROBE_DEFAULT on success, positive on failure
 237  *********************************************************************/
 238
 239 static int
 240 ixv_probe(device_t dev)
 241 {
 242         ixv_vendor_info_t *ent;
 243
 244         u16     pci_vendor_id = 0;
 245         u16     pci_device_id = 0;
 246         u16     pci_subvendor_id = 0;
 247         u16     pci_subdevice_id = 0;
 248         char    adapter_name[256];
 249
 250
 251         pci_vendor_id = pci_get_vendor(dev);
 252         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 253                 return (ENXIO);
 254
 255         pci_device_id = pci_get_device(dev);
 256         pci_subvendor_id = pci_get_subvendor(dev);
 257         pci_subdevice_id = pci_get_subdevice(dev);
 258
 259         ent = ixv_vendor_info_array;
 260         while (ent->vendor_id != 0) {
 261                 if ((pci_vendor_id == ent->vendor_id) &&
 262                     (pci_device_id == ent->device_id) &&
 263
 264                     ((pci_subvendor_id == ent->subvendor_id) ||
 265                      (ent->subvendor_id == 0)) &&
 266
 267                     ((pci_subdevice_id == ent->subdevice_id) ||
 268                      (ent->subdevice_id == 0))) {
 269                         sprintf(adapter_name, "%s, Version - %s",
 270                                 ixv_strings[ent->index],
 271                                 ixv_driver_version);
 272                         device_set_desc_copy(dev, adapter_name);
 273                         return (BUS_PROBE_DEFAULT);
 274                 }
 275                 ent++;
 276         }
 277         return (ENXIO);
 278 }
 279
 280 /*********************************************************************
 281  *  Device initialization routine
 282  *
 283  *  The attach entry point is called when the driver is being loaded.
 284  *  This routine identifies the type of hardware, allocates all resources
 285  *  and initializes the hardware.
 286  *
 287  *  return 0 on success, positive on failure
 288  *********************************************************************/
 289
 290 static int
 291 ixv_attach(device_t dev)
 292 {
 293         struct adapter *adapter;
 294         struct ixgbe_hw *hw;
 295         int             error = 0;
 296
 297         INIT_DEBUGOUT("ixv_attach: begin");
 298
 299         /* Allocate, clear, and link in our adapter structure */
 300         adapter = device_get_softc(dev);
 301         adapter->dev = adapter->osdep.dev = dev;
 302         hw = &adapter->hw;
 303
 304         /* Core Lock Init*/
 305         IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 306
 307         /* SYSCTL APIs */
 308         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 309                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 310                         OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW,
 311                         adapter, 0, ixv_sysctl_stats, "I", "Statistics");
 312
 313         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 314                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 315                         OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW,
 316                         adapter, 0, ixv_sysctl_debug, "I", "Debug Info");
 317
 318         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 319                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 320                         OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW,
 321                         adapter, 0, ixv_set_flowcntl, "I", "Flow Control");
 322
 323         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 324                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 325                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 326                         &ixv_enable_aim, 1, "Interrupt Moderation");
 327
 328         /* Set up the timer callout */
 329         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 330
 331         /* Determine hardware revision */
 332         ixv_identify_hardware(adapter);
 333
 334         /* Do base PCI setup - map BAR0 */
 335         if (ixv_allocate_pci_resources(adapter)) {
 336                 device_printf(dev, "Allocation of PCI resources failed\n");
 337                 error = ENXIO;
 338                 goto err_out;
 339         }
 340
 341         /* Do descriptor calc and sanity checks */
 342         if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 343             ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) {
 344                 device_printf(dev, "TXD config issue, using default!\n");
 345                 adapter->num_tx_desc = DEFAULT_TXD;
 346         } else
 347                 adapter->num_tx_desc = ixv_txd;
 348
 349         if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 350             ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) {
 351                 device_printf(dev, "RXD config issue, using default!\n");
 352                 adapter->num_rx_desc = DEFAULT_RXD;
 353         } else
 354                 adapter->num_rx_desc = ixv_rxd;
 355
 356         /* Allocate our TX/RX Queues */
 357         if (ixv_allocate_queues(adapter)) {
 358                 error = ENOMEM;
 359                 goto err_out;
 360         }
 361
 362         /*
 363         ** Initialize the shared code: its
 364         ** at this point the mac type is set.
 365         */
 366         error = ixgbe_init_shared_code(hw);
 367         if (error) {
 368                 device_printf(dev,"Shared Code Initialization Failure\n");
 369                 error = EIO;
 370                 goto err_late;
 371         }
 372
 373         /* Setup the mailbox */
 374         ixgbe_init_mbx_params_vf(hw);
 375
 376         ixgbe_reset_hw(hw);
 377
 378         /* Get Hardware Flow Control setting */
 379         hw->fc.requested_mode = ixgbe_fc_full;
 380         hw->fc.pause_time = IXV_FC_PAUSE;
 381         hw->fc.low_water[0] = IXV_FC_LO;
 382         hw->fc.high_water[0] = IXV_FC_HI;
 383         hw->fc.send_xon = TRUE;
 384
 385         error = ixgbe_init_hw(hw);
 386         if (error) {
 387                 device_printf(dev,"Hardware Initialization Failure\n");
 388                 error = EIO;
 389                 goto err_late;
 390         }
 391
 392         error = ixv_allocate_msix(adapter);
 393         if (error)
 394                 goto err_late;
 395
 396         /* Setup OS specific network interface */
 397         ixv_setup_interface(dev, adapter);
 398
 399         /* Sysctl for limiting the amount of work done in the taskqueue */
 400         ixv_add_rx_process_limit(adapter, "rx_processing_limit",
 401             "max number of rx packets to process", &adapter->rx_process_limit,
 402             ixv_rx_process_limit);
 403
 404         /* Do the stats setup */
 405         ixv_save_stats(adapter);
 406         ixv_init_stats(adapter);
 407
 408         /* Register for VLAN events */
 409         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 410             ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 411         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 412             ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 413
 414         INIT_DEBUGOUT("ixv_attach: end");
 415         return (0);
 416
 417 err_late:
 418         ixv_free_transmit_structures(adapter);
 419         ixv_free_receive_structures(adapter);
 420 err_out:
 421         ixv_free_pci_resources(adapter);
 422         return (error);
 423
 424 }
 425
 426 /*********************************************************************
 427  *  Device removal routine
 428  *
 429  *  The detach entry point is called when the driver is being removed.
 430  *  This routine stops the adapter and deallocates all the resources
 431  *  that were allocated for driver operation.
 432  *
 433  *  return 0 on success, positive on failure
 434  *********************************************************************/
 435
 436 static int
 437 ixv_detach(device_t dev)
 438 {
 439         struct adapter *adapter = device_get_softc(dev);
 440         struct ix_queue *que = adapter->queues;
 441
 442         INIT_DEBUGOUT("ixv_detach: begin");
 443
 444         /* Make sure VLANS are not using driver */
 445         if (adapter->ifp->if_vlantrunk != NULL) {
 446                 device_printf(dev,"Vlan in use, detach first\n");
 447                 return (EBUSY);
 448         }
 449
 450         IXV_CORE_LOCK(adapter);
 451         ixv_stop(adapter);
 452         IXV_CORE_UNLOCK(adapter);
 453
 454         for (int i = 0; i < adapter->num_queues; i++, que++) {
 455                 if (que->tq) {
 456                         taskqueue_drain(que->tq, &que->que_task);
 457                         taskqueue_free(que->tq);
 458                 }
 459         }
 460
 461         /* Drain the Link queue */
 462         if (adapter->tq) {
 463                 taskqueue_drain(adapter->tq, &adapter->mbx_task);
 464                 taskqueue_free(adapter->tq);
 465         }
 466
 467         /* Unregister VLAN events */
 468         if (adapter->vlan_attach != NULL)
 469                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 470         if (adapter->vlan_detach != NULL)
 471                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 472
 473         ether_ifdetach(adapter->ifp);
 474         callout_drain(&adapter->timer);
 475         ixv_free_pci_resources(adapter);
 476         bus_generic_detach(dev);
 477         if_free(adapter->ifp);
 478
 479         ixv_free_transmit_structures(adapter);
 480         ixv_free_receive_structures(adapter);
 481
 482         IXV_CORE_LOCK_DESTROY(adapter);
 483         return (0);
 484 }
 485
 486 /*********************************************************************
 487  *
 488  *  Shutdown entry point
 489  *
 490  **********************************************************************/
 491 static int
 492 ixv_shutdown(device_t dev)
 493 {
 494         struct adapter *adapter = device_get_softc(dev);
 495         IXV_CORE_LOCK(adapter);
 496         ixv_stop(adapter);
 497         IXV_CORE_UNLOCK(adapter);
 498         return (0);
 499 }
 500
 501 #if __FreeBSD_version < 800000
 502 /*********************************************************************
 503  *  Transmit entry point
 504  *
 505  *  ixv_start is called by the stack to initiate a transmit.
 506  *  The driver will remain in this routine as long as there are
 507  *  packets to transmit and transmit resources are available.
 508  *  In case resources are not available stack is notified and
 509  *  the packet is requeued.
 510  **********************************************************************/
 511 static void
 512 ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 513 {
 514         struct mbuf    *m_head;
 515         struct adapter *adapter = txr->adapter;
 516
 517         IXV_TX_LOCK_ASSERT(txr);
 518
 519         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 520             IFF_DRV_RUNNING)
 521                 return;
 522         if (!adapter->link_active)
 523                 return;
 524
 525         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 526
 527                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 528                 if (m_head == NULL)
 529                         break;
 530
 531                 if (ixv_xmit(txr, &m_head)) {
 532                         if (m_head == NULL)
 533                                 break;
 534                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 535                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 536                         break;
 537                 }
 538                 /* Send a copy of the frame to the BPF listener */
 539                 ETHER_BPF_MTAP(ifp, m_head);
 540
 541                 /* Set watchdog on */
 542                 txr->watchdog_check = TRUE;
 543                 txr->watchdog_time = ticks;
 544
 545         }
 546         return;
 547 }
 548
 549 /*
 550  * Legacy TX start - called by the stack, this
 551  * always uses the first tx ring, and should
 552  * not be used with multiqueue tx enabled.
 553  */
 554 static void
 555 ixv_start(struct ifnet *ifp)
 556 {
 557         struct adapter *adapter = ifp->if_softc;
 558         struct tx_ring  *txr = adapter->tx_rings;
 559
 560         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 561                 IXV_TX_LOCK(txr);
 562                 ixv_start_locked(txr, ifp);
 563                 IXV_TX_UNLOCK(txr);
 564         }
 565         return;
 566 }
 567
 568 #else
 569
 570 /*
 571 ** Multiqueue Transmit driver
 572 **
 573 */
 574 static int
 575 ixv_mq_start(struct ifnet *ifp, struct mbuf *m)
 576 {
 577         struct adapter  *adapter = ifp->if_softc;
 578         struct ix_queue *que;
 579         struct tx_ring  *txr;
 580         int             i = 0, err = 0;
 581
 582         /* Which queue to use */
 583         if ((m->m_flags & M_FLOWID) != 0)
 584                 i = m->m_pkthdr.flowid % adapter->num_queues;
 585
 586         txr = &adapter->tx_rings[i];
 587         que = &adapter->queues[i];
 588
 589         if (IXV_TX_TRYLOCK(txr)) {
 590                 err = ixv_mq_start_locked(ifp, txr, m);
 591                 IXV_TX_UNLOCK(txr);
 592         } else {
 593                 err = drbr_enqueue(ifp, txr->br, m);
 594                 taskqueue_enqueue(que->tq, &que->que_task);
 595         }
 596
 597         return (err);
 598 }
 599
 600 static int
 601 ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 602 {
 603         struct adapter  *adapter = txr->adapter;
 604         struct mbuf     *next;
 605         int             enqueued, err = 0;
 606
 607         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 608             IFF_DRV_RUNNING || adapter->link_active == 0) {
 609                 if (m != NULL)
 610                         err = drbr_enqueue(ifp, txr->br, m);
 611                 return (err);
 612         }
 613
 614         /* Do a clean if descriptors are low */
 615         if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD)
 616                 ixv_txeof(txr);
 617
 618         enqueued = 0;
 619         if (m != NULL) {
 620                 err = drbr_enqueue(ifp, txr->br, m);
 621                 if (err) {
 622                         return (err);
 623                 }
 624         }
 625         /* Process the queue */
 626         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
 627                 if ((err = ixv_xmit(txr, &next)) != 0) {
 628                         if (next == NULL) {
 629                                 drbr_advance(ifp, txr->br);
 630                         } else {
 631                                 drbr_putback(ifp, txr->br, next);
 632                         }
 633                         break;
 634                 }
 635                 drbr_advance(ifp, txr->br);
 636                 enqueued++;
 637                 ifp->if_obytes += next->m_pkthdr.len;
 638                 if (next->m_flags & M_MCAST)
 639                         ifp->if_omcasts++;
 640                 /* Send a copy of the frame to the BPF listener */
 641                 ETHER_BPF_MTAP(ifp, next);
 642                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 643                         break;
 644                 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) {
 645                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 646                         break;
 647                 }
 648         }
 649
 650         if (enqueued > 0) {
 651                 /* Set watchdog on */
 652                 txr->watchdog_check = TRUE;
 653                 txr->watchdog_time = ticks;
 654         }
 655
 656         return (err);
 657 }
 658
 659 /*
 660 ** Flush all ring buffers
 661 */
 662 static void
 663 ixv_qflush(struct ifnet *ifp)
 664 {
 665         struct adapter  *adapter = ifp->if_softc;
 666         struct tx_ring  *txr = adapter->tx_rings;
 667         struct mbuf     *m;
 668
 669         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 670                 IXV_TX_LOCK(txr);
 671                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 672                         m_freem(m);
 673                 IXV_TX_UNLOCK(txr);
 674         }
 675         if_qflush(ifp);
 676 }
 677
 678 #endif
 679
 680 /*********************************************************************
 681  *  Ioctl entry point
 682  *
 683  *  ixv_ioctl is called when the user wants to configure the
 684  *  interface.
 685  *
 686  *  return 0 on success, positive on failure
 687  **********************************************************************/
 688
 689 static int
 690 ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 691 {
 692         struct adapter  *adapter = ifp->if_softc;
 693         struct ifreq    *ifr = (struct ifreq *) data;
 694 #if defined(INET) || defined(INET6)
 695         struct ifaddr   *ifa = (struct ifaddr *) data;
 696         bool            avoid_reset = FALSE;
 697 #endif
 698         int             error = 0;
 699
 700         switch (command) {
 701
 702         case SIOCSIFADDR:
 703 #ifdef INET
 704                 if (ifa->ifa_addr->sa_family == AF_INET)
 705                         avoid_reset = TRUE;
 706 #endif
 707 #ifdef INET6
 708                 if (ifa->ifa_addr->sa_family == AF_INET6)
 709                         avoid_reset = TRUE;
 710 #endif
 711 #if defined(INET) || defined(INET6)
 712                 /*
 713                 ** Calling init results in link renegotiation,
 714                 ** so we avoid doing it when possible.
 715                 */
 716                 if (avoid_reset) {
 717                         ifp->if_flags |= IFF_UP;
 718                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 719                                 ixv_init(adapter);
 720                         if (!(ifp->if_flags & IFF_NOARP))
 721                                 arp_ifinit(ifp, ifa);
 722                 } else
 723                         error = ether_ioctl(ifp, command, data);
 724                 break;
 725 #endif
 726         case SIOCSIFMTU:
 727                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 728                 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 729                         error = EINVAL;
 730                 } else {
 731                         IXV_CORE_LOCK(adapter);
 732                         ifp->if_mtu = ifr->ifr_mtu;
 733                         adapter->max_frame_size =
 734                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 735                         ixv_init_locked(adapter);
 736                         IXV_CORE_UNLOCK(adapter);
 737                 }
 738                 break;
 739         case SIOCSIFFLAGS:
 740                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 741                 IXV_CORE_LOCK(adapter);
 742                 if (ifp->if_flags & IFF_UP) {
 743                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 744                                 ixv_init_locked(adapter);
 745                 } else
 746                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 747                                 ixv_stop(adapter);
 748                 adapter->if_flags = ifp->if_flags;
 749                 IXV_CORE_UNLOCK(adapter);
 750                 break;
 751         case SIOCADDMULTI:
 752         case SIOCDELMULTI:
 753                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 754                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 755                         IXV_CORE_LOCK(adapter);
 756                         ixv_disable_intr(adapter);
 757                         ixv_set_multi(adapter);
 758                         ixv_enable_intr(adapter);
 759                         IXV_CORE_UNLOCK(adapter);
 760                 }
 761                 break;
 762         case SIOCSIFMEDIA:
 763         case SIOCGIFMEDIA:
 764                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 765                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 766                 break;
 767         case SIOCSIFCAP:
 768         {
 769                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 770                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 771                 if (mask & IFCAP_HWCSUM)
 772                         ifp->if_capenable ^= IFCAP_HWCSUM;
 773                 if (mask & IFCAP_TSO4)
 774                         ifp->if_capenable ^= IFCAP_TSO4;
 775                 if (mask & IFCAP_LRO)
 776                         ifp->if_capenable ^= IFCAP_LRO;
 777                 if (mask & IFCAP_VLAN_HWTAGGING)
 778                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 779                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 780                         IXV_CORE_LOCK(adapter);
 781                         ixv_init_locked(adapter);
 782                         IXV_CORE_UNLOCK(adapter);
 783                 }
 784                 VLAN_CAPABILITIES(ifp);
 785                 break;
 786         }
 787
 788         default:
 789                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 790                 error = ether_ioctl(ifp, command, data);
 791                 break;
 792         }
 793
 794         return (error);
 795 }
 796
 797 /*********************************************************************
 798  *  Init entry point
 799  *
 800  *  This routine is used in two ways. It is used by the stack as
 801  *  init entry point in network interface structure. It is also used
 802  *  by the driver as a hw/sw initialization routine to get to a
 803  *  consistent state.
 804  *
 805  *  return 0 on success, positive on failure
 806  **********************************************************************/
 807 #define IXGBE_MHADD_MFS_SHIFT 16
 808
 809 static void
 810 ixv_init_locked(struct adapter *adapter)
 811 {
 812         struct ifnet    *ifp = adapter->ifp;
 813         device_t        dev = adapter->dev;
 814         struct ixgbe_hw *hw = &adapter->hw;
 815         u32             mhadd, gpie;
 816
 817         INIT_DEBUGOUT("ixv_init: begin");
 818         mtx_assert(&adapter->core_mtx, MA_OWNED);
 819         hw->adapter_stopped = FALSE;
 820         ixgbe_stop_adapter(hw);
 821         callout_stop(&adapter->timer);
 822
 823         /* reprogram the RAR[0] in case user changed it. */
 824         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
 825
 826         /* Get the latest mac address, User can use a LAA */
 827         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
 828              IXGBE_ETH_LENGTH_OF_ADDRESS);
 829         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
 830         hw->addr_ctrl.rar_used_count = 1;
 831
 832         /* Prepare transmit descriptors and buffers */
 833         if (ixv_setup_transmit_structures(adapter)) {
 834                 device_printf(dev,"Could not setup transmit structures\n");
 835                 ixv_stop(adapter);
 836                 return;
 837         }
 838
 839         ixgbe_reset_hw(hw);
 840         ixv_initialize_transmit_units(adapter);
 841
 842         /* Setup Multicast table */
 843         ixv_set_multi(adapter);
 844
 845         /*
 846         ** Determine the correct mbuf pool
 847         ** for doing jumbo/headersplit
 848         */
 849         if (ifp->if_mtu > ETHERMTU)
 850                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 851         else
 852                 adapter->rx_mbuf_sz = MCLBYTES;
 853
 854         /* Prepare receive descriptors and buffers */
 855         if (ixv_setup_receive_structures(adapter)) {
 856                 device_printf(dev,"Could not setup receive structures\n");
 857                 ixv_stop(adapter);
 858                 return;
 859         }
 860
 861         /* Configure RX settings */
 862         ixv_initialize_receive_units(adapter);
 863
 864         /* Enable Enhanced MSIX mode */
 865         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
 866         gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME;
 867         gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD;
 868         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
 869
 870         /* Set the various hardware offload abilities */
 871         ifp->if_hwassist = 0;
 872         if (ifp->if_capenable & IFCAP_TSO4)
 873                 ifp->if_hwassist |= CSUM_TSO;
 874         if (ifp->if_capenable & IFCAP_TXCSUM) {
 875                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 876 #if __FreeBSD_version >= 800000
 877                 ifp->if_hwassist |= CSUM_SCTP;
 878 #endif
 879         }
 880
 881         /* Set MTU size */
 882         if (ifp->if_mtu > ETHERMTU) {
 883                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
 884                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
 885                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
 886                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
 887         }
 888
 889         /* Set up VLAN offload and filter */
 890         ixv_setup_vlan_support(adapter);
 891
 892         callout_reset(&adapter->timer, hz, ixv_local_timer, adapter);
 893
 894         /* Set up MSI/X routing */
 895         ixv_configure_ivars(adapter);
 896
 897         /* Set up auto-mask */
 898         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE);
 899
 900         /* Set moderation on the Link interrupt */
 901         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR);
 902
 903         /* Stats init */
 904         ixv_init_stats(adapter);
 905
 906         /* Config/Enable Link */
 907         ixv_config_link(adapter);
 908
 909         /* And now turn on interrupts */
 910         ixv_enable_intr(adapter);
 911
 912         /* Now inform the stack we're ready */
 913         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 914         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 915
 916         return;
 917 }
 918
 919 static void
 920 ixv_init(void *arg)
 921 {
 922         struct adapter *adapter = arg;
 923
 924         IXV_CORE_LOCK(adapter);
 925         ixv_init_locked(adapter);
 926         IXV_CORE_UNLOCK(adapter);
 927         return;
 928 }
 929
 930
 931 /*
 932 **
 933 ** MSIX Interrupt Handlers and Tasklets
 934 **
 935 */
 936
 937 static inline void
 938 ixv_enable_queue(struct adapter *adapter, u32 vector)
 939 {
 940         struct ixgbe_hw *hw = &adapter->hw;
 941         u32     queue = 1 << vector;
 942         u32     mask;
 943
 944         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 945         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 946 }
 947
 948 static inline void
 949 ixv_disable_queue(struct adapter *adapter, u32 vector)
 950 {
 951         struct ixgbe_hw *hw = &adapter->hw;
 952         u64     queue = (u64)(1 << vector);
 953         u32     mask;
 954
 955         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 956         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
 957 }
 958
 959 static inline void
 960 ixv_rearm_queues(struct adapter *adapter, u64 queues)
 961 {
 962         u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
 963         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask);
 964 }
 965
 966
 967 static void
 968 ixv_handle_que(void *context, int pending)
 969 {
 970         struct ix_queue *que = context;
 971         struct adapter  *adapter = que->adapter;
 972         struct tx_ring  *txr = que->txr;
 973         struct ifnet    *ifp = adapter->ifp;
 974         bool            more;
 975
 976         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 977                 more = ixv_rxeof(que, adapter->rx_process_limit);
 978                 IXV_TX_LOCK(txr);
 979                 ixv_txeof(txr);
 980 #if __FreeBSD_version >= 800000
 981                 if (!drbr_empty(ifp, txr->br))
 982                         ixv_mq_start_locked(ifp, txr, NULL);
 983 #else
 984                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 985                         ixv_start_locked(txr, ifp);
 986 #endif
 987                 IXV_TX_UNLOCK(txr);
 988                 if (more) {
 989                         taskqueue_enqueue(que->tq, &que->que_task);
 990                         return;
 991                 }
 992         }
 993
 994         /* Reenable this interrupt */
 995         ixv_enable_queue(adapter, que->msix);
 996         return;
 997 }
 998
 999 /*********************************************************************
1000  *
1001  *  MSI Queue Interrupt Service routine
1002  *
1003  **********************************************************************/
1004 void
1005 ixv_msix_que(void *arg)
1006 {
1007         struct ix_queue *que = arg;
1008         struct adapter  *adapter = que->adapter;
1009         struct tx_ring  *txr = que->txr;
1010         struct rx_ring  *rxr = que->rxr;
1011         bool            more_tx, more_rx;
1012         u32             newitr = 0;
1013
1014         ixv_disable_queue(adapter, que->msix);
1015         ++que->irqs;
1016
1017         more_rx = ixv_rxeof(que, adapter->rx_process_limit);
1018
1019         IXV_TX_LOCK(txr);
1020         more_tx = ixv_txeof(txr);
1021         /*
1022         ** Make certain that if the stack
1023         ** has anything queued the task gets
1024         ** scheduled to handle it.
1025         */
1026 #if __FreeBSD_version < 800000
1027         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1028 #else
1029         if (!drbr_empty(adapter->ifp, txr->br))
1030 #endif
1031                 more_tx = 1;
1032         IXV_TX_UNLOCK(txr);
1033
1034         more_rx = ixv_rxeof(que, adapter->rx_process_limit);
1035
1036         /* Do AIM now? */
1037
1038         if (ixv_enable_aim == FALSE)
1039                 goto no_calc;
1040         /*
1041         ** Do Adaptive Interrupt Moderation:
1042         **  - Write out last calculated setting
1043         **  - Calculate based on average size over
1044         **    the last interval.
1045         */
1046         if (que->eitr_setting)
1047                 IXGBE_WRITE_REG(&adapter->hw,
1048                     IXGBE_VTEITR(que->msix),
1049                     que->eitr_setting);
1050
1051         que->eitr_setting = 0;
1052
1053         /* Idle, do nothing */
1054         if ((txr->bytes == 0) && (rxr->bytes == 0))
1055                 goto no_calc;
1056
1057         if ((txr->bytes) && (txr->packets))
1058                 newitr = txr->bytes/txr->packets;
1059         if ((rxr->bytes) && (rxr->packets))
1060                 newitr = max(newitr,
1061                     (rxr->bytes / rxr->packets));
1062         newitr += 24; /* account for hardware frame, crc */
1063
1064         /* set an upper boundary */
1065         newitr = min(newitr, 3000);
1066
1067         /* Be nice to the mid range */
1068         if ((newitr > 300) && (newitr < 1200))
1069                 newitr = (newitr / 3);
1070         else
1071                 newitr = (newitr / 2);
1072
1073         newitr |= newitr << 16;
1074
1075         /* save for next interrupt */
1076         que->eitr_setting = newitr;
1077
1078         /* Reset state */
1079         txr->bytes = 0;
1080         txr->packets = 0;
1081         rxr->bytes = 0;
1082         rxr->packets = 0;
1083
1084 no_calc:
1085         if (more_tx || more_rx)
1086                 taskqueue_enqueue(que->tq, &que->que_task);
1087         else /* Reenable this interrupt */
1088                 ixv_enable_queue(adapter, que->msix);
1089         return;
1090 }
1091
1092 static void
1093 ixv_msix_mbx(void *arg)
1094 {
1095         struct adapter  *adapter = arg;
1096         struct ixgbe_hw *hw = &adapter->hw;
1097         u32             reg;
1098
1099         ++adapter->mbx_irq;
1100
1101         /* First get the cause */
1102         reg = IXGBE_READ_REG(hw, IXGBE_VTEICS);
1103         /* Clear interrupt with write */
1104         IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg);
1105
1106         /* Link status change */
1107         if (reg & IXGBE_EICR_LSC)
1108                 taskqueue_enqueue(adapter->tq, &adapter->mbx_task);
1109
1110         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER);
1111         return;
1112 }
1113
1114 /*********************************************************************
1115  *
1116  *  Media Ioctl callback
1117  *
1118  *  This routine is called whenever the user queries the status of
1119  *  the interface using ifconfig.
1120  *
1121  **********************************************************************/
1122 static void
1123 ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1124 {
1125         struct adapter *adapter = ifp->if_softc;
1126
1127         INIT_DEBUGOUT("ixv_media_status: begin");
1128         IXV_CORE_LOCK(adapter);
1129         ixv_update_link_status(adapter);
1130
1131         ifmr->ifm_status = IFM_AVALID;
1132         ifmr->ifm_active = IFM_ETHER;
1133
1134         if (!adapter->link_active) {
1135                 IXV_CORE_UNLOCK(adapter);
1136                 return;
1137         }
1138
1139         ifmr->ifm_status |= IFM_ACTIVE;
1140
1141         switch (adapter->link_speed) {
1142                 case IXGBE_LINK_SPEED_1GB_FULL:
1143                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1144                         break;
1145                 case IXGBE_LINK_SPEED_10GB_FULL:
1146                         ifmr->ifm_active |= IFM_FDX;
1147                         break;
1148         }
1149
1150         IXV_CORE_UNLOCK(adapter);
1151
1152         return;
1153 }
1154
1155 /*********************************************************************
1156  *
1157  *  Media Ioctl callback
1158  *
1159  *  This routine is called when the user changes speed/duplex using
1160  *  media/mediopt option with ifconfig.
1161  *
1162  **********************************************************************/
1163 static int
1164 ixv_media_change(struct ifnet * ifp)
1165 {
1166         struct adapter *adapter = ifp->if_softc;
1167         struct ifmedia *ifm = &adapter->media;
1168
1169         INIT_DEBUGOUT("ixv_media_change: begin");
1170
1171         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1172                 return (EINVAL);
1173
1174         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1175         case IFM_AUTO:
1176                 break;
1177         default:
1178                 device_printf(adapter->dev, "Only auto media type\n");
1179                 return (EINVAL);
1180         }
1181
1182         return (0);
1183 }
1184
1185 /*********************************************************************
1186  *
1187  *  This routine maps the mbufs to tx descriptors, allowing the
1188  *  TX engine to transmit the packets.
1189  *      - return 0 on success, positive on failure
1190  *
1191  **********************************************************************/
1192
1193 static int
1194 ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1195 {
1196         struct adapter  *adapter = txr->adapter;
1197         u32             olinfo_status = 0, cmd_type_len;
1198         u32             paylen = 0;
1199         int             i, j, error, nsegs;
1200         int             first, last = 0;
1201         struct mbuf     *m_head;
1202         bus_dma_segment_t segs[32];
1203         bus_dmamap_t    map;
1204         struct ixv_tx_buf *txbuf, *txbuf_mapped;
1205         union ixgbe_adv_tx_desc *txd = NULL;
1206
1207         m_head = *m_headp;
1208
1209         /* Basic descriptor defines */
1210         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1211             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1212
1213         if (m_head->m_flags & M_VLANTAG)
1214                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1215
1216         /*
1217          * Important to capture the first descriptor
1218          * used because it will contain the index of
1219          * the one we tell the hardware to report back
1220          */
1221         first = txr->next_avail_desc;
1222         txbuf = &txr->tx_buffers[first];
1223         txbuf_mapped = txbuf;
1224         map = txbuf->map;
1225
1226         /*
1227          * Map the packet for DMA.
1228          */
1229         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1230             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1231
1232         if (error == EFBIG) {
1233                 struct mbuf *m;
1234
1235                 m = m_defrag(*m_headp, M_NOWAIT);
1236                 if (m == NULL) {
1237                         adapter->mbuf_defrag_failed++;
1238                         m_freem(*m_headp);
1239                         *m_headp = NULL;
1240                         return (ENOBUFS);
1241                 }
1242                 *m_headp = m;
1243
1244                 /* Try it again */
1245                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1246                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1247
1248                 if (error == ENOMEM) {
1249                         adapter->no_tx_dma_setup++;
1250                         return (error);
1251                 } else if (error != 0) {
1252                         adapter->no_tx_dma_setup++;
1253                         m_freem(*m_headp);
1254                         *m_headp = NULL;
1255                         return (error);
1256                 }
1257         } else if (error == ENOMEM) {
1258                 adapter->no_tx_dma_setup++;
1259                 return (error);
1260         } else if (error != 0) {
1261                 adapter->no_tx_dma_setup++;
1262                 m_freem(*m_headp);
1263                 *m_headp = NULL;
1264                 return (error);
1265         }
1266
1267         /* Make certain there are enough descriptors */
1268         if (nsegs > txr->tx_avail - 2) {
1269                 txr->no_desc_avail++;
1270                 error = ENOBUFS;
1271                 goto xmit_fail;
1272         }
1273         m_head = *m_headp;
1274
1275         /*
1276         ** Set up the appropriate offload context
1277         ** this becomes the first descriptor of
1278         ** a packet.
1279         */
1280         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1281                 if (ixv_tso_setup(txr, m_head, &paylen)) {
1282                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1283                         olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1284                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1285                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1286                         ++adapter->tso_tx;
1287                 } else
1288                         return (ENXIO);
1289         } else if (ixv_tx_ctx_setup(txr, m_head))
1290                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1291
1292         /* Record payload length */
1293         if (paylen == 0)
1294                 olinfo_status |= m_head->m_pkthdr.len <<
1295                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1296
1297         i = txr->next_avail_desc;
1298         for (j = 0; j < nsegs; j++) {
1299                 bus_size_t seglen;
1300                 bus_addr_t segaddr;
1301
1302                 txbuf = &txr->tx_buffers[i];
1303                 txd = &txr->tx_base[i];
1304                 seglen = segs[j].ds_len;
1305                 segaddr = htole64(segs[j].ds_addr);
1306
1307                 txd->read.buffer_addr = segaddr;
1308                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1309                     cmd_type_len |seglen);
1310                 txd->read.olinfo_status = htole32(olinfo_status);
1311                 last = i; /* descriptor that will get completion IRQ */
1312
1313                 if (++i == adapter->num_tx_desc)
1314                         i = 0;
1315
1316                 txbuf->m_head = NULL;
1317                 txbuf->eop_index = -1;
1318         }
1319
1320         txd->read.cmd_type_len |=
1321             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1322         txr->tx_avail -= nsegs;
1323         txr->next_avail_desc = i;
1324
1325         txbuf->m_head = m_head;
1326         txr->tx_buffers[first].map = txbuf->map;
1327         txbuf->map = map;
1328         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1329
1330         /* Set the index of the descriptor that will be marked done */
1331         txbuf = &txr->tx_buffers[first];
1332         txbuf->eop_index = last;
1333
1334         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1335             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1336         /*
1337          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1338          * hardware that this frame is available to transmit.
1339          */
1340         ++txr->total_packets;
1341         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i);
1342
1343         return (0);
1344
1345 xmit_fail:
1346         bus_dmamap_unload(txr->txtag, txbuf->map);
1347         return (error);
1348
1349 }
1350
1351
1352 /*********************************************************************
1353  *  Multicast Update
1354  *
1355  *  This routine is called whenever multicast address list is updated.
1356  *
1357  **********************************************************************/
1358 #define IXGBE_RAR_ENTRIES 16
1359
1360 static void
1361 ixv_set_multi(struct adapter *adapter)
1362 {
1363         u8      mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
1364         u8      *update_ptr;
1365         struct  ifmultiaddr *ifma;
1366         int     mcnt = 0;
1367         struct ifnet   *ifp = adapter->ifp;
1368
1369         IOCTL_DEBUGOUT("ixv_set_multi: begin");
1370
1371 #if __FreeBSD_version < 800000
1372         IF_ADDR_LOCK(ifp);
1373 #else
1374         if_maddr_rlock(ifp);
1375 #endif
1376         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1377                 if (ifma->ifma_addr->sa_family != AF_LINK)
1378                         continue;
1379                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1380                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1381                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1382                 mcnt++;
1383         }
1384 #if __FreeBSD_version < 800000
1385         IF_ADDR_UNLOCK(ifp);
1386 #else
1387         if_maddr_runlock(ifp);
1388 #endif
1389
1390         update_ptr = mta;
1391
1392         ixgbe_update_mc_addr_list(&adapter->hw,
1393             update_ptr, mcnt, ixv_mc_array_itr, TRUE);
1394
1395         return;
1396 }
1397
1398 /*
1399  * This is an iterator function now needed by the multicast
1400  * shared code. It simply feeds the shared code routine the
1401  * addresses in the array of ixv_set_multi() one by one.
1402  */
1403 static u8 *
1404 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1405 {
1406         u8 *addr = *update_ptr;
1407         u8 *newptr;
1408         *vmdq = 0;
1409
1410         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1411         *update_ptr = newptr;
1412         return addr;
1413 }
1414
1415 /*********************************************************************
1416  *  Timer routine
1417  *
1418  *  This routine checks for link status,updates statistics,
1419  *  and runs the watchdog check.
1420  *
1421  **********************************************************************/
1422
1423 static void
1424 ixv_local_timer(void *arg)
1425 {
1426         struct adapter  *adapter = arg;
1427         device_t        dev = adapter->dev;
1428         struct tx_ring  *txr = adapter->tx_rings;
1429         int             i;
1430
1431         mtx_assert(&adapter->core_mtx, MA_OWNED);
1432
1433         ixv_update_link_status(adapter);
1434
1435         /* Stats Update */
1436         ixv_update_stats(adapter);
1437
1438         /*
1439          * If the interface has been paused
1440          * then don't do the watchdog check
1441          */
1442         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1443                 goto out;
1444         /*
1445         ** Check for time since any descriptor was cleaned
1446         */
1447         for (i = 0; i < adapter->num_queues; i++, txr++) {
1448                 IXV_TX_LOCK(txr);
1449                 if (txr->watchdog_check == FALSE) {
1450                         IXV_TX_UNLOCK(txr);
1451                         continue;
1452                 }
1453                 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG)
1454                         goto hung;
1455                 IXV_TX_UNLOCK(txr);
1456         }
1457 out:
1458         ixv_rearm_queues(adapter, adapter->que_mask);
1459         callout_reset(&adapter->timer, hz, ixv_local_timer, adapter);
1460         return;
1461
1462 hung:
1463         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1464         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1465             IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)),
1466             IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i)));
1467         device_printf(dev,"TX(%d) desc avail = %d,"
1468             "Next TX to Clean = %d\n",
1469             txr->me, txr->tx_avail, txr->next_to_clean);
1470         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1471         adapter->watchdog_events++;
1472         IXV_TX_UNLOCK(txr);
1473         ixv_init_locked(adapter);
1474 }
1475
1476 /*
1477 ** Note: this routine updates the OS on the link state
1478 **      the real check of the hardware only happens with
1479 **      a link interrupt.
1480 */
1481 static void
1482 ixv_update_link_status(struct adapter *adapter)
1483 {
1484         struct ifnet    *ifp = adapter->ifp;
1485         struct tx_ring *txr = adapter->tx_rings;
1486         device_t dev = adapter->dev;
1487
1488
1489         if (adapter->link_up){
1490                 if (adapter->link_active == FALSE) {
1491                         if (bootverbose)
1492                                 device_printf(dev,"Link is up %d Gbps %s \n",
1493                                     ((adapter->link_speed == 128)? 10:1),
1494                                     "Full Duplex");
1495                         adapter->link_active = TRUE;
1496                         if_link_state_change(ifp, LINK_STATE_UP);
1497                 }
1498         } else { /* Link down */
1499                 if (adapter->link_active == TRUE) {
1500                         if (bootverbose)
1501                                 device_printf(dev,"Link is Down\n");
1502                         if_link_state_change(ifp, LINK_STATE_DOWN);
1503                         adapter->link_active = FALSE;
1504                         for (int i = 0; i < adapter->num_queues;
1505                             i++, txr++)
1506                                 txr->watchdog_check = FALSE;
1507                 }
1508         }
1509
1510         return;
1511 }
1512
1513
1514 /*********************************************************************
1515  *
1516  *  This routine disables all traffic on the adapter by issuing a
1517  *  global reset on the MAC and deallocates TX/RX buffers.
1518  *
1519  **********************************************************************/
1520
1521 static void
1522 ixv_stop(void *arg)
1523 {
1524         struct ifnet   *ifp;
1525         struct adapter *adapter = arg;
1526         struct ixgbe_hw *hw = &adapter->hw;
1527         ifp = adapter->ifp;
1528
1529         mtx_assert(&adapter->core_mtx, MA_OWNED);
1530
1531         INIT_DEBUGOUT("ixv_stop: begin\n");
1532         ixv_disable_intr(adapter);
1533
1534         /* Tell the stack that the interface is no longer active */
1535         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1536
1537         ixgbe_reset_hw(hw);
1538         adapter->hw.adapter_stopped = FALSE;
1539         ixgbe_stop_adapter(hw);
1540         callout_stop(&adapter->timer);
1541
1542         /* reprogram the RAR[0] in case user changed it. */
1543         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1544
1545         return;
1546 }
1547
1548
1549 /*********************************************************************
1550  *
1551  *  Determine hardware revision.
1552  *
1553  **********************************************************************/
1554 static void
1555 ixv_identify_hardware(struct adapter *adapter)
1556 {
1557         device_t        dev = adapter->dev;
1558         u16             pci_cmd_word;
1559
1560         /*
1561         ** Make sure BUSMASTER is set, on a VM under
1562         ** KVM it may not be and will break things.
1563         */
1564         pci_enable_busmaster(dev);
1565         pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1566
1567         /* Save off the information about this board */
1568         adapter->hw.vendor_id = pci_get_vendor(dev);
1569         adapter->hw.device_id = pci_get_device(dev);
1570         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
1571         adapter->hw.subsystem_vendor_id =
1572             pci_read_config(dev, PCIR_SUBVEND_0, 2);
1573         adapter->hw.subsystem_device_id =
1574             pci_read_config(dev, PCIR_SUBDEV_0, 2);
1575
1576         return;
1577 }
1578
1579 /*********************************************************************
1580  *
1581  *  Setup MSIX Interrupt resources and handlers
1582  *
1583  **********************************************************************/
1584 static int
1585 ixv_allocate_msix(struct adapter *adapter)
1586 {
1587         device_t        dev = adapter->dev;
1588         struct          ix_queue *que = adapter->queues;
1589         int             error, rid, vector = 0;
1590
1591         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
1592                 rid = vector + 1;
1593                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1594                     RF_SHAREABLE | RF_ACTIVE);
1595                 if (que->res == NULL) {
1596                         device_printf(dev,"Unable to allocate"
1597                             " bus resource: que interrupt [%d]\n", vector);
1598                         return (ENXIO);
1599                 }
1600                 /* Set the handler function */
1601                 error = bus_setup_intr(dev, que->res,
1602                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
1603                     ixv_msix_que, que, &que->tag);
1604                 if (error) {
1605                         que->res = NULL;
1606                         device_printf(dev, "Failed to register QUE handler");
1607                         return (error);
1608                 }
1609 #if __FreeBSD_version >= 800504
1610                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
1611 #endif
1612                 que->msix = vector;
1613                 adapter->que_mask |= (u64)(1 << que->msix);
1614                 /*
1615                 ** Bind the msix vector, and thus the
1616                 ** ring to the corresponding cpu.
1617                 */
1618                 if (adapter->num_queues > 1)
1619                         bus_bind_intr(dev, que->res, i);
1620
1621                 TASK_INIT(&que->que_task, 0, ixv_handle_que, que);
1622                 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT,
1623                     taskqueue_thread_enqueue, &que->tq);
1624                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1625                     device_get_nameunit(adapter->dev));
1626         }
1627
1628         /* and Mailbox */
1629         rid = vector + 1;
1630         adapter->res = bus_alloc_resource_any(dev,
1631             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
1632         if (!adapter->res) {
1633                 device_printf(dev,"Unable to allocate"
1634             " bus resource: MBX interrupt [%d]\n", rid);
1635                 return (ENXIO);
1636         }
1637         /* Set the mbx handler function */
1638         error = bus_setup_intr(dev, adapter->res,
1639             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1640             ixv_msix_mbx, adapter, &adapter->tag);
1641         if (error) {
1642                 adapter->res = NULL;
1643                 device_printf(dev, "Failed to register LINK handler");
1644                 return (error);
1645         }
1646 #if __FreeBSD_version >= 800504
1647         bus_describe_intr(dev, adapter->res, adapter->tag, "mbx");
1648 #endif
1649         adapter->mbxvec = vector;
1650         /* Tasklets for Mailbox */
1651         TASK_INIT(&adapter->mbx_task, 0, ixv_handle_mbx, adapter);
1652         adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT,
1653             taskqueue_thread_enqueue, &adapter->tq);
1654         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq",
1655             device_get_nameunit(adapter->dev));
1656         /*
1657         ** Due to a broken design QEMU will fail to properly
1658         ** enable the guest for MSIX unless the vectors in
1659         ** the table are all set up, so we must rewrite the
1660         ** ENABLE in the MSIX control register again at this
1661         ** point to cause it to successfully initialize us.
1662         */
1663         if (adapter->hw.mac.type == ixgbe_mac_82599_vf) {
1664                 int msix_ctrl;
1665                 pci_find_cap(dev, PCIY_MSIX, &rid);
1666                 rid += PCIR_MSIX_CTRL;
1667                 msix_ctrl = pci_read_config(dev, rid, 2);
1668                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1669                 pci_write_config(dev, rid, msix_ctrl, 2);
1670         }
1671
1672         return (0);
1673 }
1674
1675 /*
1676  * Setup MSIX resources, note that the VF
1677  * device MUST use MSIX, there is no fallback.
1678  */
1679 static int
1680 ixv_setup_msix(struct adapter *adapter)
1681 {
1682         device_t dev = adapter->dev;
1683         int rid, want;
1684
1685
1686         /* First try MSI/X */
1687         rid = PCIR_BAR(3);
1688         adapter->msix_mem = bus_alloc_resource_any(dev,
1689             SYS_RES_MEMORY, &rid, RF_ACTIVE);
1690         if (adapter->msix_mem == NULL) {
1691                 device_printf(adapter->dev,
1692                     "Unable to map MSIX table \n");
1693                 goto out;
1694         }
1695
1696         /*
1697         ** Want two vectors: one for a queue,
1698         ** plus an additional for mailbox.
1699         */
1700         want = 2;
1701         if (pci_alloc_msix(dev, &want) == 0) {
1702                 device_printf(adapter->dev,
1703                     "Using MSIX interrupts with %d vectors\n", want);
1704                 return (want);
1705         }
1706 out:
1707         if (adapter->msix_mem != NULL) {
1708                 bus_release_resource(dev, SYS_RES_MEMORY,
1709                     rid, adapter->msix_mem);
1710                 adapter->msix_mem = NULL;
1711         }
1712         device_printf(adapter->dev,"MSIX config error\n");
1713         return (ENXIO);
1714 }
1715
1716
1717 static int
1718 ixv_allocate_pci_resources(struct adapter *adapter)
1719 {
1720         int             rid;
1721         device_t        dev = adapter->dev;
1722
1723         rid = PCIR_BAR(0);
1724         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1725             &rid, RF_ACTIVE);
1726
1727         if (!(adapter->pci_mem)) {
1728                 device_printf(dev,"Unable to allocate bus resource: memory\n");
1729                 return (ENXIO);
1730         }
1731
1732         adapter->osdep.mem_bus_space_tag =
1733                 rman_get_bustag(adapter->pci_mem);
1734         adapter->osdep.mem_bus_space_handle =
1735                 rman_get_bushandle(adapter->pci_mem);
1736         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
1737
1738         adapter->num_queues = 1;
1739         adapter->hw.back = &adapter->osdep;
1740
1741         /*
1742         ** Now setup MSI/X, should
1743         ** return us the number of
1744         ** configured vectors.
1745         */
1746         adapter->msix = ixv_setup_msix(adapter);
1747         if (adapter->msix == ENXIO)
1748                 return (ENXIO);
1749         else
1750                 return (0);
1751 }
1752
1753 static void
1754 ixv_free_pci_resources(struct adapter * adapter)
1755 {
1756         struct          ix_queue *que = adapter->queues;
1757         device_t        dev = adapter->dev;
1758         int             rid, memrid;
1759
1760         memrid = PCIR_BAR(MSIX_BAR);
1761
1762         /*
1763         ** There is a slight possibility of a failure mode
1764         ** in attach that will result in entering this function
1765         ** before interrupt resources have been initialized, and
1766         ** in that case we do not want to execute the loops below
1767         ** We can detect this reliably by the state of the adapter
1768         ** res pointer.
1769         */
1770         if (adapter->res == NULL)
1771                 goto mem;
1772
1773         /*
1774         **  Release all msix queue resources:
1775         */
1776         for (int i = 0; i < adapter->num_queues; i++, que++) {
1777                 rid = que->msix + 1;
1778                 if (que->tag != NULL) {
1779                         bus_teardown_intr(dev, que->res, que->tag);
1780                         que->tag = NULL;
1781                 }
1782                 if (que->res != NULL)
1783                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
1784         }
1785
1786
1787         /* Clean the Legacy or Link interrupt last */
1788         if (adapter->mbxvec) /* we are doing MSIX */
1789                 rid = adapter->mbxvec + 1;
1790         else
1791                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
1792
1793         if (adapter->tag != NULL) {
1794                 bus_teardown_intr(dev, adapter->res, adapter->tag);
1795                 adapter->tag = NULL;
1796         }
1797         if (adapter->res != NULL)
1798                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
1799
1800 mem:
1801         if (adapter->msix)
1802                 pci_release_msi(dev);
1803
1804         if (adapter->msix_mem != NULL)
1805                 bus_release_resource(dev, SYS_RES_MEMORY,
1806                     memrid, adapter->msix_mem);
1807
1808         if (adapter->pci_mem != NULL)
1809                 bus_release_resource(dev, SYS_RES_MEMORY,
1810                     PCIR_BAR(0), adapter->pci_mem);
1811
1812         return;
1813 }
1814
1815 /*********************************************************************
1816  *
1817  *  Setup networking device structure and register an interface.
1818  *
1819  **********************************************************************/
1820 static void
1821 ixv_setup_interface(device_t dev, struct adapter *adapter)
1822 {
1823         struct ifnet   *ifp;
1824
1825         INIT_DEBUGOUT("ixv_setup_interface: begin");
1826
1827         ifp = adapter->ifp = if_alloc(IFT_ETHER);
1828         if (ifp == NULL)
1829                 panic("%s: can not if_alloc()\n", device_get_nameunit(dev));
1830         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1831         ifp->if_baudrate = 1000000000;
1832         ifp->if_init = ixv_init;
1833         ifp->if_softc = adapter;
1834         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1835         ifp->if_ioctl = ixv_ioctl;
1836 #if __FreeBSD_version >= 800000
1837         ifp->if_transmit = ixv_mq_start;
1838         ifp->if_qflush = ixv_qflush;
1839 #else
1840         ifp->if_start = ixv_start;
1841 #endif
1842         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
1843
1844         ether_ifattach(ifp, adapter->hw.mac.addr);
1845
1846         adapter->max_frame_size =
1847             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1848
1849         /*
1850          * Tell the upper layer(s) we support long frames.
1851          */
1852         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1853
1854         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM;
1855         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
1856         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
1857                              |  IFCAP_VLAN_HWTSO
1858                              |  IFCAP_VLAN_MTU;
1859         ifp->if_capenable = ifp->if_capabilities;
1860
1861         /* Don't enable LRO by default */
1862         ifp->if_capabilities |= IFCAP_LRO;
1863
1864         /*
1865          * Specify the media types supported by this adapter and register
1866          * callbacks to update media and link information
1867          */
1868         ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change,
1869                      ixv_media_status);
1870         ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL);
1871         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1872         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1873
1874         return;
1875 }
1876
1877 static void
1878 ixv_config_link(struct adapter *adapter)
1879 {
1880         struct ixgbe_hw *hw = &adapter->hw;
1881         u32     autoneg, err = 0;
1882
1883         if (hw->mac.ops.check_link)
1884                 err = hw->mac.ops.check_link(hw, &autoneg,
1885                     &adapter->link_up, FALSE);
1886         if (err)
1887                 goto out;
1888
1889         if (hw->mac.ops.setup_link)
1890                 err = hw->mac.ops.setup_link(hw,
1891                     autoneg, adapter->link_up);
1892 out:
1893         return;
1894 }
1895
1896 /********************************************************************
1897  * Manage DMA'able memory.
1898  *******************************************************************/
1899 static void
1900 ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
1901 {
1902         if (error)
1903                 return;
1904         *(bus_addr_t *) arg = segs->ds_addr;
1905         return;
1906 }
1907
1908 static int
1909 ixv_dma_malloc(struct adapter *adapter, bus_size_t size,
1910                 struct ixv_dma_alloc *dma, int mapflags)
1911 {
1912         device_t dev = adapter->dev;
1913         int             r;
1914
1915         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
1916                                DBA_ALIGN, 0,    /* alignment, bounds */
1917                                BUS_SPACE_MAXADDR,       /* lowaddr */
1918                                BUS_SPACE_MAXADDR,       /* highaddr */
1919                                NULL, NULL,      /* filter, filterarg */
1920                                size,    /* maxsize */
1921                                1,       /* nsegments */
1922                                size,    /* maxsegsize */
1923                                BUS_DMA_ALLOCNOW,        /* flags */
1924                                NULL,    /* lockfunc */
1925                                NULL,    /* lockfuncarg */
1926                                &dma->dma_tag);
1927         if (r != 0) {
1928                 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; "
1929                        "error %u\n", r);
1930                 goto fail_0;
1931         }
1932         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
1933                              BUS_DMA_NOWAIT, &dma->dma_map);
1934         if (r != 0) {
1935                 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; "
1936                        "error %u\n", r);
1937                 goto fail_1;
1938         }
1939         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1940                             size,
1941                             ixv_dmamap_cb,
1942                             &dma->dma_paddr,
1943                             mapflags | BUS_DMA_NOWAIT);
1944         if (r != 0) {
1945                 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; "
1946                        "error %u\n", r);
1947                 goto fail_2;
1948         }
1949         dma->dma_size = size;
1950         return (0);
1951 fail_2:
1952         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1953 fail_1:
1954         bus_dma_tag_destroy(dma->dma_tag);
1955 fail_0:
1956         dma->dma_map = NULL;
1957         dma->dma_tag = NULL;
1958         return (r);
1959 }
1960
1961 static void
1962 ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma)
1963 {
1964         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
1965             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1966         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1967         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1968         bus_dma_tag_destroy(dma->dma_tag);
1969 }
1970
1971
1972 /*********************************************************************
1973  *
1974  *  Allocate memory for the transmit and receive rings, and then
1975  *  the descriptors associated with each, called only once at attach.
1976  *
1977  **********************************************************************/
1978 static int
1979 ixv_allocate_queues(struct adapter *adapter)
1980 {
1981         device_t        dev = adapter->dev;
1982         struct ix_queue *que;
1983         struct tx_ring  *txr;
1984         struct rx_ring  *rxr;
1985         int rsize, tsize, error = 0;
1986         int txconf = 0, rxconf = 0;
1987
1988         /* First allocate the top level queue structs */
1989         if (!(adapter->queues =
1990             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
1991             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
1992                 device_printf(dev, "Unable to allocate queue memory\n");
1993                 error = ENOMEM;
1994                 goto fail;
1995         }
1996
1997         /* First allocate the TX ring struct memory */
1998         if (!(adapter->tx_rings =
1999             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2000             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2001                 device_printf(dev, "Unable to allocate TX ring memory\n");
2002                 error = ENOMEM;
2003                 goto tx_fail;
2004         }
2005
2006         /* Next allocate the RX */
2007         if (!(adapter->rx_rings =
2008             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2009             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2010                 device_printf(dev, "Unable to allocate RX ring memory\n");
2011                 error = ENOMEM;
2012                 goto rx_fail;
2013         }
2014
2015         /* For the ring itself */
2016         tsize = roundup2(adapter->num_tx_desc *
2017             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2018
2019         /*
2020          * Now set up the TX queues, txconf is needed to handle the
2021          * possibility that things fail midcourse and we need to
2022          * undo memory gracefully
2023          */
2024         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2025                 /* Set up some basics */
2026                 txr = &adapter->tx_rings[i];
2027                 txr->adapter = adapter;
2028                 txr->me = i;
2029
2030                 /* Initialize the TX side lock */
2031                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2032                     device_get_nameunit(dev), txr->me);
2033                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2034
2035                 if (ixv_dma_malloc(adapter, tsize,
2036                         &txr->txdma, BUS_DMA_NOWAIT)) {
2037                         device_printf(dev,
2038                             "Unable to allocate TX Descriptor memory\n");
2039                         error = ENOMEM;
2040                         goto err_tx_desc;
2041                 }
2042                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2043                 bzero((void *)txr->tx_base, tsize);
2044
2045                 /* Now allocate transmit buffers for the ring */
2046                 if (ixv_allocate_transmit_buffers(txr)) {
2047                         device_printf(dev,
2048                             "Critical Failure setting up transmit buffers\n");
2049                         error = ENOMEM;
2050                         goto err_tx_desc;
2051                 }
2052 #if __FreeBSD_version >= 800000
2053                 /* Allocate a buf ring */
2054                 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF,
2055                     M_WAITOK, &txr->tx_mtx);
2056                 if (txr->br == NULL) {
2057                         device_printf(dev,
2058                             "Critical Failure setting up buf ring\n");
2059                         error = ENOMEM;
2060                         goto err_tx_desc;
2061                 }
2062 #endif
2063         }
2064
2065         /*
2066          * Next the RX queues...
2067          */
2068         rsize = roundup2(adapter->num_rx_desc *
2069             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2070         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2071                 rxr = &adapter->rx_rings[i];
2072                 /* Set up some basics */
2073                 rxr->adapter = adapter;
2074                 rxr->me = i;
2075
2076                 /* Initialize the RX side lock */
2077                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2078                     device_get_nameunit(dev), rxr->me);
2079                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2080
2081                 if (ixv_dma_malloc(adapter, rsize,
2082                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2083                         device_printf(dev,
2084                             "Unable to allocate RxDescriptor memory\n");
2085                         error = ENOMEM;
2086                         goto err_rx_desc;
2087                 }
2088                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2089                 bzero((void *)rxr->rx_base, rsize);
2090
2091                 /* Allocate receive buffers for the ring*/
2092                 if (ixv_allocate_receive_buffers(rxr)) {
2093                         device_printf(dev,
2094                             "Critical Failure setting up receive buffers\n");
2095                         error = ENOMEM;
2096                         goto err_rx_desc;
2097                 }
2098         }
2099
2100         /*
2101         ** Finally set up the queue holding structs
2102         */
2103         for (int i = 0; i < adapter->num_queues; i++) {
2104                 que = &adapter->queues[i];
2105                 que->adapter = adapter;
2106                 que->txr = &adapter->tx_rings[i];
2107                 que->rxr = &adapter->rx_rings[i];
2108         }
2109
2110         return (0);
2111
2112 err_rx_desc:
2113         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2114                 ixv_dma_free(adapter, &rxr->rxdma);
2115 err_tx_desc:
2116         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2117                 ixv_dma_free(adapter, &txr->txdma);
2118         free(adapter->rx_rings, M_DEVBUF);
2119 rx_fail:
2120         free(adapter->tx_rings, M_DEVBUF);
2121 tx_fail:
2122         free(adapter->queues, M_DEVBUF);
2123 fail:
2124         return (error);
2125 }
2126
2127
2128 /*********************************************************************
2129  *
2130  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2131  *  the information needed to transmit a packet on the wire. This is
2132  *  called only once at attach, setup is done every reset.
2133  *
2134  **********************************************************************/
2135 static int
2136 ixv_allocate_transmit_buffers(struct tx_ring *txr)
2137 {
2138         struct adapter *adapter = txr->adapter;
2139         device_t dev = adapter->dev;
2140         struct ixv_tx_buf *txbuf;
2141         int error, i;
2142
2143         /*
2144          * Setup DMA descriptor areas.
2145          */
2146         if ((error = bus_dma_tag_create(
2147                                bus_get_dma_tag(adapter->dev),   /* parent */
2148                                1, 0,            /* alignment, bounds */
2149                                BUS_SPACE_MAXADDR,       /* lowaddr */
2150                                BUS_SPACE_MAXADDR,       /* highaddr */
2151                                NULL, NULL,              /* filter, filterarg */
2152                                IXV_TSO_SIZE,            /* maxsize */
2153                                32,                      /* nsegments */
2154                                PAGE_SIZE,               /* maxsegsize */
2155                                0,                       /* flags */
2156                                NULL,                    /* lockfunc */
2157                                NULL,                    /* lockfuncarg */
2158                                &txr->txtag))) {
2159                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2160                 goto fail;
2161         }
2162
2163         if (!(txr->tx_buffers =
2164             (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) *
2165             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2166                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2167                 error = ENOMEM;
2168                 goto fail;
2169         }
2170
2171         /* Create the descriptor buffer dma maps */
2172         txbuf = txr->tx_buffers;
2173         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2174                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2175                 if (error != 0) {
2176                         device_printf(dev, "Unable to create TX DMA map\n");
2177                         goto fail;
2178                 }
2179         }
2180
2181         return 0;
2182 fail:
2183         /* We free all, it handles case where we are in the middle */
2184         ixv_free_transmit_structures(adapter);
2185         return (error);
2186 }
2187
2188 /*********************************************************************
2189  *
2190  *  Initialize a transmit ring.
2191  *
2192  **********************************************************************/
2193 static void
2194 ixv_setup_transmit_ring(struct tx_ring *txr)
2195 {
2196         struct adapter *adapter = txr->adapter;
2197         struct ixv_tx_buf *txbuf;
2198         int i;
2199
2200         /* Clear the old ring contents */
2201         IXV_TX_LOCK(txr);
2202         bzero((void *)txr->tx_base,
2203               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2204         /* Reset indices */
2205         txr->next_avail_desc = 0;
2206         txr->next_to_clean = 0;
2207
2208         /* Free any existing tx buffers. */
2209         txbuf = txr->tx_buffers;
2210         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2211                 if (txbuf->m_head != NULL) {
2212                         bus_dmamap_sync(txr->txtag, txbuf->map,
2213                             BUS_DMASYNC_POSTWRITE);
2214                         bus_dmamap_unload(txr->txtag, txbuf->map);
2215                         m_freem(txbuf->m_head);
2216                         txbuf->m_head = NULL;
2217                 }
2218                 /* Clear the EOP index */
2219                 txbuf->eop_index = -1;
2220         }
2221
2222         /* Set number of descriptors available */
2223         txr->tx_avail = adapter->num_tx_desc;
2224
2225         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2226             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2227         IXV_TX_UNLOCK(txr);
2228 }
2229
2230 /*********************************************************************
2231  *
2232  *  Initialize all transmit rings.
2233  *
2234  **********************************************************************/
2235 static int
2236 ixv_setup_transmit_structures(struct adapter *adapter)
2237 {
2238         struct tx_ring *txr = adapter->tx_rings;
2239
2240         for (int i = 0; i < adapter->num_queues; i++, txr++)
2241                 ixv_setup_transmit_ring(txr);
2242
2243         return (0);
2244 }
2245
2246 /*********************************************************************
2247  *
2248  *  Enable transmit unit.
2249  *
2250  **********************************************************************/
2251 static void
2252 ixv_initialize_transmit_units(struct adapter *adapter)
2253 {
2254         struct tx_ring  *txr = adapter->tx_rings;
2255         struct ixgbe_hw *hw = &adapter->hw;
2256
2257
2258         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2259                 u64     tdba = txr->txdma.dma_paddr;
2260                 u32     txctrl, txdctl;
2261
2262                 /* Set WTHRESH to 8, burst writeback */
2263                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
2264                 txdctl |= (8 << 16);
2265                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
2266                 /* Now enable */
2267                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
2268                 txdctl |= IXGBE_TXDCTL_ENABLE;
2269                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
2270
2271                 /* Set the HW Tx Head and Tail indices */
2272                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0);
2273                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0);
2274
2275                 /* Setup Transmit Descriptor Cmd Settings */
2276                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2277                 txr->watchdog_check = FALSE;
2278
2279                 /* Set Ring parameters */
2280                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
2281                        (tdba & 0x00000000ffffffffULL));
2282                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32));
2283                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
2284                     adapter->num_tx_desc *
2285                     sizeof(struct ixgbe_legacy_tx_desc));
2286                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i));
2287                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2288                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl);
2289                 break;
2290         }
2291
2292         return;
2293 }
2294
2295 /*********************************************************************
2296  *
2297  *  Free all transmit rings.
2298  *
2299  **********************************************************************/
2300 static void
2301 ixv_free_transmit_structures(struct adapter *adapter)
2302 {
2303         struct tx_ring *txr = adapter->tx_rings;
2304
2305         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2306                 IXV_TX_LOCK(txr);
2307                 ixv_free_transmit_buffers(txr);
2308                 ixv_dma_free(adapter, &txr->txdma);
2309                 IXV_TX_UNLOCK(txr);
2310                 IXV_TX_LOCK_DESTROY(txr);
2311         }
2312         free(adapter->tx_rings, M_DEVBUF);
2313 }
2314
2315 /*********************************************************************
2316  *
2317  *  Free transmit ring related data structures.
2318  *
2319  **********************************************************************/
2320 static void
2321 ixv_free_transmit_buffers(struct tx_ring *txr)
2322 {
2323         struct adapter *adapter = txr->adapter;
2324         struct ixv_tx_buf *tx_buffer;
2325         int             i;
2326
2327         INIT_DEBUGOUT("free_transmit_ring: begin");
2328
2329         if (txr->tx_buffers == NULL)
2330                 return;
2331
2332         tx_buffer = txr->tx_buffers;
2333         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2334                 if (tx_buffer->m_head != NULL) {
2335                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2336                             BUS_DMASYNC_POSTWRITE);
2337                         bus_dmamap_unload(txr->txtag,
2338                             tx_buffer->map);
2339                         m_freem(tx_buffer->m_head);
2340                         tx_buffer->m_head = NULL;
2341                         if (tx_buffer->map != NULL) {
2342                                 bus_dmamap_destroy(txr->txtag,
2343                                     tx_buffer->map);
2344                                 tx_buffer->map = NULL;
2345                         }
2346                 } else if (tx_buffer->map != NULL) {
2347                         bus_dmamap_unload(txr->txtag,
2348                             tx_buffer->map);
2349                         bus_dmamap_destroy(txr->txtag,
2350                             tx_buffer->map);
2351                         tx_buffer->map = NULL;
2352                 }
2353         }
2354 #if __FreeBSD_version >= 800000
2355         if (txr->br != NULL)
2356                 buf_ring_free(txr->br, M_DEVBUF);
2357 #endif
2358         if (txr->tx_buffers != NULL) {
2359                 free(txr->tx_buffers, M_DEVBUF);
2360                 txr->tx_buffers = NULL;
2361         }
2362         if (txr->txtag != NULL) {
2363                 bus_dma_tag_destroy(txr->txtag);
2364                 txr->txtag = NULL;
2365         }
2366         return;
2367 }
2368
2369 /*********************************************************************
2370  *
2371  *  Advanced Context Descriptor setup for VLAN or CSUM
2372  *
2373  **********************************************************************/
2374
2375 static bool
2376 ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
2377 {
2378         struct adapter *adapter = txr->adapter;
2379         struct ixgbe_adv_tx_context_desc *TXD;
2380         struct ixv_tx_buf        *tx_buffer;
2381         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2382         struct ether_vlan_header *eh;
2383         struct ip *ip;
2384         struct ip6_hdr *ip6;
2385         int  ehdrlen, ip_hlen = 0;
2386         u16     etype;
2387         u8      ipproto = 0;
2388         bool    offload = TRUE;
2389         int ctxd = txr->next_avail_desc;
2390         u16 vtag = 0;
2391
2392
2393         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
2394                 offload = FALSE;
2395
2396
2397         tx_buffer = &txr->tx_buffers[ctxd];
2398         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2399
2400         /*
2401         ** In advanced descriptors the vlan tag must
2402         ** be placed into the descriptor itself.
2403         */
2404         if (mp->m_flags & M_VLANTAG) {
2405                 vtag = htole16(mp->m_pkthdr.ether_vtag);
2406                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2407         } else if (offload == FALSE)
2408                 return FALSE;
2409
2410         /*
2411          * Determine where frame payload starts.
2412          * Jump over vlan headers if already present,
2413          * helpful for QinQ too.
2414          */
2415         eh = mtod(mp, struct ether_vlan_header *);
2416         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2417                 etype = ntohs(eh->evl_proto);
2418                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2419         } else {
2420                 etype = ntohs(eh->evl_encap_proto);
2421                 ehdrlen = ETHER_HDR_LEN;
2422         }
2423
2424         /* Set the ether header length */
2425         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2426
2427         switch (etype) {
2428                 case ETHERTYPE_IP:
2429                         ip = (struct ip *)(mp->m_data + ehdrlen);
2430                         ip_hlen = ip->ip_hl << 2;
2431                         if (mp->m_len < ehdrlen + ip_hlen)
2432                                 return (FALSE);
2433                         ipproto = ip->ip_p;
2434                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2435                         break;
2436                 case ETHERTYPE_IPV6:
2437                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
2438                         ip_hlen = sizeof(struct ip6_hdr);
2439                         if (mp->m_len < ehdrlen + ip_hlen)
2440                                 return (FALSE);
2441                         ipproto = ip6->ip6_nxt;
2442                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
2443                         break;
2444                 default:
2445                         offload = FALSE;
2446                         break;
2447         }
2448
2449         vlan_macip_lens |= ip_hlen;
2450         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2451
2452         switch (ipproto) {
2453                 case IPPROTO_TCP:
2454                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
2455                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2456                         break;
2457
2458                 case IPPROTO_UDP:
2459                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
2460                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
2461                         break;
2462
2463 #if __FreeBSD_version >= 800000
2464                 case IPPROTO_SCTP:
2465                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
2466                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2467                         break;
2468 #endif
2469                 default:
2470                         offload = FALSE;
2471                         break;
2472         }
2473
2474         /* Now copy bits into descriptor */
2475         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
2476         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
2477         TXD->seqnum_seed = htole32(0);
2478         TXD->mss_l4len_idx = htole32(0);
2479
2480         tx_buffer->m_head = NULL;
2481         tx_buffer->eop_index = -1;
2482
2483         /* We've consumed the first desc, adjust counters */
2484         if (++ctxd == adapter->num_tx_desc)
2485                 ctxd = 0;
2486         txr->next_avail_desc = ctxd;
2487         --txr->tx_avail;
2488
2489         return (offload);
2490 }
2491
2492 /**********************************************************************
2493  *
2494  *  Setup work for hardware segmentation offload (TSO) on
2495  *  adapters using advanced tx descriptors
2496  *
2497  **********************************************************************/
2498 static bool
2499 ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
2500 {
2501         struct adapter *adapter = txr->adapter;
2502         struct ixgbe_adv_tx_context_desc *TXD;
2503         struct ixv_tx_buf        *tx_buffer;
2504         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2505         u32 mss_l4len_idx = 0;
2506         u16 vtag = 0;
2507         int ctxd, ehdrlen,  hdrlen, ip_hlen, tcp_hlen;
2508         struct ether_vlan_header *eh;
2509         struct ip *ip;
2510         struct tcphdr *th;
2511
2512
2513         /*
2514          * Determine where frame payload starts.
2515          * Jump over vlan headers if already present
2516          */
2517         eh = mtod(mp, struct ether_vlan_header *);
2518         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
2519                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2520         else
2521                 ehdrlen = ETHER_HDR_LEN;
2522
2523         /* Ensure we have at least the IP+TCP header in the first mbuf. */
2524         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
2525                 return FALSE;
2526
2527         ctxd = txr->next_avail_desc;
2528         tx_buffer = &txr->tx_buffers[ctxd];
2529         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2530
2531         ip = (struct ip *)(mp->m_data + ehdrlen);
2532         if (ip->ip_p != IPPROTO_TCP)
2533                 return FALSE;   /* 0 */
2534         ip->ip_sum = 0;
2535         ip_hlen = ip->ip_hl << 2;
2536         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
2537         th->th_sum = in_pseudo(ip->ip_src.s_addr,
2538             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
2539         tcp_hlen = th->th_off << 2;
2540         hdrlen = ehdrlen + ip_hlen + tcp_hlen;
2541
2542         /* This is used in the transmit desc in encap */
2543         *paylen = mp->m_pkthdr.len - hdrlen;
2544
2545         /* VLAN MACLEN IPLEN */
2546         if (mp->m_flags & M_VLANTAG) {
2547                 vtag = htole16(mp->m_pkthdr.ether_vtag);
2548                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2549         }
2550
2551         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2552         vlan_macip_lens |= ip_hlen;
2553         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
2554
2555         /* ADV DTYPE TUCMD */
2556         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2557         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2558         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2559         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
2560
2561
2562         /* MSS L4LEN IDX */
2563         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
2564         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
2565         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
2566
2567         TXD->seqnum_seed = htole32(0);
2568         tx_buffer->m_head = NULL;
2569         tx_buffer->eop_index = -1;
2570
2571         if (++ctxd == adapter->num_tx_desc)
2572                 ctxd = 0;
2573
2574         txr->tx_avail--;
2575         txr->next_avail_desc = ctxd;
2576         return TRUE;
2577 }
2578
2579
2580 /**********************************************************************
2581  *
2582  *  Examine each tx_buffer in the used queue. If the hardware is done
2583  *  processing the packet then free associated resources. The
2584  *  tx_buffer is put back on the free queue.
2585  *
2586  **********************************************************************/
2587 static bool
2588 ixv_txeof(struct tx_ring *txr)
2589 {
2590         struct adapter  *adapter = txr->adapter;
2591         struct ifnet    *ifp = adapter->ifp;
2592         u32     first, last, done;
2593         struct ixv_tx_buf *tx_buffer;
2594         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
2595
2596         mtx_assert(&txr->tx_mtx, MA_OWNED);
2597
2598         if (txr->tx_avail == adapter->num_tx_desc)
2599                 return FALSE;
2600
2601         first = txr->next_to_clean;
2602         tx_buffer = &txr->tx_buffers[first];
2603         /* For cleanup we just use legacy struct */
2604         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
2605         last = tx_buffer->eop_index;
2606         if (last == -1)
2607                 return FALSE;
2608         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
2609
2610         /*
2611         ** Get the index of the first descriptor
2612         ** BEYOND the EOP and call that 'done'.
2613         ** I do this so the comparison in the
2614         ** inner while loop below can be simple
2615         */
2616         if (++last == adapter->num_tx_desc) last = 0;
2617         done = last;
2618
2619         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2620             BUS_DMASYNC_POSTREAD);
2621         /*
2622         ** Only the EOP descriptor of a packet now has the DD
2623         ** bit set, this is what we look for...
2624         */
2625         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
2626                 /* We clean the range of the packet */
2627                 while (first != done) {
2628                         tx_desc->upper.data = 0;
2629                         tx_desc->lower.data = 0;
2630                         tx_desc->buffer_addr = 0;
2631                         ++txr->tx_avail;
2632
2633                         if (tx_buffer->m_head) {
2634                                 bus_dmamap_sync(txr->txtag,
2635                                     tx_buffer->map,
2636                                     BUS_DMASYNC_POSTWRITE);
2637                                 bus_dmamap_unload(txr->txtag,
2638                                     tx_buffer->map);
2639                                 m_freem(tx_buffer->m_head);
2640                                 tx_buffer->m_head = NULL;
2641                                 tx_buffer->map = NULL;
2642                         }
2643                         tx_buffer->eop_index = -1;
2644                         txr->watchdog_time = ticks;
2645
2646                         if (++first == adapter->num_tx_desc)
2647                                 first = 0;
2648
2649                         tx_buffer = &txr->tx_buffers[first];
2650                         tx_desc =
2651                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
2652                 }
2653                 ++ifp->if_opackets;
2654                 /* See if there is more work now */
2655                 last = tx_buffer->eop_index;
2656                 if (last != -1) {
2657                         eop_desc =
2658                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
2659                         /* Get next done point */
2660                         if (++last == adapter->num_tx_desc) last = 0;
2661                         done = last;
2662                 } else
2663                         break;
2664         }
2665         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2666             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2667
2668         txr->next_to_clean = first;
2669
2670         /*
2671          * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that
2672          * it is OK to send packets. If there are no pending descriptors,
2673          * clear the timeout. Otherwise, if some descriptors have been freed,
2674          * restart the timeout.
2675          */
2676         if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) {
2677                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2678                 if (txr->tx_avail == adapter->num_tx_desc) {
2679                         txr->watchdog_check = FALSE;
2680                         return FALSE;
2681                 }
2682         }
2683
2684         return TRUE;
2685 }
2686
2687 /*********************************************************************
2688  *
2689  *  Refresh mbuf buffers for RX descriptor rings
2690  *   - now keeps its own state so discards due to resource
2691  *     exhaustion are unnecessary, if an mbuf cannot be obtained
2692  *     it just returns, keeping its placeholder, thus it can simply
2693  *     be recalled to try again.
2694  *
2695  **********************************************************************/
2696 static void
2697 ixv_refresh_mbufs(struct rx_ring *rxr, int limit)
2698 {
2699         struct adapter          *adapter = rxr->adapter;
2700         bus_dma_segment_t       hseg[1];
2701         bus_dma_segment_t       pseg[1];
2702         struct ixv_rx_buf       *rxbuf;
2703         struct mbuf             *mh, *mp;
2704         int                     i, j, nsegs, error;
2705         bool                    refreshed = FALSE;
2706
2707         i = j = rxr->next_to_refresh;
2708         /* Get the control variable, one beyond refresh point */
2709         if (++j == adapter->num_rx_desc)
2710                 j = 0;
2711         while (j != limit) {
2712                 rxbuf = &rxr->rx_buffers[i];
2713                 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) {
2714                         mh = m_gethdr(M_NOWAIT, MT_DATA);
2715                         if (mh == NULL)
2716                                 goto update;
2717                         mh->m_pkthdr.len = mh->m_len = MHLEN;
2718                         mh->m_len = MHLEN;
2719                         mh->m_flags |= M_PKTHDR;
2720                         m_adj(mh, ETHER_ALIGN);
2721                         /* Get the memory mapping */
2722                         error = bus_dmamap_load_mbuf_sg(rxr->htag,
2723                             rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
2724                         if (error != 0) {
2725                                 printf("GET BUF: dmamap load"
2726                                     " failure - %d\n", error);
2727                                 m_free(mh);
2728                                 goto update;
2729                         }
2730                         rxbuf->m_head = mh;
2731                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
2732                             BUS_DMASYNC_PREREAD);
2733                         rxr->rx_base[i].read.hdr_addr =
2734                             htole64(hseg[0].ds_addr);
2735                 }
2736
2737                 if (rxbuf->m_pack == NULL) {
2738                         mp = m_getjcl(M_NOWAIT, MT_DATA,
2739                             M_PKTHDR, adapter->rx_mbuf_sz);
2740                         if (mp == NULL)
2741                                 goto update;
2742                 } else
2743                         mp = rxbuf->m_pack;
2744
2745                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
2746                 /* Get the memory mapping */
2747                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
2748                     rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
2749                 if (error != 0) {
2750                         printf("GET BUF: dmamap load"
2751                             " failure - %d\n", error);
2752                         m_free(mp);
2753                         rxbuf->m_pack = NULL;
2754                         goto update;
2755                 }
2756                 rxbuf->m_pack = mp;
2757                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
2758                     BUS_DMASYNC_PREREAD);
2759                 rxr->rx_base[i].read.pkt_addr =
2760                     htole64(pseg[0].ds_addr);
2761
2762                 refreshed = TRUE;
2763                 rxr->next_to_refresh = i = j;
2764                 /* Calculate next index */
2765                 if (++j == adapter->num_rx_desc)
2766                         j = 0;
2767         }
2768 update:
2769         if (refreshed) /* update tail index */
2770                 IXGBE_WRITE_REG(&adapter->hw,
2771                     IXGBE_VFRDT(rxr->me), rxr->next_to_refresh);
2772         return;
2773 }
2774
2775 /*********************************************************************
2776  *
2777  *  Allocate memory for rx_buffer structures. Since we use one
2778  *  rx_buffer per received packet, the maximum number of rx_buffer's
2779  *  that we'll need is equal to the number of receive descriptors
2780  *  that we've allocated.
2781  *
2782  **********************************************************************/
2783 static int
2784 ixv_allocate_receive_buffers(struct rx_ring *rxr)
2785 {
2786         struct  adapter         *adapter = rxr->adapter;
2787         device_t                dev = adapter->dev;
2788         struct ixv_rx_buf       *rxbuf;
2789         int                     i, bsize, error;
2790
2791         bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc;
2792         if (!(rxr->rx_buffers =
2793             (struct ixv_rx_buf *) malloc(bsize,
2794             M_DEVBUF, M_NOWAIT | M_ZERO))) {
2795                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
2796                 error = ENOMEM;
2797                 goto fail;
2798         }
2799
2800         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
2801                                    1, 0,        /* alignment, bounds */
2802                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2803                                    BUS_SPACE_MAXADDR,   /* highaddr */
2804                                    NULL, NULL,          /* filter, filterarg */
2805                                    MSIZE,               /* maxsize */
2806                                    1,                   /* nsegments */
2807                                    MSIZE,               /* maxsegsize */
2808                                    0,                   /* flags */
2809                                    NULL,                /* lockfunc */
2810                                    NULL,                /* lockfuncarg */
2811                                    &rxr->htag))) {
2812                 device_printf(dev, "Unable to create RX DMA tag\n");
2813                 goto fail;
2814         }
2815
2816         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
2817                                    1, 0,        /* alignment, bounds */
2818                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2819                                    BUS_SPACE_MAXADDR,   /* highaddr */
2820                                    NULL, NULL,          /* filter, filterarg */
2821                                    MJUMPAGESIZE,        /* maxsize */
2822                                    1,                   /* nsegments */
2823                                    MJUMPAGESIZE,        /* maxsegsize */
2824                                    0,                   /* flags */
2825                                    NULL,                /* lockfunc */
2826                                    NULL,                /* lockfuncarg */
2827                                    &rxr->ptag))) {
2828                 device_printf(dev, "Unable to create RX DMA tag\n");
2829                 goto fail;
2830         }
2831
2832         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
2833                 rxbuf = &rxr->rx_buffers[i];
2834                 error = bus_dmamap_create(rxr->htag,
2835                     BUS_DMA_NOWAIT, &rxbuf->hmap);
2836                 if (error) {
2837                         device_printf(dev, "Unable to create RX head map\n");
2838                         goto fail;
2839                 }
2840                 error = bus_dmamap_create(rxr->ptag,
2841                     BUS_DMA_NOWAIT, &rxbuf->pmap);
2842                 if (error) {
2843                         device_printf(dev, "Unable to create RX pkt map\n");
2844                         goto fail;
2845                 }
2846         }
2847
2848         return (0);
2849
2850 fail:
2851         /* Frees all, but can handle partial completion */
2852         ixv_free_receive_structures(adapter);
2853         return (error);
2854 }
2855
2856 static void
2857 ixv_free_receive_ring(struct rx_ring *rxr)
2858 {
2859         struct  adapter         *adapter;
2860         struct ixv_rx_buf       *rxbuf;
2861         int i;
2862
2863         adapter = rxr->adapter;
2864         for (i = 0; i < adapter->num_rx_desc; i++) {
2865                 rxbuf = &rxr->rx_buffers[i];
2866                 if (rxbuf->m_head != NULL) {
2867                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
2868                             BUS_DMASYNC_POSTREAD);
2869                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
2870                         rxbuf->m_head->m_flags |= M_PKTHDR;
2871                         m_freem(rxbuf->m_head);
2872                 }
2873                 if (rxbuf->m_pack != NULL) {
2874                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
2875                             BUS_DMASYNC_POSTREAD);
2876                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
2877                         rxbuf->m_pack->m_flags |= M_PKTHDR;
2878                         m_freem(rxbuf->m_pack);
2879                 }
2880                 rxbuf->m_head = NULL;
2881                 rxbuf->m_pack = NULL;
2882         }
2883 }
2884
2885
2886 /*********************************************************************
2887  *
2888  *  Initialize a receive ring and its buffers.
2889  *
2890  **********************************************************************/
2891 static int
2892 ixv_setup_receive_ring(struct rx_ring *rxr)
2893 {
2894         struct  adapter         *adapter;
2895         struct ifnet            *ifp;
2896         device_t                dev;
2897         struct ixv_rx_buf       *rxbuf;
2898         bus_dma_segment_t       pseg[1], hseg[1];
2899         struct lro_ctrl         *lro = &rxr->lro;
2900         int                     rsize, nsegs, error = 0;
2901
2902         adapter = rxr->adapter;
2903         ifp = adapter->ifp;
2904         dev = adapter->dev;
2905
2906         /* Clear the ring contents */
2907         IXV_RX_LOCK(rxr);
2908         rsize = roundup2(adapter->num_rx_desc *
2909             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2910         bzero((void *)rxr->rx_base, rsize);
2911
2912         /* Free current RX buffer structs and their mbufs */
2913         ixv_free_receive_ring(rxr);
2914
2915         /* Configure header split? */
2916         if (ixv_header_split)
2917                 rxr->hdr_split = TRUE;
2918
2919         /* Now replenish the mbufs */
2920         for (int j = 0; j != adapter->num_rx_desc; ++j) {
2921                 struct mbuf     *mh, *mp;
2922
2923                 rxbuf = &rxr->rx_buffers[j];
2924                 /*
2925                 ** Dont allocate mbufs if not
2926                 ** doing header split, its wasteful
2927                 */
2928                 if (rxr->hdr_split == FALSE)
2929                         goto skip_head;
2930
2931                 /* First the header */
2932                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
2933                 if (rxbuf->m_head == NULL) {
2934                         error = ENOBUFS;
2935                         goto fail;
2936                 }
2937                 m_adj(rxbuf->m_head, ETHER_ALIGN);
2938                 mh = rxbuf->m_head;
2939                 mh->m_len = mh->m_pkthdr.len = MHLEN;
2940                 mh->m_flags |= M_PKTHDR;
2941                 /* Get the memory mapping */
2942                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
2943                     rxbuf->hmap, rxbuf->m_head, hseg,
2944                     &nsegs, BUS_DMA_NOWAIT);
2945                 if (error != 0) /* Nothing elegant to do here */
2946                         goto fail;
2947                 bus_dmamap_sync(rxr->htag,
2948                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
2949                 /* Update descriptor */
2950                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
2951
2952 skip_head:
2953                 /* Now the payload cluster */
2954                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
2955                     M_PKTHDR, adapter->rx_mbuf_sz);
2956                 if (rxbuf->m_pack == NULL) {
2957                         error = ENOBUFS;
2958                         goto fail;
2959                 }
2960                 mp = rxbuf->m_pack;
2961                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
2962                 /* Get the memory mapping */
2963                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
2964                     rxbuf->pmap, mp, pseg,
2965                     &nsegs, BUS_DMA_NOWAIT);
2966                 if (error != 0)
2967                         goto fail;
2968                 bus_dmamap_sync(rxr->ptag,
2969                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
2970                 /* Update descriptor */
2971                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
2972         }
2973
2974
2975         /* Setup our descriptor indices */
2976         rxr->next_to_check = 0;
2977         rxr->next_to_refresh = 0;
2978         rxr->lro_enabled = FALSE;
2979         rxr->rx_split_packets = 0;
2980         rxr->rx_bytes = 0;
2981         rxr->discard = FALSE;
2982
2983         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
2984             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2985
2986         /*
2987         ** Now set up the LRO interface:
2988         */
2989         if (ifp->if_capenable & IFCAP_LRO) {
2990                 int err = tcp_lro_init(lro);
2991                 if (err) {
2992                         device_printf(dev, "LRO Initialization failed!\n");
2993                         goto fail;
2994                 }
2995                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
2996                 rxr->lro_enabled = TRUE;
2997                 lro->ifp = adapter->ifp;
2998         }
2999
3000         IXV_RX_UNLOCK(rxr);
3001         return (0);
3002
3003 fail:
3004         ixv_free_receive_ring(rxr);
3005         IXV_RX_UNLOCK(rxr);
3006         return (error);
3007 }
3008
3009 /*********************************************************************
3010  *
3011  *  Initialize all receive rings.
3012  *
3013  **********************************************************************/
3014 static int
3015 ixv_setup_receive_structures(struct adapter *adapter)
3016 {
3017         struct rx_ring *rxr = adapter->rx_rings;
3018         int j;
3019
3020         for (j = 0; j < adapter->num_queues; j++, rxr++)
3021                 if (ixv_setup_receive_ring(rxr))
3022                         goto fail;
3023
3024         return (0);
3025 fail:
3026         /*
3027          * Free RX buffers allocated so far, we will only handle
3028          * the rings that completed, the failing case will have
3029          * cleaned up for itself. 'j' failed, so its the terminus.
3030          */
3031         for (int i = 0; i < j; ++i) {
3032                 rxr = &adapter->rx_rings[i];
3033                 ixv_free_receive_ring(rxr);
3034         }
3035
3036         return (ENOBUFS);
3037 }
3038
3039 /*********************************************************************
3040  *
3041  *  Setup receive registers and features.
3042  *
3043  **********************************************************************/
3044 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3045
3046 static void
3047 ixv_initialize_receive_units(struct adapter *adapter)
3048 {
3049         struct  rx_ring *rxr = adapter->rx_rings;
3050         struct ixgbe_hw *hw = &adapter->hw;
3051         struct ifnet   *ifp = adapter->ifp;
3052         u32             bufsz, fctrl, rxcsum, hlreg;
3053
3054
3055         /* Enable broadcasts */
3056         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3057         fctrl |= IXGBE_FCTRL_BAM;
3058         fctrl |= IXGBE_FCTRL_DPF;
3059         fctrl |= IXGBE_FCTRL_PMCF;
3060         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3061
3062         /* Set for Jumbo Frames? */
3063         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3064         if (ifp->if_mtu > ETHERMTU) {
3065                 hlreg |= IXGBE_HLREG0_JUMBOEN;
3066                 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3067         } else {
3068                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
3069                 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3070         }
3071         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
3072
3073         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3074                 u64 rdba = rxr->rxdma.dma_paddr;
3075                 u32 reg, rxdctl;
3076
3077                 /* Do the queue enabling first */
3078                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
3079                 rxdctl |= IXGBE_RXDCTL_ENABLE;
3080                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
3081                 for (int k = 0; k < 10; k++) {
3082                         if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) &
3083                             IXGBE_RXDCTL_ENABLE)
3084                                 break;
3085                         else
3086                                 msec_delay(1);
3087                 }
3088                 wmb();
3089
3090                 /* Setup the Base and Length of the Rx Descriptor Ring */
3091                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
3092                     (rdba & 0x00000000ffffffffULL));
3093                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
3094                     (rdba >> 32));
3095                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
3096                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
3097
3098                 /* Set up the SRRCTL register */
3099                 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i));
3100                 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
3101                 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
3102                 reg |= bufsz;
3103                 if (rxr->hdr_split) {
3104                         /* Use a standard mbuf for the header */
3105                         reg |= ((IXV_RX_HDR <<
3106                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
3107                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
3108                         reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3109                 } else
3110                         reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
3111                 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg);
3112
3113                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
3114                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
3115                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
3116                     adapter->num_rx_desc - 1);
3117         }
3118
3119         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3120
3121         if (ifp->if_capenable & IFCAP_RXCSUM)
3122                 rxcsum |= IXGBE_RXCSUM_PCSD;
3123
3124         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
3125                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
3126
3127         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3128
3129         return;
3130 }
3131
3132 /*********************************************************************
3133  *
3134  *  Free all receive rings.
3135  *
3136  **********************************************************************/
3137 static void
3138 ixv_free_receive_structures(struct adapter *adapter)
3139 {
3140         struct rx_ring *rxr = adapter->rx_rings;
3141
3142         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3143                 struct lro_ctrl         *lro = &rxr->lro;
3144                 ixv_free_receive_buffers(rxr);
3145                 /* Free LRO memory */
3146                 tcp_lro_free(lro);
3147                 /* Free the ring memory as well */
3148                 ixv_dma_free(adapter, &rxr->rxdma);
3149         }
3150
3151         free(adapter->rx_rings, M_DEVBUF);
3152 }
3153
3154
3155 /*********************************************************************
3156  *
3157  *  Free receive ring data structures
3158  *
3159  **********************************************************************/
3160 static void
3161 ixv_free_receive_buffers(struct rx_ring *rxr)
3162 {
3163         struct adapter          *adapter = rxr->adapter;
3164         struct ixv_rx_buf       *rxbuf;
3165
3166         INIT_DEBUGOUT("free_receive_structures: begin");
3167
3168         /* Cleanup any existing buffers */
3169         if (rxr->rx_buffers != NULL) {
3170                 for (int i = 0; i < adapter->num_rx_desc; i++) {
3171                         rxbuf = &rxr->rx_buffers[i];
3172                         if (rxbuf->m_head != NULL) {
3173                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3174                                     BUS_DMASYNC_POSTREAD);
3175                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3176                                 rxbuf->m_head->m_flags |= M_PKTHDR;
3177                                 m_freem(rxbuf->m_head);
3178                         }
3179                         if (rxbuf->m_pack != NULL) {
3180                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3181                                     BUS_DMASYNC_POSTREAD);
3182                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3183                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
3184                                 m_freem(rxbuf->m_pack);
3185                         }
3186                         rxbuf->m_head = NULL;
3187                         rxbuf->m_pack = NULL;
3188                         if (rxbuf->hmap != NULL) {
3189                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
3190                                 rxbuf->hmap = NULL;
3191                         }
3192                         if (rxbuf->pmap != NULL) {
3193                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
3194                                 rxbuf->pmap = NULL;
3195                         }
3196                 }
3197                 if (rxr->rx_buffers != NULL) {
3198                         free(rxr->rx_buffers, M_DEVBUF);
3199                         rxr->rx_buffers = NULL;
3200                 }
3201         }
3202
3203         if (rxr->htag != NULL) {
3204                 bus_dma_tag_destroy(rxr->htag);
3205                 rxr->htag = NULL;
3206         }
3207         if (rxr->ptag != NULL) {
3208                 bus_dma_tag_destroy(rxr->ptag);
3209                 rxr->ptag = NULL;
3210         }
3211
3212         return;
3213 }
3214
3215 static __inline void
3216 ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
3217 {
3218
3219         /*
3220          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
3221          * should be computed by hardware. Also it should not have VLAN tag in
3222          * ethernet header.
3223          */
3224         if (rxr->lro_enabled &&
3225             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
3226             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
3227             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
3228             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) &&
3229             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
3230             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
3231                 /*
3232                  * Send to the stack if:
3233                  **  - LRO not enabled, or
3234                  **  - no LRO resources, or
3235                  **  - lro enqueue fails
3236                  */
3237                 if (rxr->lro.lro_cnt != 0)
3238                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
3239                                 return;
3240         }
3241         IXV_RX_UNLOCK(rxr);
3242         (*ifp->if_input)(ifp, m);
3243         IXV_RX_LOCK(rxr);
3244 }
3245
3246 static __inline void
3247 ixv_rx_discard(struct rx_ring *rxr, int i)
3248 {
3249         struct ixv_rx_buf       *rbuf;
3250
3251         rbuf = &rxr->rx_buffers[i];
3252
3253         if (rbuf->fmp != NULL) {/* Partial chain ? */
3254                 rbuf->fmp->m_flags |= M_PKTHDR;
3255                 m_freem(rbuf->fmp);
3256                 rbuf->fmp = NULL;
3257         }
3258
3259         /*
3260         ** With advanced descriptors the writeback
3261         ** clobbers the buffer addrs, so its easier
3262         ** to just free the existing mbufs and take
3263         ** the normal refresh path to get new buffers
3264         ** and mapping.
3265         */
3266         if (rbuf->m_head) {
3267                 m_free(rbuf->m_head);
3268                 rbuf->m_head = NULL;
3269         }
3270
3271         if (rbuf->m_pack) {
3272                 m_free(rbuf->m_pack);
3273                 rbuf->m_pack = NULL;
3274         }
3275
3276         return;
3277 }
3278
3279
3280 /*********************************************************************
3281  *
3282  *  This routine executes in interrupt context. It replenishes
3283  *  the mbufs in the descriptor and sends data which has been
3284  *  dma'ed into host memory to upper layer.
3285  *
3286  *  We loop at most count times if count is > 0, or until done if
3287  *  count < 0.
3288  *
3289  *  Return TRUE for more work, FALSE for all clean.
3290  *********************************************************************/
3291 static bool
3292 ixv_rxeof(struct ix_queue *que, int count)
3293 {
3294         struct adapter          *adapter = que->adapter;
3295         struct rx_ring          *rxr = que->rxr;
3296         struct ifnet            *ifp = adapter->ifp;
3297         struct lro_ctrl         *lro = &rxr->lro;
3298         struct lro_entry        *queued;
3299         int                     i, nextp, processed = 0;
3300         u32                     staterr = 0;
3301         union ixgbe_adv_rx_desc *cur;
3302         struct ixv_rx_buf       *rbuf, *nbuf;
3303
3304         IXV_RX_LOCK(rxr);
3305
3306         for (i = rxr->next_to_check; count != 0;) {
3307                 struct mbuf     *sendmp, *mh, *mp;
3308                 u32             rsc, ptype;
3309                 u16             hlen, plen, hdr, vtag;
3310                 bool            eop;
3311
3312                 /* Sync the ring. */
3313                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3314                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3315
3316                 cur = &rxr->rx_base[i];
3317                 staterr = le32toh(cur->wb.upper.status_error);
3318
3319                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
3320                         break;
3321                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
3322                         break;
3323
3324                 count--;
3325                 sendmp = NULL;
3326                 nbuf = NULL;
3327                 rsc = 0;
3328                 cur->wb.upper.status_error = 0;
3329                 rbuf = &rxr->rx_buffers[i];
3330                 mh = rbuf->m_head;
3331                 mp = rbuf->m_pack;
3332
3333                 plen = le16toh(cur->wb.upper.length);
3334                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
3335                     IXGBE_RXDADV_PKTTYPE_MASK;
3336                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
3337                 vtag = le16toh(cur->wb.upper.vlan);
3338                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
3339
3340                 /* Make sure all parts of a bad packet are discarded */
3341                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
3342                     (rxr->discard)) {
3343                         ifp->if_ierrors++;
3344                         rxr->rx_discarded++;
3345                         if (!eop)
3346                                 rxr->discard = TRUE;
3347                         else
3348                                 rxr->discard = FALSE;
3349                         ixv_rx_discard(rxr, i);
3350                         goto next_desc;
3351                 }
3352
3353                 if (!eop) {
3354                         nextp = i + 1;
3355                         if (nextp == adapter->num_rx_desc)
3356                                 nextp = 0;
3357                         nbuf = &rxr->rx_buffers[nextp];
3358                         prefetch(nbuf);
3359                 }
3360                 /*
3361                 ** The header mbuf is ONLY used when header
3362                 ** split is enabled, otherwise we get normal
3363                 ** behavior, ie, both header and payload
3364                 ** are DMA'd into the payload buffer.
3365                 **
3366                 ** Rather than using the fmp/lmp global pointers
3367                 ** we now keep the head of a packet chain in the
3368                 ** buffer struct and pass this along from one
3369                 ** descriptor to the next, until we get EOP.
3370                 */
3371                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
3372                         /* This must be an initial descriptor */
3373                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
3374                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
3375                         if (hlen > IXV_RX_HDR)
3376                                 hlen = IXV_RX_HDR;
3377                         mh->m_len = hlen;
3378                         mh->m_flags |= M_PKTHDR;
3379                         mh->m_next = NULL;
3380                         mh->m_pkthdr.len = mh->m_len;
3381                         /* Null buf pointer so it is refreshed */
3382                         rbuf->m_head = NULL;
3383                         /*
3384                         ** Check the payload length, this
3385                         ** could be zero if its a small
3386                         ** packet.
3387                         */
3388                         if (plen > 0) {
3389                                 mp->m_len = plen;
3390                                 mp->m_next = NULL;
3391                                 mp->m_flags &= ~M_PKTHDR;
3392                                 mh->m_next = mp;
3393                                 mh->m_pkthdr.len += mp->m_len;
3394                                 /* Null buf pointer so it is refreshed */
3395                                 rbuf->m_pack = NULL;
3396                                 rxr->rx_split_packets++;
3397                         }
3398                         /*
3399                         ** Now create the forward
3400                         ** chain so when complete
3401                         ** we wont have to.
3402                         */
3403                         if (eop == 0) {
3404                                 /* stash the chain head */
3405                                 nbuf->fmp = mh;
3406                                 /* Make forward chain */
3407                                 if (plen)
3408                                         mp->m_next = nbuf->m_pack;
3409                                 else
3410                                         mh->m_next = nbuf->m_pack;
3411                         } else {
3412                                 /* Singlet, prepare to send */
3413                                 sendmp = mh;
3414                                 if ((adapter->num_vlans) &&
3415                                     (staterr & IXGBE_RXD_STAT_VP)) {
3416                                         sendmp->m_pkthdr.ether_vtag = vtag;
3417                                         sendmp->m_flags |= M_VLANTAG;
3418                                 }
3419                         }
3420                 } else {
3421                         /*
3422                         ** Either no header split, or a
3423                         ** secondary piece of a fragmented
3424                         ** split packet.
3425                         */
3426                         mp->m_len = plen;
3427                         /*
3428                         ** See if there is a stored head
3429                         ** that determines what we are
3430                         */
3431                         sendmp = rbuf->fmp;
3432                         rbuf->m_pack = rbuf->fmp = NULL;
3433
3434                         if (sendmp != NULL) /* secondary frag */
3435                                 sendmp->m_pkthdr.len += mp->m_len;
3436                         else {
3437                                 /* first desc of a non-ps chain */
3438                                 sendmp = mp;
3439                                 sendmp->m_flags |= M_PKTHDR;
3440                                 sendmp->m_pkthdr.len = mp->m_len;
3441                                 if (staterr & IXGBE_RXD_STAT_VP) {
3442                                         sendmp->m_pkthdr.ether_vtag = vtag;
3443                                         sendmp->m_flags |= M_VLANTAG;
3444                                 }
3445                         }
3446                         /* Pass the head pointer on */
3447                         if (eop == 0) {
3448                                 nbuf->fmp = sendmp;
3449                                 sendmp = NULL;
3450                                 mp->m_next = nbuf->m_pack;
3451                         }
3452                 }
3453                 ++processed;
3454                 /* Sending this frame? */
3455                 if (eop) {
3456                         sendmp->m_pkthdr.rcvif = ifp;
3457                         ifp->if_ipackets++;
3458                         rxr->rx_packets++;
3459                         /* capture data for AIM */
3460                         rxr->bytes += sendmp->m_pkthdr.len;
3461                         rxr->rx_bytes += sendmp->m_pkthdr.len;
3462                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
3463                                 ixv_rx_checksum(staterr, sendmp, ptype);
3464 #if __FreeBSD_version >= 800000
3465                         sendmp->m_pkthdr.flowid = que->msix;
3466                         sendmp->m_flags |= M_FLOWID;
3467 #endif
3468                 }
3469 next_desc:
3470                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3471                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3472
3473                 /* Advance our pointers to the next descriptor. */
3474                 if (++i == adapter->num_rx_desc)
3475                         i = 0;
3476
3477                 /* Now send to the stack or do LRO */
3478                 if (sendmp != NULL)
3479                         ixv_rx_input(rxr, ifp, sendmp, ptype);
3480
3481                /* Every 8 descriptors we go to refresh mbufs */
3482                 if (processed == 8) {
3483                         ixv_refresh_mbufs(rxr, i);
3484                         processed = 0;
3485                 }
3486         }
3487
3488         /* Refresh any remaining buf structs */
3489         if (ixv_rx_unrefreshed(rxr))
3490                 ixv_refresh_mbufs(rxr, i);
3491
3492         rxr->next_to_check = i;
3493
3494         /*
3495          * Flush any outstanding LRO work
3496          */
3497         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
3498                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
3499                 tcp_lro_flush(lro, queued);
3500         }
3501
3502         IXV_RX_UNLOCK(rxr);
3503
3504         /*
3505         ** We still have cleaning to do?
3506         ** Schedule another interrupt if so.
3507         */
3508         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
3509                 ixv_rearm_queues(adapter, (u64)(1 << que->msix));
3510                 return (TRUE);
3511         }
3512
3513         return (FALSE);
3514 }
3515
3516
3517 /*********************************************************************
3518  *
3519  *  Verify that the hardware indicated that the checksum is valid.
3520  *  Inform the stack about the status of checksum so that stack
3521  *  doesn't spend time verifying the checksum.
3522  *
3523  *********************************************************************/
3524 static void
3525 ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
3526 {
3527         u16     status = (u16) staterr;
3528         u8      errors = (u8) (staterr >> 24);
3529         bool    sctp = FALSE;
3530
3531         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
3532             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
3533                 sctp = TRUE;
3534
3535         if (status & IXGBE_RXD_STAT_IPCS) {
3536                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
3537                         /* IP Checksum Good */
3538                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3539                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3540
3541                 } else
3542                         mp->m_pkthdr.csum_flags = 0;
3543         }
3544         if (status & IXGBE_RXD_STAT_L4CS) {
3545                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3546 #if __FreeBSD_version >= 800000
3547                 if (sctp)
3548                         type = CSUM_SCTP_VALID;
3549 #endif
3550                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
3551                         mp->m_pkthdr.csum_flags |= type;
3552                         if (!sctp)
3553                                 mp->m_pkthdr.csum_data = htons(0xffff);
3554                 }
3555         }
3556         return;
3557 }
3558
3559 static void
3560 ixv_setup_vlan_support(struct adapter *adapter)
3561 {
3562         struct ixgbe_hw *hw = &adapter->hw;
3563         u32             ctrl, vid, vfta, retry;
3564
3565
3566         /*
3567         ** We get here thru init_locked, meaning
3568         ** a soft reset, this has already cleared
3569         ** the VFTA and other state, so if there
3570         ** have been no vlan's registered do nothing.
3571         */
3572         if (adapter->num_vlans == 0)
3573                 return;
3574
3575         /* Enable the queues */
3576         for (int i = 0; i < adapter->num_queues; i++) {
3577                 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
3578                 ctrl |= IXGBE_RXDCTL_VME;
3579                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl);
3580         }
3581
3582         /*
3583         ** A soft reset zero's out the VFTA, so
3584         ** we need to repopulate it now.
3585         */
3586         for (int i = 0; i < VFTA_SIZE; i++) {
3587                 if (ixv_shadow_vfta[i] == 0)
3588                         continue;
3589                 vfta = ixv_shadow_vfta[i];
3590                 /*
3591                 ** Reconstruct the vlan id's
3592                 ** based on the bits set in each
3593                 ** of the array ints.
3594                 */
3595                 for ( int j = 0; j < 32; j++) {
3596                         retry = 0;
3597                         if ((vfta & (1 << j)) == 0)
3598                                 continue;
3599                         vid = (i * 32) + j;
3600                         /* Call the shared code mailbox routine */
3601                         while (ixgbe_set_vfta(hw, vid, 0, TRUE)) {
3602                                 if (++retry > 5)
3603                                         break;
3604                         }
3605                 }
3606         }
3607 }
3608
3609 /*
3610 ** This routine is run via an vlan config EVENT,
3611 ** it enables us to use the HW Filter table since
3612 ** we can get the vlan id. This just creates the
3613 ** entry in the soft version of the VFTA, init will
3614 ** repopulate the real table.
3615 */
3616 static void
3617 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3618 {
3619         struct adapter  *adapter = ifp->if_softc;
3620         u16             index, bit;
3621
3622         if (ifp->if_softc !=  arg)   /* Not our event */
3623                 return;
3624
3625         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3626                 return;
3627
3628         IXV_CORE_LOCK(adapter);
3629         index = (vtag >> 5) & 0x7F;
3630         bit = vtag & 0x1F;
3631         ixv_shadow_vfta[index] |= (1 << bit);
3632         ++adapter->num_vlans;
3633         /* Re-init to load the changes */
3634         ixv_init_locked(adapter);
3635         IXV_CORE_UNLOCK(adapter);
3636 }
3637
3638 /*
3639 ** This routine is run via an vlan
3640 ** unconfig EVENT, remove our entry
3641 ** in the soft vfta.
3642 */
3643 static void
3644 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3645 {
3646         struct adapter  *adapter = ifp->if_softc;
3647         u16             index, bit;
3648
3649         if (ifp->if_softc !=  arg)
3650                 return;
3651
3652         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3653                 return;
3654
3655         IXV_CORE_LOCK(adapter);
3656         index = (vtag >> 5) & 0x7F;
3657         bit = vtag & 0x1F;
3658         ixv_shadow_vfta[index] &= ~(1 << bit);
3659         --adapter->num_vlans;
3660         /* Re-init to load the changes */
3661         ixv_init_locked(adapter);
3662         IXV_CORE_UNLOCK(adapter);
3663 }
3664
3665 static void
3666 ixv_enable_intr(struct adapter *adapter)
3667 {
3668         struct ixgbe_hw *hw = &adapter->hw;
3669         struct ix_queue *que = adapter->queues;
3670         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3671
3672
3673         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
3674
3675         mask = IXGBE_EIMS_ENABLE_MASK;
3676         mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
3677         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
3678
3679         for (int i = 0; i < adapter->num_queues; i++, que++)
3680                 ixv_enable_queue(adapter, que->msix);
3681
3682         IXGBE_WRITE_FLUSH(hw);
3683
3684         return;
3685 }
3686
3687 static void
3688 ixv_disable_intr(struct adapter *adapter)
3689 {
3690         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0);
3691         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0);
3692         IXGBE_WRITE_FLUSH(&adapter->hw);
3693         return;
3694 }
3695
3696 /*
3697 ** Setup the correct IVAR register for a particular MSIX interrupt
3698 **  - entry is the register array entry
3699 **  - vector is the MSIX vector for this queue
3700 **  - type is RX/TX/MISC
3701 */
3702 static void
3703 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
3704 {
3705         struct ixgbe_hw *hw = &adapter->hw;
3706         u32 ivar, index;
3707
3708         vector |= IXGBE_IVAR_ALLOC_VAL;
3709
3710         if (type == -1) { /* MISC IVAR */
3711                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
3712                 ivar &= ~0xFF;
3713                 ivar |= vector;
3714                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
3715         } else {        /* RX/TX IVARS */
3716                 index = (16 * (entry & 1)) + (8 * type);
3717                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
3718                 ivar &= ~(0xFF << index);
3719                 ivar |= (vector << index);
3720                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
3721         }
3722 }
3723
3724 static void
3725 ixv_configure_ivars(struct adapter *adapter)
3726 {
3727         struct  ix_queue *que = adapter->queues;
3728
3729         for (int i = 0; i < adapter->num_queues; i++, que++) {
3730                 /* First the RX queue entry */
3731                 ixv_set_ivar(adapter, i, que->msix, 0);
3732                 /* ... and the TX */
3733                 ixv_set_ivar(adapter, i, que->msix, 1);
3734                 /* Set an initial value in EITR */
3735                 IXGBE_WRITE_REG(&adapter->hw,
3736                     IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT);
3737         }
3738
3739         /* For the Link interrupt */
3740         ixv_set_ivar(adapter, 1, adapter->mbxvec, -1);
3741 }
3742
3743
3744 /*
3745 ** Tasklet handler for MSIX MBX interrupts
3746 **  - do outside interrupt since it might sleep
3747 */
3748 static void
3749 ixv_handle_mbx(void *context, int pending)
3750 {
3751         struct adapter  *adapter = context;
3752
3753         ixgbe_check_link(&adapter->hw,
3754             &adapter->link_speed, &adapter->link_up, 0);
3755         ixv_update_link_status(adapter);
3756 }
3757
3758 /*
3759 ** The VF stats registers never have a truely virgin
3760 ** starting point, so this routine tries to make an
3761 ** artificial one, marking ground zero on attach as
3762 ** it were.
3763 */
3764 static void
3765 ixv_save_stats(struct adapter *adapter)
3766 {
3767         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
3768                 adapter->stats.saved_reset_vfgprc +=
3769                     adapter->stats.vfgprc - adapter->stats.base_vfgprc;
3770                 adapter->stats.saved_reset_vfgptc +=
3771                     adapter->stats.vfgptc - adapter->stats.base_vfgptc;
3772                 adapter->stats.saved_reset_vfgorc +=
3773                     adapter->stats.vfgorc - adapter->stats.base_vfgorc;
3774                 adapter->stats.saved_reset_vfgotc +=
3775                     adapter->stats.vfgotc - adapter->stats.base_vfgotc;
3776                 adapter->stats.saved_reset_vfmprc +=
3777                     adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3778         }
3779 }
3780
3781 static void
3782 ixv_init_stats(struct adapter *adapter)
3783 {
3784         struct ixgbe_hw *hw = &adapter->hw;
3785
3786         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
3787         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
3788         adapter->stats.last_vfgorc |=
3789             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
3790
3791         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
3792         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
3793         adapter->stats.last_vfgotc |=
3794             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
3795
3796         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
3797
3798         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
3799         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
3800         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
3801         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
3802         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
3803 }
3804
3805 #define UPDATE_STAT_32(reg, last, count)                \
3806 {                                                       \
3807         u32 current = IXGBE_READ_REG(hw, reg);          \
3808         if (current < last)                             \
3809                 count += 0x100000000LL;                 \
3810         last = current;                                 \
3811         count &= 0xFFFFFFFF00000000LL;                  \
3812         count |= current;                               \
3813 }
3814
3815 #define UPDATE_STAT_36(lsb, msb, last, count)           \
3816 {                                                       \
3817         u64 cur_lsb = IXGBE_READ_REG(hw, lsb);          \
3818         u64 cur_msb = IXGBE_READ_REG(hw, msb);          \
3819         u64 current = ((cur_msb << 32) | cur_lsb);      \
3820         if (current < last)                             \
3821                 count += 0x1000000000LL;                \
3822         last = current;                                 \
3823         count &= 0xFFFFFFF000000000LL;                  \
3824         count |= current;                               \
3825 }
3826
3827 /*
3828 ** ixv_update_stats - Update the board statistics counters.
3829 */
3830 void
3831 ixv_update_stats(struct adapter *adapter)
3832 {
3833         struct ixgbe_hw *hw = &adapter->hw;
3834
3835         UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
3836             adapter->stats.vfgprc);
3837         UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
3838             adapter->stats.vfgptc);
3839         UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
3840             adapter->stats.last_vfgorc, adapter->stats.vfgorc);
3841         UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
3842             adapter->stats.last_vfgotc, adapter->stats.vfgotc);
3843         UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
3844             adapter->stats.vfmprc);
3845 }
3846
3847 /**********************************************************************
3848  *
3849  *  This routine is called only when ixgbe_display_debug_stats is enabled.
3850  *  This routine provides a way to take a look at important statistics
3851  *  maintained by the driver and hardware.
3852  *
3853  **********************************************************************/
3854 static void
3855 ixv_print_hw_stats(struct adapter * adapter)
3856 {
3857         device_t dev = adapter->dev;
3858
3859         device_printf(dev,"Std Mbuf Failed = %lu\n",
3860                adapter->mbuf_defrag_failed);
3861         device_printf(dev,"Driver dropped packets = %lu\n",
3862                adapter->dropped_pkts);
3863         device_printf(dev, "watchdog timeouts = %ld\n",
3864                adapter->watchdog_events);
3865
3866         device_printf(dev,"Good Packets Rcvd = %llu\n",
3867                (long long)adapter->stats.vfgprc);
3868         device_printf(dev,"Good Packets Xmtd = %llu\n",
3869                (long long)adapter->stats.vfgptc);
3870         device_printf(dev,"TSO Transmissions = %lu\n",
3871                adapter->tso_tx);
3872
3873 }
3874
3875 /**********************************************************************
3876  *
3877  *  This routine is called only when em_display_debug_stats is enabled.
3878  *  This routine provides a way to take a look at important statistics
3879  *  maintained by the driver and hardware.
3880  *
3881  **********************************************************************/
3882 static void
3883 ixv_print_debug_info(struct adapter *adapter)
3884 {
3885         device_t dev = adapter->dev;
3886         struct ixgbe_hw         *hw = &adapter->hw;
3887         struct ix_queue         *que = adapter->queues;
3888         struct rx_ring          *rxr;
3889         struct tx_ring          *txr;
3890         struct lro_ctrl         *lro;
3891
3892         device_printf(dev,"Error Byte Count = %u \n",
3893             IXGBE_READ_REG(hw, IXGBE_ERRBC));
3894
3895         for (int i = 0; i < adapter->num_queues; i++, que++) {
3896                 txr = que->txr;
3897                 rxr = que->rxr;
3898                 lro = &rxr->lro;
3899                 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n",
3900                     que->msix, (long)que->irqs);
3901                 device_printf(dev,"RX(%d) Packets Received: %lld\n",
3902                     rxr->me, (long long)rxr->rx_packets);
3903                 device_printf(dev,"RX(%d) Split RX Packets: %lld\n",
3904                     rxr->me, (long long)rxr->rx_split_packets);
3905                 device_printf(dev,"RX(%d) Bytes Received: %lu\n",
3906                     rxr->me, (long)rxr->rx_bytes);
3907                 device_printf(dev,"RX(%d) LRO Queued= %d\n",
3908                     rxr->me, lro->lro_queued);
3909                 device_printf(dev,"RX(%d) LRO Flushed= %d\n",
3910                     rxr->me, lro->lro_flushed);
3911                 device_printf(dev,"TX(%d) Packets Sent: %lu\n",
3912                     txr->me, (long)txr->total_packets);
3913                 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n",
3914                     txr->me, (long)txr->no_desc_avail);
3915         }
3916
3917         device_printf(dev,"MBX IRQ Handled: %lu\n",
3918             (long)adapter->mbx_irq);
3919         return;
3920 }
3921
3922 static int
3923 ixv_sysctl_stats(SYSCTL_HANDLER_ARGS)
3924 {
3925         int             error;
3926         int             result;
3927         struct adapter *adapter;
3928
3929         result = -1;
3930         error = sysctl_handle_int(oidp, &result, 0, req);
3931
3932         if (error || !req->newptr)
3933                 return (error);
3934
3935         if (result == 1) {
3936                 adapter = (struct adapter *) arg1;
3937                 ixv_print_hw_stats(adapter);
3938         }
3939         return error;
3940 }
3941
3942 static int
3943 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)
3944 {
3945         int error, result;
3946         struct adapter *adapter;
3947
3948         result = -1;
3949         error = sysctl_handle_int(oidp, &result, 0, req);
3950
3951         if (error || !req->newptr)
3952                 return (error);
3953
3954         if (result == 1) {
3955                 adapter = (struct adapter *) arg1;
3956                 ixv_print_debug_info(adapter);
3957         }
3958         return error;
3959 }
3960
3961 /*
3962 ** Set flow control using sysctl:
3963 ** Flow control values:
3964 **      0 - off
3965 **      1 - rx pause
3966 **      2 - tx pause
3967 **      3 - full
3968 */
3969 static int
3970 ixv_set_flowcntl(SYSCTL_HANDLER_ARGS)
3971 {
3972         int error;
3973         struct adapter *adapter;
3974
3975         error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req);
3976
3977         if (error)
3978                 return (error);
3979
3980         adapter = (struct adapter *) arg1;
3981         switch (ixv_flow_control) {
3982                 case ixgbe_fc_rx_pause:
3983                 case ixgbe_fc_tx_pause:
3984                 case ixgbe_fc_full:
3985                         adapter->hw.fc.requested_mode = ixv_flow_control;
3986                         break;
3987                 case ixgbe_fc_none:
3988                 default:
3989                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
3990         }
3991
3992         ixgbe_fc_enable(&adapter->hw);
3993         return error;
3994 }
3995
3996 static void
3997 ixv_add_rx_process_limit(struct adapter *adapter, const char *name,
3998         const char *description, int *limit, int value)
3999 {
4000         *limit = value;
4001         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4002             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4003             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
4004 }
4005