sys/dev/ixgbe/ixv.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_inet.h"
  37 #include "opt_inet6.h"
  38 #endif
  39
  40 #include "ixv.h"
  41
  42 /*********************************************************************
  43  *  Driver version
  44  *********************************************************************/
  45 char ixv_driver_version[] = "1.1.4";
  46
  47 /*********************************************************************
  48  *  PCI Device ID Table
  49  *
  50  *  Used by probe to select devices to load on
  51  *  Last field stores an index into ixv_strings
  52  *  Last entry must be all 0s
  53  *
  54  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  55  *********************************************************************/
  56
  57 static ixv_vendor_info_t ixv_vendor_info_array[] =
  58 {
  59         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0},
  60         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0},
  61         /* required last entry */
  62         {0, 0, 0, 0, 0}
  63 };
  64
  65 /*********************************************************************
  66  *  Table of branding strings
  67  *********************************************************************/
  68
  69 static char    *ixv_strings[] = {
  70         "Intel(R) PRO/10GbE Virtual Function Network Driver"
  71 };
  72
  73 /*********************************************************************
  74  *  Function prototypes
  75  *********************************************************************/
  76 static int      ixv_probe(device_t);
  77 static int      ixv_attach(device_t);
  78 static int      ixv_detach(device_t);
  79 static int      ixv_shutdown(device_t);
  80 #if __FreeBSD_version < 800000
  81 static void     ixv_start(struct ifnet *);
  82 static void     ixv_start_locked(struct tx_ring *, struct ifnet *);
  83 #else
  84 static int      ixv_mq_start(struct ifnet *, struct mbuf *);
  85 static int      ixv_mq_start_locked(struct ifnet *,
  86                     struct tx_ring *, struct mbuf *);
  87 static void     ixv_qflush(struct ifnet *);
  88 #endif
  89 static int      ixv_ioctl(struct ifnet *, u_long, caddr_t);
  90 static void     ixv_init(void *);
  91 static void     ixv_init_locked(struct adapter *);
  92 static void     ixv_stop(void *);
  93 static void     ixv_media_status(struct ifnet *, struct ifmediareq *);
  94 static int      ixv_media_change(struct ifnet *);
  95 static void     ixv_identify_hardware(struct adapter *);
  96 static int      ixv_allocate_pci_resources(struct adapter *);
  97 static int      ixv_allocate_msix(struct adapter *);
  98 static int      ixv_allocate_queues(struct adapter *);
  99 static int      ixv_setup_msix(struct adapter *);
 100 static void     ixv_free_pci_resources(struct adapter *);
 101 static void     ixv_local_timer(void *);
 102 static void     ixv_setup_interface(device_t, struct adapter *);
 103 static void     ixv_config_link(struct adapter *);
 104
 105 static int      ixv_allocate_transmit_buffers(struct tx_ring *);
 106 static int      ixv_setup_transmit_structures(struct adapter *);
 107 static void     ixv_setup_transmit_ring(struct tx_ring *);
 108 static void     ixv_initialize_transmit_units(struct adapter *);
 109 static void     ixv_free_transmit_structures(struct adapter *);
 110 static void     ixv_free_transmit_buffers(struct tx_ring *);
 111
 112 static int      ixv_allocate_receive_buffers(struct rx_ring *);
 113 static int      ixv_setup_receive_structures(struct adapter *);
 114 static int      ixv_setup_receive_ring(struct rx_ring *);
 115 static void     ixv_initialize_receive_units(struct adapter *);
 116 static void     ixv_free_receive_structures(struct adapter *);
 117 static void     ixv_free_receive_buffers(struct rx_ring *);
 118
 119 static void     ixv_enable_intr(struct adapter *);
 120 static void     ixv_disable_intr(struct adapter *);
 121 static bool     ixv_txeof(struct tx_ring *);
 122 static bool     ixv_rxeof(struct ix_queue *, int);
 123 static void     ixv_rx_checksum(u32, struct mbuf *, u32);
 124 static void     ixv_set_multi(struct adapter *);
 125 static void     ixv_update_link_status(struct adapter *);
 126 static void     ixv_refresh_mbufs(struct rx_ring *, int);
 127 static int      ixv_xmit(struct tx_ring *, struct mbuf **);
 128 static int      ixv_sysctl_stats(SYSCTL_HANDLER_ARGS);
 129 static int      ixv_sysctl_debug(SYSCTL_HANDLER_ARGS);
 130 static int      ixv_set_flowcntl(SYSCTL_HANDLER_ARGS);
 131 static int      ixv_dma_malloc(struct adapter *, bus_size_t,
 132                     struct ixv_dma_alloc *, int);
 133 static void     ixv_dma_free(struct adapter *, struct ixv_dma_alloc *);
 134 static void     ixv_add_rx_process_limit(struct adapter *, const char *,
 135                     const char *, int *, int);
 136 static bool     ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 137 static bool     ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 138 static void     ixv_set_ivar(struct adapter *, u8, u8, s8);
 139 static void     ixv_configure_ivars(struct adapter *);
 140 static u8 *     ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 141
 142 static void     ixv_setup_vlan_support(struct adapter *);
 143 static void     ixv_register_vlan(void *, struct ifnet *, u16);
 144 static void     ixv_unregister_vlan(void *, struct ifnet *, u16);
 145
 146 static void     ixv_save_stats(struct adapter *);
 147 static void     ixv_init_stats(struct adapter *);
 148 static void     ixv_update_stats(struct adapter *);
 149
 150 static __inline void ixv_rx_discard(struct rx_ring *, int);
 151 static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *,
 152                     struct mbuf *, u32);
 153
 154 /* The MSI/X Interrupt handlers */
 155 static void     ixv_msix_que(void *);
 156 static void     ixv_msix_mbx(void *);
 157
 158 /* Deferred interrupt tasklets */
 159 static void     ixv_handle_que(void *, int);
 160 static void     ixv_handle_mbx(void *, int);
 161
 162 /*********************************************************************
 163  *  FreeBSD Device Interface Entry Points
 164  *********************************************************************/
 165
 166 static device_method_t ixv_methods[] = {
 167         /* Device interface */
 168         DEVMETHOD(device_probe, ixv_probe),
 169         DEVMETHOD(device_attach, ixv_attach),
 170         DEVMETHOD(device_detach, ixv_detach),
 171         DEVMETHOD(device_shutdown, ixv_shutdown),
 172         {0, 0}
 173 };
 174
 175 static driver_t ixv_driver = {
 176         "ix", ixv_methods, sizeof(struct adapter),
 177 };
 178
 179 extern devclass_t ixgbe_devclass;
 180 DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0);
 181 MODULE_DEPEND(ixv, pci, 1, 1, 1);
 182 MODULE_DEPEND(ixv, ether, 1, 1, 1);
 183
 184 /*
 185 ** TUNEABLE PARAMETERS:
 186 */
 187
 188 /*
 189 ** AIM: Adaptive Interrupt Moderation
 190 ** which means that the interrupt rate
 191 ** is varied over time based on the
 192 ** traffic for that interrupt vector
 193 */
 194 static int ixv_enable_aim = FALSE;
 195 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim);
 196
 197 /* How many packets rxeof tries to clean at a time */
 198 static int ixv_rx_process_limit = 128;
 199 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit);
 200
 201 /* Flow control setting, default to full */
 202 static int ixv_flow_control = ixgbe_fc_full;
 203 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control);
 204
 205 /*
 206  * Header split: this causes the hardware to DMA
 207  * the header into a seperate mbuf from the payload,
 208  * it can be a performance win in some workloads, but
 209  * in others it actually hurts, its off by default.
 210  */
 211 static int ixv_header_split = FALSE;
 212 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split);
 213
 214 /*
 215 ** Number of TX descriptors per ring,
 216 ** setting higher than RX as this seems
 217 ** the better performing choice.
 218 */
 219 static int ixv_txd = DEFAULT_TXD;
 220 TUNABLE_INT("hw.ixv.txd", &ixv_txd);
 221
 222 /* Number of RX descriptors per ring */
 223 static int ixv_rxd = DEFAULT_RXD;
 224 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd);
 225
 226 /*
 227 ** Shadow VFTA table, this is needed because
 228 ** the real filter table gets cleared during
 229 ** a soft reset and we need to repopulate it.
 230 */
 231 static u32 ixv_shadow_vfta[VFTA_SIZE];
 232
 233 /*********************************************************************
 234  *  Device identification routine
 235  *
 236  *  ixv_probe determines if the driver should be loaded on
 237  *  adapter based on PCI vendor/device id of the adapter.
 238  *
 239  *  return BUS_PROBE_DEFAULT on success, positive on failure
 240  *********************************************************************/
 241
 242 static int
 243 ixv_probe(device_t dev)
 244 {
 245         ixv_vendor_info_t *ent;
 246
 247         u16     pci_vendor_id = 0;
 248         u16     pci_device_id = 0;
 249         u16     pci_subvendor_id = 0;
 250         u16     pci_subdevice_id = 0;
 251         char    adapter_name[256];
 252
 253
 254         pci_vendor_id = pci_get_vendor(dev);
 255         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 256                 return (ENXIO);
 257
 258         pci_device_id = pci_get_device(dev);
 259         pci_subvendor_id = pci_get_subvendor(dev);
 260         pci_subdevice_id = pci_get_subdevice(dev);
 261
 262         ent = ixv_vendor_info_array;
 263         while (ent->vendor_id != 0) {
 264                 if ((pci_vendor_id == ent->vendor_id) &&
 265                     (pci_device_id == ent->device_id) &&
 266
 267                     ((pci_subvendor_id == ent->subvendor_id) ||
 268                      (ent->subvendor_id == 0)) &&
 269
 270                     ((pci_subdevice_id == ent->subdevice_id) ||
 271                      (ent->subdevice_id == 0))) {
 272                         sprintf(adapter_name, "%s, Version - %s",
 273                                 ixv_strings[ent->index],
 274                                 ixv_driver_version);
 275                         device_set_desc_copy(dev, adapter_name);
 276                         return (BUS_PROBE_DEFAULT);
 277                 }
 278                 ent++;
 279         }
 280         return (ENXIO);
 281 }
 282
 283 /*********************************************************************
 284  *  Device initialization routine
 285  *
 286  *  The attach entry point is called when the driver is being loaded.
 287  *  This routine identifies the type of hardware, allocates all resources
 288  *  and initializes the hardware.
 289  *
 290  *  return 0 on success, positive on failure
 291  *********************************************************************/
 292
 293 static int
 294 ixv_attach(device_t dev)
 295 {
 296         struct adapter *adapter;
 297         struct ixgbe_hw *hw;
 298         int             error = 0;
 299
 300         INIT_DEBUGOUT("ixv_attach: begin");
 301
 302         /* Allocate, clear, and link in our adapter structure */
 303         adapter = device_get_softc(dev);
 304         adapter->dev = adapter->osdep.dev = dev;
 305         hw = &adapter->hw;
 306
 307         /* Core Lock Init*/
 308         IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 309
 310         /* SYSCTL APIs */
 311         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 312                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 313                         OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW,
 314                         adapter, 0, ixv_sysctl_stats, "I", "Statistics");
 315
 316         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 317                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 318                         OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW,
 319                         adapter, 0, ixv_sysctl_debug, "I", "Debug Info");
 320
 321         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 322                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 323                         OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW,
 324                         adapter, 0, ixv_set_flowcntl, "I", "Flow Control");
 325
 326         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 327                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 328                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 329                         &ixv_enable_aim, 1, "Interrupt Moderation");
 330
 331         /* Set up the timer callout */
 332         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 333
 334         /* Determine hardware revision */
 335         ixv_identify_hardware(adapter);
 336
 337         /* Do base PCI setup - map BAR0 */
 338         if (ixv_allocate_pci_resources(adapter)) {
 339                 device_printf(dev, "Allocation of PCI resources failed\n");
 340                 error = ENXIO;
 341                 goto err_out;
 342         }
 343
 344         /* Do descriptor calc and sanity checks */
 345         if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 346             ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) {
 347                 device_printf(dev, "TXD config issue, using default!\n");
 348                 adapter->num_tx_desc = DEFAULT_TXD;
 349         } else
 350                 adapter->num_tx_desc = ixv_txd;
 351
 352         if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 353             ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) {
 354                 device_printf(dev, "RXD config issue, using default!\n");
 355                 adapter->num_rx_desc = DEFAULT_RXD;
 356         } else
 357                 adapter->num_rx_desc = ixv_rxd;
 358
 359         /* Allocate our TX/RX Queues */
 360         if (ixv_allocate_queues(adapter)) {
 361                 error = ENOMEM;
 362                 goto err_out;
 363         }
 364
 365         /*
 366         ** Initialize the shared code: its
 367         ** at this point the mac type is set.
 368         */
 369         error = ixgbe_init_shared_code(hw);
 370         if (error) {
 371                 device_printf(dev,"Shared Code Initialization Failure\n");
 372                 error = EIO;
 373                 goto err_late;
 374         }
 375
 376         /* Setup the mailbox */
 377         ixgbe_init_mbx_params_vf(hw);
 378
 379         ixgbe_reset_hw(hw);
 380
 381         /* Get Hardware Flow Control setting */
 382         hw->fc.requested_mode = ixgbe_fc_full;
 383         hw->fc.pause_time = IXV_FC_PAUSE;
 384         hw->fc.low_water[0] = IXV_FC_LO;
 385         hw->fc.high_water[0] = IXV_FC_HI;
 386         hw->fc.send_xon = TRUE;
 387
 388         error = ixgbe_init_hw(hw);
 389         if (error) {
 390                 device_printf(dev,"Hardware Initialization Failure\n");
 391                 error = EIO;
 392                 goto err_late;
 393         }
 394
 395         error = ixv_allocate_msix(adapter);
 396         if (error)
 397                 goto err_late;
 398
 399         /* Setup OS specific network interface */
 400         ixv_setup_interface(dev, adapter);
 401
 402         /* Sysctl for limiting the amount of work done in the taskqueue */
 403         ixv_add_rx_process_limit(adapter, "rx_processing_limit",
 404             "max number of rx packets to process", &adapter->rx_process_limit,
 405             ixv_rx_process_limit);
 406
 407         /* Do the stats setup */
 408         ixv_save_stats(adapter);
 409         ixv_init_stats(adapter);
 410
 411         /* Register for VLAN events */
 412         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 413             ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 414         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 415             ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 416
 417         INIT_DEBUGOUT("ixv_attach: end");
 418         return (0);
 419
 420 err_late:
 421         ixv_free_transmit_structures(adapter);
 422         ixv_free_receive_structures(adapter);
 423 err_out:
 424         ixv_free_pci_resources(adapter);
 425         return (error);
 426
 427 }
 428
 429 /*********************************************************************
 430  *  Device removal routine
 431  *
 432  *  The detach entry point is called when the driver is being removed.
 433  *  This routine stops the adapter and deallocates all the resources
 434  *  that were allocated for driver operation.
 435  *
 436  *  return 0 on success, positive on failure
 437  *********************************************************************/
 438
 439 static int
 440 ixv_detach(device_t dev)
 441 {
 442         struct adapter *adapter = device_get_softc(dev);
 443         struct ix_queue *que = adapter->queues;
 444
 445         INIT_DEBUGOUT("ixv_detach: begin");
 446
 447         /* Make sure VLANS are not using driver */
 448         if (adapter->ifp->if_vlantrunk != NULL) {
 449                 device_printf(dev,"Vlan in use, detach first\n");
 450                 return (EBUSY);
 451         }
 452
 453         IXV_CORE_LOCK(adapter);
 454         ixv_stop(adapter);
 455         IXV_CORE_UNLOCK(adapter);
 456
 457         for (int i = 0; i < adapter->num_queues; i++, que++) {
 458                 if (que->tq) {
 459                         taskqueue_drain(que->tq, &que->que_task);
 460                         taskqueue_free(que->tq);
 461                 }
 462         }
 463
 464         /* Drain the Link queue */
 465         if (adapter->tq) {
 466                 taskqueue_drain(adapter->tq, &adapter->mbx_task);
 467                 taskqueue_free(adapter->tq);
 468         }
 469
 470         /* Unregister VLAN events */
 471         if (adapter->vlan_attach != NULL)
 472                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 473         if (adapter->vlan_detach != NULL)
 474                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 475
 476         ether_ifdetach(adapter->ifp);
 477         callout_drain(&adapter->timer);
 478         ixv_free_pci_resources(adapter);
 479         bus_generic_detach(dev);
 480         if_free(adapter->ifp);
 481
 482         ixv_free_transmit_structures(adapter);
 483         ixv_free_receive_structures(adapter);
 484
 485         IXV_CORE_LOCK_DESTROY(adapter);
 486         return (0);
 487 }
 488
 489 /*********************************************************************
 490  *
 491  *  Shutdown entry point
 492  *
 493  **********************************************************************/
 494 static int
 495 ixv_shutdown(device_t dev)
 496 {
 497         struct adapter *adapter = device_get_softc(dev);
 498         IXV_CORE_LOCK(adapter);
 499         ixv_stop(adapter);
 500         IXV_CORE_UNLOCK(adapter);
 501         return (0);
 502 }
 503
 504 #if __FreeBSD_version < 800000
 505 /*********************************************************************
 506  *  Transmit entry point
 507  *
 508  *  ixv_start is called by the stack to initiate a transmit.
 509  *  The driver will remain in this routine as long as there are
 510  *  packets to transmit and transmit resources are available.
 511  *  In case resources are not available stack is notified and
 512  *  the packet is requeued.
 513  **********************************************************************/
 514 static void
 515 ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 516 {
 517         struct mbuf    *m_head;
 518         struct adapter *adapter = txr->adapter;
 519
 520         IXV_TX_LOCK_ASSERT(txr);
 521
 522         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 523             IFF_DRV_RUNNING)
 524                 return;
 525         if (!adapter->link_active)
 526                 return;
 527
 528         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 529
 530                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 531                 if (m_head == NULL)
 532                         break;
 533
 534                 if (ixv_xmit(txr, &m_head)) {
 535                         if (m_head == NULL)
 536                                 break;
 537                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 538                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 539                         break;
 540                 }
 541                 /* Send a copy of the frame to the BPF listener */
 542                 ETHER_BPF_MTAP(ifp, m_head);
 543
 544                 /* Set watchdog on */
 545                 txr->watchdog_check = TRUE;
 546                 txr->watchdog_time = ticks;
 547
 548         }
 549         return;
 550 }
 551
 552 /*
 553  * Legacy TX start - called by the stack, this
 554  * always uses the first tx ring, and should
 555  * not be used with multiqueue tx enabled.
 556  */
 557 static void
 558 ixv_start(struct ifnet *ifp)
 559 {
 560         struct adapter *adapter = ifp->if_softc;
 561         struct tx_ring  *txr = adapter->tx_rings;
 562
 563         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 564                 IXV_TX_LOCK(txr);
 565                 ixv_start_locked(txr, ifp);
 566                 IXV_TX_UNLOCK(txr);
 567         }
 568         return;
 569 }
 570
 571 #else
 572
 573 /*
 574 ** Multiqueue Transmit driver
 575 **
 576 */
 577 static int
 578 ixv_mq_start(struct ifnet *ifp, struct mbuf *m)
 579 {
 580         struct adapter  *adapter = ifp->if_softc;
 581         struct ix_queue *que;
 582         struct tx_ring  *txr;
 583         int             i = 0, err = 0;
 584
 585         /* Which queue to use */
 586         if ((m->m_flags & M_FLOWID) != 0)
 587                 i = m->m_pkthdr.flowid % adapter->num_queues;
 588
 589         txr = &adapter->tx_rings[i];
 590         que = &adapter->queues[i];
 591
 592         if (IXV_TX_TRYLOCK(txr)) {
 593                 err = ixv_mq_start_locked(ifp, txr, m);
 594                 IXV_TX_UNLOCK(txr);
 595         } else {
 596                 err = drbr_enqueue(ifp, txr->br, m);
 597                 taskqueue_enqueue(que->tq, &que->que_task);
 598         }
 599
 600         return (err);
 601 }
 602
 603 static int
 604 ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 605 {
 606         struct adapter  *adapter = txr->adapter;
 607         struct mbuf     *next;
 608         int             enqueued, err = 0;
 609
 610         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 611             IFF_DRV_RUNNING || adapter->link_active == 0) {
 612                 if (m != NULL)
 613                         err = drbr_enqueue(ifp, txr->br, m);
 614                 return (err);
 615         }
 616
 617         /* Do a clean if descriptors are low */
 618         if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD)
 619                 ixv_txeof(txr);
 620
 621         enqueued = 0;
 622         if (m == NULL) {
 623                 next = drbr_dequeue(ifp, txr->br);
 624         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 625                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 626                         return (err);
 627                 next = drbr_dequeue(ifp, txr->br);
 628         } else
 629                 next = m;
 630
 631         /* Process the queue */
 632         while (next != NULL) {
 633                 if ((err = ixv_xmit(txr, &next)) != 0) {
 634                         if (next != NULL)
 635                                 err = drbr_enqueue(ifp, txr->br, next);
 636                         break;
 637                 }
 638                 enqueued++;
 639                 ifp->if_obytes += next->m_pkthdr.len;
 640                 if (next->m_flags & M_MCAST)
 641                         ifp->if_omcasts++;
 642                 /* Send a copy of the frame to the BPF listener */
 643                 ETHER_BPF_MTAP(ifp, next);
 644                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 645                         break;
 646                 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) {
 647                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 648                         break;
 649                 }
 650                 next = drbr_dequeue(ifp, txr->br);
 651         }
 652
 653         if (enqueued > 0) {
 654                 /* Set watchdog on */
 655                 txr->watchdog_check = TRUE;
 656                 txr->watchdog_time = ticks;
 657         }
 658
 659         return (err);
 660 }
 661
 662 /*
 663 ** Flush all ring buffers
 664 */
 665 static void
 666 ixv_qflush(struct ifnet *ifp)
 667 {
 668         struct adapter  *adapter = ifp->if_softc;
 669         struct tx_ring  *txr = adapter->tx_rings;
 670         struct mbuf     *m;
 671
 672         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 673                 IXV_TX_LOCK(txr);
 674                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 675                         m_freem(m);
 676                 IXV_TX_UNLOCK(txr);
 677         }
 678         if_qflush(ifp);
 679 }
 680
 681 #endif
 682
 683 /*********************************************************************
 684  *  Ioctl entry point
 685  *
 686  *  ixv_ioctl is called when the user wants to configure the
 687  *  interface.
 688  *
 689  *  return 0 on success, positive on failure
 690  **********************************************************************/
 691
 692 static int
 693 ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 694 {
 695         struct adapter  *adapter = ifp->if_softc;
 696         struct ifreq    *ifr = (struct ifreq *) data;
 697 #if defined(INET) || defined(INET6)
 698         struct ifaddr   *ifa = (struct ifaddr *) data;
 699         bool            avoid_reset = FALSE;
 700 #endif
 701         int             error = 0;
 702
 703         switch (command) {
 704
 705         case SIOCSIFADDR:
 706 #ifdef INET
 707                 if (ifa->ifa_addr->sa_family == AF_INET)
 708                         avoid_reset = TRUE;
 709 #endif
 710 #ifdef INET6
 711                 if (ifa->ifa_addr->sa_family == AF_INET6)
 712                         avoid_reset = TRUE;
 713 #endif
 714 #if defined(INET) || defined(INET6)
 715                 /*
 716                 ** Calling init results in link renegotiation,
 717                 ** so we avoid doing it when possible.
 718                 */
 719                 if (avoid_reset) {
 720                         ifp->if_flags |= IFF_UP;
 721                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 722                                 ixv_init(adapter);
 723                         if (!(ifp->if_flags & IFF_NOARP))
 724                                 arp_ifinit(ifp, ifa);
 725                 } else
 726                         error = ether_ioctl(ifp, command, data);
 727                 break;
 728 #endif
 729         case SIOCSIFMTU:
 730                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 731                 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 732                         error = EINVAL;
 733                 } else {
 734                         IXV_CORE_LOCK(adapter);
 735                         ifp->if_mtu = ifr->ifr_mtu;
 736                         adapter->max_frame_size =
 737                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 738                         ixv_init_locked(adapter);
 739                         IXV_CORE_UNLOCK(adapter);
 740                 }
 741                 break;
 742         case SIOCSIFFLAGS:
 743                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 744                 IXV_CORE_LOCK(adapter);
 745                 if (ifp->if_flags & IFF_UP) {
 746                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 747                                 ixv_init_locked(adapter);
 748                 } else
 749                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 750                                 ixv_stop(adapter);
 751                 adapter->if_flags = ifp->if_flags;
 752                 IXV_CORE_UNLOCK(adapter);
 753                 break;
 754         case SIOCADDMULTI:
 755         case SIOCDELMULTI:
 756                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 757                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 758                         IXV_CORE_LOCK(adapter);
 759                         ixv_disable_intr(adapter);
 760                         ixv_set_multi(adapter);
 761                         ixv_enable_intr(adapter);
 762                         IXV_CORE_UNLOCK(adapter);
 763                 }
 764                 break;
 765         case SIOCSIFMEDIA:
 766         case SIOCGIFMEDIA:
 767                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
 768                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 769                 break;
 770         case SIOCSIFCAP:
 771         {
 772                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 773                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
 774                 if (mask & IFCAP_HWCSUM)
 775                         ifp->if_capenable ^= IFCAP_HWCSUM;
 776                 if (mask & IFCAP_TSO4)
 777                         ifp->if_capenable ^= IFCAP_TSO4;
 778                 if (mask & IFCAP_LRO)
 779                         ifp->if_capenable ^= IFCAP_LRO;
 780                 if (mask & IFCAP_VLAN_HWTAGGING)
 781                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 782                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 783                         IXV_CORE_LOCK(adapter);
 784                         ixv_init_locked(adapter);
 785                         IXV_CORE_UNLOCK(adapter);
 786                 }
 787                 VLAN_CAPABILITIES(ifp);
 788                 break;
 789         }
 790
 791         default:
 792                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 793                 error = ether_ioctl(ifp, command, data);
 794                 break;
 795         }
 796
 797         return (error);
 798 }
 799
 800 /*********************************************************************
 801  *  Init entry point
 802  *
 803  *  This routine is used in two ways. It is used by the stack as
 804  *  init entry point in network interface structure. It is also used
 805  *  by the driver as a hw/sw initialization routine to get to a
 806  *  consistent state.
 807  *
 808  *  return 0 on success, positive on failure
 809  **********************************************************************/
 810 #define IXGBE_MHADD_MFS_SHIFT 16
 811
 812 static void
 813 ixv_init_locked(struct adapter *adapter)
 814 {
 815         struct ifnet    *ifp = adapter->ifp;
 816         device_t        dev = adapter->dev;
 817         struct ixgbe_hw *hw = &adapter->hw;
 818         u32             mhadd, gpie;
 819
 820         INIT_DEBUGOUT("ixv_init: begin");
 821         mtx_assert(&adapter->core_mtx, MA_OWNED);
 822         hw->adapter_stopped = FALSE;
 823         ixgbe_stop_adapter(hw);
 824         callout_stop(&adapter->timer);
 825
 826         /* reprogram the RAR[0] in case user changed it. */
 827         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
 828
 829         /* Get the latest mac address, User can use a LAA */
 830         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
 831              IXGBE_ETH_LENGTH_OF_ADDRESS);
 832         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
 833         hw->addr_ctrl.rar_used_count = 1;
 834
 835         /* Prepare transmit descriptors and buffers */
 836         if (ixv_setup_transmit_structures(adapter)) {
 837                 device_printf(dev,"Could not setup transmit structures\n");
 838                 ixv_stop(adapter);
 839                 return;
 840         }
 841
 842         ixgbe_reset_hw(hw);
 843         ixv_initialize_transmit_units(adapter);
 844
 845         /* Setup Multicast table */
 846         ixv_set_multi(adapter);
 847
 848         /*
 849         ** Determine the correct mbuf pool
 850         ** for doing jumbo/headersplit
 851         */
 852         if (ifp->if_mtu > ETHERMTU)
 853                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 854         else
 855                 adapter->rx_mbuf_sz = MCLBYTES;
 856
 857         /* Prepare receive descriptors and buffers */
 858         if (ixv_setup_receive_structures(adapter)) {
 859                 device_printf(dev,"Could not setup receive structures\n");
 860                 ixv_stop(adapter);
 861                 return;
 862         }
 863
 864         /* Configure RX settings */
 865         ixv_initialize_receive_units(adapter);
 866
 867         /* Enable Enhanced MSIX mode */
 868         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
 869         gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME;
 870         gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD;
 871         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
 872
 873         /* Set the various hardware offload abilities */
 874         ifp->if_hwassist = 0;
 875         if (ifp->if_capenable & IFCAP_TSO4)
 876                 ifp->if_hwassist |= CSUM_TSO;
 877         if (ifp->if_capenable & IFCAP_TXCSUM) {
 878                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 879 #if __FreeBSD_version >= 800000
 880                 ifp->if_hwassist |= CSUM_SCTP;
 881 #endif
 882         }
 883
 884         /* Set MTU size */
 885         if (ifp->if_mtu > ETHERMTU) {
 886                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
 887                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
 888                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
 889                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
 890         }
 891
 892         /* Set up VLAN offload and filter */
 893         ixv_setup_vlan_support(adapter);
 894
 895         callout_reset(&adapter->timer, hz, ixv_local_timer, adapter);
 896
 897         /* Set up MSI/X routing */
 898         ixv_configure_ivars(adapter);
 899
 900         /* Set up auto-mask */
 901         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE);
 902
 903         /* Set moderation on the Link interrupt */
 904         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR);
 905
 906         /* Stats init */
 907         ixv_init_stats(adapter);
 908
 909         /* Config/Enable Link */
 910         ixv_config_link(adapter);
 911
 912         /* And now turn on interrupts */
 913         ixv_enable_intr(adapter);
 914
 915         /* Now inform the stack we're ready */
 916         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 917         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 918
 919         return;
 920 }
 921
 922 static void
 923 ixv_init(void *arg)
 924 {
 925         struct adapter *adapter = arg;
 926
 927         IXV_CORE_LOCK(adapter);
 928         ixv_init_locked(adapter);
 929         IXV_CORE_UNLOCK(adapter);
 930         return;
 931 }
 932
 933
 934 /*
 935 **
 936 ** MSIX Interrupt Handlers and Tasklets
 937 **
 938 */
 939
 940 static inline void
 941 ixv_enable_queue(struct adapter *adapter, u32 vector)
 942 {
 943         struct ixgbe_hw *hw = &adapter->hw;
 944         u32     queue = 1 << vector;
 945         u32     mask;
 946
 947         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 948         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 949 }
 950
 951 static inline void
 952 ixv_disable_queue(struct adapter *adapter, u32 vector)
 953 {
 954         struct ixgbe_hw *hw = &adapter->hw;
 955         u64     queue = (u64)(1 << vector);
 956         u32     mask;
 957
 958         mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 959         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
 960 }
 961
 962 static inline void
 963 ixv_rearm_queues(struct adapter *adapter, u64 queues)
 964 {
 965         u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
 966         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask);
 967 }
 968
 969
 970 static void
 971 ixv_handle_que(void *context, int pending)
 972 {
 973         struct ix_queue *que = context;
 974         struct adapter  *adapter = que->adapter;
 975         struct tx_ring  *txr = que->txr;
 976         struct ifnet    *ifp = adapter->ifp;
 977         bool            more;
 978
 979         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 980                 more = ixv_rxeof(que, adapter->rx_process_limit);
 981                 IXV_TX_LOCK(txr);
 982                 ixv_txeof(txr);
 983 #if __FreeBSD_version >= 800000
 984                 if (!drbr_empty(ifp, txr->br))
 985                         ixv_mq_start_locked(ifp, txr, NULL);
 986 #else
 987                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 988                         ixv_start_locked(txr, ifp);
 989 #endif
 990                 IXV_TX_UNLOCK(txr);
 991                 if (more) {
 992                         taskqueue_enqueue(que->tq, &que->que_task);
 993                         return;
 994                 }
 995         }
 996
 997         /* Reenable this interrupt */
 998         ixv_enable_queue(adapter, que->msix);
 999         return;
1000 }
1001
1002 /*********************************************************************
1003  *
1004  *  MSI Queue Interrupt Service routine
1005  *
1006  **********************************************************************/
1007 void
1008 ixv_msix_que(void *arg)
1009 {
1010         struct ix_queue *que = arg;
1011         struct adapter  *adapter = que->adapter;
1012         struct tx_ring  *txr = que->txr;
1013         struct rx_ring  *rxr = que->rxr;
1014         bool            more_tx, more_rx;
1015         u32             newitr = 0;
1016
1017         ixv_disable_queue(adapter, que->msix);
1018         ++que->irqs;
1019
1020         more_rx = ixv_rxeof(que, adapter->rx_process_limit);
1021
1022         IXV_TX_LOCK(txr);
1023         more_tx = ixv_txeof(txr);
1024         /*
1025         ** Make certain that if the stack
1026         ** has anything queued the task gets
1027         ** scheduled to handle it.
1028         */
1029 #if __FreeBSD_version < 800000
1030         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1031 #else
1032         if (!drbr_empty(adapter->ifp, txr->br))
1033 #endif
1034                 more_tx = 1;
1035         IXV_TX_UNLOCK(txr);
1036
1037         more_rx = ixv_rxeof(que, adapter->rx_process_limit);
1038
1039         /* Do AIM now? */
1040
1041         if (ixv_enable_aim == FALSE)
1042                 goto no_calc;
1043         /*
1044         ** Do Adaptive Interrupt Moderation:
1045         **  - Write out last calculated setting
1046         **  - Calculate based on average size over
1047         **    the last interval.
1048         */
1049         if (que->eitr_setting)
1050                 IXGBE_WRITE_REG(&adapter->hw,
1051                     IXGBE_VTEITR(que->msix),
1052                     que->eitr_setting);
1053
1054         que->eitr_setting = 0;
1055
1056         /* Idle, do nothing */
1057         if ((txr->bytes == 0) && (rxr->bytes == 0))
1058                 goto no_calc;
1059
1060         if ((txr->bytes) && (txr->packets))
1061                 newitr = txr->bytes/txr->packets;
1062         if ((rxr->bytes) && (rxr->packets))
1063                 newitr = max(newitr,
1064                     (rxr->bytes / rxr->packets));
1065         newitr += 24; /* account for hardware frame, crc */
1066
1067         /* set an upper boundary */
1068         newitr = min(newitr, 3000);
1069
1070         /* Be nice to the mid range */
1071         if ((newitr > 300) && (newitr < 1200))
1072                 newitr = (newitr / 3);
1073         else
1074                 newitr = (newitr / 2);
1075
1076         newitr |= newitr << 16;
1077
1078         /* save for next interrupt */
1079         que->eitr_setting = newitr;
1080
1081         /* Reset state */
1082         txr->bytes = 0;
1083         txr->packets = 0;
1084         rxr->bytes = 0;
1085         rxr->packets = 0;
1086
1087 no_calc:
1088         if (more_tx || more_rx)
1089                 taskqueue_enqueue(que->tq, &que->que_task);
1090         else /* Reenable this interrupt */
1091                 ixv_enable_queue(adapter, que->msix);
1092         return;
1093 }
1094
1095 static void
1096 ixv_msix_mbx(void *arg)
1097 {
1098         struct adapter  *adapter = arg;
1099         struct ixgbe_hw *hw = &adapter->hw;
1100         u32             reg;
1101
1102         ++adapter->mbx_irq;
1103
1104         /* First get the cause */
1105         reg = IXGBE_READ_REG(hw, IXGBE_VTEICS);
1106         /* Clear interrupt with write */
1107         IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg);
1108
1109         /* Link status change */
1110         if (reg & IXGBE_EICR_LSC)
1111                 taskqueue_enqueue(adapter->tq, &adapter->mbx_task);
1112
1113         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER);
1114         return;
1115 }
1116
1117 /*********************************************************************
1118  *
1119  *  Media Ioctl callback
1120  *
1121  *  This routine is called whenever the user queries the status of
1122  *  the interface using ifconfig.
1123  *
1124  **********************************************************************/
1125 static void
1126 ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1127 {
1128         struct adapter *adapter = ifp->if_softc;
1129
1130         INIT_DEBUGOUT("ixv_media_status: begin");
1131         IXV_CORE_LOCK(adapter);
1132         ixv_update_link_status(adapter);
1133
1134         ifmr->ifm_status = IFM_AVALID;
1135         ifmr->ifm_active = IFM_ETHER;
1136
1137         if (!adapter->link_active) {
1138                 IXV_CORE_UNLOCK(adapter);
1139                 return;
1140         }
1141
1142         ifmr->ifm_status |= IFM_ACTIVE;
1143
1144         switch (adapter->link_speed) {
1145                 case IXGBE_LINK_SPEED_1GB_FULL:
1146                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1147                         break;
1148                 case IXGBE_LINK_SPEED_10GB_FULL:
1149                         ifmr->ifm_active |= IFM_FDX;
1150                         break;
1151         }
1152
1153         IXV_CORE_UNLOCK(adapter);
1154
1155         return;
1156 }
1157
1158 /*********************************************************************
1159  *
1160  *  Media Ioctl callback
1161  *
1162  *  This routine is called when the user changes speed/duplex using
1163  *  media/mediopt option with ifconfig.
1164  *
1165  **********************************************************************/
1166 static int
1167 ixv_media_change(struct ifnet * ifp)
1168 {
1169         struct adapter *adapter = ifp->if_softc;
1170         struct ifmedia *ifm = &adapter->media;
1171
1172         INIT_DEBUGOUT("ixv_media_change: begin");
1173
1174         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1175                 return (EINVAL);
1176
1177         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1178         case IFM_AUTO:
1179                 break;
1180         default:
1181                 device_printf(adapter->dev, "Only auto media type\n");
1182                 return (EINVAL);
1183         }
1184
1185         return (0);
1186 }
1187
1188 /*********************************************************************
1189  *
1190  *  This routine maps the mbufs to tx descriptors, allowing the
1191  *  TX engine to transmit the packets.
1192  *      - return 0 on success, positive on failure
1193  *
1194  **********************************************************************/
1195
1196 static int
1197 ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1198 {
1199         struct adapter  *adapter = txr->adapter;
1200         u32             olinfo_status = 0, cmd_type_len;
1201         u32             paylen = 0;
1202         int             i, j, error, nsegs;
1203         int             first, last = 0;
1204         struct mbuf     *m_head;
1205         bus_dma_segment_t segs[32];
1206         bus_dmamap_t    map;
1207         struct ixv_tx_buf *txbuf, *txbuf_mapped;
1208         union ixgbe_adv_tx_desc *txd = NULL;
1209
1210         m_head = *m_headp;
1211
1212         /* Basic descriptor defines */
1213         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1214             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1215
1216         if (m_head->m_flags & M_VLANTAG)
1217                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1218
1219         /*
1220          * Important to capture the first descriptor
1221          * used because it will contain the index of
1222          * the one we tell the hardware to report back
1223          */
1224         first = txr->next_avail_desc;
1225         txbuf = &txr->tx_buffers[first];
1226         txbuf_mapped = txbuf;
1227         map = txbuf->map;
1228
1229         /*
1230          * Map the packet for DMA.
1231          */
1232         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1233             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1234
1235         if (error == EFBIG) {
1236                 struct mbuf *m;
1237
1238                 m = m_defrag(*m_headp, M_DONTWAIT);
1239                 if (m == NULL) {
1240                         adapter->mbuf_defrag_failed++;
1241                         m_freem(*m_headp);
1242                         *m_headp = NULL;
1243                         return (ENOBUFS);
1244                 }
1245                 *m_headp = m;
1246
1247                 /* Try it again */
1248                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1249                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1250
1251                 if (error == ENOMEM) {
1252                         adapter->no_tx_dma_setup++;
1253                         return (error);
1254                 } else if (error != 0) {
1255                         adapter->no_tx_dma_setup++;
1256                         m_freem(*m_headp);
1257                         *m_headp = NULL;
1258                         return (error);
1259                 }
1260         } else if (error == ENOMEM) {
1261                 adapter->no_tx_dma_setup++;
1262                 return (error);
1263         } else if (error != 0) {
1264                 adapter->no_tx_dma_setup++;
1265                 m_freem(*m_headp);
1266                 *m_headp = NULL;
1267                 return (error);
1268         }
1269
1270         /* Make certain there are enough descriptors */
1271         if (nsegs > txr->tx_avail - 2) {
1272                 txr->no_desc_avail++;
1273                 error = ENOBUFS;
1274                 goto xmit_fail;
1275         }
1276         m_head = *m_headp;
1277
1278         /*
1279         ** Set up the appropriate offload context
1280         ** this becomes the first descriptor of
1281         ** a packet.
1282         */
1283         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1284                 if (ixv_tso_setup(txr, m_head, &paylen)) {
1285                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1286                         olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
1287                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1288                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1289                         ++adapter->tso_tx;
1290                 } else
1291                         return (ENXIO);
1292         } else if (ixv_tx_ctx_setup(txr, m_head))
1293                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1294
1295         /* Record payload length */
1296         if (paylen == 0)
1297                 olinfo_status |= m_head->m_pkthdr.len <<
1298                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1299
1300         i = txr->next_avail_desc;
1301         for (j = 0; j < nsegs; j++) {
1302                 bus_size_t seglen;
1303                 bus_addr_t segaddr;
1304
1305                 txbuf = &txr->tx_buffers[i];
1306                 txd = &txr->tx_base[i];
1307                 seglen = segs[j].ds_len;
1308                 segaddr = htole64(segs[j].ds_addr);
1309
1310                 txd->read.buffer_addr = segaddr;
1311                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1312                     cmd_type_len |seglen);
1313                 txd->read.olinfo_status = htole32(olinfo_status);
1314                 last = i; /* descriptor that will get completion IRQ */
1315
1316                 if (++i == adapter->num_tx_desc)
1317                         i = 0;
1318
1319                 txbuf->m_head = NULL;
1320                 txbuf->eop_index = -1;
1321         }
1322
1323         txd->read.cmd_type_len |=
1324             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1325         txr->tx_avail -= nsegs;
1326         txr->next_avail_desc = i;
1327
1328         txbuf->m_head = m_head;
1329         txr->tx_buffers[first].map = txbuf->map;
1330         txbuf->map = map;
1331         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1332
1333         /* Set the index of the descriptor that will be marked done */
1334         txbuf = &txr->tx_buffers[first];
1335         txbuf->eop_index = last;
1336
1337         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1338             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1339         /*
1340          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1341          * hardware that this frame is available to transmit.
1342          */
1343         ++txr->total_packets;
1344         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i);
1345
1346         return (0);
1347
1348 xmit_fail:
1349         bus_dmamap_unload(txr->txtag, txbuf->map);
1350         return (error);
1351
1352 }
1353
1354
1355 /*********************************************************************
1356  *  Multicast Update
1357  *
1358  *  This routine is called whenever multicast address list is updated.
1359  *
1360  **********************************************************************/
1361 #define IXGBE_RAR_ENTRIES 16
1362
1363 static void
1364 ixv_set_multi(struct adapter *adapter)
1365 {
1366         u8      mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
1367         u8      *update_ptr;
1368         struct  ifmultiaddr *ifma;
1369         int     mcnt = 0;
1370         struct ifnet   *ifp = adapter->ifp;
1371
1372         IOCTL_DEBUGOUT("ixv_set_multi: begin");
1373
1374 #if __FreeBSD_version < 800000
1375         IF_ADDR_LOCK(ifp);
1376 #else
1377         if_maddr_rlock(ifp);
1378 #endif
1379         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1380                 if (ifma->ifma_addr->sa_family != AF_LINK)
1381                         continue;
1382                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1383                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1384                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1385                 mcnt++;
1386         }
1387 #if __FreeBSD_version < 800000
1388         IF_ADDR_UNLOCK(ifp);
1389 #else
1390         if_maddr_runlock(ifp);
1391 #endif
1392
1393         update_ptr = mta;
1394
1395         ixgbe_update_mc_addr_list(&adapter->hw,
1396             update_ptr, mcnt, ixv_mc_array_itr, TRUE);
1397
1398         return;
1399 }
1400
1401 /*
1402  * This is an iterator function now needed by the multicast
1403  * shared code. It simply feeds the shared code routine the
1404  * addresses in the array of ixv_set_multi() one by one.
1405  */
1406 static u8 *
1407 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1408 {
1409         u8 *addr = *update_ptr;
1410         u8 *newptr;
1411         *vmdq = 0;
1412
1413         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1414         *update_ptr = newptr;
1415         return addr;
1416 }
1417
1418 /*********************************************************************
1419  *  Timer routine
1420  *
1421  *  This routine checks for link status,updates statistics,
1422  *  and runs the watchdog check.
1423  *
1424  **********************************************************************/
1425
1426 static void
1427 ixv_local_timer(void *arg)
1428 {
1429         struct adapter  *adapter = arg;
1430         device_t        dev = adapter->dev;
1431         struct tx_ring  *txr = adapter->tx_rings;
1432         int             i;
1433
1434         mtx_assert(&adapter->core_mtx, MA_OWNED);
1435
1436         ixv_update_link_status(adapter);
1437
1438         /* Stats Update */
1439         ixv_update_stats(adapter);
1440
1441         /*
1442          * If the interface has been paused
1443          * then don't do the watchdog check
1444          */
1445         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1446                 goto out;
1447         /*
1448         ** Check for time since any descriptor was cleaned
1449         */
1450         for (i = 0; i < adapter->num_queues; i++, txr++) {
1451                 IXV_TX_LOCK(txr);
1452                 if (txr->watchdog_check == FALSE) {
1453                         IXV_TX_UNLOCK(txr);
1454                         continue;
1455                 }
1456                 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG)
1457                         goto hung;
1458                 IXV_TX_UNLOCK(txr);
1459         }
1460 out:
1461         ixv_rearm_queues(adapter, adapter->que_mask);
1462         callout_reset(&adapter->timer, hz, ixv_local_timer, adapter);
1463         return;
1464
1465 hung:
1466         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1467         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1468             IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)),
1469             IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i)));
1470         device_printf(dev,"TX(%d) desc avail = %d,"
1471             "Next TX to Clean = %d\n",
1472             txr->me, txr->tx_avail, txr->next_to_clean);
1473         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1474         adapter->watchdog_events++;
1475         IXV_TX_UNLOCK(txr);
1476         ixv_init_locked(adapter);
1477 }
1478
1479 /*
1480 ** Note: this routine updates the OS on the link state
1481 **      the real check of the hardware only happens with
1482 **      a link interrupt.
1483 */
1484 static void
1485 ixv_update_link_status(struct adapter *adapter)
1486 {
1487         struct ifnet    *ifp = adapter->ifp;
1488         struct tx_ring *txr = adapter->tx_rings;
1489         device_t dev = adapter->dev;
1490
1491
1492         if (adapter->link_up){
1493                 if (adapter->link_active == FALSE) {
1494                         if (bootverbose)
1495                                 device_printf(dev,"Link is up %d Gbps %s \n",
1496                                     ((adapter->link_speed == 128)? 10:1),
1497                                     "Full Duplex");
1498                         adapter->link_active = TRUE;
1499                         if_link_state_change(ifp, LINK_STATE_UP);
1500                 }
1501         } else { /* Link down */
1502                 if (adapter->link_active == TRUE) {
1503                         if (bootverbose)
1504                                 device_printf(dev,"Link is Down\n");
1505                         if_link_state_change(ifp, LINK_STATE_DOWN);
1506                         adapter->link_active = FALSE;
1507                         for (int i = 0; i < adapter->num_queues;
1508                             i++, txr++)
1509                                 txr->watchdog_check = FALSE;
1510                 }
1511         }
1512
1513         return;
1514 }
1515
1516
1517 /*********************************************************************
1518  *
1519  *  This routine disables all traffic on the adapter by issuing a
1520  *  global reset on the MAC and deallocates TX/RX buffers.
1521  *
1522  **********************************************************************/
1523
1524 static void
1525 ixv_stop(void *arg)
1526 {
1527         struct ifnet   *ifp;
1528         struct adapter *adapter = arg;
1529         struct ixgbe_hw *hw = &adapter->hw;
1530         ifp = adapter->ifp;
1531
1532         mtx_assert(&adapter->core_mtx, MA_OWNED);
1533
1534         INIT_DEBUGOUT("ixv_stop: begin\n");
1535         ixv_disable_intr(adapter);
1536
1537         /* Tell the stack that the interface is no longer active */
1538         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1539
1540         ixgbe_reset_hw(hw);
1541         adapter->hw.adapter_stopped = FALSE;
1542         ixgbe_stop_adapter(hw);
1543         callout_stop(&adapter->timer);
1544
1545         /* reprogram the RAR[0] in case user changed it. */
1546         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1547
1548         return;
1549 }
1550
1551
1552 /*********************************************************************
1553  *
1554  *  Determine hardware revision.
1555  *
1556  **********************************************************************/
1557 static void
1558 ixv_identify_hardware(struct adapter *adapter)
1559 {
1560         device_t        dev = adapter->dev;
1561         u16             pci_cmd_word;
1562
1563         /*
1564         ** Make sure BUSMASTER is set, on a VM under
1565         ** KVM it may not be and will break things.
1566         */
1567         pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1568         if (!((pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1569             (pci_cmd_word & PCIM_CMD_MEMEN))) {
1570                 INIT_DEBUGOUT("Memory Access and/or Bus Master "
1571                     "bits were not set!\n");
1572                 pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1573                 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
1574         }
1575
1576         /* Save off the information about this board */
1577         adapter->hw.vendor_id = pci_get_vendor(dev);
1578         adapter->hw.device_id = pci_get_device(dev);
1579         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
1580         adapter->hw.subsystem_vendor_id =
1581             pci_read_config(dev, PCIR_SUBVEND_0, 2);
1582         adapter->hw.subsystem_device_id =
1583             pci_read_config(dev, PCIR_SUBDEV_0, 2);
1584
1585         return;
1586 }
1587
1588 /*********************************************************************
1589  *
1590  *  Setup MSIX Interrupt resources and handlers
1591  *
1592  **********************************************************************/
1593 static int
1594 ixv_allocate_msix(struct adapter *adapter)
1595 {
1596         device_t        dev = adapter->dev;
1597         struct          ix_queue *que = adapter->queues;
1598         int             error, rid, vector = 0;
1599
1600         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
1601                 rid = vector + 1;
1602                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1603                     RF_SHAREABLE | RF_ACTIVE);
1604                 if (que->res == NULL) {
1605                         device_printf(dev,"Unable to allocate"
1606                             " bus resource: que interrupt [%d]\n", vector);
1607                         return (ENXIO);
1608                 }
1609                 /* Set the handler function */
1610                 error = bus_setup_intr(dev, que->res,
1611                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
1612                     ixv_msix_que, que, &que->tag);
1613                 if (error) {
1614                         que->res = NULL;
1615                         device_printf(dev, "Failed to register QUE handler");
1616                         return (error);
1617                 }
1618 #if __FreeBSD_version >= 800504
1619                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
1620 #endif
1621                 que->msix = vector;
1622                 adapter->que_mask |= (u64)(1 << que->msix);
1623                 /*
1624                 ** Bind the msix vector, and thus the
1625                 ** ring to the corresponding cpu.
1626                 */
1627                 if (adapter->num_queues > 1)
1628                         bus_bind_intr(dev, que->res, i);
1629
1630                 TASK_INIT(&que->que_task, 0, ixv_handle_que, que);
1631                 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT,
1632                     taskqueue_thread_enqueue, &que->tq);
1633                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1634                     device_get_nameunit(adapter->dev));
1635         }
1636
1637         /* and Mailbox */
1638         rid = vector + 1;
1639         adapter->res = bus_alloc_resource_any(dev,
1640             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
1641         if (!adapter->res) {
1642                 device_printf(dev,"Unable to allocate"
1643             " bus resource: MBX interrupt [%d]\n", rid);
1644                 return (ENXIO);
1645         }
1646         /* Set the mbx handler function */
1647         error = bus_setup_intr(dev, adapter->res,
1648             INTR_TYPE_NET | INTR_MPSAFE, NULL,
1649             ixv_msix_mbx, adapter, &adapter->tag);
1650         if (error) {
1651                 adapter->res = NULL;
1652                 device_printf(dev, "Failed to register LINK handler");
1653                 return (error);
1654         }
1655 #if __FreeBSD_version >= 800504
1656         bus_describe_intr(dev, adapter->res, adapter->tag, "mbx");
1657 #endif
1658         adapter->mbxvec = vector;
1659         /* Tasklets for Mailbox */
1660         TASK_INIT(&adapter->mbx_task, 0, ixv_handle_mbx, adapter);
1661         adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT,
1662             taskqueue_thread_enqueue, &adapter->tq);
1663         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq",
1664             device_get_nameunit(adapter->dev));
1665         /*
1666         ** Due to a broken design QEMU will fail to properly
1667         ** enable the guest for MSIX unless the vectors in
1668         ** the table are all set up, so we must rewrite the
1669         ** ENABLE in the MSIX control register again at this
1670         ** point to cause it to successfully initialize us.
1671         */
1672         if (adapter->hw.mac.type == ixgbe_mac_82599_vf) {
1673                 int msix_ctrl;
1674                 pci_find_cap(dev, PCIY_MSIX, &rid);
1675                 rid += PCIR_MSIX_CTRL;
1676                 msix_ctrl = pci_read_config(dev, rid, 2);
1677                 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1678                 pci_write_config(dev, rid, msix_ctrl, 2);
1679         }
1680
1681         return (0);
1682 }
1683
1684 /*
1685  * Setup MSIX resources, note that the VF
1686  * device MUST use MSIX, there is no fallback.
1687  */
1688 static int
1689 ixv_setup_msix(struct adapter *adapter)
1690 {
1691         device_t dev = adapter->dev;
1692         int rid, vectors, want = 2;
1693
1694
1695         /* First try MSI/X */
1696         rid = PCIR_BAR(3);
1697         adapter->msix_mem = bus_alloc_resource_any(dev,
1698             SYS_RES_MEMORY, &rid, RF_ACTIVE);
1699         if (!adapter->msix_mem) {
1700                 device_printf(adapter->dev,
1701                     "Unable to map MSIX table \n");
1702                 goto out;
1703         }
1704
1705         vectors = pci_msix_count(dev);
1706         if (vectors < 2) {
1707                 bus_release_resource(dev, SYS_RES_MEMORY,
1708                     rid, adapter->msix_mem);
1709                 adapter->msix_mem = NULL;
1710                 goto out;
1711         }
1712
1713         /*
1714         ** Want two vectors: one for a queue,
1715         ** plus an additional for mailbox.
1716         */
1717         if (pci_alloc_msix(dev, &want) == 0) {
1718                 device_printf(adapter->dev,
1719                     "Using MSIX interrupts with %d vectors\n", want);
1720                 return (want);
1721         }
1722 out:
1723         device_printf(adapter->dev,"MSIX config error\n");
1724         return (ENXIO);
1725 }
1726
1727
1728 static int
1729 ixv_allocate_pci_resources(struct adapter *adapter)
1730 {
1731         int             rid;
1732         device_t        dev = adapter->dev;
1733
1734         rid = PCIR_BAR(0);
1735         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1736             &rid, RF_ACTIVE);
1737
1738         if (!(adapter->pci_mem)) {
1739                 device_printf(dev,"Unable to allocate bus resource: memory\n");
1740                 return (ENXIO);
1741         }
1742
1743         adapter->osdep.mem_bus_space_tag =
1744                 rman_get_bustag(adapter->pci_mem);
1745         adapter->osdep.mem_bus_space_handle =
1746                 rman_get_bushandle(adapter->pci_mem);
1747         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
1748
1749         adapter->num_queues = 1;
1750         adapter->hw.back = &adapter->osdep;
1751
1752         /*
1753         ** Now setup MSI/X, should
1754         ** return us the number of
1755         ** configured vectors.
1756         */
1757         adapter->msix = ixv_setup_msix(adapter);
1758         if (adapter->msix == ENXIO)
1759                 return (ENXIO);
1760         else
1761                 return (0);
1762 }
1763
1764 static void
1765 ixv_free_pci_resources(struct adapter * adapter)
1766 {
1767         struct          ix_queue *que = adapter->queues;
1768         device_t        dev = adapter->dev;
1769         int             rid, memrid;
1770
1771         memrid = PCIR_BAR(MSIX_BAR);
1772
1773         /*
1774         ** There is a slight possibility of a failure mode
1775         ** in attach that will result in entering this function
1776         ** before interrupt resources have been initialized, and
1777         ** in that case we do not want to execute the loops below
1778         ** We can detect this reliably by the state of the adapter
1779         ** res pointer.
1780         */
1781         if (adapter->res == NULL)
1782                 goto mem;
1783
1784         /*
1785         **  Release all msix queue resources:
1786         */
1787         for (int i = 0; i < adapter->num_queues; i++, que++) {
1788                 rid = que->msix + 1;
1789                 if (que->tag != NULL) {
1790                         bus_teardown_intr(dev, que->res, que->tag);
1791                         que->tag = NULL;
1792                 }
1793                 if (que->res != NULL)
1794                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
1795         }
1796
1797
1798         /* Clean the Legacy or Link interrupt last */
1799         if (adapter->mbxvec) /* we are doing MSIX */
1800                 rid = adapter->mbxvec + 1;
1801         else
1802                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
1803
1804         if (adapter->tag != NULL) {
1805                 bus_teardown_intr(dev, adapter->res, adapter->tag);
1806                 adapter->tag = NULL;
1807         }
1808         if (adapter->res != NULL)
1809                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
1810
1811 mem:
1812         if (adapter->msix)
1813                 pci_release_msi(dev);
1814
1815         if (adapter->msix_mem != NULL)
1816                 bus_release_resource(dev, SYS_RES_MEMORY,
1817                     memrid, adapter->msix_mem);
1818
1819         if (adapter->pci_mem != NULL)
1820                 bus_release_resource(dev, SYS_RES_MEMORY,
1821                     PCIR_BAR(0), adapter->pci_mem);
1822
1823         return;
1824 }
1825
1826 /*********************************************************************
1827  *
1828  *  Setup networking device structure and register an interface.
1829  *
1830  **********************************************************************/
1831 static void
1832 ixv_setup_interface(device_t dev, struct adapter *adapter)
1833 {
1834         struct ifnet   *ifp;
1835
1836         INIT_DEBUGOUT("ixv_setup_interface: begin");
1837
1838         ifp = adapter->ifp = if_alloc(IFT_ETHER);
1839         if (ifp == NULL)
1840                 panic("%s: can not if_alloc()\n", device_get_nameunit(dev));
1841         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1842         ifp->if_baudrate = 1000000000;
1843         ifp->if_init = ixv_init;
1844         ifp->if_softc = adapter;
1845         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1846         ifp->if_ioctl = ixv_ioctl;
1847 #if __FreeBSD_version >= 800000
1848         ifp->if_transmit = ixv_mq_start;
1849         ifp->if_qflush = ixv_qflush;
1850 #else
1851         ifp->if_start = ixv_start;
1852 #endif
1853         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
1854
1855         ether_ifattach(ifp, adapter->hw.mac.addr);
1856
1857         adapter->max_frame_size =
1858             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1859
1860         /*
1861          * Tell the upper layer(s) we support long frames.
1862          */
1863         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1864
1865         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM;
1866         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
1867         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
1868                              |  IFCAP_VLAN_HWTSO
1869                              |  IFCAP_VLAN_MTU;
1870         ifp->if_capenable = ifp->if_capabilities;
1871
1872         /* Don't enable LRO by default */
1873         ifp->if_capabilities |= IFCAP_LRO;
1874
1875         /*
1876          * Specify the media types supported by this adapter and register
1877          * callbacks to update media and link information
1878          */
1879         ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change,
1880                      ixv_media_status);
1881         ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL);
1882         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1883         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1884
1885         return;
1886 }
1887
1888 static void
1889 ixv_config_link(struct adapter *adapter)
1890 {
1891         struct ixgbe_hw *hw = &adapter->hw;
1892         u32     autoneg, err = 0;
1893         bool    negotiate = TRUE;
1894
1895         if (hw->mac.ops.check_link)
1896                 err = hw->mac.ops.check_link(hw, &autoneg,
1897                     &adapter->link_up, FALSE);
1898         if (err)
1899                 goto out;
1900
1901         if (hw->mac.ops.setup_link)
1902                 err = hw->mac.ops.setup_link(hw, autoneg,
1903                     negotiate, adapter->link_up);
1904 out:
1905         return;
1906 }
1907
1908 /********************************************************************
1909  * Manage DMA'able memory.
1910  *******************************************************************/
1911 static void
1912 ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
1913 {
1914         if (error)
1915                 return;
1916         *(bus_addr_t *) arg = segs->ds_addr;
1917         return;
1918 }
1919
1920 static int
1921 ixv_dma_malloc(struct adapter *adapter, bus_size_t size,
1922                 struct ixv_dma_alloc *dma, int mapflags)
1923 {
1924         device_t dev = adapter->dev;
1925         int             r;
1926
1927         r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev),   /* parent */
1928                                DBA_ALIGN, 0,    /* alignment, bounds */
1929                                BUS_SPACE_MAXADDR,       /* lowaddr */
1930                                BUS_SPACE_MAXADDR,       /* highaddr */
1931                                NULL, NULL,      /* filter, filterarg */
1932                                size,    /* maxsize */
1933                                1,       /* nsegments */
1934                                size,    /* maxsegsize */
1935                                BUS_DMA_ALLOCNOW,        /* flags */
1936                                NULL,    /* lockfunc */
1937                                NULL,    /* lockfuncarg */
1938                                &dma->dma_tag);
1939         if (r != 0) {
1940                 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; "
1941                        "error %u\n", r);
1942                 goto fail_0;
1943         }
1944         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
1945                              BUS_DMA_NOWAIT, &dma->dma_map);
1946         if (r != 0) {
1947                 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; "
1948                        "error %u\n", r);
1949                 goto fail_1;
1950         }
1951         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1952                             size,
1953                             ixv_dmamap_cb,
1954                             &dma->dma_paddr,
1955                             mapflags | BUS_DMA_NOWAIT);
1956         if (r != 0) {
1957                 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; "
1958                        "error %u\n", r);
1959                 goto fail_2;
1960         }
1961         dma->dma_size = size;
1962         return (0);
1963 fail_2:
1964         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1965 fail_1:
1966         bus_dma_tag_destroy(dma->dma_tag);
1967 fail_0:
1968         dma->dma_map = NULL;
1969         dma->dma_tag = NULL;
1970         return (r);
1971 }
1972
1973 static void
1974 ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma)
1975 {
1976         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
1977             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1978         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1979         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1980         bus_dma_tag_destroy(dma->dma_tag);
1981 }
1982
1983
1984 /*********************************************************************
1985  *
1986  *  Allocate memory for the transmit and receive rings, and then
1987  *  the descriptors associated with each, called only once at attach.
1988  *
1989  **********************************************************************/
1990 static int
1991 ixv_allocate_queues(struct adapter *adapter)
1992 {
1993         device_t        dev = adapter->dev;
1994         struct ix_queue *que;
1995         struct tx_ring  *txr;
1996         struct rx_ring  *rxr;
1997         int rsize, tsize, error = 0;
1998         int txconf = 0, rxconf = 0;
1999
2000         /* First allocate the top level queue structs */
2001         if (!(adapter->queues =
2002             (struct ix_queue *) malloc(sizeof(struct ix_queue) *
2003             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2004                 device_printf(dev, "Unable to allocate queue memory\n");
2005                 error = ENOMEM;
2006                 goto fail;
2007         }
2008
2009         /* First allocate the TX ring struct memory */
2010         if (!(adapter->tx_rings =
2011             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2012             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2013                 device_printf(dev, "Unable to allocate TX ring memory\n");
2014                 error = ENOMEM;
2015                 goto tx_fail;
2016         }
2017
2018         /* Next allocate the RX */
2019         if (!(adapter->rx_rings =
2020             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2021             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2022                 device_printf(dev, "Unable to allocate RX ring memory\n");
2023                 error = ENOMEM;
2024                 goto rx_fail;
2025         }
2026
2027         /* For the ring itself */
2028         tsize = roundup2(adapter->num_tx_desc *
2029             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2030
2031         /*
2032          * Now set up the TX queues, txconf is needed to handle the
2033          * possibility that things fail midcourse and we need to
2034          * undo memory gracefully
2035          */
2036         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2037                 /* Set up some basics */
2038                 txr = &adapter->tx_rings[i];
2039                 txr->adapter = adapter;
2040                 txr->me = i;
2041
2042                 /* Initialize the TX side lock */
2043                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2044                     device_get_nameunit(dev), txr->me);
2045                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2046
2047                 if (ixv_dma_malloc(adapter, tsize,
2048                         &txr->txdma, BUS_DMA_NOWAIT)) {
2049                         device_printf(dev,
2050                             "Unable to allocate TX Descriptor memory\n");
2051                         error = ENOMEM;
2052                         goto err_tx_desc;
2053                 }
2054                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2055                 bzero((void *)txr->tx_base, tsize);
2056
2057                 /* Now allocate transmit buffers for the ring */
2058                 if (ixv_allocate_transmit_buffers(txr)) {
2059                         device_printf(dev,
2060                             "Critical Failure setting up transmit buffers\n");
2061                         error = ENOMEM;
2062                         goto err_tx_desc;
2063                 }
2064 #if __FreeBSD_version >= 800000
2065                 /* Allocate a buf ring */
2066                 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF,
2067                     M_WAITOK, &txr->tx_mtx);
2068                 if (txr->br == NULL) {
2069                         device_printf(dev,
2070                             "Critical Failure setting up buf ring\n");
2071                         error = ENOMEM;
2072                         goto err_tx_desc;
2073                 }
2074 #endif
2075         }
2076
2077         /*
2078          * Next the RX queues...
2079          */
2080         rsize = roundup2(adapter->num_rx_desc *
2081             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2082         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2083                 rxr = &adapter->rx_rings[i];
2084                 /* Set up some basics */
2085                 rxr->adapter = adapter;
2086                 rxr->me = i;
2087
2088                 /* Initialize the RX side lock */
2089                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2090                     device_get_nameunit(dev), rxr->me);
2091                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2092
2093                 if (ixv_dma_malloc(adapter, rsize,
2094                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2095                         device_printf(dev,
2096                             "Unable to allocate RxDescriptor memory\n");
2097                         error = ENOMEM;
2098                         goto err_rx_desc;
2099                 }
2100                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2101                 bzero((void *)rxr->rx_base, rsize);
2102
2103                 /* Allocate receive buffers for the ring*/
2104                 if (ixv_allocate_receive_buffers(rxr)) {
2105                         device_printf(dev,
2106                             "Critical Failure setting up receive buffers\n");
2107                         error = ENOMEM;
2108                         goto err_rx_desc;
2109                 }
2110         }
2111
2112         /*
2113         ** Finally set up the queue holding structs
2114         */
2115         for (int i = 0; i < adapter->num_queues; i++) {
2116                 que = &adapter->queues[i];
2117                 que->adapter = adapter;
2118                 que->txr = &adapter->tx_rings[i];
2119                 que->rxr = &adapter->rx_rings[i];
2120         }
2121
2122         return (0);
2123
2124 err_rx_desc:
2125         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2126                 ixv_dma_free(adapter, &rxr->rxdma);
2127 err_tx_desc:
2128         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2129                 ixv_dma_free(adapter, &txr->txdma);
2130         free(adapter->rx_rings, M_DEVBUF);
2131 rx_fail:
2132         free(adapter->tx_rings, M_DEVBUF);
2133 tx_fail:
2134         free(adapter->queues, M_DEVBUF);
2135 fail:
2136         return (error);
2137 }
2138
2139
2140 /*********************************************************************
2141  *
2142  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2143  *  the information needed to transmit a packet on the wire. This is
2144  *  called only once at attach, setup is done every reset.
2145  *
2146  **********************************************************************/
2147 static int
2148 ixv_allocate_transmit_buffers(struct tx_ring *txr)
2149 {
2150         struct adapter *adapter = txr->adapter;
2151         device_t dev = adapter->dev;
2152         struct ixv_tx_buf *txbuf;
2153         int error, i;
2154
2155         /*
2156          * Setup DMA descriptor areas.
2157          */
2158         if ((error = bus_dma_tag_create(
2159                                bus_get_dma_tag(adapter->dev),   /* parent */
2160                                1, 0,            /* alignment, bounds */
2161                                BUS_SPACE_MAXADDR,       /* lowaddr */
2162                                BUS_SPACE_MAXADDR,       /* highaddr */
2163                                NULL, NULL,              /* filter, filterarg */
2164                                IXV_TSO_SIZE,            /* maxsize */
2165                                32,                      /* nsegments */
2166                                PAGE_SIZE,               /* maxsegsize */
2167                                0,                       /* flags */
2168                                NULL,                    /* lockfunc */
2169                                NULL,                    /* lockfuncarg */
2170                                &txr->txtag))) {
2171                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2172                 goto fail;
2173         }
2174
2175         if (!(txr->tx_buffers =
2176             (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) *
2177             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2178                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2179                 error = ENOMEM;
2180                 goto fail;
2181         }
2182
2183         /* Create the descriptor buffer dma maps */
2184         txbuf = txr->tx_buffers;
2185         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2186                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2187                 if (error != 0) {
2188                         device_printf(dev, "Unable to create TX DMA map\n");
2189                         goto fail;
2190                 }
2191         }
2192
2193         return 0;
2194 fail:
2195         /* We free all, it handles case where we are in the middle */
2196         ixv_free_transmit_structures(adapter);
2197         return (error);
2198 }
2199
2200 /*********************************************************************
2201  *
2202  *  Initialize a transmit ring.
2203  *
2204  **********************************************************************/
2205 static void
2206 ixv_setup_transmit_ring(struct tx_ring *txr)
2207 {
2208         struct adapter *adapter = txr->adapter;
2209         struct ixv_tx_buf *txbuf;
2210         int i;
2211
2212         /* Clear the old ring contents */
2213         IXV_TX_LOCK(txr);
2214         bzero((void *)txr->tx_base,
2215               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2216         /* Reset indices */
2217         txr->next_avail_desc = 0;
2218         txr->next_to_clean = 0;
2219
2220         /* Free any existing tx buffers. */
2221         txbuf = txr->tx_buffers;
2222         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2223                 if (txbuf->m_head != NULL) {
2224                         bus_dmamap_sync(txr->txtag, txbuf->map,
2225                             BUS_DMASYNC_POSTWRITE);
2226                         bus_dmamap_unload(txr->txtag, txbuf->map);
2227                         m_freem(txbuf->m_head);
2228                         txbuf->m_head = NULL;
2229                 }
2230                 /* Clear the EOP index */
2231                 txbuf->eop_index = -1;
2232         }
2233
2234         /* Set number of descriptors available */
2235         txr->tx_avail = adapter->num_tx_desc;
2236
2237         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2238             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2239         IXV_TX_UNLOCK(txr);
2240 }
2241
2242 /*********************************************************************
2243  *
2244  *  Initialize all transmit rings.
2245  *
2246  **********************************************************************/
2247 static int
2248 ixv_setup_transmit_structures(struct adapter *adapter)
2249 {
2250         struct tx_ring *txr = adapter->tx_rings;
2251
2252         for (int i = 0; i < adapter->num_queues; i++, txr++)
2253                 ixv_setup_transmit_ring(txr);
2254
2255         return (0);
2256 }
2257
2258 /*********************************************************************
2259  *
2260  *  Enable transmit unit.
2261  *
2262  **********************************************************************/
2263 static void
2264 ixv_initialize_transmit_units(struct adapter *adapter)
2265 {
2266         struct tx_ring  *txr = adapter->tx_rings;
2267         struct ixgbe_hw *hw = &adapter->hw;
2268
2269
2270         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2271                 u64     tdba = txr->txdma.dma_paddr;
2272                 u32     txctrl, txdctl;
2273
2274                 /* Set WTHRESH to 8, burst writeback */
2275                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
2276                 txdctl |= (8 << 16);
2277                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
2278                 /* Now enable */
2279                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
2280                 txdctl |= IXGBE_TXDCTL_ENABLE;
2281                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
2282
2283                 /* Set the HW Tx Head and Tail indices */
2284                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0);
2285                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0);
2286
2287                 /* Setup Transmit Descriptor Cmd Settings */
2288                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2289                 txr->watchdog_check = FALSE;
2290
2291                 /* Set Ring parameters */
2292                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
2293                        (tdba & 0x00000000ffffffffULL));
2294                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32));
2295                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
2296                     adapter->num_tx_desc *
2297                     sizeof(struct ixgbe_legacy_tx_desc));
2298                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i));
2299                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2300                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl);
2301                 break;
2302         }
2303
2304         return;
2305 }
2306
2307 /*********************************************************************
2308  *
2309  *  Free all transmit rings.
2310  *
2311  **********************************************************************/
2312 static void
2313 ixv_free_transmit_structures(struct adapter *adapter)
2314 {
2315         struct tx_ring *txr = adapter->tx_rings;
2316
2317         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2318                 IXV_TX_LOCK(txr);
2319                 ixv_free_transmit_buffers(txr);
2320                 ixv_dma_free(adapter, &txr->txdma);
2321                 IXV_TX_UNLOCK(txr);
2322                 IXV_TX_LOCK_DESTROY(txr);
2323         }
2324         free(adapter->tx_rings, M_DEVBUF);
2325 }
2326
2327 /*********************************************************************
2328  *
2329  *  Free transmit ring related data structures.
2330  *
2331  **********************************************************************/
2332 static void
2333 ixv_free_transmit_buffers(struct tx_ring *txr)
2334 {
2335         struct adapter *adapter = txr->adapter;
2336         struct ixv_tx_buf *tx_buffer;
2337         int             i;
2338
2339         INIT_DEBUGOUT("free_transmit_ring: begin");
2340
2341         if (txr->tx_buffers == NULL)
2342                 return;
2343
2344         tx_buffer = txr->tx_buffers;
2345         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2346                 if (tx_buffer->m_head != NULL) {
2347                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2348                             BUS_DMASYNC_POSTWRITE);
2349                         bus_dmamap_unload(txr->txtag,
2350                             tx_buffer->map);
2351                         m_freem(tx_buffer->m_head);
2352                         tx_buffer->m_head = NULL;
2353                         if (tx_buffer->map != NULL) {
2354                                 bus_dmamap_destroy(txr->txtag,
2355                                     tx_buffer->map);
2356                                 tx_buffer->map = NULL;
2357                         }
2358                 } else if (tx_buffer->map != NULL) {
2359                         bus_dmamap_unload(txr->txtag,
2360                             tx_buffer->map);
2361                         bus_dmamap_destroy(txr->txtag,
2362                             tx_buffer->map);
2363                         tx_buffer->map = NULL;
2364                 }
2365         }
2366 #if __FreeBSD_version >= 800000
2367         if (txr->br != NULL)
2368                 buf_ring_free(txr->br, M_DEVBUF);
2369 #endif
2370         if (txr->tx_buffers != NULL) {
2371                 free(txr->tx_buffers, M_DEVBUF);
2372                 txr->tx_buffers = NULL;
2373         }
2374         if (txr->txtag != NULL) {
2375                 bus_dma_tag_destroy(txr->txtag);
2376                 txr->txtag = NULL;
2377         }
2378         return;
2379 }
2380
2381 /*********************************************************************
2382  *
2383  *  Advanced Context Descriptor setup for VLAN or CSUM
2384  *
2385  **********************************************************************/
2386
2387 static bool
2388 ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
2389 {
2390         struct adapter *adapter = txr->adapter;
2391         struct ixgbe_adv_tx_context_desc *TXD;
2392         struct ixv_tx_buf        *tx_buffer;
2393         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2394         struct ether_vlan_header *eh;
2395         struct ip *ip;
2396         struct ip6_hdr *ip6;
2397         int  ehdrlen, ip_hlen = 0;
2398         u16     etype;
2399         u8      ipproto = 0;
2400         bool    offload = TRUE;
2401         int ctxd = txr->next_avail_desc;
2402         u16 vtag = 0;
2403
2404
2405         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
2406                 offload = FALSE;
2407
2408
2409         tx_buffer = &txr->tx_buffers[ctxd];
2410         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2411
2412         /*
2413         ** In advanced descriptors the vlan tag must
2414         ** be placed into the descriptor itself.
2415         */
2416         if (mp->m_flags & M_VLANTAG) {
2417                 vtag = htole16(mp->m_pkthdr.ether_vtag);
2418                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2419         } else if (offload == FALSE)
2420                 return FALSE;
2421
2422         /*
2423          * Determine where frame payload starts.
2424          * Jump over vlan headers if already present,
2425          * helpful for QinQ too.
2426          */
2427         eh = mtod(mp, struct ether_vlan_header *);
2428         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2429                 etype = ntohs(eh->evl_proto);
2430                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2431         } else {
2432                 etype = ntohs(eh->evl_encap_proto);
2433                 ehdrlen = ETHER_HDR_LEN;
2434         }
2435
2436         /* Set the ether header length */
2437         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2438
2439         switch (etype) {
2440                 case ETHERTYPE_IP:
2441                         ip = (struct ip *)(mp->m_data + ehdrlen);
2442                         ip_hlen = ip->ip_hl << 2;
2443                         if (mp->m_len < ehdrlen + ip_hlen)
2444                                 return (FALSE);
2445                         ipproto = ip->ip_p;
2446                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2447                         break;
2448                 case ETHERTYPE_IPV6:
2449                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
2450                         ip_hlen = sizeof(struct ip6_hdr);
2451                         if (mp->m_len < ehdrlen + ip_hlen)
2452                                 return (FALSE);
2453                         ipproto = ip6->ip6_nxt;
2454                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
2455                         break;
2456                 default:
2457                         offload = FALSE;
2458                         break;
2459         }
2460
2461         vlan_macip_lens |= ip_hlen;
2462         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2463
2464         switch (ipproto) {
2465                 case IPPROTO_TCP:
2466                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
2467                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2468                         break;
2469
2470                 case IPPROTO_UDP:
2471                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
2472                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
2473                         break;
2474
2475 #if __FreeBSD_version >= 800000
2476                 case IPPROTO_SCTP:
2477                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
2478                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2479                         break;
2480 #endif
2481                 default:
2482                         offload = FALSE;
2483                         break;
2484         }
2485
2486         /* Now copy bits into descriptor */
2487         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
2488         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
2489         TXD->seqnum_seed = htole32(0);
2490         TXD->mss_l4len_idx = htole32(0);
2491
2492         tx_buffer->m_head = NULL;
2493         tx_buffer->eop_index = -1;
2494
2495         /* We've consumed the first desc, adjust counters */
2496         if (++ctxd == adapter->num_tx_desc)
2497                 ctxd = 0;
2498         txr->next_avail_desc = ctxd;
2499         --txr->tx_avail;
2500
2501         return (offload);
2502 }
2503
2504 /**********************************************************************
2505  *
2506  *  Setup work for hardware segmentation offload (TSO) on
2507  *  adapters using advanced tx descriptors
2508  *
2509  **********************************************************************/
2510 static bool
2511 ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
2512 {
2513         struct adapter *adapter = txr->adapter;
2514         struct ixgbe_adv_tx_context_desc *TXD;
2515         struct ixv_tx_buf        *tx_buffer;
2516         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2517         u32 mss_l4len_idx = 0;
2518         u16 vtag = 0;
2519         int ctxd, ehdrlen,  hdrlen, ip_hlen, tcp_hlen;
2520         struct ether_vlan_header *eh;
2521         struct ip *ip;
2522         struct tcphdr *th;
2523
2524
2525         /*
2526          * Determine where frame payload starts.
2527          * Jump over vlan headers if already present
2528          */
2529         eh = mtod(mp, struct ether_vlan_header *);
2530         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
2531                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2532         else
2533                 ehdrlen = ETHER_HDR_LEN;
2534
2535         /* Ensure we have at least the IP+TCP header in the first mbuf. */
2536         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
2537                 return FALSE;
2538
2539         ctxd = txr->next_avail_desc;
2540         tx_buffer = &txr->tx_buffers[ctxd];
2541         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
2542
2543         ip = (struct ip *)(mp->m_data + ehdrlen);
2544         if (ip->ip_p != IPPROTO_TCP)
2545                 return FALSE;   /* 0 */
2546         ip->ip_sum = 0;
2547         ip_hlen = ip->ip_hl << 2;
2548         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
2549         th->th_sum = in_pseudo(ip->ip_src.s_addr,
2550             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
2551         tcp_hlen = th->th_off << 2;
2552         hdrlen = ehdrlen + ip_hlen + tcp_hlen;
2553
2554         /* This is used in the transmit desc in encap */
2555         *paylen = mp->m_pkthdr.len - hdrlen;
2556
2557         /* VLAN MACLEN IPLEN */
2558         if (mp->m_flags & M_VLANTAG) {
2559                 vtag = htole16(mp->m_pkthdr.ether_vtag);
2560                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2561         }
2562
2563         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
2564         vlan_macip_lens |= ip_hlen;
2565         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
2566
2567         /* ADV DTYPE TUCMD */
2568         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2569         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2570         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2571         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
2572
2573
2574         /* MSS L4LEN IDX */
2575         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
2576         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
2577         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
2578
2579         TXD->seqnum_seed = htole32(0);
2580         tx_buffer->m_head = NULL;
2581         tx_buffer->eop_index = -1;
2582
2583         if (++ctxd == adapter->num_tx_desc)
2584                 ctxd = 0;
2585
2586         txr->tx_avail--;
2587         txr->next_avail_desc = ctxd;
2588         return TRUE;
2589 }
2590
2591
2592 /**********************************************************************
2593  *
2594  *  Examine each tx_buffer in the used queue. If the hardware is done
2595  *  processing the packet then free associated resources. The
2596  *  tx_buffer is put back on the free queue.
2597  *
2598  **********************************************************************/
2599 static bool
2600 ixv_txeof(struct tx_ring *txr)
2601 {
2602         struct adapter  *adapter = txr->adapter;
2603         struct ifnet    *ifp = adapter->ifp;
2604         u32     first, last, done;
2605         struct ixv_tx_buf *tx_buffer;
2606         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
2607
2608         mtx_assert(&txr->tx_mtx, MA_OWNED);
2609
2610         if (txr->tx_avail == adapter->num_tx_desc)
2611                 return FALSE;
2612
2613         first = txr->next_to_clean;
2614         tx_buffer = &txr->tx_buffers[first];
2615         /* For cleanup we just use legacy struct */
2616         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
2617         last = tx_buffer->eop_index;
2618         if (last == -1)
2619                 return FALSE;
2620         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
2621
2622         /*
2623         ** Get the index of the first descriptor
2624         ** BEYOND the EOP and call that 'done'.
2625         ** I do this so the comparison in the
2626         ** inner while loop below can be simple
2627         */
2628         if (++last == adapter->num_tx_desc) last = 0;
2629         done = last;
2630
2631         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2632             BUS_DMASYNC_POSTREAD);
2633         /*
2634         ** Only the EOP descriptor of a packet now has the DD
2635         ** bit set, this is what we look for...
2636         */
2637         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
2638                 /* We clean the range of the packet */
2639                 while (first != done) {
2640                         tx_desc->upper.data = 0;
2641                         tx_desc->lower.data = 0;
2642                         tx_desc->buffer_addr = 0;
2643                         ++txr->tx_avail;
2644
2645                         if (tx_buffer->m_head) {
2646                                 bus_dmamap_sync(txr->txtag,
2647                                     tx_buffer->map,
2648                                     BUS_DMASYNC_POSTWRITE);
2649                                 bus_dmamap_unload(txr->txtag,
2650                                     tx_buffer->map);
2651                                 m_freem(tx_buffer->m_head);
2652                                 tx_buffer->m_head = NULL;
2653                                 tx_buffer->map = NULL;
2654                         }
2655                         tx_buffer->eop_index = -1;
2656                         txr->watchdog_time = ticks;
2657
2658                         if (++first == adapter->num_tx_desc)
2659                                 first = 0;
2660
2661                         tx_buffer = &txr->tx_buffers[first];
2662                         tx_desc =
2663                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
2664                 }
2665                 ++ifp->if_opackets;
2666                 /* See if there is more work now */
2667                 last = tx_buffer->eop_index;
2668                 if (last != -1) {
2669                         eop_desc =
2670                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
2671                         /* Get next done point */
2672                         if (++last == adapter->num_tx_desc) last = 0;
2673                         done = last;
2674                 } else
2675                         break;
2676         }
2677         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2678             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2679
2680         txr->next_to_clean = first;
2681
2682         /*
2683          * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that
2684          * it is OK to send packets. If there are no pending descriptors,
2685          * clear the timeout. Otherwise, if some descriptors have been freed,
2686          * restart the timeout.
2687          */
2688         if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) {
2689                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2690                 if (txr->tx_avail == adapter->num_tx_desc) {
2691                         txr->watchdog_check = FALSE;
2692                         return FALSE;
2693                 }
2694         }
2695
2696         return TRUE;
2697 }
2698
2699 /*********************************************************************
2700  *
2701  *  Refresh mbuf buffers for RX descriptor rings
2702  *   - now keeps its own state so discards due to resource
2703  *     exhaustion are unnecessary, if an mbuf cannot be obtained
2704  *     it just returns, keeping its placeholder, thus it can simply
2705  *     be recalled to try again.
2706  *
2707  **********************************************************************/
2708 static void
2709 ixv_refresh_mbufs(struct rx_ring *rxr, int limit)
2710 {
2711         struct adapter          *adapter = rxr->adapter;
2712         bus_dma_segment_t       hseg[1];
2713         bus_dma_segment_t       pseg[1];
2714         struct ixv_rx_buf       *rxbuf;
2715         struct mbuf             *mh, *mp;
2716         int                     i, j, nsegs, error;
2717         bool                    refreshed = FALSE;
2718
2719         i = j = rxr->next_to_refresh;
2720         /* Get the control variable, one beyond refresh point */
2721         if (++j == adapter->num_rx_desc)
2722                 j = 0;
2723         while (j != limit) {
2724                 rxbuf = &rxr->rx_buffers[i];
2725                 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) {
2726                         mh = m_gethdr(M_DONTWAIT, MT_DATA);
2727                         if (mh == NULL)
2728                                 goto update;
2729                         mh->m_pkthdr.len = mh->m_len = MHLEN;
2730                         mh->m_len = MHLEN;
2731                         mh->m_flags |= M_PKTHDR;
2732                         m_adj(mh, ETHER_ALIGN);
2733                         /* Get the memory mapping */
2734                         error = bus_dmamap_load_mbuf_sg(rxr->htag,
2735                             rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
2736                         if (error != 0) {
2737                                 printf("GET BUF: dmamap load"
2738                                     " failure - %d\n", error);
2739                                 m_free(mh);
2740                                 goto update;
2741                         }
2742                         rxbuf->m_head = mh;
2743                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
2744                             BUS_DMASYNC_PREREAD);
2745                         rxr->rx_base[i].read.hdr_addr =
2746                             htole64(hseg[0].ds_addr);
2747                 }
2748
2749                 if (rxbuf->m_pack == NULL) {
2750                         mp = m_getjcl(M_DONTWAIT, MT_DATA,
2751                             M_PKTHDR, adapter->rx_mbuf_sz);
2752                         if (mp == NULL)
2753                                 goto update;
2754                 } else
2755                         mp = rxbuf->m_pack;
2756
2757                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
2758                 /* Get the memory mapping */
2759                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
2760                     rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
2761                 if (error != 0) {
2762                         printf("GET BUF: dmamap load"
2763                             " failure - %d\n", error);
2764                         m_free(mp);
2765                         rxbuf->m_pack = NULL;
2766                         goto update;
2767                 }
2768                 rxbuf->m_pack = mp;
2769                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
2770                     BUS_DMASYNC_PREREAD);
2771                 rxr->rx_base[i].read.pkt_addr =
2772                     htole64(pseg[0].ds_addr);
2773
2774                 refreshed = TRUE;
2775                 rxr->next_to_refresh = i = j;
2776                 /* Calculate next index */
2777                 if (++j == adapter->num_rx_desc)
2778                         j = 0;
2779         }
2780 update:
2781         if (refreshed) /* update tail index */
2782                 IXGBE_WRITE_REG(&adapter->hw,
2783                     IXGBE_VFRDT(rxr->me), rxr->next_to_refresh);
2784         return;
2785 }
2786
2787 /*********************************************************************
2788  *
2789  *  Allocate memory for rx_buffer structures. Since we use one
2790  *  rx_buffer per received packet, the maximum number of rx_buffer's
2791  *  that we'll need is equal to the number of receive descriptors
2792  *  that we've allocated.
2793  *
2794  **********************************************************************/
2795 static int
2796 ixv_allocate_receive_buffers(struct rx_ring *rxr)
2797 {
2798         struct  adapter         *adapter = rxr->adapter;
2799         device_t                dev = adapter->dev;
2800         struct ixv_rx_buf       *rxbuf;
2801         int                     i, bsize, error;
2802
2803         bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc;
2804         if (!(rxr->rx_buffers =
2805             (struct ixv_rx_buf *) malloc(bsize,
2806             M_DEVBUF, M_NOWAIT | M_ZERO))) {
2807                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
2808                 error = ENOMEM;
2809                 goto fail;
2810         }
2811
2812         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
2813                                    1, 0,        /* alignment, bounds */
2814                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2815                                    BUS_SPACE_MAXADDR,   /* highaddr */
2816                                    NULL, NULL,          /* filter, filterarg */
2817                                    MSIZE,               /* maxsize */
2818                                    1,                   /* nsegments */
2819                                    MSIZE,               /* maxsegsize */
2820                                    0,                   /* flags */
2821                                    NULL,                /* lockfunc */
2822                                    NULL,                /* lockfuncarg */
2823                                    &rxr->htag))) {
2824                 device_printf(dev, "Unable to create RX DMA tag\n");
2825                 goto fail;
2826         }
2827
2828         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),   /* parent */
2829                                    1, 0,        /* alignment, bounds */
2830                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2831                                    BUS_SPACE_MAXADDR,   /* highaddr */
2832                                    NULL, NULL,          /* filter, filterarg */
2833                                    MJUMPAGESIZE,        /* maxsize */
2834                                    1,                   /* nsegments */
2835                                    MJUMPAGESIZE,        /* maxsegsize */
2836                                    0,                   /* flags */
2837                                    NULL,                /* lockfunc */
2838                                    NULL,                /* lockfuncarg */
2839                                    &rxr->ptag))) {
2840                 device_printf(dev, "Unable to create RX DMA tag\n");
2841                 goto fail;
2842         }
2843
2844         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
2845                 rxbuf = &rxr->rx_buffers[i];
2846                 error = bus_dmamap_create(rxr->htag,
2847                     BUS_DMA_NOWAIT, &rxbuf->hmap);
2848                 if (error) {
2849                         device_printf(dev, "Unable to create RX head map\n");
2850                         goto fail;
2851                 }
2852                 error = bus_dmamap_create(rxr->ptag,
2853                     BUS_DMA_NOWAIT, &rxbuf->pmap);
2854                 if (error) {
2855                         device_printf(dev, "Unable to create RX pkt map\n");
2856                         goto fail;
2857                 }
2858         }
2859
2860         return (0);
2861
2862 fail:
2863         /* Frees all, but can handle partial completion */
2864         ixv_free_receive_structures(adapter);
2865         return (error);
2866 }
2867
2868 static void
2869 ixv_free_receive_ring(struct rx_ring *rxr)
2870 {
2871         struct  adapter         *adapter;
2872         struct ixv_rx_buf       *rxbuf;
2873         int i;
2874
2875         adapter = rxr->adapter;
2876         for (i = 0; i < adapter->num_rx_desc; i++) {
2877                 rxbuf = &rxr->rx_buffers[i];
2878                 if (rxbuf->m_head != NULL) {
2879                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
2880                             BUS_DMASYNC_POSTREAD);
2881                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
2882                         rxbuf->m_head->m_flags |= M_PKTHDR;
2883                         m_freem(rxbuf->m_head);
2884                 }
2885                 if (rxbuf->m_pack != NULL) {
2886                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
2887                             BUS_DMASYNC_POSTREAD);
2888                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
2889                         rxbuf->m_pack->m_flags |= M_PKTHDR;
2890                         m_freem(rxbuf->m_pack);
2891                 }
2892                 rxbuf->m_head = NULL;
2893                 rxbuf->m_pack = NULL;
2894         }
2895 }
2896
2897
2898 /*********************************************************************
2899  *
2900  *  Initialize a receive ring and its buffers.
2901  *
2902  **********************************************************************/
2903 static int
2904 ixv_setup_receive_ring(struct rx_ring *rxr)
2905 {
2906         struct  adapter         *adapter;
2907         struct ifnet            *ifp;
2908         device_t                dev;
2909         struct ixv_rx_buf       *rxbuf;
2910         bus_dma_segment_t       pseg[1], hseg[1];
2911         struct lro_ctrl         *lro = &rxr->lro;
2912         int                     rsize, nsegs, error = 0;
2913
2914         adapter = rxr->adapter;
2915         ifp = adapter->ifp;
2916         dev = adapter->dev;
2917
2918         /* Clear the ring contents */
2919         IXV_RX_LOCK(rxr);
2920         rsize = roundup2(adapter->num_rx_desc *
2921             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2922         bzero((void *)rxr->rx_base, rsize);
2923
2924         /* Free current RX buffer structs and their mbufs */
2925         ixv_free_receive_ring(rxr);
2926
2927         /* Configure header split? */
2928         if (ixv_header_split)
2929                 rxr->hdr_split = TRUE;
2930
2931         /* Now replenish the mbufs */
2932         for (int j = 0; j != adapter->num_rx_desc; ++j) {
2933                 struct mbuf     *mh, *mp;
2934
2935                 rxbuf = &rxr->rx_buffers[j];
2936                 /*
2937                 ** Dont allocate mbufs if not
2938                 ** doing header split, its wasteful
2939                 */
2940                 if (rxr->hdr_split == FALSE)
2941                         goto skip_head;
2942
2943                 /* First the header */
2944                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
2945                 if (rxbuf->m_head == NULL) {
2946                         error = ENOBUFS;
2947                         goto fail;
2948                 }
2949                 m_adj(rxbuf->m_head, ETHER_ALIGN);
2950                 mh = rxbuf->m_head;
2951                 mh->m_len = mh->m_pkthdr.len = MHLEN;
2952                 mh->m_flags |= M_PKTHDR;
2953                 /* Get the memory mapping */
2954                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
2955                     rxbuf->hmap, rxbuf->m_head, hseg,
2956                     &nsegs, BUS_DMA_NOWAIT);
2957                 if (error != 0) /* Nothing elegant to do here */
2958                         goto fail;
2959                 bus_dmamap_sync(rxr->htag,
2960                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
2961                 /* Update descriptor */
2962                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
2963
2964 skip_head:
2965                 /* Now the payload cluster */
2966                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
2967                     M_PKTHDR, adapter->rx_mbuf_sz);
2968                 if (rxbuf->m_pack == NULL) {
2969                         error = ENOBUFS;
2970                         goto fail;
2971                 }
2972                 mp = rxbuf->m_pack;
2973                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
2974                 /* Get the memory mapping */
2975                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
2976                     rxbuf->pmap, mp, pseg,
2977                     &nsegs, BUS_DMA_NOWAIT);
2978                 if (error != 0)
2979                         goto fail;
2980                 bus_dmamap_sync(rxr->ptag,
2981                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
2982                 /* Update descriptor */
2983                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
2984         }
2985
2986
2987         /* Setup our descriptor indices */
2988         rxr->next_to_check = 0;
2989         rxr->next_to_refresh = 0;
2990         rxr->lro_enabled = FALSE;
2991         rxr->rx_split_packets = 0;
2992         rxr->rx_bytes = 0;
2993         rxr->discard = FALSE;
2994
2995         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
2996             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2997
2998         /*
2999         ** Now set up the LRO interface:
3000         */
3001         if (ifp->if_capenable & IFCAP_LRO) {
3002                 int err = tcp_lro_init(lro);
3003                 if (err) {
3004                         device_printf(dev, "LRO Initialization failed!\n");
3005                         goto fail;
3006                 }
3007                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3008                 rxr->lro_enabled = TRUE;
3009                 lro->ifp = adapter->ifp;
3010         }
3011
3012         IXV_RX_UNLOCK(rxr);
3013         return (0);
3014
3015 fail:
3016         ixv_free_receive_ring(rxr);
3017         IXV_RX_UNLOCK(rxr);
3018         return (error);
3019 }
3020
3021 /*********************************************************************
3022  *
3023  *  Initialize all receive rings.
3024  *
3025  **********************************************************************/
3026 static int
3027 ixv_setup_receive_structures(struct adapter *adapter)
3028 {
3029         struct rx_ring *rxr = adapter->rx_rings;
3030         int j;
3031
3032         for (j = 0; j < adapter->num_queues; j++, rxr++)
3033                 if (ixv_setup_receive_ring(rxr))
3034                         goto fail;
3035
3036         return (0);
3037 fail:
3038         /*
3039          * Free RX buffers allocated so far, we will only handle
3040          * the rings that completed, the failing case will have
3041          * cleaned up for itself. 'j' failed, so its the terminus.
3042          */
3043         for (int i = 0; i < j; ++i) {
3044                 rxr = &adapter->rx_rings[i];
3045                 ixv_free_receive_ring(rxr);
3046         }
3047
3048         return (ENOBUFS);
3049 }
3050
3051 /*********************************************************************
3052  *
3053  *  Setup receive registers and features.
3054  *
3055  **********************************************************************/
3056 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3057
3058 static void
3059 ixv_initialize_receive_units(struct adapter *adapter)
3060 {
3061         struct  rx_ring *rxr = adapter->rx_rings;
3062         struct ixgbe_hw *hw = &adapter->hw;
3063         struct ifnet   *ifp = adapter->ifp;
3064         u32             bufsz, fctrl, rxcsum, hlreg;
3065
3066
3067         /* Enable broadcasts */
3068         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3069         fctrl |= IXGBE_FCTRL_BAM;
3070         fctrl |= IXGBE_FCTRL_DPF;
3071         fctrl |= IXGBE_FCTRL_PMCF;
3072         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3073
3074         /* Set for Jumbo Frames? */
3075         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3076         if (ifp->if_mtu > ETHERMTU) {
3077                 hlreg |= IXGBE_HLREG0_JUMBOEN;
3078                 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3079         } else {
3080                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
3081                 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3082         }
3083         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
3084
3085         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3086                 u64 rdba = rxr->rxdma.dma_paddr;
3087                 u32 reg, rxdctl;
3088
3089                 /* Do the queue enabling first */
3090                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
3091                 rxdctl |= IXGBE_RXDCTL_ENABLE;
3092                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
3093                 for (int k = 0; k < 10; k++) {
3094                         if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) &
3095                             IXGBE_RXDCTL_ENABLE)
3096                                 break;
3097                         else
3098                                 msec_delay(1);
3099                 }
3100                 wmb();
3101
3102                 /* Setup the Base and Length of the Rx Descriptor Ring */
3103                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
3104                     (rdba & 0x00000000ffffffffULL));
3105                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
3106                     (rdba >> 32));
3107                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
3108                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
3109
3110                 /* Set up the SRRCTL register */
3111                 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i));
3112                 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
3113                 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
3114                 reg |= bufsz;
3115                 if (rxr->hdr_split) {
3116                         /* Use a standard mbuf for the header */
3117                         reg |= ((IXV_RX_HDR <<
3118                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
3119                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
3120                         reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3121                 } else
3122                         reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
3123                 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg);
3124
3125                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
3126                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
3127                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
3128                     adapter->num_rx_desc - 1);
3129         }
3130
3131         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3132
3133         if (ifp->if_capenable & IFCAP_RXCSUM)
3134                 rxcsum |= IXGBE_RXCSUM_PCSD;
3135
3136         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
3137                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
3138
3139         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3140
3141         return;
3142 }
3143
3144 /*********************************************************************
3145  *
3146  *  Free all receive rings.
3147  *
3148  **********************************************************************/
3149 static void
3150 ixv_free_receive_structures(struct adapter *adapter)
3151 {
3152         struct rx_ring *rxr = adapter->rx_rings;
3153
3154         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3155                 struct lro_ctrl         *lro = &rxr->lro;
3156                 ixv_free_receive_buffers(rxr);
3157                 /* Free LRO memory */
3158                 tcp_lro_free(lro);
3159                 /* Free the ring memory as well */
3160                 ixv_dma_free(adapter, &rxr->rxdma);
3161         }
3162
3163         free(adapter->rx_rings, M_DEVBUF);
3164 }
3165
3166
3167 /*********************************************************************
3168  *
3169  *  Free receive ring data structures
3170  *
3171  **********************************************************************/
3172 static void
3173 ixv_free_receive_buffers(struct rx_ring *rxr)
3174 {
3175         struct adapter          *adapter = rxr->adapter;
3176         struct ixv_rx_buf       *rxbuf;
3177
3178         INIT_DEBUGOUT("free_receive_structures: begin");
3179
3180         /* Cleanup any existing buffers */
3181         if (rxr->rx_buffers != NULL) {
3182                 for (int i = 0; i < adapter->num_rx_desc; i++) {
3183                         rxbuf = &rxr->rx_buffers[i];
3184                         if (rxbuf->m_head != NULL) {
3185                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3186                                     BUS_DMASYNC_POSTREAD);
3187                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3188                                 rxbuf->m_head->m_flags |= M_PKTHDR;
3189                                 m_freem(rxbuf->m_head);
3190                         }
3191                         if (rxbuf->m_pack != NULL) {
3192                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3193                                     BUS_DMASYNC_POSTREAD);
3194                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3195                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
3196                                 m_freem(rxbuf->m_pack);
3197                         }
3198                         rxbuf->m_head = NULL;
3199                         rxbuf->m_pack = NULL;
3200                         if (rxbuf->hmap != NULL) {
3201                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
3202                                 rxbuf->hmap = NULL;
3203                         }
3204                         if (rxbuf->pmap != NULL) {
3205                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
3206                                 rxbuf->pmap = NULL;
3207                         }
3208                 }
3209                 if (rxr->rx_buffers != NULL) {
3210                         free(rxr->rx_buffers, M_DEVBUF);
3211                         rxr->rx_buffers = NULL;
3212                 }
3213         }
3214
3215         if (rxr->htag != NULL) {
3216                 bus_dma_tag_destroy(rxr->htag);
3217                 rxr->htag = NULL;
3218         }
3219         if (rxr->ptag != NULL) {
3220                 bus_dma_tag_destroy(rxr->ptag);
3221                 rxr->ptag = NULL;
3222         }
3223
3224         return;
3225 }
3226
3227 static __inline void
3228 ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
3229 {
3230
3231         /*
3232          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
3233          * should be computed by hardware. Also it should not have VLAN tag in
3234          * ethernet header.
3235          */
3236         if (rxr->lro_enabled &&
3237             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
3238             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
3239             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
3240             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) &&
3241             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
3242             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
3243                 /*
3244                  * Send to the stack if:
3245                  **  - LRO not enabled, or
3246                  **  - no LRO resources, or
3247                  **  - lro enqueue fails
3248                  */
3249                 if (rxr->lro.lro_cnt != 0)
3250                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
3251                                 return;
3252         }
3253         IXV_RX_UNLOCK(rxr);
3254         (*ifp->if_input)(ifp, m);
3255         IXV_RX_LOCK(rxr);
3256 }
3257
3258 static __inline void
3259 ixv_rx_discard(struct rx_ring *rxr, int i)
3260 {
3261         struct ixv_rx_buf       *rbuf;
3262
3263         rbuf = &rxr->rx_buffers[i];
3264
3265         if (rbuf->fmp != NULL) {/* Partial chain ? */
3266                 rbuf->fmp->m_flags |= M_PKTHDR;
3267                 m_freem(rbuf->fmp);
3268                 rbuf->fmp = NULL;
3269         }
3270
3271         /*
3272         ** With advanced descriptors the writeback
3273         ** clobbers the buffer addrs, so its easier
3274         ** to just free the existing mbufs and take
3275         ** the normal refresh path to get new buffers
3276         ** and mapping.
3277         */
3278         if (rbuf->m_head) {
3279                 m_free(rbuf->m_head);
3280                 rbuf->m_head = NULL;
3281         }
3282
3283         if (rbuf->m_pack) {
3284                 m_free(rbuf->m_pack);
3285                 rbuf->m_pack = NULL;
3286         }
3287
3288         return;
3289 }
3290
3291
3292 /*********************************************************************
3293  *
3294  *  This routine executes in interrupt context. It replenishes
3295  *  the mbufs in the descriptor and sends data which has been
3296  *  dma'ed into host memory to upper layer.
3297  *
3298  *  We loop at most count times if count is > 0, or until done if
3299  *  count < 0.
3300  *
3301  *  Return TRUE for more work, FALSE for all clean.
3302  *********************************************************************/
3303 static bool
3304 ixv_rxeof(struct ix_queue *que, int count)
3305 {
3306         struct adapter          *adapter = que->adapter;
3307         struct rx_ring          *rxr = que->rxr;
3308         struct ifnet            *ifp = adapter->ifp;
3309         struct lro_ctrl         *lro = &rxr->lro;
3310         struct lro_entry        *queued;
3311         int                     i, nextp, processed = 0;
3312         u32                     staterr = 0;
3313         union ixgbe_adv_rx_desc *cur;
3314         struct ixv_rx_buf       *rbuf, *nbuf;
3315
3316         IXV_RX_LOCK(rxr);
3317
3318         for (i = rxr->next_to_check; count != 0;) {
3319                 struct mbuf     *sendmp, *mh, *mp;
3320                 u32             rsc, ptype;
3321                 u16             hlen, plen, hdr, vtag;
3322                 bool            eop;
3323
3324                 /* Sync the ring. */
3325                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3326                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3327
3328                 cur = &rxr->rx_base[i];
3329                 staterr = le32toh(cur->wb.upper.status_error);
3330
3331                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
3332                         break;
3333                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
3334                         break;
3335
3336                 count--;
3337                 sendmp = NULL;
3338                 nbuf = NULL;
3339                 rsc = 0;
3340                 cur->wb.upper.status_error = 0;
3341                 rbuf = &rxr->rx_buffers[i];
3342                 mh = rbuf->m_head;
3343                 mp = rbuf->m_pack;
3344
3345                 plen = le16toh(cur->wb.upper.length);
3346                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
3347                     IXGBE_RXDADV_PKTTYPE_MASK;
3348                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
3349                 vtag = le16toh(cur->wb.upper.vlan);
3350                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
3351
3352                 /* Make sure all parts of a bad packet are discarded */
3353                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
3354                     (rxr->discard)) {
3355                         ifp->if_ierrors++;
3356                         rxr->rx_discarded++;
3357                         if (!eop)
3358                                 rxr->discard = TRUE;
3359                         else
3360                                 rxr->discard = FALSE;
3361                         ixv_rx_discard(rxr, i);
3362                         goto next_desc;
3363                 }
3364
3365                 if (!eop) {
3366                         nextp = i + 1;
3367                         if (nextp == adapter->num_rx_desc)
3368                                 nextp = 0;
3369                         nbuf = &rxr->rx_buffers[nextp];
3370                         prefetch(nbuf);
3371                 }
3372                 /*
3373                 ** The header mbuf is ONLY used when header
3374                 ** split is enabled, otherwise we get normal
3375                 ** behavior, ie, both header and payload
3376                 ** are DMA'd into the payload buffer.
3377                 **
3378                 ** Rather than using the fmp/lmp global pointers
3379                 ** we now keep the head of a packet chain in the
3380                 ** buffer struct and pass this along from one
3381                 ** descriptor to the next, until we get EOP.
3382                 */
3383                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
3384                         /* This must be an initial descriptor */
3385                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
3386                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
3387                         if (hlen > IXV_RX_HDR)
3388                                 hlen = IXV_RX_HDR;
3389                         mh->m_len = hlen;
3390                         mh->m_flags |= M_PKTHDR;
3391                         mh->m_next = NULL;
3392                         mh->m_pkthdr.len = mh->m_len;
3393                         /* Null buf pointer so it is refreshed */
3394                         rbuf->m_head = NULL;
3395                         /*
3396                         ** Check the payload length, this
3397                         ** could be zero if its a small
3398                         ** packet.
3399                         */
3400                         if (plen > 0) {
3401                                 mp->m_len = plen;
3402                                 mp->m_next = NULL;
3403                                 mp->m_flags &= ~M_PKTHDR;
3404                                 mh->m_next = mp;
3405                                 mh->m_pkthdr.len += mp->m_len;
3406                                 /* Null buf pointer so it is refreshed */
3407                                 rbuf->m_pack = NULL;
3408                                 rxr->rx_split_packets++;
3409                         }
3410                         /*
3411                         ** Now create the forward
3412                         ** chain so when complete
3413                         ** we wont have to.
3414                         */
3415                         if (eop == 0) {
3416                                 /* stash the chain head */
3417                                 nbuf->fmp = mh;
3418                                 /* Make forward chain */
3419                                 if (plen)
3420                                         mp->m_next = nbuf->m_pack;
3421                                 else
3422                                         mh->m_next = nbuf->m_pack;
3423                         } else {
3424                                 /* Singlet, prepare to send */
3425                                 sendmp = mh;
3426                                 if ((adapter->num_vlans) &&
3427                                     (staterr & IXGBE_RXD_STAT_VP)) {
3428                                         sendmp->m_pkthdr.ether_vtag = vtag;
3429                                         sendmp->m_flags |= M_VLANTAG;
3430                                 }
3431                         }
3432                 } else {
3433                         /*
3434                         ** Either no header split, or a
3435                         ** secondary piece of a fragmented
3436                         ** split packet.
3437                         */
3438                         mp->m_len = plen;
3439                         /*
3440                         ** See if there is a stored head
3441                         ** that determines what we are
3442                         */
3443                         sendmp = rbuf->fmp;
3444                         rbuf->m_pack = rbuf->fmp = NULL;
3445
3446                         if (sendmp != NULL) /* secondary frag */
3447                                 sendmp->m_pkthdr.len += mp->m_len;
3448                         else {
3449                                 /* first desc of a non-ps chain */
3450                                 sendmp = mp;
3451                                 sendmp->m_flags |= M_PKTHDR;
3452                                 sendmp->m_pkthdr.len = mp->m_len;
3453                                 if (staterr & IXGBE_RXD_STAT_VP) {
3454                                         sendmp->m_pkthdr.ether_vtag = vtag;
3455                                         sendmp->m_flags |= M_VLANTAG;
3456                                 }
3457                         }
3458                         /* Pass the head pointer on */
3459                         if (eop == 0) {
3460                                 nbuf->fmp = sendmp;
3461                                 sendmp = NULL;
3462                                 mp->m_next = nbuf->m_pack;
3463                         }
3464                 }
3465                 ++processed;
3466                 /* Sending this frame? */
3467                 if (eop) {
3468                         sendmp->m_pkthdr.rcvif = ifp;
3469                         ifp->if_ipackets++;
3470                         rxr->rx_packets++;
3471                         /* capture data for AIM */
3472                         rxr->bytes += sendmp->m_pkthdr.len;
3473                         rxr->rx_bytes += sendmp->m_pkthdr.len;
3474                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
3475                                 ixv_rx_checksum(staterr, sendmp, ptype);
3476 #if __FreeBSD_version >= 800000
3477                         sendmp->m_pkthdr.flowid = que->msix;
3478                         sendmp->m_flags |= M_FLOWID;
3479 #endif
3480                 }
3481 next_desc:
3482                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3483                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3484
3485                 /* Advance our pointers to the next descriptor. */
3486                 if (++i == adapter->num_rx_desc)
3487                         i = 0;
3488
3489                 /* Now send to the stack or do LRO */
3490                 if (sendmp != NULL)
3491                         ixv_rx_input(rxr, ifp, sendmp, ptype);
3492
3493                /* Every 8 descriptors we go to refresh mbufs */
3494                 if (processed == 8) {
3495                         ixv_refresh_mbufs(rxr, i);
3496                         processed = 0;
3497                 }
3498         }
3499
3500         /* Refresh any remaining buf structs */
3501         if (ixv_rx_unrefreshed(rxr))
3502                 ixv_refresh_mbufs(rxr, i);
3503
3504         rxr->next_to_check = i;
3505
3506         /*
3507          * Flush any outstanding LRO work
3508          */
3509         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
3510                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
3511                 tcp_lro_flush(lro, queued);
3512         }
3513
3514         IXV_RX_UNLOCK(rxr);
3515
3516         /*
3517         ** We still have cleaning to do?
3518         ** Schedule another interrupt if so.
3519         */
3520         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
3521                 ixv_rearm_queues(adapter, (u64)(1 << que->msix));
3522                 return (TRUE);
3523         }
3524
3525         return (FALSE);
3526 }
3527
3528
3529 /*********************************************************************
3530  *
3531  *  Verify that the hardware indicated that the checksum is valid.
3532  *  Inform the stack about the status of checksum so that stack
3533  *  doesn't spend time verifying the checksum.
3534  *
3535  *********************************************************************/
3536 static void
3537 ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
3538 {
3539         u16     status = (u16) staterr;
3540         u8      errors = (u8) (staterr >> 24);
3541         bool    sctp = FALSE;
3542
3543         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
3544             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
3545                 sctp = TRUE;
3546
3547         if (status & IXGBE_RXD_STAT_IPCS) {
3548                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
3549                         /* IP Checksum Good */
3550                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3551                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3552
3553                 } else
3554                         mp->m_pkthdr.csum_flags = 0;
3555         }
3556         if (status & IXGBE_RXD_STAT_L4CS) {
3557                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3558 #if __FreeBSD_version >= 800000
3559                 if (sctp)
3560                         type = CSUM_SCTP_VALID;
3561 #endif
3562                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
3563                         mp->m_pkthdr.csum_flags |= type;
3564                         if (!sctp)
3565                                 mp->m_pkthdr.csum_data = htons(0xffff);
3566                 }
3567         }
3568         return;
3569 }
3570
3571 static void
3572 ixv_setup_vlan_support(struct adapter *adapter)
3573 {
3574         struct ixgbe_hw *hw = &adapter->hw;
3575         u32             ctrl, vid, vfta, retry;
3576
3577
3578         /*
3579         ** We get here thru init_locked, meaning
3580         ** a soft reset, this has already cleared
3581         ** the VFTA and other state, so if there
3582         ** have been no vlan's registered do nothing.
3583         */
3584         if (adapter->num_vlans == 0)
3585                 return;
3586
3587         /* Enable the queues */
3588         for (int i = 0; i < adapter->num_queues; i++) {
3589                 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
3590                 ctrl |= IXGBE_RXDCTL_VME;
3591                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl);
3592         }
3593
3594         /*
3595         ** A soft reset zero's out the VFTA, so
3596         ** we need to repopulate it now.
3597         */
3598         for (int i = 0; i < VFTA_SIZE; i++) {
3599                 if (ixv_shadow_vfta[i] == 0)
3600                         continue;
3601                 vfta = ixv_shadow_vfta[i];
3602                 /*
3603                 ** Reconstruct the vlan id's
3604                 ** based on the bits set in each
3605                 ** of the array ints.
3606                 */
3607                 for ( int j = 0; j < 32; j++) {
3608                         retry = 0;
3609                         if ((vfta & (1 << j)) == 0)
3610                                 continue;
3611                         vid = (i * 32) + j;
3612                         /* Call the shared code mailbox routine */
3613                         while (ixgbe_set_vfta(hw, vid, 0, TRUE)) {
3614                                 if (++retry > 5)
3615                                         break;
3616                         }
3617                 }
3618         }
3619 }
3620
3621 /*
3622 ** This routine is run via an vlan config EVENT,
3623 ** it enables us to use the HW Filter table since
3624 ** we can get the vlan id. This just creates the
3625 ** entry in the soft version of the VFTA, init will
3626 ** repopulate the real table.
3627 */
3628 static void
3629 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3630 {
3631         struct adapter  *adapter = ifp->if_softc;
3632         u16             index, bit;
3633
3634         if (ifp->if_softc !=  arg)   /* Not our event */
3635                 return;
3636
3637         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3638                 return;
3639
3640         IXV_CORE_LOCK(adapter);
3641         index = (vtag >> 5) & 0x7F;
3642         bit = vtag & 0x1F;
3643         ixv_shadow_vfta[index] |= (1 << bit);
3644         ++adapter->num_vlans;
3645         /* Re-init to load the changes */
3646         ixv_init_locked(adapter);
3647         IXV_CORE_UNLOCK(adapter);
3648 }
3649
3650 /*
3651 ** This routine is run via an vlan
3652 ** unconfig EVENT, remove our entry
3653 ** in the soft vfta.
3654 */
3655 static void
3656 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3657 {
3658         struct adapter  *adapter = ifp->if_softc;
3659         u16             index, bit;
3660
3661         if (ifp->if_softc !=  arg)
3662                 return;
3663
3664         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
3665                 return;
3666
3667         IXV_CORE_LOCK(adapter);
3668         index = (vtag >> 5) & 0x7F;
3669         bit = vtag & 0x1F;
3670         ixv_shadow_vfta[index] &= ~(1 << bit);
3671         --adapter->num_vlans;
3672         /* Re-init to load the changes */
3673         ixv_init_locked(adapter);
3674         IXV_CORE_UNLOCK(adapter);
3675 }
3676
3677 static void
3678 ixv_enable_intr(struct adapter *adapter)
3679 {
3680         struct ixgbe_hw *hw = &adapter->hw;
3681         struct ix_queue *que = adapter->queues;
3682         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3683
3684
3685         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
3686
3687         mask = IXGBE_EIMS_ENABLE_MASK;
3688         mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
3689         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
3690
3691         for (int i = 0; i < adapter->num_queues; i++, que++)
3692                 ixv_enable_queue(adapter, que->msix);
3693
3694         IXGBE_WRITE_FLUSH(hw);
3695
3696         return;
3697 }
3698
3699 static void
3700 ixv_disable_intr(struct adapter *adapter)
3701 {
3702         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0);
3703         IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0);
3704         IXGBE_WRITE_FLUSH(&adapter->hw);
3705         return;
3706 }
3707
3708 /*
3709 ** Setup the correct IVAR register for a particular MSIX interrupt
3710 **  - entry is the register array entry
3711 **  - vector is the MSIX vector for this queue
3712 **  - type is RX/TX/MISC
3713 */
3714 static void
3715 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
3716 {
3717         struct ixgbe_hw *hw = &adapter->hw;
3718         u32 ivar, index;
3719
3720         vector |= IXGBE_IVAR_ALLOC_VAL;
3721
3722         if (type == -1) { /* MISC IVAR */
3723                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
3724                 ivar &= ~0xFF;
3725                 ivar |= vector;
3726                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
3727         } else {        /* RX/TX IVARS */
3728                 index = (16 * (entry & 1)) + (8 * type);
3729                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
3730                 ivar &= ~(0xFF << index);
3731                 ivar |= (vector << index);
3732                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
3733         }
3734 }
3735
3736 static void
3737 ixv_configure_ivars(struct adapter *adapter)
3738 {
3739         struct  ix_queue *que = adapter->queues;
3740
3741         for (int i = 0; i < adapter->num_queues; i++, que++) {
3742                 /* First the RX queue entry */
3743                 ixv_set_ivar(adapter, i, que->msix, 0);
3744                 /* ... and the TX */
3745                 ixv_set_ivar(adapter, i, que->msix, 1);
3746                 /* Set an initial value in EITR */
3747                 IXGBE_WRITE_REG(&adapter->hw,
3748                     IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT);
3749         }
3750
3751         /* For the Link interrupt */
3752         ixv_set_ivar(adapter, 1, adapter->mbxvec, -1);
3753 }
3754
3755
3756 /*
3757 ** Tasklet handler for MSIX MBX interrupts
3758 **  - do outside interrupt since it might sleep
3759 */
3760 static void
3761 ixv_handle_mbx(void *context, int pending)
3762 {
3763         struct adapter  *adapter = context;
3764
3765         ixgbe_check_link(&adapter->hw,
3766             &adapter->link_speed, &adapter->link_up, 0);
3767         ixv_update_link_status(adapter);
3768 }
3769
3770 /*
3771 ** The VF stats registers never have a truely virgin
3772 ** starting point, so this routine tries to make an
3773 ** artificial one, marking ground zero on attach as
3774 ** it were.
3775 */
3776 static void
3777 ixv_save_stats(struct adapter *adapter)
3778 {
3779         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
3780                 adapter->stats.saved_reset_vfgprc +=
3781                     adapter->stats.vfgprc - adapter->stats.base_vfgprc;
3782                 adapter->stats.saved_reset_vfgptc +=
3783                     adapter->stats.vfgptc - adapter->stats.base_vfgptc;
3784                 adapter->stats.saved_reset_vfgorc +=
3785                     adapter->stats.vfgorc - adapter->stats.base_vfgorc;
3786                 adapter->stats.saved_reset_vfgotc +=
3787                     adapter->stats.vfgotc - adapter->stats.base_vfgotc;
3788                 adapter->stats.saved_reset_vfmprc +=
3789                     adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3790         }
3791 }
3792
3793 static void
3794 ixv_init_stats(struct adapter *adapter)
3795 {
3796         struct ixgbe_hw *hw = &adapter->hw;
3797
3798         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
3799         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
3800         adapter->stats.last_vfgorc |=
3801             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
3802
3803         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
3804         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
3805         adapter->stats.last_vfgotc |=
3806             (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
3807
3808         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
3809
3810         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
3811         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
3812         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
3813         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
3814         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
3815 }
3816
3817 #define UPDATE_STAT_32(reg, last, count)                \
3818 {                                                       \
3819         u32 current = IXGBE_READ_REG(hw, reg);          \
3820         if (current < last)                             \
3821                 count += 0x100000000LL;                 \
3822         last = current;                                 \
3823         count &= 0xFFFFFFFF00000000LL;                  \
3824         count |= current;                               \
3825 }
3826
3827 #define UPDATE_STAT_36(lsb, msb, last, count)           \
3828 {                                                       \
3829         u64 cur_lsb = IXGBE_READ_REG(hw, lsb);          \
3830         u64 cur_msb = IXGBE_READ_REG(hw, msb);          \
3831         u64 current = ((cur_msb << 32) | cur_lsb);      \
3832         if (current < last)                             \
3833                 count += 0x1000000000LL;                \
3834         last = current;                                 \
3835         count &= 0xFFFFFFF000000000LL;                  \
3836         count |= current;                               \
3837 }
3838
3839 /*
3840 ** ixv_update_stats - Update the board statistics counters.
3841 */
3842 void
3843 ixv_update_stats(struct adapter *adapter)
3844 {
3845         struct ixgbe_hw *hw = &adapter->hw;
3846
3847         UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
3848             adapter->stats.vfgprc);
3849         UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
3850             adapter->stats.vfgptc);
3851         UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
3852             adapter->stats.last_vfgorc, adapter->stats.vfgorc);
3853         UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
3854             adapter->stats.last_vfgotc, adapter->stats.vfgotc);
3855         UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
3856             adapter->stats.vfmprc);
3857 }
3858
3859 /**********************************************************************
3860  *
3861  *  This routine is called only when ixgbe_display_debug_stats is enabled.
3862  *  This routine provides a way to take a look at important statistics
3863  *  maintained by the driver and hardware.
3864  *
3865  **********************************************************************/
3866 static void
3867 ixv_print_hw_stats(struct adapter * adapter)
3868 {
3869         device_t dev = adapter->dev;
3870
3871         device_printf(dev,"Std Mbuf Failed = %lu\n",
3872                adapter->mbuf_defrag_failed);
3873         device_printf(dev,"Driver dropped packets = %lu\n",
3874                adapter->dropped_pkts);
3875         device_printf(dev, "watchdog timeouts = %ld\n",
3876                adapter->watchdog_events);
3877
3878         device_printf(dev,"Good Packets Rcvd = %llu\n",
3879                (long long)adapter->stats.vfgprc);
3880         device_printf(dev,"Good Packets Xmtd = %llu\n",
3881                (long long)adapter->stats.vfgptc);
3882         device_printf(dev,"TSO Transmissions = %lu\n",
3883                adapter->tso_tx);
3884
3885 }
3886
3887 /**********************************************************************
3888  *
3889  *  This routine is called only when em_display_debug_stats is enabled.
3890  *  This routine provides a way to take a look at important statistics
3891  *  maintained by the driver and hardware.
3892  *
3893  **********************************************************************/
3894 static void
3895 ixv_print_debug_info(struct adapter *adapter)
3896 {
3897         device_t dev = adapter->dev;
3898         struct ixgbe_hw         *hw = &adapter->hw;
3899         struct ix_queue         *que = adapter->queues;
3900         struct rx_ring          *rxr;
3901         struct tx_ring          *txr;
3902         struct lro_ctrl         *lro;
3903
3904         device_printf(dev,"Error Byte Count = %u \n",
3905             IXGBE_READ_REG(hw, IXGBE_ERRBC));
3906
3907         for (int i = 0; i < adapter->num_queues; i++, que++) {
3908                 txr = que->txr;
3909                 rxr = que->rxr;
3910                 lro = &rxr->lro;
3911                 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n",
3912                     que->msix, (long)que->irqs);
3913                 device_printf(dev,"RX(%d) Packets Received: %lld\n",
3914                     rxr->me, (long long)rxr->rx_packets);
3915                 device_printf(dev,"RX(%d) Split RX Packets: %lld\n",
3916                     rxr->me, (long long)rxr->rx_split_packets);
3917                 device_printf(dev,"RX(%d) Bytes Received: %lu\n",
3918                     rxr->me, (long)rxr->rx_bytes);
3919                 device_printf(dev,"RX(%d) LRO Queued= %d\n",
3920                     rxr->me, lro->lro_queued);
3921                 device_printf(dev,"RX(%d) LRO Flushed= %d\n",
3922                     rxr->me, lro->lro_flushed);
3923                 device_printf(dev,"TX(%d) Packets Sent: %lu\n",
3924                     txr->me, (long)txr->total_packets);
3925                 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n",
3926                     txr->me, (long)txr->no_desc_avail);
3927         }
3928
3929         device_printf(dev,"MBX IRQ Handled: %lu\n",
3930             (long)adapter->mbx_irq);
3931         return;
3932 }
3933
3934 static int
3935 ixv_sysctl_stats(SYSCTL_HANDLER_ARGS)
3936 {
3937         int             error;
3938         int             result;
3939         struct adapter *adapter;
3940
3941         result = -1;
3942         error = sysctl_handle_int(oidp, &result, 0, req);
3943
3944         if (error || !req->newptr)
3945                 return (error);
3946
3947         if (result == 1) {
3948                 adapter = (struct adapter *) arg1;
3949                 ixv_print_hw_stats(adapter);
3950         }
3951         return error;
3952 }
3953
3954 static int
3955 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)
3956 {
3957         int error, result;
3958         struct adapter *adapter;
3959
3960         result = -1;
3961         error = sysctl_handle_int(oidp, &result, 0, req);
3962
3963         if (error || !req->newptr)
3964                 return (error);
3965
3966         if (result == 1) {
3967                 adapter = (struct adapter *) arg1;
3968                 ixv_print_debug_info(adapter);
3969         }
3970         return error;
3971 }
3972
3973 /*
3974 ** Set flow control using sysctl:
3975 ** Flow control values:
3976 **      0 - off
3977 **      1 - rx pause
3978 **      2 - tx pause
3979 **      3 - full
3980 */
3981 static int
3982 ixv_set_flowcntl(SYSCTL_HANDLER_ARGS)
3983 {
3984         int error;
3985         struct adapter *adapter;
3986
3987         error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req);
3988
3989         if (error)
3990                 return (error);
3991
3992         adapter = (struct adapter *) arg1;
3993         switch (ixv_flow_control) {
3994                 case ixgbe_fc_rx_pause:
3995                 case ixgbe_fc_tx_pause:
3996                 case ixgbe_fc_full:
3997                         adapter->hw.fc.requested_mode = ixv_flow_control;
3998                         break;
3999                 case ixgbe_fc_none:
4000                 default:
4001                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
4002         }
4003
4004         ixgbe_fc_enable(&adapter->hw);
4005         return error;
4006 }
4007
4008 static void
4009 ixv_add_rx_process_limit(struct adapter *adapter, const char *name,
4010         const char *description, int *limit, int value)
4011 {
4012         *limit = value;
4013         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4014             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4015             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
4016 }
4017