1 /******************************************************************************
3 Copyright (c) 2001-2017, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
37 #include "opt_inet6.h"
42 #include <net/netmap.h>
43 #include <dev/netmap/netmap_kern.h>
45 /************************************************************************
47 ************************************************************************/
48 char ixv_driver_version[] = "2.0.1-k";
50 /************************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixv_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 ************************************************************************/
59 static pci_vendor_info_t ixv_vendor_info_array[] =
61 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
62 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
63 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
64 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
65 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
66 /* required last entry */
70 /************************************************************************
72 ************************************************************************/
73 static void *ixv_register(device_t dev);
74 static int ixv_if_attach_pre(if_ctx_t ctx);
75 static int ixv_if_attach_post(if_ctx_t ctx);
76 static int ixv_if_detach(if_ctx_t ctx);
78 static int ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid);
79 static int ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
80 static int ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
81 static void ixv_if_queues_free(if_ctx_t ctx);
82 static void ixv_identify_hardware(if_ctx_t ctx);
83 static void ixv_init_device_features(struct adapter *);
84 static int ixv_allocate_pci_resources(if_ctx_t ctx);
85 static void ixv_free_pci_resources(if_ctx_t ctx);
86 static int ixv_setup_interface(if_ctx_t ctx);
87 static void ixv_if_media_status(if_ctx_t , struct ifmediareq *);
88 static int ixv_if_media_change(if_ctx_t ctx);
89 static void ixv_if_update_admin_status(if_ctx_t ctx);
90 static int ixv_if_msix_intr_assign(if_ctx_t ctx, int msix);
92 static int ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
93 static void ixv_if_init(if_ctx_t ctx);
94 static void ixv_if_local_timer(if_ctx_t ctx, uint16_t qid);
95 static void ixv_if_stop(if_ctx_t ctx);
96 static int ixv_negotiate_api(struct adapter *);
98 static void ixv_initialize_transmit_units(if_ctx_t ctx);
99 static void ixv_initialize_receive_units(if_ctx_t ctx);
100 static void ixv_initialize_rss_mapping(struct adapter *);
102 static void ixv_setup_vlan_support(if_ctx_t ctx);
103 static void ixv_configure_ivars(struct adapter *);
104 static void ixv_if_enable_intr(if_ctx_t ctx);
105 static void ixv_if_disable_intr(if_ctx_t ctx);
106 static void ixv_if_multi_set(if_ctx_t ctx);
108 static void ixv_if_register_vlan(if_ctx_t, u16);
109 static void ixv_if_unregister_vlan(if_ctx_t, u16);
111 static uint64_t ixv_if_get_counter(if_ctx_t, ift_counter);
113 static void ixv_save_stats(struct adapter *);
114 static void ixv_init_stats(struct adapter *);
115 static void ixv_update_stats(struct adapter *);
116 static void ixv_add_stats_sysctls(struct adapter *adapter);
118 static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS);
119 static void ixv_set_ivar(struct adapter *, u8, u8, s8);
121 static u8 *ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
123 /* The MSI-X Interrupt handlers */
124 static int ixv_msix_que(void *);
125 static int ixv_msix_mbx(void *);
127 /************************************************************************
128 * FreeBSD Device Interface Entry Points
129 ************************************************************************/
130 static device_method_t ixv_methods[] = {
131 /* Device interface */
132 DEVMETHOD(device_register, ixv_register),
133 DEVMETHOD(device_probe, iflib_device_probe),
134 DEVMETHOD(device_attach, iflib_device_attach),
135 DEVMETHOD(device_detach, iflib_device_detach),
136 DEVMETHOD(device_shutdown, iflib_device_shutdown),
140 static driver_t ixv_driver = {
141 "ixv", ixv_methods, sizeof(struct adapter),
144 devclass_t ixv_devclass;
145 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0);
146 MODULE_DEPEND(ixv, pci, 1, 1, 1);
147 MODULE_DEPEND(ixv, ether, 1, 1, 1);
149 MODULE_DEPEND(ixv, netmap, 1, 1, 1);
150 #endif /* DEV_NETMAP */
152 static device_method_t ixv_if_methods[] = {
153 DEVMETHOD(ifdi_attach_pre, ixv_if_attach_pre),
154 DEVMETHOD(ifdi_attach_post, ixv_if_attach_post),
155 DEVMETHOD(ifdi_detach, ixv_if_detach),
156 DEVMETHOD(ifdi_init, ixv_if_init),
157 DEVMETHOD(ifdi_stop, ixv_if_stop),
158 DEVMETHOD(ifdi_msix_intr_assign, ixv_if_msix_intr_assign),
159 DEVMETHOD(ifdi_intr_enable, ixv_if_enable_intr),
160 DEVMETHOD(ifdi_intr_disable, ixv_if_disable_intr),
161 DEVMETHOD(ifdi_tx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
162 DEVMETHOD(ifdi_rx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
163 DEVMETHOD(ifdi_tx_queues_alloc, ixv_if_tx_queues_alloc),
164 DEVMETHOD(ifdi_rx_queues_alloc, ixv_if_rx_queues_alloc),
165 DEVMETHOD(ifdi_queues_free, ixv_if_queues_free),
166 DEVMETHOD(ifdi_update_admin_status, ixv_if_update_admin_status),
167 DEVMETHOD(ifdi_multi_set, ixv_if_multi_set),
168 DEVMETHOD(ifdi_mtu_set, ixv_if_mtu_set),
169 DEVMETHOD(ifdi_media_status, ixv_if_media_status),
170 DEVMETHOD(ifdi_media_change, ixv_if_media_change),
171 DEVMETHOD(ifdi_timer, ixv_if_local_timer),
172 DEVMETHOD(ifdi_vlan_register, ixv_if_register_vlan),
173 DEVMETHOD(ifdi_vlan_unregister, ixv_if_unregister_vlan),
174 DEVMETHOD(ifdi_get_counter, ixv_if_get_counter),
178 static driver_t ixv_if_driver = {
179 "ixv_if", ixv_if_methods, sizeof(struct adapter)
183 * TUNEABLE PARAMETERS:
186 /* Flow control setting, default to full */
187 static int ixv_flow_control = ixgbe_fc_full;
188 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control);
191 * Header split: this causes the hardware to DMA
192 * the header into a separate mbuf from the payload,
193 * it can be a performance win in some workloads, but
194 * in others it actually hurts, its off by default.
196 static int ixv_header_split = FALSE;
197 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split);
200 * Shadow VFTA table, this is needed because
201 * the real filter table gets cleared during
202 * a soft reset and we need to repopulate it.
204 static u32 ixv_shadow_vfta[IXGBE_VFTA_SIZE];
205 extern struct if_txrx ixgbe_txrx;
207 static struct if_shared_ctx ixv_sctx_init = {
208 .isc_magic = IFLIB_MAGIC,
209 .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
210 .isc_tx_maxsize = IXGBE_TSO_SIZE,
212 .isc_tx_maxsegsize = PAGE_SIZE,
214 .isc_rx_maxsize = MJUM16BYTES,
215 .isc_rx_nsegments = 1,
216 .isc_rx_maxsegsize = MJUM16BYTES,
220 .isc_admin_intrcnt = 1,
221 .isc_vendor_info = ixv_vendor_info_array,
222 .isc_driver_version = ixv_driver_version,
223 .isc_driver = &ixv_if_driver,
225 .isc_nrxd_min = {MIN_RXD},
226 .isc_ntxd_min = {MIN_TXD},
227 .isc_nrxd_max = {MAX_RXD},
228 .isc_ntxd_max = {MAX_TXD},
229 .isc_nrxd_default = {DEFAULT_RXD},
230 .isc_ntxd_default = {DEFAULT_TXD},
233 if_shared_ctx_t ixv_sctx = &ixv_sctx_init;
236 ixv_register(device_t dev)
241 /************************************************************************
242 * ixv_if_tx_queues_alloc
243 ************************************************************************/
245 ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
246 int ntxqs, int ntxqsets)
248 struct adapter *adapter = iflib_get_softc(ctx);
249 if_softc_ctx_t scctx = adapter->shared;
250 struct ix_tx_queue *que;
253 MPASS(adapter->num_tx_queues == ntxqsets);
256 /* Allocate queue structure memory */
258 (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * ntxqsets,
259 M_DEVBUF, M_NOWAIT | M_ZERO);
260 if (!adapter->tx_queues) {
261 device_printf(iflib_get_dev(ctx),
262 "Unable to allocate TX ring memory\n");
266 for (i = 0, que = adapter->tx_queues; i < ntxqsets; i++, que++) {
267 struct tx_ring *txr = &que->txr;
270 txr->adapter = que->adapter = adapter;
271 adapter->active_queues |= (u64)1 << txr->me;
273 /* Allocate report status array */
274 if (!(txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) {
278 for (j = 0; j < scctx->isc_ntxd[0]; j++)
279 txr->tx_rsq[j] = QIDX_INVALID;
280 /* get the virtual and physical address of the hardware queues */
281 txr->tail = IXGBE_VFTDT(txr->me);
282 txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i*ntxqs];
283 txr->tx_paddr = paddrs[i*ntxqs];
286 txr->total_packets = 0;
290 device_printf(iflib_get_dev(ctx), "allocated for %d queues\n",
291 adapter->num_tx_queues);
296 ixv_if_queues_free(ctx);
299 } /* ixv_if_tx_queues_alloc */
301 /************************************************************************
302 * ixv_if_rx_queues_alloc
303 ************************************************************************/
305 ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
306 int nrxqs, int nrxqsets)
308 struct adapter *adapter = iflib_get_softc(ctx);
309 struct ix_rx_queue *que;
312 MPASS(adapter->num_rx_queues == nrxqsets);
315 /* Allocate queue structure memory */
317 (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue) * nrxqsets,
318 M_DEVBUF, M_NOWAIT | M_ZERO);
319 if (!adapter->rx_queues) {
320 device_printf(iflib_get_dev(ctx),
321 "Unable to allocate TX ring memory\n");
326 for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
327 struct rx_ring *rxr = &que->rxr;
329 rxr->adapter = que->adapter = adapter;
332 /* get the virtual and physical address of the hw queues */
333 rxr->tail = IXGBE_VFRDT(rxr->me);
334 rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];
335 rxr->rx_paddr = paddrs[i*nrxqs];
340 device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n",
341 adapter->num_rx_queues);
346 ixv_if_queues_free(ctx);
349 } /* ixv_if_rx_queues_alloc */
351 /************************************************************************
353 ************************************************************************/
355 ixv_if_queues_free(if_ctx_t ctx)
357 struct adapter *adapter = iflib_get_softc(ctx);
358 struct ix_tx_queue *que = adapter->tx_queues;
364 for (i = 0; i < adapter->num_tx_queues; i++, que++) {
365 struct tx_ring *txr = &que->txr;
366 if (txr->tx_rsq == NULL)
369 free(txr->tx_rsq, M_DEVBUF);
372 if (adapter->tx_queues != NULL)
373 free(adapter->tx_queues, M_DEVBUF);
375 if (adapter->rx_queues != NULL)
376 free(adapter->rx_queues, M_DEVBUF);
377 adapter->tx_queues = NULL;
378 adapter->rx_queues = NULL;
379 } /* ixv_if_queues_free */
381 /************************************************************************
382 * ixv_if_attach_pre - Device initialization routine
384 * Called when the driver is being loaded.
385 * Identifies the type of hardware, allocates all resources
386 * and initializes the hardware.
388 * return 0 on success, positive on failure
389 ************************************************************************/
391 ixv_if_attach_pre(if_ctx_t ctx)
393 struct adapter *adapter;
395 if_softc_ctx_t scctx;
399 INIT_DEBUGOUT("ixv_attach: begin");
401 /* Allocate, clear, and link in our adapter structure */
402 dev = iflib_get_dev(ctx);
403 adapter = iflib_get_softc(ctx);
406 adapter->hw.back = adapter;
407 scctx = adapter->shared = iflib_get_softc_ctx(ctx);
408 adapter->media = iflib_get_media(ctx);
411 /* Do base PCI setup - map BAR0 */
412 if (ixv_allocate_pci_resources(ctx)) {
413 device_printf(dev, "ixv_allocate_pci_resources() failed!\n");
419 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
420 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
421 CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixv_sysctl_debug, "I",
424 /* Determine hardware revision */
425 ixv_identify_hardware(ctx);
426 ixv_init_device_features(adapter);
428 /* Initialize the shared code */
429 error = ixgbe_init_ops_vf(hw);
431 device_printf(dev, "ixgbe_init_ops_vf() failed!\n");
436 /* Setup the mailbox */
437 ixgbe_init_mbx_params_vf(hw);
439 error = hw->mac.ops.reset_hw(hw);
440 if (error == IXGBE_ERR_RESET_FAILED)
441 device_printf(dev, "...reset_hw() failure: Reset Failed!\n");
443 device_printf(dev, "...reset_hw() failed with error %d\n",
450 error = hw->mac.ops.init_hw(hw);
452 device_printf(dev, "...init_hw() failed with error %d\n",
458 /* Negotiate mailbox API version */
459 error = ixv_negotiate_api(adapter);
462 "Mailbox API negotiation failed during attach!\n");
466 /* If no mac address was assigned, make a random one */
467 if (!ixv_check_ether_addr(hw->mac.addr)) {
468 u8 addr[ETHER_ADDR_LEN];
469 arc4rand(&addr, sizeof(addr), 0);
472 bcopy(addr, hw->mac.addr, sizeof(addr));
473 bcopy(addr, hw->mac.perm_addr, sizeof(addr));
476 /* Most of the iflib initialization... */
478 iflib_set_mac(ctx, hw->mac.addr);
479 switch (adapter->hw.mac.type) {
480 case ixgbe_mac_X550_vf:
481 case ixgbe_mac_X550EM_x_vf:
482 case ixgbe_mac_X550EM_a_vf:
483 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 2;
486 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 1;
488 scctx->isc_txqsizes[0] =
489 roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +
490 sizeof(u32), DBA_ALIGN);
491 scctx->isc_rxqsizes[0] =
492 roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),
495 scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
496 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO;
497 scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;
498 scctx->isc_msix_bar = PCIR_BAR(MSIX_82598_BAR);
499 scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
500 scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;
501 scctx->isc_tx_tso_segsize_max = PAGE_SIZE;
503 scctx->isc_txrx = &ixgbe_txrx;
506 * Tell the upper layer(s) we support everything the PF
507 * driver does except...
511 scctx->isc_capenable = IXGBE_CAPS;
512 scctx->isc_capenable ^= IFCAP_HWSTATS | IFCAP_WOL;
514 INIT_DEBUGOUT("ixv_if_attach_pre: end");
519 ixv_free_pci_resources(ctx);
522 } /* ixv_if_attach_pre */
525 ixv_if_attach_post(if_ctx_t ctx)
527 struct adapter *adapter = iflib_get_softc(ctx);
528 device_t dev = iflib_get_dev(ctx);
531 /* Setup OS specific network interface */
532 error = ixv_setup_interface(ctx);
534 device_printf(dev, "Interface setup failed: %d\n", error);
538 /* Do the stats setup */
539 ixv_save_stats(adapter);
540 ixv_init_stats(adapter);
541 ixv_add_stats_sysctls(adapter);
545 } /* ixv_if_attach_post */
547 /************************************************************************
548 * ixv_detach - Device removal routine
550 * Called when the driver is being removed.
551 * Stops the adapter and deallocates all the resources
552 * that were allocated for driver operation.
554 * return 0 on success, positive on failure
555 ************************************************************************/
557 ixv_if_detach(if_ctx_t ctx)
559 INIT_DEBUGOUT("ixv_detach: begin");
561 ixv_free_pci_resources(ctx);
564 } /* ixv_if_detach */
566 /************************************************************************
568 ************************************************************************/
570 ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
572 struct adapter *adapter = iflib_get_softc(ctx);
573 struct ifnet *ifp = iflib_get_ifp(ctx);
576 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
577 if (mtu > IXGBE_MAX_FRAME_SIZE - IXGBE_MTU_HDR) {
581 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR;
585 } /* ixv_if_mtu_set */
587 /************************************************************************
588 * ixv_if_init - Init entry point
590 * Used in two ways: It is used by the stack as an init entry
591 * point in network interface structure. It is also used
592 * by the driver as a hw/sw initialization routine to get
593 * to a consistent state.
595 * return 0 on success, positive on failure
596 ************************************************************************/
598 ixv_if_init(if_ctx_t ctx)
600 struct adapter *adapter = iflib_get_softc(ctx);
601 struct ifnet *ifp = iflib_get_ifp(ctx);
602 device_t dev = iflib_get_dev(ctx);
603 struct ixgbe_hw *hw = &adapter->hw;
606 INIT_DEBUGOUT("ixv_if_init: begin");
607 hw->adapter_stopped = FALSE;
608 hw->mac.ops.stop_adapter(hw);
610 /* reprogram the RAR[0] in case user changed it. */
611 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
613 /* Get the latest mac address, User can use a LAA */
614 bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
615 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1);
617 /* Reset VF and renegotiate mailbox API version */
618 hw->mac.ops.reset_hw(hw);
619 hw->mac.ops.start_hw(hw);
620 error = ixv_negotiate_api(adapter);
623 "Mailbox API negotiation failed in if_init!\n");
627 ixv_initialize_transmit_units(ctx);
629 /* Setup Multicast table */
630 ixv_if_multi_set(ctx);
633 * Determine the correct mbuf pool
634 * for doing jumbo/headersplit
636 if (ifp->if_mtu > ETHERMTU)
637 adapter->rx_mbuf_sz = MJUMPAGESIZE;
639 adapter->rx_mbuf_sz = MCLBYTES;
641 /* Configure RX settings */
642 ixv_initialize_receive_units(ctx);
644 /* Set up VLAN offload and filter */
645 ixv_setup_vlan_support(ctx);
647 /* Set up MSI-X routing */
648 ixv_configure_ivars(adapter);
650 /* Set up auto-mask */
651 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE);
653 /* Set moderation on the Link interrupt */
654 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->vector), IXGBE_LINK_ITR);
657 ixv_init_stats(adapter);
659 /* Config/Enable Link */
660 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up,
663 /* And now turn on interrupts */
664 ixv_if_enable_intr(ctx);
669 /************************************************************************
671 ************************************************************************/
673 ixv_enable_queue(struct adapter *adapter, u32 vector)
675 struct ixgbe_hw *hw = &adapter->hw;
676 u32 queue = 1 << vector;
679 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
680 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
681 } /* ixv_enable_queue */
683 /************************************************************************
685 ************************************************************************/
687 ixv_disable_queue(struct adapter *adapter, u32 vector)
689 struct ixgbe_hw *hw = &adapter->hw;
690 u64 queue = (u64)(1 << vector);
693 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
694 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
695 } /* ixv_disable_queue */
698 /************************************************************************
699 * ixv_msix_que - MSI-X Queue Interrupt Service routine
700 ************************************************************************/
702 ixv_msix_que(void *arg)
704 struct ix_rx_queue *que = arg;
705 struct adapter *adapter = que->adapter;
707 ixv_disable_queue(adapter, que->msix);
710 return (FILTER_SCHEDULE_THREAD);
713 /************************************************************************
715 ************************************************************************/
717 ixv_msix_mbx(void *arg)
719 struct adapter *adapter = arg;
720 struct ixgbe_hw *hw = &adapter->hw;
725 /* First get the cause */
726 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS);
727 /* Clear interrupt with write */
728 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg);
730 /* Link status change */
731 if (reg & IXGBE_EICR_LSC)
732 iflib_admin_intr_deferred(adapter->ctx);
734 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER);
736 return (FILTER_HANDLED);
739 /************************************************************************
740 * ixv_media_status - Media Ioctl callback
742 * Called whenever the user queries the status of
743 * the interface using ifconfig.
744 ************************************************************************/
746 ixv_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
748 struct adapter *adapter = iflib_get_softc(ctx);
750 INIT_DEBUGOUT("ixv_media_status: begin");
752 iflib_admin_intr_deferred(ctx);
754 ifmr->ifm_status = IFM_AVALID;
755 ifmr->ifm_active = IFM_ETHER;
757 if (!adapter->link_active)
760 ifmr->ifm_status |= IFM_ACTIVE;
762 switch (adapter->link_speed) {
763 case IXGBE_LINK_SPEED_1GB_FULL:
764 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
766 case IXGBE_LINK_SPEED_10GB_FULL:
767 ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
769 case IXGBE_LINK_SPEED_100_FULL:
770 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
772 case IXGBE_LINK_SPEED_10_FULL:
773 ifmr->ifm_active |= IFM_10_T | IFM_FDX;
776 } /* ixv_if_media_status */
778 /************************************************************************
779 * ixv_if_media_change - Media Ioctl callback
781 * Called when the user changes speed/duplex using
782 * media/mediopt option with ifconfig.
783 ************************************************************************/
785 ixv_if_media_change(if_ctx_t ctx)
787 struct adapter *adapter = iflib_get_softc(ctx);
788 struct ifmedia *ifm = iflib_get_media(ctx);
790 INIT_DEBUGOUT("ixv_media_change: begin");
792 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
795 switch (IFM_SUBTYPE(ifm->ifm_media)) {
799 device_printf(adapter->dev, "Only auto media type\n");
804 } /* ixv_if_media_change */
807 /************************************************************************
810 * Negotiate the Mailbox API with the PF;
811 * start with the most featured API first.
812 ************************************************************************/
814 ixv_negotiate_api(struct adapter *adapter)
816 struct ixgbe_hw *hw = &adapter->hw;
817 int mbx_api[] = { ixgbe_mbox_api_11,
819 ixgbe_mbox_api_unknown };
822 while (mbx_api[i] != ixgbe_mbox_api_unknown) {
823 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0)
829 } /* ixv_negotiate_api */
832 /************************************************************************
833 * ixv_if_multi_set - Multicast Update
835 * Called whenever multicast address list is updated.
836 ************************************************************************/
838 ixv_if_multi_set(if_ctx_t ctx)
840 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
841 struct adapter *adapter = iflib_get_softc(ctx);
843 struct ifmultiaddr *ifma;
844 if_t ifp = iflib_get_ifp(ctx);
847 IOCTL_DEBUGOUT("ixv_if_multi_set: begin");
849 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
850 if (ifma->ifma_addr->sa_family != AF_LINK)
852 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
853 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
854 IXGBE_ETH_LENGTH_OF_ADDRESS);
860 adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, update_ptr, mcnt,
861 ixv_mc_array_itr, TRUE);
862 } /* ixv_if_multi_set */
864 /************************************************************************
867 * An iterator function needed by the multicast shared code.
868 * It feeds the shared code routine the addresses in the
869 * array of ixv_set_multi() one by one.
870 ************************************************************************/
872 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
874 u8 *addr = *update_ptr;
879 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
880 *update_ptr = newptr;
883 } /* ixv_mc_array_itr */
885 /************************************************************************
886 * ixv_if_local_timer - Timer routine
888 * Checks for link status, updates statistics,
889 * and runs the watchdog check.
890 ************************************************************************/
892 ixv_if_local_timer(if_ctx_t ctx, uint16_t qid)
897 /* Fire off the adminq task */
898 iflib_admin_intr_deferred(ctx);
899 } /* ixv_if_local_timer */
901 /************************************************************************
902 * ixv_if_update_admin_status - Update OS on link state
904 * Note: Only updates the OS on the cached link state.
905 * The real check of the hardware only happens with
907 ************************************************************************/
909 ixv_if_update_admin_status(if_ctx_t ctx)
911 struct adapter *adapter = iflib_get_softc(ctx);
912 device_t dev = iflib_get_dev(ctx);
915 adapter->hw.mac.get_link_status = TRUE;
917 status = ixgbe_check_link(&adapter->hw, &adapter->link_speed,
918 &adapter->link_up, FALSE);
920 if (status != IXGBE_SUCCESS && adapter->hw.adapter_stopped == FALSE) {
921 /* Mailbox's Clear To Send status is lost or timeout occurred.
922 * We need reinitialization. */
923 iflib_get_ifp(ctx)->if_init(ctx);
926 if (adapter->link_up) {
927 if (adapter->link_active == FALSE) {
929 device_printf(dev, "Link is up %d Gbps %s \n",
930 ((adapter->link_speed == 128) ? 10 : 1),
932 adapter->link_active = TRUE;
933 iflib_link_state_change(ctx, LINK_STATE_UP,
936 } else { /* Link down */
937 if (adapter->link_active == TRUE) {
939 device_printf(dev, "Link is Down\n");
940 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
941 adapter->link_active = FALSE;
946 ixv_update_stats(adapter);
947 } /* ixv_if_update_admin_status */
950 /************************************************************************
951 * ixv_if_stop - Stop the hardware
953 * Disables all traffic on the adapter by issuing a
954 * global reset on the MAC and deallocates TX/RX buffers.
955 ************************************************************************/
957 ixv_if_stop(if_ctx_t ctx)
959 struct adapter *adapter = iflib_get_softc(ctx);
960 struct ixgbe_hw *hw = &adapter->hw;
962 INIT_DEBUGOUT("ixv_stop: begin\n");
964 ixv_if_disable_intr(ctx);
966 hw->mac.ops.reset_hw(hw);
967 adapter->hw.adapter_stopped = FALSE;
968 hw->mac.ops.stop_adapter(hw);
970 /* Update the stack */
971 adapter->link_up = FALSE;
972 ixv_if_update_admin_status(ctx);
974 /* reprogram the RAR[0] in case user changed it. */
975 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
979 /************************************************************************
980 * ixv_identify_hardware - Determine hardware revision.
981 ************************************************************************/
983 ixv_identify_hardware(if_ctx_t ctx)
985 struct adapter *adapter = iflib_get_softc(ctx);
986 device_t dev = iflib_get_dev(ctx);
987 struct ixgbe_hw *hw = &adapter->hw;
989 /* Save off the information about this board */
990 hw->vendor_id = pci_get_vendor(dev);
991 hw->device_id = pci_get_device(dev);
992 hw->revision_id = pci_get_revid(dev);
993 hw->subsystem_vendor_id = pci_get_subvendor(dev);
994 hw->subsystem_device_id = pci_get_subdevice(dev);
996 /* A subset of set_mac_type */
997 switch (hw->device_id) {
998 case IXGBE_DEV_ID_82599_VF:
999 hw->mac.type = ixgbe_mac_82599_vf;
1001 case IXGBE_DEV_ID_X540_VF:
1002 hw->mac.type = ixgbe_mac_X540_vf;
1004 case IXGBE_DEV_ID_X550_VF:
1005 hw->mac.type = ixgbe_mac_X550_vf;
1007 case IXGBE_DEV_ID_X550EM_X_VF:
1008 hw->mac.type = ixgbe_mac_X550EM_x_vf;
1010 case IXGBE_DEV_ID_X550EM_A_VF:
1011 hw->mac.type = ixgbe_mac_X550EM_a_vf;
1014 device_printf(dev, "unknown mac type\n");
1015 hw->mac.type = ixgbe_mac_unknown;
1018 } /* ixv_identify_hardware */
1020 /************************************************************************
1021 * ixv_if_msix_intr_assign - Setup MSI-X Interrupt resources and handlers
1022 ************************************************************************/
1024 ixv_if_msix_intr_assign(if_ctx_t ctx, int msix)
1026 struct adapter *adapter = iflib_get_softc(ctx);
1027 device_t dev = iflib_get_dev(ctx);
1028 struct ix_rx_queue *rx_que = adapter->rx_queues;
1029 struct ix_tx_queue *tx_que;
1030 int error, rid, vector = 0;
1033 for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rx_que++) {
1036 snprintf(buf, sizeof(buf), "rxq%d", i);
1037 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
1038 IFLIB_INTR_RX, ixv_msix_que, rx_que, rx_que->rxr.me, buf);
1041 device_printf(iflib_get_dev(ctx),
1042 "Failed to allocate que int %d err: %d", i, error);
1043 adapter->num_rx_queues = i + 1;
1047 rx_que->msix = vector;
1048 adapter->active_queues |= (u64)(1 << rx_que->msix);
1052 for (int i = 0; i < adapter->num_tx_queues; i++) {
1053 snprintf(buf, sizeof(buf), "txq%d", i);
1054 tx_que = &adapter->tx_queues[i];
1055 tx_que->msix = i % adapter->num_rx_queues;
1056 iflib_softirq_alloc_generic(ctx,
1057 &adapter->rx_queues[tx_que->msix].que_irq,
1058 IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
1061 error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid,
1062 IFLIB_INTR_ADMIN, ixv_msix_mbx, adapter, 0, "aq");
1064 device_printf(iflib_get_dev(ctx),
1065 "Failed to register admin handler");
1069 adapter->vector = vector;
1071 * Due to a broken design QEMU will fail to properly
1072 * enable the guest for MSIX unless the vectors in
1073 * the table are all set up, so we must rewrite the
1074 * ENABLE in the MSIX control register again at this
1075 * point to cause it to successfully initialize us.
1077 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) {
1079 pci_find_cap(dev, PCIY_MSIX, &rid);
1080 rid += PCIR_MSIX_CTRL;
1081 msix_ctrl = pci_read_config(dev, rid, 2);
1082 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1083 pci_write_config(dev, rid, msix_ctrl, 2);
1089 iflib_irq_free(ctx, &adapter->irq);
1090 rx_que = adapter->rx_queues;
1091 for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++)
1092 iflib_irq_free(ctx, &rx_que->que_irq);
1095 } /* ixv_if_msix_intr_assign */
1097 /************************************************************************
1098 * ixv_allocate_pci_resources
1099 ************************************************************************/
1101 ixv_allocate_pci_resources(if_ctx_t ctx)
1103 struct adapter *adapter = iflib_get_softc(ctx);
1104 device_t dev = iflib_get_dev(ctx);
1108 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1111 if (!(adapter->pci_mem)) {
1112 device_printf(dev, "Unable to allocate bus resource: memory\n");
1116 adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->pci_mem);
1117 adapter->osdep.mem_bus_space_handle =
1118 rman_get_bushandle(adapter->pci_mem);
1119 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
1122 } /* ixv_allocate_pci_resources */
1124 /************************************************************************
1125 * ixv_free_pci_resources
1126 ************************************************************************/
1128 ixv_free_pci_resources(if_ctx_t ctx)
1130 struct adapter *adapter = iflib_get_softc(ctx);
1131 struct ix_rx_queue *que = adapter->rx_queues;
1132 device_t dev = iflib_get_dev(ctx);
1134 /* Release all msix queue resources */
1135 if (adapter->intr_type == IFLIB_INTR_MSIX)
1136 iflib_irq_free(ctx, &adapter->irq);
1139 for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
1140 iflib_irq_free(ctx, &que->que_irq);
1144 /* Clean the Legacy or Link interrupt last */
1145 if (adapter->pci_mem != NULL)
1146 bus_release_resource(dev, SYS_RES_MEMORY,
1147 PCIR_BAR(0), adapter->pci_mem);
1148 } /* ixv_free_pci_resources */
1150 /************************************************************************
1151 * ixv_setup_interface
1153 * Setup networking device structure and register an interface.
1154 ************************************************************************/
1156 ixv_setup_interface(if_ctx_t ctx)
1158 struct adapter *adapter = iflib_get_softc(ctx);
1159 if_softc_ctx_t scctx = adapter->shared;
1160 struct ifnet *ifp = iflib_get_ifp(ctx);
1162 INIT_DEBUGOUT("ixv_setup_interface: begin");
1164 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
1165 if_setbaudrate(ifp, IF_Gbps(10));
1166 ifp->if_snd.ifq_maxlen = scctx->isc_ntxd[0] - 2;
1169 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR;
1170 ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1171 ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
1174 } /* ixv_setup_interface */
1176 /************************************************************************
1177 * ixv_if_get_counter
1178 ************************************************************************/
1180 ixv_if_get_counter(if_ctx_t ctx, ift_counter cnt)
1182 struct adapter *adapter = iflib_get_softc(ctx);
1183 if_t ifp = iflib_get_ifp(ctx);
1186 case IFCOUNTER_IPACKETS:
1187 return (adapter->ipackets);
1188 case IFCOUNTER_OPACKETS:
1189 return (adapter->opackets);
1190 case IFCOUNTER_IBYTES:
1191 return (adapter->ibytes);
1192 case IFCOUNTER_OBYTES:
1193 return (adapter->obytes);
1194 case IFCOUNTER_IMCASTS:
1195 return (adapter->imcasts);
1197 return (if_get_counter_default(ifp, cnt));
1199 } /* ixv_if_get_counter */
1201 /************************************************************************
1202 * ixv_initialize_transmit_units - Enable transmit unit.
1203 ************************************************************************/
1205 ixv_initialize_transmit_units(if_ctx_t ctx)
1207 struct adapter *adapter = iflib_get_softc(ctx);
1208 struct ixgbe_hw *hw = &adapter->hw;
1209 if_softc_ctx_t scctx = adapter->shared;
1210 struct ix_tx_queue *que = adapter->tx_queues;
1213 for (i = 0; i < adapter->num_tx_queues; i++, que++) {
1214 struct tx_ring *txr = &que->txr;
1215 u64 tdba = txr->tx_paddr;
1219 /* Set WTHRESH to 8, burst writeback */
1220 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1221 txdctl |= (8 << 16);
1222 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1224 /* Set the HW Tx Head and Tail indices */
1225 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0);
1226 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0);
1228 /* Set Tx Tail register */
1229 txr->tail = IXGBE_VFTDT(j);
1231 txr->tx_rs_cidx = txr->tx_rs_pidx = txr->tx_cidx_processed = 0;
1232 for (int k = 0; k < scctx->isc_ntxd[0]; k++)
1233 txr->tx_rsq[k] = QIDX_INVALID;
1235 /* Set Ring parameters */
1236 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1237 (tdba & 0x00000000ffffffffULL));
1238 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1239 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j),
1240 scctx->isc_ntxd[0] * sizeof(struct ixgbe_legacy_tx_desc));
1241 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1242 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
1243 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1246 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1247 txdctl |= IXGBE_TXDCTL_ENABLE;
1248 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1252 } /* ixv_initialize_transmit_units */
1254 /************************************************************************
1255 * ixv_initialize_rss_mapping
1256 ************************************************************************/
1258 ixv_initialize_rss_mapping(struct adapter *adapter)
1260 struct ixgbe_hw *hw = &adapter->hw;
1261 u32 reta = 0, mrqc, rss_key[10];
1264 u32 rss_hash_config;
1266 if (adapter->feat_en & IXGBE_FEATURE_RSS) {
1267 /* Fetch the configured RSS key */
1268 rss_getkey((uint8_t *)&rss_key);
1270 /* set up random bits */
1271 arc4rand(&rss_key, sizeof(rss_key), 0);
1274 /* Now fill out hash function seeds */
1275 for (i = 0; i < 10; i++)
1276 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]);
1278 /* Set up the redirection table */
1279 for (i = 0, j = 0; i < 64; i++, j++) {
1280 if (j == adapter->num_rx_queues)
1283 if (adapter->feat_en & IXGBE_FEATURE_RSS) {
1285 * Fetch the RSS bucket id for the given indirection
1286 * entry. Cap it at the number of configured buckets
1287 * (which is num_rx_queues.)
1289 queue_id = rss_get_indirection_to_bucket(i);
1290 queue_id = queue_id % adapter->num_rx_queues;
1295 * The low 8 bits are for hash value (n+0);
1296 * The next 8 bits are for hash value (n+1), etc.
1299 reta |= ((uint32_t)queue_id) << 24;
1301 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta);
1306 /* Perform hash on these packet types */
1307 if (adapter->feat_en & IXGBE_FEATURE_RSS)
1308 rss_hash_config = rss_gethashconfig();
1311 * Disable UDP - IP fragments aren't currently being handled
1312 * and so we end up with a mix of 2-tuple and 4-tuple
1315 rss_hash_config = RSS_HASHTYPE_RSS_IPV4
1316 | RSS_HASHTYPE_RSS_TCP_IPV4
1317 | RSS_HASHTYPE_RSS_IPV6
1318 | RSS_HASHTYPE_RSS_TCP_IPV6;
1321 mrqc = IXGBE_MRQC_RSSEN;
1322 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
1323 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
1324 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
1325 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
1326 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
1327 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
1328 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
1329 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
1330 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
1331 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n",
1333 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
1334 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n",
1336 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
1337 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
1338 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
1339 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
1340 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
1341 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n",
1343 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc);
1344 } /* ixv_initialize_rss_mapping */
1347 /************************************************************************
1348 * ixv_initialize_receive_units - Setup receive registers and features.
1349 ************************************************************************/
1351 ixv_initialize_receive_units(if_ctx_t ctx)
1353 struct adapter *adapter = iflib_get_softc(ctx);
1354 if_softc_ctx_t scctx;
1355 struct ixgbe_hw *hw = &adapter->hw;
1356 struct ifnet *ifp = iflib_get_ifp(ctx);
1357 struct ix_rx_queue *que = adapter->rx_queues;
1360 if (ifp->if_mtu > ETHERMTU)
1361 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1363 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1365 psrtype = IXGBE_PSRTYPE_TCPHDR
1366 | IXGBE_PSRTYPE_UDPHDR
1367 | IXGBE_PSRTYPE_IPV4HDR
1368 | IXGBE_PSRTYPE_IPV6HDR
1369 | IXGBE_PSRTYPE_L2HDR;
1371 if (adapter->num_rx_queues > 1)
1374 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1376 /* Tell PF our max_frame size */
1377 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) {
1378 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n");
1380 scctx = adapter->shared;
1382 for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
1383 struct rx_ring *rxr = &que->rxr;
1384 u64 rdba = rxr->rx_paddr;
1388 /* Disable the queue */
1389 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1390 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1391 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1392 for (int k = 0; k < 10; k++) {
1393 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
1394 IXGBE_RXDCTL_ENABLE)
1400 /* Setup the Base and Length of the Rx Descriptor Ring */
1401 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1402 (rdba & 0x00000000ffffffffULL));
1403 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1404 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j),
1405 scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));
1407 /* Reset the ring indices */
1408 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
1409 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0);
1411 /* Set up the SRRCTL register */
1412 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j));
1413 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
1414 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
1416 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1417 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg);
1419 /* Capture Rx Tail index */
1420 rxr->tail = IXGBE_VFRDT(rxr->me);
1422 /* Do the queue enabling last */
1423 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1424 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1425 for (int l = 0; l < 10; l++) {
1426 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
1427 IXGBE_RXDCTL_ENABLE)
1433 /* Set the Tail Pointer */
1436 * In netmap mode, we must preserve the buffers made
1437 * available to userspace before the if_init()
1438 * (this is true by default on the TX side, because
1439 * init makes all buffers available to userspace).
1441 * netmap_reset() and the device specific routines
1442 * (e.g. ixgbe_setup_receive_rings()) map these
1443 * buffers at the end of the NIC ring, so here we
1444 * must set the RDT (tail) register to make sure
1445 * they are not overwritten.
1447 * In this driver the NIC ring starts at RDH = 0,
1448 * RDT points to the last slot available for reception (?),
1449 * so RDT = num_rx_desc - 1 means the whole ring is available.
1451 if (ifp->if_capenable & IFCAP_NETMAP) {
1452 struct netmap_adapter *na = NA(ifp);
1453 struct netmap_kring *kring = na->rx_rings[j];
1454 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1456 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t);
1458 #endif /* DEV_NETMAP */
1459 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
1460 scctx->isc_nrxd[0] - 1);
1463 ixv_initialize_rss_mapping(adapter);
1464 } /* ixv_initialize_receive_units */
1466 /************************************************************************
1467 * ixv_setup_vlan_support
1468 ************************************************************************/
1470 ixv_setup_vlan_support(if_ctx_t ctx)
1472 struct adapter *adapter = iflib_get_softc(ctx);
1473 struct ixgbe_hw *hw = &adapter->hw;
1474 u32 ctrl, vid, vfta, retry;
1477 * We get here thru if_init, meaning
1478 * a soft reset, this has already cleared
1479 * the VFTA and other state, so if there
1480 * have been no vlan's registered do nothing.
1482 if (adapter->num_vlans == 0)
1485 /* Enable the queues */
1486 for (int i = 0; i < adapter->num_rx_queues; i++) {
1487 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
1488 ctrl |= IXGBE_RXDCTL_VME;
1489 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl);
1491 * Let Rx path know that it needs to store VLAN tag
1492 * as part of extra mbuf info.
1494 adapter->rx_queues[i].rxr.vtag_strip = TRUE;
1498 * A soft reset zero's out the VFTA, so
1499 * we need to repopulate it now.
1501 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) {
1502 if (ixv_shadow_vfta[i] == 0)
1504 vfta = ixv_shadow_vfta[i];
1506 * Reconstruct the vlan id's
1507 * based on the bits set in each
1508 * of the array ints.
1510 for (int j = 0; j < 32; j++) {
1512 if ((vfta & (1 << j)) == 0)
1515 /* Call the shared code mailbox routine */
1516 while (hw->mac.ops.set_vfta(hw, vid, 0, TRUE, FALSE)) {
1522 } /* ixv_setup_vlan_support */
1524 /************************************************************************
1525 * ixv_if_register_vlan
1527 * Run via a vlan config EVENT, it enables us to use the
1528 * HW Filter table since we can get the vlan id. This just
1529 * creates the entry in the soft version of the VFTA, init
1530 * will repopulate the real table.
1531 ************************************************************************/
1533 ixv_if_register_vlan(if_ctx_t ctx, u16 vtag)
1535 struct adapter *adapter = iflib_get_softc(ctx);
1538 index = (vtag >> 5) & 0x7F;
1540 ixv_shadow_vfta[index] |= (1 << bit);
1541 ++adapter->num_vlans;
1542 } /* ixv_if_register_vlan */
1544 /************************************************************************
1545 * ixv_if_unregister_vlan
1547 * Run via a vlan unconfig EVENT, remove our entry
1549 ************************************************************************/
1551 ixv_if_unregister_vlan(if_ctx_t ctx, u16 vtag)
1553 struct adapter *adapter = iflib_get_softc(ctx);
1556 index = (vtag >> 5) & 0x7F;
1558 ixv_shadow_vfta[index] &= ~(1 << bit);
1559 --adapter->num_vlans;
1560 } /* ixv_if_unregister_vlan */
1562 /************************************************************************
1563 * ixv_if_enable_intr
1564 ************************************************************************/
1566 ixv_if_enable_intr(if_ctx_t ctx)
1568 struct adapter *adapter = iflib_get_softc(ctx);
1569 struct ixgbe_hw *hw = &adapter->hw;
1570 struct ix_rx_queue *que = adapter->rx_queues;
1571 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1573 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
1575 mask = IXGBE_EIMS_ENABLE_MASK;
1576 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1577 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
1579 for (int i = 0; i < adapter->num_rx_queues; i++, que++)
1580 ixv_enable_queue(adapter, que->msix);
1582 IXGBE_WRITE_FLUSH(hw);
1583 } /* ixv_if_enable_intr */
1585 /************************************************************************
1586 * ixv_if_disable_intr
1587 ************************************************************************/
1589 ixv_if_disable_intr(if_ctx_t ctx)
1591 struct adapter *adapter = iflib_get_softc(ctx);
1592 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0);
1593 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0);
1594 IXGBE_WRITE_FLUSH(&adapter->hw);
1595 } /* ixv_if_disable_intr */
1597 /************************************************************************
1598 * ixv_if_rx_queue_intr_enable
1599 ************************************************************************/
1601 ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
1603 struct adapter *adapter = iflib_get_softc(ctx);
1604 struct ix_rx_queue *que = &adapter->rx_queues[rxqid];
1606 ixv_enable_queue(adapter, que->rxr.me);
1609 } /* ixv_if_rx_queue_intr_enable */
1611 /************************************************************************
1614 * Setup the correct IVAR register for a particular MSI-X interrupt
1615 * - entry is the register array entry
1616 * - vector is the MSI-X vector for this queue
1617 * - type is RX/TX/MISC
1618 ************************************************************************/
1620 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
1622 struct ixgbe_hw *hw = &adapter->hw;
1625 vector |= IXGBE_IVAR_ALLOC_VAL;
1627 if (type == -1) { /* MISC IVAR */
1628 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
1631 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
1632 } else { /* RX/TX IVARS */
1633 index = (16 * (entry & 1)) + (8 * type);
1634 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
1635 ivar &= ~(0xFF << index);
1636 ivar |= (vector << index);
1637 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
1639 } /* ixv_set_ivar */
1641 /************************************************************************
1642 * ixv_configure_ivars
1643 ************************************************************************/
1645 ixv_configure_ivars(struct adapter *adapter)
1647 struct ix_rx_queue *que = adapter->rx_queues;
1649 MPASS(adapter->num_rx_queues == adapter->num_tx_queues);
1651 for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
1652 /* First the RX queue entry */
1653 ixv_set_ivar(adapter, i, que->msix, 0);
1654 /* ... and the TX */
1655 ixv_set_ivar(adapter, i, que->msix, 1);
1656 /* Set an initial value in EITR */
1657 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(que->msix),
1658 IXGBE_EITR_DEFAULT);
1661 /* For the mailbox interrupt */
1662 ixv_set_ivar(adapter, 1, adapter->vector, -1);
1663 } /* ixv_configure_ivars */
1665 /************************************************************************
1668 * The VF stats registers never have a truly virgin
1669 * starting point, so this routine tries to make an
1670 * artificial one, marking ground zero on attach as
1672 ************************************************************************/
1674 ixv_save_stats(struct adapter *adapter)
1676 if (adapter->stats.vf.vfgprc || adapter->stats.vf.vfgptc) {
1677 adapter->stats.vf.saved_reset_vfgprc +=
1678 adapter->stats.vf.vfgprc - adapter->stats.vf.base_vfgprc;
1679 adapter->stats.vf.saved_reset_vfgptc +=
1680 adapter->stats.vf.vfgptc - adapter->stats.vf.base_vfgptc;
1681 adapter->stats.vf.saved_reset_vfgorc +=
1682 adapter->stats.vf.vfgorc - adapter->stats.vf.base_vfgorc;
1683 adapter->stats.vf.saved_reset_vfgotc +=
1684 adapter->stats.vf.vfgotc - adapter->stats.vf.base_vfgotc;
1685 adapter->stats.vf.saved_reset_vfmprc +=
1686 adapter->stats.vf.vfmprc - adapter->stats.vf.base_vfmprc;
1688 } /* ixv_save_stats */
1690 /************************************************************************
1692 ************************************************************************/
1694 ixv_init_stats(struct adapter *adapter)
1696 struct ixgbe_hw *hw = &adapter->hw;
1698 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1699 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1700 adapter->stats.vf.last_vfgorc |=
1701 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1703 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1704 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1705 adapter->stats.vf.last_vfgotc |=
1706 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1708 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1710 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc;
1711 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc;
1712 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc;
1713 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc;
1714 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc;
1715 } /* ixv_init_stats */
1717 #define UPDATE_STAT_32(reg, last, count) \
1719 u32 current = IXGBE_READ_REG(hw, reg); \
1720 if (current < last) \
1721 count += 0x100000000LL; \
1723 count &= 0xFFFFFFFF00000000LL; \
1727 #define UPDATE_STAT_36(lsb, msb, last, count) \
1729 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \
1730 u64 cur_msb = IXGBE_READ_REG(hw, msb); \
1731 u64 current = ((cur_msb << 32) | cur_lsb); \
1732 if (current < last) \
1733 count += 0x1000000000LL; \
1735 count &= 0xFFFFFFF000000000LL; \
1739 /************************************************************************
1740 * ixv_update_stats - Update the board statistics counters.
1741 ************************************************************************/
1743 ixv_update_stats(struct adapter *adapter)
1745 struct ixgbe_hw *hw = &adapter->hw;
1746 struct ixgbevf_hw_stats *stats = &adapter->stats.vf;
1748 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.vf.last_vfgprc,
1749 adapter->stats.vf.vfgprc);
1750 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.vf.last_vfgptc,
1751 adapter->stats.vf.vfgptc);
1752 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1753 adapter->stats.vf.last_vfgorc, adapter->stats.vf.vfgorc);
1754 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1755 adapter->stats.vf.last_vfgotc, adapter->stats.vf.vfgotc);
1756 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.vf.last_vfmprc,
1757 adapter->stats.vf.vfmprc);
1759 /* Fill out the OS statistics structure */
1760 IXGBE_SET_IPACKETS(adapter, stats->vfgprc);
1761 IXGBE_SET_OPACKETS(adapter, stats->vfgptc);
1762 IXGBE_SET_IBYTES(adapter, stats->vfgorc);
1763 IXGBE_SET_OBYTES(adapter, stats->vfgotc);
1764 IXGBE_SET_IMCASTS(adapter, stats->vfmprc);
1765 } /* ixv_update_stats */
1767 /************************************************************************
1768 * ixv_add_stats_sysctls - Add statistic sysctls for the VF.
1769 ************************************************************************/
1771 ixv_add_stats_sysctls(struct adapter *adapter)
1773 device_t dev = adapter->dev;
1774 struct ix_tx_queue *tx_que = adapter->tx_queues;
1775 struct ix_rx_queue *rx_que = adapter->rx_queues;
1776 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
1777 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
1778 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
1779 struct ixgbevf_hw_stats *stats = &adapter->stats.vf;
1780 struct sysctl_oid *stat_node, *queue_node;
1781 struct sysctl_oid_list *stat_list, *queue_list;
1783 #define QUEUE_NAME_LEN 32
1784 char namebuf[QUEUE_NAME_LEN];
1786 /* Driver Statistics */
1787 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
1788 CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts");
1789 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
1790 CTLFLAG_RD, &adapter->link_irq, "Link MSI-X IRQ Handled");
1792 for (int i = 0; i < adapter->num_tx_queues; i++, tx_que++) {
1793 struct tx_ring *txr = &tx_que->txr;
1794 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1795 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1796 CTLFLAG_RD, NULL, "Queue Name");
1797 queue_list = SYSCTL_CHILDREN(queue_node);
1799 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
1800 CTLFLAG_RD, &(txr->tso_tx), "TSO Packets");
1801 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
1802 CTLFLAG_RD, &(txr->total_packets), "TX Packets");
1805 for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++) {
1806 struct rx_ring *rxr = &rx_que->rxr;
1807 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1808 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1809 CTLFLAG_RD, NULL, "Queue Name");
1810 queue_list = SYSCTL_CHILDREN(queue_node);
1812 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
1813 CTLFLAG_RD, &(rx_que->irqs), "IRQs on queue");
1814 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
1815 CTLFLAG_RD, &(rxr->rx_packets), "RX packets");
1816 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
1817 CTLFLAG_RD, &(rxr->rx_bytes), "RX bytes");
1818 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
1819 CTLFLAG_RD, &(rxr->rx_discarded), "Discarded RX packets");
1822 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
1823 CTLFLAG_RD, NULL, "VF Statistics (read from HW registers)");
1824 stat_list = SYSCTL_CHILDREN(stat_node);
1826 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
1827 CTLFLAG_RD, &stats->vfgprc, "Good Packets Received");
1828 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
1829 CTLFLAG_RD, &stats->vfgorc, "Good Octets Received");
1830 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
1831 CTLFLAG_RD, &stats->vfmprc, "Multicast Packets Received");
1832 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
1833 CTLFLAG_RD, &stats->vfgptc, "Good Packets Transmitted");
1834 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
1835 CTLFLAG_RD, &stats->vfgotc, "Good Octets Transmitted");
1836 } /* ixv_add_stats_sysctls */
1838 /************************************************************************
1839 * ixv_print_debug_info
1841 * Called only when em_display_debug_stats is enabled.
1842 * Provides a way to take a look at important statistics
1843 * maintained by the driver and hardware.
1844 ************************************************************************/
1846 ixv_print_debug_info(struct adapter *adapter)
1848 device_t dev = adapter->dev;
1849 struct ixgbe_hw *hw = &adapter->hw;
1851 device_printf(dev, "Error Byte Count = %u \n",
1852 IXGBE_READ_REG(hw, IXGBE_ERRBC));
1854 device_printf(dev, "MBX IRQ Handled: %lu\n", (long)adapter->link_irq);
1855 } /* ixv_print_debug_info */
1857 /************************************************************************
1859 ************************************************************************/
1861 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)
1863 struct adapter *adapter;
1867 error = sysctl_handle_int(oidp, &result, 0, req);
1869 if (error || !req->newptr)
1873 adapter = (struct adapter *)arg1;
1874 ixv_print_debug_info(adapter);
1878 } /* ixv_sysctl_debug */
1880 /************************************************************************
1881 * ixv_init_device_features
1882 ************************************************************************/
1884 ixv_init_device_features(struct adapter *adapter)
1886 adapter->feat_cap = IXGBE_FEATURE_NETMAP
1889 | IXGBE_FEATURE_LEGACY_TX;
1891 /* A tad short on feature flags for VFs, atm. */
1892 switch (adapter->hw.mac.type) {
1893 case ixgbe_mac_82599_vf:
1895 case ixgbe_mac_X540_vf:
1897 case ixgbe_mac_X550_vf:
1898 case ixgbe_mac_X550EM_x_vf:
1899 case ixgbe_mac_X550EM_a_vf:
1900 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD;
1906 /* Enabled by default... */
1907 /* Is a virtual function (VF) */
1908 if (adapter->feat_cap & IXGBE_FEATURE_VF)
1909 adapter->feat_en |= IXGBE_FEATURE_VF;
1911 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP)
1912 adapter->feat_en |= IXGBE_FEATURE_NETMAP;
1913 /* Receive-Side Scaling (RSS) */
1914 if (adapter->feat_cap & IXGBE_FEATURE_RSS)
1915 adapter->feat_en |= IXGBE_FEATURE_RSS;
1916 /* Needs advanced context descriptor regardless of offloads req'd */
1917 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD)
1918 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD;
1919 } /* ixv_init_device_features */