sys/dev/ixgbe/if_ix.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2017, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37 #include "opt_rss.h"
  38
  39 #include "ixgbe.h"
  40 #include "ixgbe_sriov.h"
  41 #include "ifdi_if.h"
  42
  43 #include <net/netmap.h>
  44 #include <dev/netmap/netmap_kern.h>
  45
  46 /************************************************************************
  47  * Driver version
  48  ************************************************************************/
  49 char ixgbe_driver_version[] = "4.0.1-k";
  50
  51 /************************************************************************
  52  * PCI Device ID Table
  53  *
  54  *   Used by probe to select devices to load on
  55  *   Last field stores an index into ixgbe_strings
  56  *   Last entry must be all 0s
  57  *
  58  *   { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  59  ************************************************************************/
  60 static pci_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT,  "Intel(R) 82598EB AF (Dual Fiber)"),
  63   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT,  "Intel(R) 82598EB AF (Fiber)"),
  64   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4,  "Intel(R) 82598EB AT (CX4)"),
  65   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT,  "Intel(R) 82598EB AT"),
  66   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2,  "Intel(R) 82598EB AT2"),
  67   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598,  "Intel(R) 82598"),
  68   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT,  "Intel(R) 82598EB AF DA (Dual Fiber)"),
  69   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT,  "Intel(R) 82598EB AT (Dual CX4)"),
  70   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR,  "Intel(R) 82598EB AF (Dual Fiber LR)"),
  71   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM,  "Intel(R) 82598EB AF (Dual Fiber SR)"),
  72   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM,  "Intel(R) 82598EB LOM"),
  73   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4,  "Intel(R) X520 82599 (KX4)"),
  74   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ,  "Intel(R) X520 82599 (KX4 Mezzanine)"),
  75   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP,  "Intel(R) X520 82599ES (SFI/SFP+)"),
  76   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM,  "Intel(R) X520 82599 (XAUI/BX4)"),
  77   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4,  "Intel(R) X520 82599 (Dual CX4)"),
  78   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM,  "Intel(R) X520-T 82599 LOM"),
  79   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE,  "Intel(R) X520 82599 (Combined Backplane)"),
  80   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE,  "Intel(R) X520 82599 (Backplane w/FCoE)"),
  81   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2,  "Intel(R) X520 82599 (Dual SFP+)"),
  82   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE,  "Intel(R) X520 82599 (Dual SFP+ w/FCoE)"),
  83   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP,  "Intel(R) X520-1 82599EN (SFP+)"),
  84   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP,  "Intel(R) X520-4 82599 (Quad SFP+)"),
  85   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP,  "Intel(R) X520-Q1 82599 (QSFP+)"),
  86   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T,  "Intel(R) X540-AT2"),
  87   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1,  "Intel(R) X540-T1"),
  88   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T,  "Intel(R) X550-T2"),
  89   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, "Intel(R) X550-T1"),
  90   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR,  "Intel(R) X552 (KR Backplane)"),
  91   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4,  "Intel(R) X552 (KX4 Backplane)"),
  92   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T,  "Intel(R) X552/X557-AT (10GBASE-T)"),
  93   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T,  "Intel(R) X552 (1000BASE-T)"),
  94   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, "Intel(R) X552 (SFP+)"),
  95   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, "Intel(R) X553 (KR Backplane)"),
  96   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, "Intel(R) X553 L (KR Backplane)"),
  97   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, "Intel(R) X553 (SFP+)"),
  98   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, "Intel(R) X553 N (SFP+)"),
  99   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, "Intel(R) X553 (1GbE SGMII)"),
 100   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, "Intel(R) X553 L (1GbE SGMII)"),
 101   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, "Intel(R) X553/X557-AT (10GBASE-T)"),
 102   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, "Intel(R) X553 (1GbE)"),
 103   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, "Intel(R) X553 L (1GbE)"),
 104   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, "Intel(R) X540-T2 (Bypass)"),
 105   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, "Intel(R) X520 82599 (Bypass)"),
 106         /* required last entry */
 107   PVID_END
 108 };
 109
 110 static void *ixgbe_register(device_t);
 111 static int  ixgbe_if_attach_pre(if_ctx_t);
 112 static int  ixgbe_if_attach_post(if_ctx_t);
 113 static int  ixgbe_if_detach(if_ctx_t);
 114 static int  ixgbe_if_shutdown(if_ctx_t);
 115 static int  ixgbe_if_suspend(if_ctx_t);
 116 static int  ixgbe_if_resume(if_ctx_t);
 117
 118 static void ixgbe_if_stop(if_ctx_t);
 119 void ixgbe_if_enable_intr(if_ctx_t);
 120 static void ixgbe_if_disable_intr(if_ctx_t);
 121 static void ixgbe_link_intr_enable(if_ctx_t);
 122 static int  ixgbe_if_rx_queue_intr_enable(if_ctx_t, uint16_t);
 123 static void ixgbe_if_media_status(if_ctx_t, struct ifmediareq *);
 124 static int  ixgbe_if_media_change(if_ctx_t);
 125 static int  ixgbe_if_msix_intr_assign(if_ctx_t, int);
 126 static int  ixgbe_if_mtu_set(if_ctx_t, uint32_t);
 127 static void ixgbe_if_crcstrip_set(if_ctx_t, int, int);
 128 static void ixgbe_if_multi_set(if_ctx_t);
 129 static int  ixgbe_if_promisc_set(if_ctx_t, int);
 130 static int  ixgbe_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
 131 static int  ixgbe_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
 132 static void ixgbe_if_queues_free(if_ctx_t);
 133 static void ixgbe_if_timer(if_ctx_t, uint16_t);
 134 static void ixgbe_if_update_admin_status(if_ctx_t);
 135 static void ixgbe_if_vlan_register(if_ctx_t, u16);
 136 static void ixgbe_if_vlan_unregister(if_ctx_t, u16);
 137 static int  ixgbe_if_i2c_req(if_ctx_t, struct ifi2creq *);
 138 static bool ixgbe_if_needs_restart(if_ctx_t, enum iflib_restart_event);
 139 int ixgbe_intr(void *);
 140
 141 /************************************************************************
 142  * Function prototypes
 143  ************************************************************************/
 144 static uint64_t ixgbe_if_get_counter(if_ctx_t, ift_counter);
 145
 146 static void ixgbe_enable_queue(struct ixgbe_softc *, u32);
 147 static void ixgbe_disable_queue(struct ixgbe_softc *, u32);
 148 static void ixgbe_add_device_sysctls(if_ctx_t);
 149 static int  ixgbe_allocate_pci_resources(if_ctx_t);
 150 static int  ixgbe_setup_low_power_mode(if_ctx_t);
 151
 152 static void ixgbe_config_dmac(struct ixgbe_softc *);
 153 static void ixgbe_configure_ivars(struct ixgbe_softc *);
 154 static void ixgbe_set_ivar(struct ixgbe_softc *, u8, u8, s8);
 155 static u8   *ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 156 static bool ixgbe_sfp_probe(if_ctx_t);
 157
 158 static void ixgbe_free_pci_resources(if_ctx_t);
 159
 160 static int  ixgbe_msix_link(void *);
 161 static int  ixgbe_msix_que(void *);
 162 static void ixgbe_initialize_rss_mapping(struct ixgbe_softc *);
 163 static void ixgbe_initialize_receive_units(if_ctx_t);
 164 static void ixgbe_initialize_transmit_units(if_ctx_t);
 165
 166 static int  ixgbe_setup_interface(if_ctx_t);
 167 static void ixgbe_init_device_features(struct ixgbe_softc *);
 168 static void ixgbe_check_fan_failure(struct ixgbe_softc *, u32, bool);
 169 static void ixgbe_sbuf_fw_version(struct ixgbe_hw *, struct sbuf *);
 170 static void ixgbe_print_fw_version(if_ctx_t);
 171 static void ixgbe_add_media_types(if_ctx_t);
 172 static void ixgbe_update_stats_counters(struct ixgbe_softc *);
 173 static void ixgbe_config_link(if_ctx_t);
 174 static void ixgbe_get_slot_info(struct ixgbe_softc *);
 175 static void ixgbe_check_wol_support(struct ixgbe_softc *);
 176 static void ixgbe_enable_rx_drop(struct ixgbe_softc *);
 177 static void ixgbe_disable_rx_drop(struct ixgbe_softc *);
 178
 179 static void ixgbe_add_hw_stats(struct ixgbe_softc *);
 180 static int  ixgbe_set_flowcntl(struct ixgbe_softc *, int);
 181 static int  ixgbe_set_advertise(struct ixgbe_softc *, int);
 182 static int  ixgbe_get_default_advertise(struct ixgbe_softc *);
 183 static void ixgbe_setup_vlan_hw_support(if_ctx_t);
 184 static void ixgbe_config_gpie(struct ixgbe_softc *);
 185 static void ixgbe_config_delay_values(struct ixgbe_softc *);
 186
 187 /* Sysctl handlers */
 188 static int  ixgbe_sysctl_flowcntl(SYSCTL_HANDLER_ARGS);
 189 static int  ixgbe_sysctl_advertise(SYSCTL_HANDLER_ARGS);
 190 static int  ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS);
 191 static int  ixgbe_sysctl_dmac(SYSCTL_HANDLER_ARGS);
 192 static int  ixgbe_sysctl_phy_temp(SYSCTL_HANDLER_ARGS);
 193 static int  ixgbe_sysctl_phy_overtemp_occurred(SYSCTL_HANDLER_ARGS);
 194 static int  ixgbe_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS);
 195 #ifdef IXGBE_DEBUG
 196 static int  ixgbe_sysctl_power_state(SYSCTL_HANDLER_ARGS);
 197 static int  ixgbe_sysctl_print_rss_config(SYSCTL_HANDLER_ARGS);
 198 #endif
 199 static int  ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS);
 200 static int  ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS);
 201 static int  ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS);
 202 static int  ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS);
 203 static int  ixgbe_sysctl_eee_state(SYSCTL_HANDLER_ARGS);
 204 static int  ixgbe_sysctl_wol_enable(SYSCTL_HANDLER_ARGS);
 205 static int  ixgbe_sysctl_wufc(SYSCTL_HANDLER_ARGS);
 206
 207 /* Deferred interrupt tasklets */
 208 static void ixgbe_handle_msf(void *);
 209 static void ixgbe_handle_mod(void *);
 210 static void ixgbe_handle_phy(void *);
 211
 212 /************************************************************************
 213  *  FreeBSD Device Interface Entry Points
 214  ************************************************************************/
 215 static device_method_t ix_methods[] = {
 216         /* Device interface */
 217         DEVMETHOD(device_register, ixgbe_register),
 218         DEVMETHOD(device_probe, iflib_device_probe),
 219         DEVMETHOD(device_attach, iflib_device_attach),
 220         DEVMETHOD(device_detach, iflib_device_detach),
 221         DEVMETHOD(device_shutdown, iflib_device_shutdown),
 222         DEVMETHOD(device_suspend, iflib_device_suspend),
 223         DEVMETHOD(device_resume, iflib_device_resume),
 224 #ifdef PCI_IOV
 225         DEVMETHOD(pci_iov_init, iflib_device_iov_init),
 226         DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit),
 227         DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf),
 228 #endif /* PCI_IOV */
 229         DEVMETHOD_END
 230 };
 231
 232 static driver_t ix_driver = {
 233         "ix", ix_methods, sizeof(struct ixgbe_softc),
 234 };
 235
 236 devclass_t ix_devclass;
 237 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0);
 238 IFLIB_PNP_INFO(pci, ix_driver, ixgbe_vendor_info_array);
 239 MODULE_DEPEND(ix, pci, 1, 1, 1);
 240 MODULE_DEPEND(ix, ether, 1, 1, 1);
 241 MODULE_DEPEND(ix, iflib, 1, 1, 1);
 242
 243 static device_method_t ixgbe_if_methods[] = {
 244         DEVMETHOD(ifdi_attach_pre, ixgbe_if_attach_pre),
 245         DEVMETHOD(ifdi_attach_post, ixgbe_if_attach_post),
 246         DEVMETHOD(ifdi_detach, ixgbe_if_detach),
 247         DEVMETHOD(ifdi_shutdown, ixgbe_if_shutdown),
 248         DEVMETHOD(ifdi_suspend, ixgbe_if_suspend),
 249         DEVMETHOD(ifdi_resume, ixgbe_if_resume),
 250         DEVMETHOD(ifdi_init, ixgbe_if_init),
 251         DEVMETHOD(ifdi_stop, ixgbe_if_stop),
 252         DEVMETHOD(ifdi_msix_intr_assign, ixgbe_if_msix_intr_assign),
 253         DEVMETHOD(ifdi_intr_enable, ixgbe_if_enable_intr),
 254         DEVMETHOD(ifdi_intr_disable, ixgbe_if_disable_intr),
 255         DEVMETHOD(ifdi_link_intr_enable, ixgbe_link_intr_enable),
 256         DEVMETHOD(ifdi_tx_queue_intr_enable, ixgbe_if_rx_queue_intr_enable),
 257         DEVMETHOD(ifdi_rx_queue_intr_enable, ixgbe_if_rx_queue_intr_enable),
 258         DEVMETHOD(ifdi_tx_queues_alloc, ixgbe_if_tx_queues_alloc),
 259         DEVMETHOD(ifdi_rx_queues_alloc, ixgbe_if_rx_queues_alloc),
 260         DEVMETHOD(ifdi_queues_free, ixgbe_if_queues_free),
 261         DEVMETHOD(ifdi_update_admin_status, ixgbe_if_update_admin_status),
 262         DEVMETHOD(ifdi_multi_set, ixgbe_if_multi_set),
 263         DEVMETHOD(ifdi_mtu_set, ixgbe_if_mtu_set),
 264         DEVMETHOD(ifdi_crcstrip_set, ixgbe_if_crcstrip_set),
 265         DEVMETHOD(ifdi_media_status, ixgbe_if_media_status),
 266         DEVMETHOD(ifdi_media_change, ixgbe_if_media_change),
 267         DEVMETHOD(ifdi_promisc_set, ixgbe_if_promisc_set),
 268         DEVMETHOD(ifdi_timer, ixgbe_if_timer),
 269         DEVMETHOD(ifdi_vlan_register, ixgbe_if_vlan_register),
 270         DEVMETHOD(ifdi_vlan_unregister, ixgbe_if_vlan_unregister),
 271         DEVMETHOD(ifdi_get_counter, ixgbe_if_get_counter),
 272         DEVMETHOD(ifdi_i2c_req, ixgbe_if_i2c_req),
 273         DEVMETHOD(ifdi_needs_restart, ixgbe_if_needs_restart),
 274 #ifdef PCI_IOV
 275         DEVMETHOD(ifdi_iov_init, ixgbe_if_iov_init),
 276         DEVMETHOD(ifdi_iov_uninit, ixgbe_if_iov_uninit),
 277         DEVMETHOD(ifdi_iov_vf_add, ixgbe_if_iov_vf_add),
 278 #endif /* PCI_IOV */
 279         DEVMETHOD_END
 280 };
 281
 282 /*
 283  * TUNEABLE PARAMETERS:
 284  */
 285
 286 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 287     "IXGBE driver parameters");
 288 static driver_t ixgbe_if_driver = {
 289   "ixgbe_if", ixgbe_if_methods, sizeof(struct ixgbe_softc)
 290 };
 291
 292 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 293 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
 294     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
 295
 296 /* Flow control setting, default to full */
 297 static int ixgbe_flow_control = ixgbe_fc_full;
 298 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN,
 299     &ixgbe_flow_control, 0, "Default flow control used for all adapters");
 300
 301 /* Advertise Speed, default to 0 (auto) */
 302 static int ixgbe_advertise_speed = 0;
 303 SYSCTL_INT(_hw_ix, OID_AUTO, advertise_speed, CTLFLAG_RDTUN,
 304     &ixgbe_advertise_speed, 0, "Default advertised speed for all adapters");
 305
 306 /*
 307  * Smart speed setting, default to on
 308  * this only works as a compile option
 309  * right now as its during attach, set
 310  * this to 'ixgbe_smart_speed_off' to
 311  * disable.
 312  */
 313 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 314
 315 /*
 316  * MSI-X should be the default for best performance,
 317  * but this allows it to be forced off for testing.
 318  */
 319 static int ixgbe_enable_msix = 1;
 320 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
 321     "Enable MSI-X interrupts");
 322
 323 /*
 324  * Defining this on will allow the use
 325  * of unsupported SFP+ modules, note that
 326  * doing so you are on your own :)
 327  */
 328 static int allow_unsupported_sfp = false;
 329 SYSCTL_INT(_hw_ix, OID_AUTO, unsupported_sfp, CTLFLAG_RDTUN,
 330     &allow_unsupported_sfp, 0,
 331     "Allow unsupported SFP modules...use at your own risk");
 332
 333 /*
 334  * Not sure if Flow Director is fully baked,
 335  * so we'll default to turning it off.
 336  */
 337 static int ixgbe_enable_fdir = 0;
 338 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0,
 339     "Enable Flow Director");
 340
 341 /* Receive-Side Scaling */
 342 static int ixgbe_enable_rss = 1;
 343 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0,
 344     "Enable Receive-Side Scaling (RSS)");
 345
 346 /*
 347  * AIM: Adaptive Interrupt Moderation
 348  * which means that the interrupt rate
 349  * is varied over time based on the
 350  * traffic for that interrupt vector
 351  */
 352 static int ixgbe_enable_aim = false;
 353 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &ixgbe_enable_aim, 0,
 354     "Enable adaptive interrupt moderation");
 355
 356 #if 0
 357 /* Keep running tab on them for sanity check */
 358 static int ixgbe_total_ports;
 359 #endif
 360
 361 MALLOC_DEFINE(M_IXGBE, "ix", "ix driver allocations");
 362
 363 /*
 364  * For Flow Director: this is the number of TX packets we sample
 365  * for the filter pool, this means every 20th packet will be probed.
 366  *
 367  * This feature can be disabled by setting this to 0.
 368  */
 369 static int atr_sample_rate = 20;
 370
 371 extern struct if_txrx ixgbe_txrx;
 372
 373 static struct if_shared_ctx ixgbe_sctx_init = {
 374         .isc_magic = IFLIB_MAGIC,
 375         .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
 376         .isc_tx_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 377         .isc_tx_maxsegsize = PAGE_SIZE,
 378         .isc_tso_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 379         .isc_tso_maxsegsize = PAGE_SIZE,
 380         .isc_rx_maxsize = PAGE_SIZE*4,
 381         .isc_rx_nsegments = 1,
 382         .isc_rx_maxsegsize = PAGE_SIZE*4,
 383         .isc_nfl = 1,
 384         .isc_ntxqs = 1,
 385         .isc_nrxqs = 1,
 386
 387         .isc_admin_intrcnt = 1,
 388         .isc_vendor_info = ixgbe_vendor_info_array,
 389         .isc_driver_version = ixgbe_driver_version,
 390         .isc_driver = &ixgbe_if_driver,
 391         .isc_flags = IFLIB_TSO_INIT_IP,
 392
 393         .isc_nrxd_min = {MIN_RXD},
 394         .isc_ntxd_min = {MIN_TXD},
 395         .isc_nrxd_max = {MAX_RXD},
 396         .isc_ntxd_max = {MAX_TXD},
 397         .isc_nrxd_default = {DEFAULT_RXD},
 398         .isc_ntxd_default = {DEFAULT_TXD},
 399 };
 400
 401 /************************************************************************
 402  * ixgbe_if_tx_queues_alloc
 403  ************************************************************************/
 404 static int
 405 ixgbe_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
 406     int ntxqs, int ntxqsets)
 407 {
 408         struct ixgbe_softc *sc = iflib_get_softc(ctx);
 409         if_softc_ctx_t     scctx = sc->shared;
 410         struct ix_tx_queue *que;
 411         int                i, j, error;
 412
 413         MPASS(sc->num_tx_queues > 0);
 414         MPASS(sc->num_tx_queues == ntxqsets);
 415         MPASS(ntxqs == 1);
 416
 417         /* Allocate queue structure memory */
 418         sc->tx_queues =
 419             (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * ntxqsets,
 420                                          M_IXGBE, M_NOWAIT | M_ZERO);
 421         if (!sc->tx_queues) {
 422                 device_printf(iflib_get_dev(ctx),
 423                     "Unable to allocate TX ring memory\n");
 424                 return (ENOMEM);
 425         }
 426
 427         for (i = 0, que = sc->tx_queues; i < ntxqsets; i++, que++) {
 428                 struct tx_ring *txr = &que->txr;
 429
 430                 /* In case SR-IOV is enabled, align the index properly */
 431                 txr->me = ixgbe_vf_que_index(sc->iov_mode, sc->pool,
 432                     i);
 433
 434                 txr->sc = que->sc = sc;
 435
 436                 /* Allocate report status array */
 437                 txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_IXGBE, M_NOWAIT | M_ZERO);
 438                 if (txr->tx_rsq == NULL) {
 439                         error = ENOMEM;
 440                         goto fail;
 441                 }
 442                 for (j = 0; j < scctx->isc_ntxd[0]; j++)
 443                         txr->tx_rsq[j] = QIDX_INVALID;
 444                 /* get the virtual and physical address of the hardware queues */
 445                 txr->tail = IXGBE_TDT(txr->me);
 446                 txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i];
 447                 txr->tx_paddr = paddrs[i];
 448
 449                 txr->bytes = 0;
 450                 txr->total_packets = 0;
 451
 452                 /* Set the rate at which we sample packets */
 453                 if (sc->feat_en & IXGBE_FEATURE_FDIR)
 454                         txr->atr_sample = atr_sample_rate;
 455
 456         }
 457
 458         device_printf(iflib_get_dev(ctx), "allocated for %d queues\n",
 459             sc->num_tx_queues);
 460
 461         return (0);
 462
 463 fail:
 464         ixgbe_if_queues_free(ctx);
 465
 466         return (error);
 467 } /* ixgbe_if_tx_queues_alloc */
 468
 469 /************************************************************************
 470  * ixgbe_if_rx_queues_alloc
 471  ************************************************************************/
 472 static int
 473 ixgbe_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
 474     int nrxqs, int nrxqsets)
 475 {
 476         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 477         struct ix_rx_queue *que;
 478         int                i;
 479
 480         MPASS(sc->num_rx_queues > 0);
 481         MPASS(sc->num_rx_queues == nrxqsets);
 482         MPASS(nrxqs == 1);
 483
 484         /* Allocate queue structure memory */
 485         sc->rx_queues =
 486             (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue)*nrxqsets,
 487                                          M_IXGBE, M_NOWAIT | M_ZERO);
 488         if (!sc->rx_queues) {
 489                 device_printf(iflib_get_dev(ctx),
 490                     "Unable to allocate TX ring memory\n");
 491                 return (ENOMEM);
 492         }
 493
 494         for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) {
 495                 struct rx_ring *rxr = &que->rxr;
 496
 497                 /* In case SR-IOV is enabled, align the index properly */
 498                 rxr->me = ixgbe_vf_que_index(sc->iov_mode, sc->pool,
 499                     i);
 500
 501                 rxr->sc = que->sc = sc;
 502
 503                 /* get the virtual and physical address of the hw queues */
 504                 rxr->tail = IXGBE_RDT(rxr->me);
 505                 rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];
 506                 rxr->rx_paddr = paddrs[i];
 507                 rxr->bytes = 0;
 508                 rxr->que = que;
 509         }
 510
 511         device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n",
 512             sc->num_rx_queues);
 513
 514         return (0);
 515 } /* ixgbe_if_rx_queues_alloc */
 516
 517 /************************************************************************
 518  * ixgbe_if_queues_free
 519  ************************************************************************/
 520 static void
 521 ixgbe_if_queues_free(if_ctx_t ctx)
 522 {
 523         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 524         struct ix_tx_queue *tx_que = sc->tx_queues;
 525         struct ix_rx_queue *rx_que = sc->rx_queues;
 526         int                i;
 527
 528         if (tx_que != NULL) {
 529                 for (i = 0; i < sc->num_tx_queues; i++, tx_que++) {
 530                         struct tx_ring *txr = &tx_que->txr;
 531                         if (txr->tx_rsq == NULL)
 532                                 break;
 533
 534                         free(txr->tx_rsq, M_IXGBE);
 535                         txr->tx_rsq = NULL;
 536                 }
 537
 538                 free(sc->tx_queues, M_IXGBE);
 539                 sc->tx_queues = NULL;
 540         }
 541         if (rx_que != NULL) {
 542                 free(sc->rx_queues, M_IXGBE);
 543                 sc->rx_queues = NULL;
 544         }
 545 } /* ixgbe_if_queues_free */
 546
 547 /************************************************************************
 548  * ixgbe_initialize_rss_mapping
 549  ************************************************************************/
 550 static void
 551 ixgbe_initialize_rss_mapping(struct ixgbe_softc *sc)
 552 {
 553         struct ixgbe_hw *hw = &sc->hw;
 554         u32             reta = 0, mrqc, rss_key[10];
 555         int             queue_id, table_size, index_mult;
 556         int             i, j;
 557         u32             rss_hash_config;
 558
 559         if (sc->feat_en & IXGBE_FEATURE_RSS) {
 560                 /* Fetch the configured RSS key */
 561                 rss_getkey((uint8_t *)&rss_key);
 562         } else {
 563                 /* set up random bits */
 564                 arc4rand(&rss_key, sizeof(rss_key), 0);
 565         }
 566
 567         /* Set multiplier for RETA setup and table size based on MAC */
 568         index_mult = 0x1;
 569         table_size = 128;
 570         switch (sc->hw.mac.type) {
 571         case ixgbe_mac_82598EB:
 572                 index_mult = 0x11;
 573                 break;
 574         case ixgbe_mac_X550:
 575         case ixgbe_mac_X550EM_x:
 576         case ixgbe_mac_X550EM_a:
 577                 table_size = 512;
 578                 break;
 579         default:
 580                 break;
 581         }
 582
 583         /* Set up the redirection table */
 584         for (i = 0, j = 0; i < table_size; i++, j++) {
 585                 if (j == sc->num_rx_queues)
 586                         j = 0;
 587
 588                 if (sc->feat_en & IXGBE_FEATURE_RSS) {
 589                         /*
 590                          * Fetch the RSS bucket id for the given indirection
 591                          * entry. Cap it at the number of configured buckets
 592                          * (which is num_rx_queues.)
 593                          */
 594                         queue_id = rss_get_indirection_to_bucket(i);
 595                         queue_id = queue_id % sc->num_rx_queues;
 596                 } else
 597                         queue_id = (j * index_mult);
 598
 599                 /*
 600                  * The low 8 bits are for hash value (n+0);
 601                  * The next 8 bits are for hash value (n+1), etc.
 602                  */
 603                 reta = reta >> 8;
 604                 reta = reta | (((uint32_t)queue_id) << 24);
 605                 if ((i & 3) == 3) {
 606                         if (i < 128)
 607                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
 608                         else
 609                                 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
 610                                     reta);
 611                         reta = 0;
 612                 }
 613         }
 614
 615         /* Now fill our hash function seeds */
 616         for (i = 0; i < 10; i++)
 617                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
 618
 619         /* Perform hash on these packet types */
 620         if (sc->feat_en & IXGBE_FEATURE_RSS)
 621                 rss_hash_config = rss_gethashconfig();
 622         else {
 623                 /*
 624                  * Disable UDP - IP fragments aren't currently being handled
 625                  * and so we end up with a mix of 2-tuple and 4-tuple
 626                  * traffic.
 627                  */
 628                 rss_hash_config = RSS_HASHTYPE_RSS_IPV4
 629                                 | RSS_HASHTYPE_RSS_TCP_IPV4
 630                                 | RSS_HASHTYPE_RSS_IPV6
 631                                 | RSS_HASHTYPE_RSS_TCP_IPV6
 632                                 | RSS_HASHTYPE_RSS_IPV6_EX
 633                                 | RSS_HASHTYPE_RSS_TCP_IPV6_EX;
 634         }
 635
 636         mrqc = IXGBE_MRQC_RSSEN;
 637         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
 638                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
 639         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
 640                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
 641         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
 642                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
 643         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
 644                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
 645         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
 646                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
 647         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
 648                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
 649         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
 650                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
 651         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
 652                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
 653         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
 654                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
 655         mrqc |= ixgbe_get_mrqc(sc->iov_mode);
 656         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
 657 } /* ixgbe_initialize_rss_mapping */
 658
 659 /************************************************************************
 660  * ixgbe_initialize_receive_units - Setup receive registers and features.
 661  ************************************************************************/
 662 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
 663
 664 static void
 665 ixgbe_initialize_receive_units(if_ctx_t ctx)
 666 {
 667         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 668         if_softc_ctx_t     scctx = sc->shared;
 669         struct ixgbe_hw    *hw = &sc->hw;
 670         struct ifnet       *ifp = iflib_get_ifp(ctx);
 671         struct ix_rx_queue *que;
 672         int                i, j;
 673         u32                bufsz, fctrl, srrctl, rxcsum;
 674         u32                hlreg;
 675
 676         /*
 677          * Make sure receives are disabled while
 678          * setting up the descriptor ring
 679          */
 680         ixgbe_disable_rx(hw);
 681
 682         /* Enable broadcasts */
 683         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
 684         fctrl |= IXGBE_FCTRL_BAM;
 685         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
 686                 fctrl |= IXGBE_FCTRL_DPF;
 687                 fctrl |= IXGBE_FCTRL_PMCF;
 688         }
 689         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
 690
 691         /* Set for Jumbo Frames? */
 692         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
 693         if (ifp->if_mtu > ETHERMTU)
 694                 hlreg |= IXGBE_HLREG0_JUMBOEN;
 695         else
 696                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
 697         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
 698
 699         bufsz = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
 700             IXGBE_SRRCTL_BSIZEPKT_SHIFT;
 701
 702         /* Setup the Base and Length of the Rx Descriptor Ring */
 703         for (i = 0, que = sc->rx_queues; i < sc->num_rx_queues; i++, que++) {
 704                 struct rx_ring *rxr = &que->rxr;
 705                 u64            rdba = rxr->rx_paddr;
 706
 707                 j = rxr->me;
 708
 709                 /* Setup the Base and Length of the Rx Descriptor Ring */
 710                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j),
 711                     (rdba & 0x00000000ffffffffULL));
 712                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32));
 713                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j),
 714                      scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));
 715
 716                 /* Set up the SRRCTL register */
 717                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j));
 718                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
 719                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
 720                 srrctl |= bufsz;
 721                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
 722
 723                 /*
 724                  * Set DROP_EN iff we have no flow control and >1 queue.
 725                  * Note that srrctl was cleared shortly before during reset,
 726                  * so we do not need to clear the bit, but do it just in case
 727                  * this code is moved elsewhere.
 728                  */
 729                 if (sc->num_rx_queues > 1 &&
 730                     sc->hw.fc.requested_mode == ixgbe_fc_none) {
 731                         srrctl |= IXGBE_SRRCTL_DROP_EN;
 732                 } else {
 733                         srrctl &= ~IXGBE_SRRCTL_DROP_EN;
 734                 }
 735
 736                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl);
 737
 738                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
 739                 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0);
 740                 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0);
 741
 742                 /* Set the driver rx tail address */
 743                 rxr->tail =  IXGBE_RDT(rxr->me);
 744         }
 745
 746         if (sc->hw.mac.type != ixgbe_mac_82598EB) {
 747                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR
 748                             | IXGBE_PSRTYPE_UDPHDR
 749                             | IXGBE_PSRTYPE_IPV4HDR
 750                             | IXGBE_PSRTYPE_IPV6HDR;
 751                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
 752         }
 753
 754         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
 755
 756         ixgbe_initialize_rss_mapping(sc);
 757
 758         if (sc->num_rx_queues > 1) {
 759                 /* RSS and RX IPP Checksum are mutually exclusive */
 760                 rxcsum |= IXGBE_RXCSUM_PCSD;
 761         }
 762
 763         if (ifp->if_capenable & IFCAP_RXCSUM)
 764                 rxcsum |= IXGBE_RXCSUM_PCSD;
 765
 766         /* This is useful for calculating UDP/IP fragment checksums */
 767         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
 768                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
 769
 770         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
 771
 772 } /* ixgbe_initialize_receive_units */
 773
 774 /************************************************************************
 775  * ixgbe_initialize_transmit_units - Enable transmit units.
 776  ************************************************************************/
 777 static void
 778 ixgbe_initialize_transmit_units(if_ctx_t ctx)
 779 {
 780         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 781         struct ixgbe_hw    *hw = &sc->hw;
 782         if_softc_ctx_t     scctx = sc->shared;
 783         struct ix_tx_queue *que;
 784         int i;
 785
 786         /* Setup the Base and Length of the Tx Descriptor Ring */
 787         for (i = 0, que = sc->tx_queues; i < sc->num_tx_queues;
 788             i++, que++) {
 789                 struct tx_ring     *txr = &que->txr;
 790                 u64 tdba = txr->tx_paddr;
 791                 u32 txctrl = 0;
 792                 int j = txr->me;
 793
 794                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j),
 795                     (tdba & 0x00000000ffffffffULL));
 796                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32));
 797                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j),
 798                     scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc));
 799
 800                 /* Setup the HW Tx Head and Tail descriptor pointers */
 801                 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0);
 802                 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0);
 803
 804                 /* Cache the tail address */
 805                 txr->tail = IXGBE_TDT(txr->me);
 806
 807                 txr->tx_rs_cidx = txr->tx_rs_pidx;
 808                 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
 809                 for (int k = 0; k < scctx->isc_ntxd[0]; k++)
 810                         txr->tx_rsq[k] = QIDX_INVALID;
 811
 812                 /* Disable Head Writeback */
 813                 /*
 814                  * Note: for X550 series devices, these registers are actually
 815                  * prefixed with TPH_ isntead of DCA_, but the addresses and
 816                  * fields remain the same.
 817                  */
 818                 switch (hw->mac.type) {
 819                 case ixgbe_mac_82598EB:
 820                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j));
 821                         break;
 822                 default:
 823                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j));
 824                         break;
 825                 }
 826                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
 827                 switch (hw->mac.type) {
 828                 case ixgbe_mac_82598EB:
 829                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl);
 830                         break;
 831                 default:
 832                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl);
 833                         break;
 834                 }
 835
 836         }
 837
 838         if (hw->mac.type != ixgbe_mac_82598EB) {
 839                 u32 dmatxctl, rttdcs;
 840
 841                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
 842                 dmatxctl |= IXGBE_DMATXCTL_TE;
 843                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
 844                 /* Disable arbiter to set MTQC */
 845                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
 846                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
 847                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
 848                 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
 849                     ixgbe_get_mtqc(sc->iov_mode));
 850                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
 851                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
 852         }
 853
 854 } /* ixgbe_initialize_transmit_units */
 855
 856 /************************************************************************
 857  * ixgbe_register
 858  ************************************************************************/
 859 static void *
 860 ixgbe_register(device_t dev)
 861 {
 862         return (&ixgbe_sctx_init);
 863 } /* ixgbe_register */
 864
 865 /************************************************************************
 866  * ixgbe_if_attach_pre - Device initialization routine, part 1
 867  *
 868  *   Called when the driver is being loaded.
 869  *   Identifies the type of hardware, initializes the hardware,
 870  *   and initializes iflib structures.
 871  *
 872  *   return 0 on success, positive on failure
 873  ************************************************************************/
 874 static int
 875 ixgbe_if_attach_pre(if_ctx_t ctx)
 876 {
 877         struct ixgbe_softc  *sc;
 878         device_t        dev;
 879         if_softc_ctx_t  scctx;
 880         struct ixgbe_hw *hw;
 881         int             error = 0;
 882         u32             ctrl_ext;
 883
 884         INIT_DEBUGOUT("ixgbe_attach: begin");
 885
 886         /* Allocate, clear, and link in our adapter structure */
 887         dev = iflib_get_dev(ctx);
 888         sc = iflib_get_softc(ctx);
 889         sc->hw.back = sc;
 890         sc->ctx = ctx;
 891         sc->dev = dev;
 892         scctx = sc->shared = iflib_get_softc_ctx(ctx);
 893         sc->media = iflib_get_media(ctx);
 894         hw = &sc->hw;
 895
 896         /* Determine hardware revision */
 897         hw->vendor_id = pci_get_vendor(dev);
 898         hw->device_id = pci_get_device(dev);
 899         hw->revision_id = pci_get_revid(dev);
 900         hw->subsystem_vendor_id = pci_get_subvendor(dev);
 901         hw->subsystem_device_id = pci_get_subdevice(dev);
 902
 903         /* Do base PCI setup - map BAR0 */
 904         if (ixgbe_allocate_pci_resources(ctx)) {
 905                 device_printf(dev, "Allocation of PCI resources failed\n");
 906                 return (ENXIO);
 907         }
 908
 909         /* let hardware know driver is loaded */
 910         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 911         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 912         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 913
 914         /*
 915          * Initialize the shared code
 916          */
 917         if (ixgbe_init_shared_code(hw) != 0) {
 918                 device_printf(dev, "Unable to initialize the shared code\n");
 919                 error = ENXIO;
 920                 goto err_pci;
 921         }
 922
 923         if (hw->mbx.ops.init_params)
 924                 hw->mbx.ops.init_params(hw);
 925
 926         hw->allow_unsupported_sfp = allow_unsupported_sfp;
 927
 928         if (hw->mac.type != ixgbe_mac_82598EB)
 929                 hw->phy.smart_speed = ixgbe_smart_speed;
 930
 931         ixgbe_init_device_features(sc);
 932
 933         /* Enable WoL (if supported) */
 934         ixgbe_check_wol_support(sc);
 935
 936         /* Verify adapter fan is still functional (if applicable) */
 937         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL) {
 938                 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
 939                 ixgbe_check_fan_failure(sc, esdp, false);
 940         }
 941
 942         /* Ensure SW/FW semaphore is free */
 943         ixgbe_init_swfw_semaphore(hw);
 944
 945         /* Set an initial default flow control value */
 946         hw->fc.requested_mode = ixgbe_flow_control;
 947
 948         hw->phy.reset_if_overtemp = true;
 949         error = ixgbe_reset_hw(hw);
 950         hw->phy.reset_if_overtemp = false;
 951         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 952                 /*
 953                  * No optics in this port, set up
 954                  * so the timer routine will probe
 955                  * for later insertion.
 956                  */
 957                 sc->sfp_probe = true;
 958                 error = 0;
 959         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 960                 device_printf(dev, "Unsupported SFP+ module detected!\n");
 961                 error = EIO;
 962                 goto err_pci;
 963         } else if (error) {
 964                 device_printf(dev, "Hardware initialization failed\n");
 965                 error = EIO;
 966                 goto err_pci;
 967         }
 968
 969         /* Make sure we have a good EEPROM before we read from it */
 970         if (ixgbe_validate_eeprom_checksum(&sc->hw, NULL) < 0) {
 971                 device_printf(dev, "The EEPROM Checksum Is Not Valid\n");
 972                 error = EIO;
 973                 goto err_pci;
 974         }
 975
 976         error = ixgbe_start_hw(hw);
 977         switch (error) {
 978         case IXGBE_ERR_EEPROM_VERSION:
 979                 device_printf(dev, "This device is a pre-production adapter/LOM.  Please be aware there may be issues associated with your hardware.\nIf you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
 980                 break;
 981         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 982                 device_printf(dev, "Unsupported SFP+ Module\n");
 983                 error = EIO;
 984                 goto err_pci;
 985         case IXGBE_ERR_SFP_NOT_PRESENT:
 986                 device_printf(dev, "No SFP+ Module found\n");
 987                 /* falls thru */
 988         default:
 989                 break;
 990         }
 991
 992         /* Most of the iflib initialization... */
 993
 994         iflib_set_mac(ctx, hw->mac.addr);
 995         switch (sc->hw.mac.type) {
 996         case ixgbe_mac_X550:
 997         case ixgbe_mac_X550EM_x:
 998         case ixgbe_mac_X550EM_a:
 999                 scctx->isc_rss_table_size = 512;
1000                 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 64;
1001                 break;
1002         default:
1003                 scctx->isc_rss_table_size = 128;
1004                 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 16;
1005         }
1006
1007         /* Allow legacy interrupts */
1008         ixgbe_txrx.ift_legacy_intr = ixgbe_intr;
1009
1010         scctx->isc_txqsizes[0] =
1011             roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +
1012             sizeof(u32), DBA_ALIGN),
1013         scctx->isc_rxqsizes[0] =
1014             roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),
1015             DBA_ALIGN);
1016
1017         /* XXX */
1018         scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
1019             CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO;
1020         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
1021                 scctx->isc_tx_nsegments = IXGBE_82598_SCATTER;
1022         } else {
1023                 scctx->isc_tx_csum_flags |= CSUM_SCTP |CSUM_IP6_SCTP;
1024                 scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;
1025         }
1026
1027         scctx->isc_msix_bar = pci_msix_table_bar(dev);
1028
1029         scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
1030         scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;
1031         scctx->isc_tx_tso_segsize_max = PAGE_SIZE;
1032
1033         scctx->isc_txrx = &ixgbe_txrx;
1034
1035         scctx->isc_capabilities = scctx->isc_capenable = IXGBE_CAPS;
1036
1037         return (0);
1038
1039 err_pci:
1040         ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
1041         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
1042         IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
1043         ixgbe_free_pci_resources(ctx);
1044
1045         return (error);
1046 } /* ixgbe_if_attach_pre */
1047
1048  /*********************************************************************
1049  * ixgbe_if_attach_post - Device initialization routine, part 2
1050  *
1051  *   Called during driver load, but after interrupts and
1052  *   resources have been allocated and configured.
1053  *   Sets up some data structures not relevant to iflib.
1054  *
1055  *   return 0 on success, positive on failure
1056  *********************************************************************/
1057 static int
1058 ixgbe_if_attach_post(if_ctx_t ctx)
1059 {
1060         device_t dev;
1061         struct ixgbe_softc  *sc;
1062         struct ixgbe_hw *hw;
1063         int             error = 0;
1064
1065         dev = iflib_get_dev(ctx);
1066         sc = iflib_get_softc(ctx);
1067         hw = &sc->hw;
1068
1069
1070         if (sc->intr_type == IFLIB_INTR_LEGACY &&
1071                 (sc->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) == 0) {
1072                 device_printf(dev, "Device does not support legacy interrupts");
1073                 error = ENXIO;
1074                 goto err;
1075         }
1076
1077         /* Allocate multicast array memory. */
1078         sc->mta = malloc(sizeof(*sc->mta) *
1079                               MAX_NUM_MULTICAST_ADDRESSES, M_IXGBE, M_NOWAIT);
1080         if (sc->mta == NULL) {
1081                 device_printf(dev, "Can not allocate multicast setup array\n");
1082                 error = ENOMEM;
1083                 goto err;
1084         }
1085
1086         /* hw.ix defaults init */
1087         ixgbe_set_advertise(sc, ixgbe_advertise_speed);
1088
1089         /* Enable the optics for 82599 SFP+ fiber */
1090         ixgbe_enable_tx_laser(hw);
1091
1092         /* Enable power to the phy. */
1093         ixgbe_set_phy_power(hw, true);
1094
1095         ixgbe_initialize_iov(sc);
1096
1097         error = ixgbe_setup_interface(ctx);
1098         if (error) {
1099                 device_printf(dev, "Interface setup failed: %d\n", error);
1100                 goto err;
1101         }
1102
1103         ixgbe_if_update_admin_status(ctx);
1104
1105         /* Initialize statistics */
1106         ixgbe_update_stats_counters(sc);
1107         ixgbe_add_hw_stats(sc);
1108
1109         /* Check PCIE slot type/speed/width */
1110         ixgbe_get_slot_info(sc);
1111
1112         /*
1113          * Do time init and sysctl init here, but
1114          * only on the first port of a bypass sc.
1115          */
1116         ixgbe_bypass_init(sc);
1117
1118         /* Display NVM and Option ROM versions */
1119         ixgbe_print_fw_version(ctx);
1120
1121         /* Set an initial dmac value */
1122         sc->dmac = 0;
1123         /* Set initial advertised speeds (if applicable) */
1124         sc->advertise = ixgbe_get_default_advertise(sc);
1125
1126         if (sc->feat_cap & IXGBE_FEATURE_SRIOV)
1127                 ixgbe_define_iov_schemas(dev, &error);
1128
1129         /* Add sysctls */
1130         ixgbe_add_device_sysctls(ctx);
1131
1132         return (0);
1133 err:
1134         return (error);
1135 } /* ixgbe_if_attach_post */
1136
1137 /************************************************************************
1138  * ixgbe_check_wol_support
1139  *
1140  *   Checks whether the adapter's ports are capable of
1141  *   Wake On LAN by reading the adapter's NVM.
1142  *
1143  *   Sets each port's hw->wol_enabled value depending
1144  *   on the value read here.
1145  ************************************************************************/
1146 static void
1147 ixgbe_check_wol_support(struct ixgbe_softc *sc)
1148 {
1149         struct ixgbe_hw *hw = &sc->hw;
1150         u16             dev_caps = 0;
1151
1152         /* Find out WoL support for port */
1153         sc->wol_support = hw->wol_enabled = 0;
1154         ixgbe_get_device_caps(hw, &dev_caps);
1155         if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
1156             ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) &&
1157              hw->bus.func == 0))
1158                 sc->wol_support = hw->wol_enabled = 1;
1159
1160         /* Save initial wake up filter configuration */
1161         sc->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC);
1162
1163         return;
1164 } /* ixgbe_check_wol_support */
1165
1166 /************************************************************************
1167  * ixgbe_setup_interface
1168  *
1169  *   Setup networking device structure and register an interface.
1170  ************************************************************************/
1171 static int
1172 ixgbe_setup_interface(if_ctx_t ctx)
1173 {
1174         struct ifnet   *ifp = iflib_get_ifp(ctx);
1175         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1176
1177         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
1178
1179         if_setbaudrate(ifp, IF_Gbps(10));
1180
1181         sc->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1182
1183         sc->phy_layer = ixgbe_get_supported_physical_layer(&sc->hw);
1184
1185         ixgbe_add_media_types(ctx);
1186
1187         /* Autoselect media by default */
1188         ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO);
1189
1190         return (0);
1191 } /* ixgbe_setup_interface */
1192
1193 /************************************************************************
1194  * ixgbe_if_get_counter
1195  ************************************************************************/
1196 static uint64_t
1197 ixgbe_if_get_counter(if_ctx_t ctx, ift_counter cnt)
1198 {
1199         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1200         if_t           ifp = iflib_get_ifp(ctx);
1201
1202         switch (cnt) {
1203         case IFCOUNTER_IPACKETS:
1204                 return (sc->ipackets);
1205         case IFCOUNTER_OPACKETS:
1206                 return (sc->opackets);
1207         case IFCOUNTER_IBYTES:
1208                 return (sc->ibytes);
1209         case IFCOUNTER_OBYTES:
1210                 return (sc->obytes);
1211         case IFCOUNTER_IMCASTS:
1212                 return (sc->imcasts);
1213         case IFCOUNTER_OMCASTS:
1214                 return (sc->omcasts);
1215         case IFCOUNTER_COLLISIONS:
1216                 return (0);
1217         case IFCOUNTER_IQDROPS:
1218                 return (sc->iqdrops);
1219         case IFCOUNTER_OQDROPS:
1220                 return (0);
1221         case IFCOUNTER_IERRORS:
1222                 return (sc->ierrors);
1223         default:
1224                 return (if_get_counter_default(ifp, cnt));
1225         }
1226 } /* ixgbe_if_get_counter */
1227
1228 /************************************************************************
1229  * ixgbe_if_i2c_req
1230  ************************************************************************/
1231 static int
1232 ixgbe_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req)
1233 {
1234         struct ixgbe_softc              *sc = iflib_get_softc(ctx);
1235         struct ixgbe_hw         *hw = &sc->hw;
1236         int                     i;
1237
1238
1239         if (hw->phy.ops.read_i2c_byte == NULL)
1240                 return (ENXIO);
1241         for (i = 0; i < req->len; i++)
1242                 hw->phy.ops.read_i2c_byte(hw, req->offset + i,
1243                     req->dev_addr, &req->data[i]);
1244         return (0);
1245 } /* ixgbe_if_i2c_req */
1246
1247 /* ixgbe_if_needs_restart - Tell iflib when the driver needs to be reinitialized
1248  * @ctx: iflib context
1249  * @event: event code to check
1250  *
1251  * Defaults to returning true for unknown events.
1252  *
1253  * @returns true if iflib needs to reinit the interface
1254  */
1255 static bool
1256 ixgbe_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event)
1257 {
1258         switch (event) {
1259         case IFLIB_RESTART_VLAN_CONFIG:
1260                 return (false);
1261         default:
1262                 return (true);
1263         }
1264 }
1265
1266 /************************************************************************
1267  * ixgbe_add_media_types
1268  ************************************************************************/
1269 static void
1270 ixgbe_add_media_types(if_ctx_t ctx)
1271 {
1272         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
1273         struct ixgbe_hw *hw = &sc->hw;
1274         device_t        dev = iflib_get_dev(ctx);
1275         u64             layer;
1276
1277         layer = sc->phy_layer = ixgbe_get_supported_physical_layer(hw);
1278
1279         /* Media types with matching FreeBSD media defines */
1280         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T)
1281                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_T, 0, NULL);
1282         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T)
1283                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1284         if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
1285                 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL);
1286         if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
1287                 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL);
1288
1289         if (hw->mac.type == ixgbe_mac_X550) {
1290                 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_T, 0, NULL);
1291                 ifmedia_add(sc->media, IFM_ETHER | IFM_5000_T, 0, NULL);
1292         }
1293
1294         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
1295             layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
1296                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_TWINAX, 0,
1297                     NULL);
1298
1299         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
1300                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
1301                 if (hw->phy.multispeed_fiber)
1302                         ifmedia_add(sc->media, IFM_ETHER | IFM_1000_LX, 0,
1303                             NULL);
1304         }
1305         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
1306                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
1307                 if (hw->phy.multispeed_fiber)
1308                         ifmedia_add(sc->media, IFM_ETHER | IFM_1000_SX, 0,
1309                             NULL);
1310         } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
1311                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
1312         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
1313                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
1314
1315 #ifdef IFM_ETH_XTYPE
1316         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
1317                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
1318         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4)
1319                 ifmedia_add( sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
1320         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
1321                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
1322         if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX)
1323                 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL);
1324 #else
1325         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
1326                 device_printf(dev, "Media supported: 10GbaseKR\n");
1327                 device_printf(dev, "10GbaseKR mapped to 10GbaseSR\n");
1328                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
1329         }
1330         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) {
1331                 device_printf(dev, "Media supported: 10GbaseKX4\n");
1332                 device_printf(dev, "10GbaseKX4 mapped to 10GbaseCX4\n");
1333                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
1334         }
1335         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) {
1336                 device_printf(dev, "Media supported: 1000baseKX\n");
1337                 device_printf(dev, "1000baseKX mapped to 1000baseCX\n");
1338                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
1339         }
1340         if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) {
1341                 device_printf(dev, "Media supported: 2500baseKX\n");
1342                 device_printf(dev, "2500baseKX mapped to 2500baseSX\n");
1343                 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_SX, 0, NULL);
1344         }
1345 #endif
1346         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX)
1347                 device_printf(dev, "Media supported: 1000baseBX\n");
1348
1349         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
1350                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
1351                     0, NULL);
1352                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1353         }
1354
1355         ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1356 } /* ixgbe_add_media_types */
1357
1358 /************************************************************************
1359  * ixgbe_is_sfp
1360  ************************************************************************/
1361 static inline bool
1362 ixgbe_is_sfp(struct ixgbe_hw *hw)
1363 {
1364         switch (hw->mac.type) {
1365         case ixgbe_mac_82598EB:
1366                 if (hw->phy.type == ixgbe_phy_nl)
1367                         return (true);
1368                 return (false);
1369         case ixgbe_mac_82599EB:
1370                 switch (hw->mac.ops.get_media_type(hw)) {
1371                 case ixgbe_media_type_fiber:
1372                 case ixgbe_media_type_fiber_qsfp:
1373                         return (true);
1374                 default:
1375                         return (false);
1376                 }
1377         case ixgbe_mac_X550EM_x:
1378         case ixgbe_mac_X550EM_a:
1379                 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber)
1380                         return (true);
1381                 return (false);
1382         default:
1383                 return (false);
1384         }
1385 } /* ixgbe_is_sfp */
1386
1387 /************************************************************************
1388  * ixgbe_config_link
1389  ************************************************************************/
1390 static void
1391 ixgbe_config_link(if_ctx_t ctx)
1392 {
1393         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
1394         struct ixgbe_hw *hw = &sc->hw;
1395         u32             autoneg, err = 0;
1396         bool            sfp, negotiate;
1397
1398         sfp = ixgbe_is_sfp(hw);
1399
1400         if (sfp) {
1401                 sc->task_requests |= IXGBE_REQUEST_TASK_MOD;
1402                 iflib_admin_intr_deferred(ctx);
1403         } else {
1404                 if (hw->mac.ops.check_link)
1405                         err = ixgbe_check_link(hw, &sc->link_speed,
1406                             &sc->link_up, false);
1407                 if (err)
1408                         return;
1409                 autoneg = hw->phy.autoneg_advertised;
1410                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
1411                         err = hw->mac.ops.get_link_capabilities(hw, &autoneg,
1412                             &negotiate);
1413                 if (err)
1414                         return;
1415
1416                 if (hw->mac.type == ixgbe_mac_X550 &&
1417                     hw->phy.autoneg_advertised == 0) {
1418                         /*
1419                          * 2.5G and 5G autonegotiation speeds on X550
1420                          * are disabled by default due to reported
1421                          * interoperability issues with some switches.
1422                          *
1423                          * The second condition checks if any operations
1424                          * involving setting autonegotiation speeds have
1425                          * been performed prior to this ixgbe_config_link()
1426                          * call.
1427                          *
1428                          * If hw->phy.autoneg_advertised does not
1429                          * equal 0, this means that the user might have
1430                          * set autonegotiation speeds via the sysctl
1431                          * before bringing the interface up. In this
1432                          * case, we should not disable 2.5G and 5G
1433                          * since that speeds might be selected by the
1434                          * user.
1435                          *
1436                          * Otherwise (i.e. if hw->phy.autoneg_advertised
1437                          * is set to 0), it is the first time we set
1438                          * autonegotiation preferences and the default
1439                          * set of speeds should exclude 2.5G and 5G.
1440                          */
1441                         autoneg &= ~(IXGBE_LINK_SPEED_2_5GB_FULL |
1442                             IXGBE_LINK_SPEED_5GB_FULL);
1443                 }
1444
1445                 if (hw->mac.ops.setup_link)
1446                         err = hw->mac.ops.setup_link(hw, autoneg,
1447                             sc->link_up);
1448         }
1449 } /* ixgbe_config_link */
1450
1451 /************************************************************************
1452  * ixgbe_update_stats_counters - Update board statistics counters.
1453  ************************************************************************/
1454 static void
1455 ixgbe_update_stats_counters(struct ixgbe_softc *sc)
1456 {
1457         struct ixgbe_hw       *hw = &sc->hw;
1458         struct ixgbe_hw_stats *stats = &sc->stats.pf;
1459         u32                   missed_rx = 0, bprc, lxon, lxoff, total;
1460         u32                   lxoffrxc;
1461         u64                   total_missed_rx = 0;
1462
1463         stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1464         stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1465         stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1466         stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1467         stats->mpc[0] += IXGBE_READ_REG(hw, IXGBE_MPC(0));
1468
1469         for (int i = 0; i < 16; i++) {
1470                 stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1471                 stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1472                 stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1473         }
1474         stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1475         stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1476         stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1477
1478         /* Hardware workaround, gprc counts missed packets */
1479         stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1480         stats->gprc -= missed_rx;
1481
1482         if (hw->mac.type != ixgbe_mac_82598EB) {
1483                 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
1484                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1485                 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
1486                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1487                 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
1488                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1489                 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1490                 lxoffrxc = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1491                 stats->lxoffrxc += lxoffrxc;
1492         } else {
1493                 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1494                 lxoffrxc = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1495                 stats->lxoffrxc += lxoffrxc;
1496                 /* 82598 only has a counter in the high register */
1497                 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1498                 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1499                 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1500         }
1501
1502         /*
1503          * For watchdog management we need to know if we have been paused
1504          * during the last interval, so capture that here.
1505         */
1506         if (lxoffrxc)
1507                 sc->shared->isc_pause_frames = 1;
1508
1509         /*
1510          * Workaround: mprc hardware is incorrectly counting
1511          * broadcasts, so for now we subtract those.
1512          */
1513         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1514         stats->bprc += bprc;
1515         stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1516         if (hw->mac.type == ixgbe_mac_82598EB)
1517                 stats->mprc -= bprc;
1518
1519         stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1520         stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1521         stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1522         stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1523         stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1524         stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1525
1526         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1527         stats->lxontxc += lxon;
1528         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1529         stats->lxofftxc += lxoff;
1530         total = lxon + lxoff;
1531
1532         stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1533         stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1534         stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1535         stats->gptc -= total;
1536         stats->mptc -= total;
1537         stats->ptc64 -= total;
1538         stats->gotc -= total * ETHER_MIN_LEN;
1539
1540         stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1541         stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1542         stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1543         stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1544         stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1545         stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1546         stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1547         stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1548         stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1549         stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1550         stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1551         stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1552         stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1553         stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1554         stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1555         stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1556         stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1557         stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1558         /* Only read FCOE on 82599 */
1559         if (hw->mac.type != ixgbe_mac_82598EB) {
1560                 stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1561                 stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1562                 stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1563                 stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1564                 stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1565         }
1566
1567         /* Fill out the OS statistics structure */
1568         IXGBE_SET_IPACKETS(sc, stats->gprc);
1569         IXGBE_SET_OPACKETS(sc, stats->gptc);
1570         IXGBE_SET_IBYTES(sc, stats->gorc);
1571         IXGBE_SET_OBYTES(sc, stats->gotc);
1572         IXGBE_SET_IMCASTS(sc, stats->mprc);
1573         IXGBE_SET_OMCASTS(sc, stats->mptc);
1574         IXGBE_SET_COLLISIONS(sc, 0);
1575         IXGBE_SET_IQDROPS(sc, total_missed_rx);
1576
1577         /*
1578          * Aggregate following types of errors as RX errors:
1579          * - CRC error count,
1580          * - illegal byte error count,
1581          * - checksum error count,
1582          * - missed packets count,
1583          * - length error count,
1584          * - undersized packets count,
1585          * - fragmented packets count,
1586          * - oversized packets count,
1587          * - jabber count.
1588          */
1589         IXGBE_SET_IERRORS(sc, stats->crcerrs + stats->illerrc + stats->xec +
1590             stats->mpc[0] + stats->rlec + stats->ruc + stats->rfc + stats->roc +
1591             stats->rjc);
1592 } /* ixgbe_update_stats_counters */
1593
1594 /************************************************************************
1595  * ixgbe_add_hw_stats
1596  *
1597  *   Add sysctl variables, one per statistic, to the system.
1598  ************************************************************************/
1599 static void
1600 ixgbe_add_hw_stats(struct ixgbe_softc *sc)
1601 {
1602         device_t               dev = iflib_get_dev(sc->ctx);
1603         struct ix_rx_queue     *rx_que;
1604         struct ix_tx_queue     *tx_que;
1605         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
1606         struct sysctl_oid      *tree = device_get_sysctl_tree(dev);
1607         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
1608         struct ixgbe_hw_stats  *stats = &sc->stats.pf;
1609         struct sysctl_oid      *stat_node, *queue_node;
1610         struct sysctl_oid_list *stat_list, *queue_list;
1611         int                    i;
1612
1613 #define QUEUE_NAME_LEN 32
1614         char                   namebuf[QUEUE_NAME_LEN];
1615
1616         /* Driver Statistics */
1617         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
1618             CTLFLAG_RD, &sc->dropped_pkts, "Driver dropped packets");
1619         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
1620             CTLFLAG_RD, &sc->watchdog_events, "Watchdog timeouts");
1621         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
1622             CTLFLAG_RD, &sc->link_irq, "Link MSI-X IRQ Handled");
1623
1624         for (i = 0, tx_que = sc->tx_queues; i < sc->num_tx_queues; i++, tx_que++) {
1625                 struct tx_ring *txr = &tx_que->txr;
1626                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1627                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1628                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name");
1629                 queue_list = SYSCTL_CHILDREN(queue_node);
1630
1631                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
1632                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, txr, 0,
1633                     ixgbe_sysctl_tdh_handler, "IU", "Transmit Descriptor Head");
1634                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
1635                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, txr, 0,
1636                     ixgbe_sysctl_tdt_handler, "IU", "Transmit Descriptor Tail");
1637                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
1638                     CTLFLAG_RD, &txr->tso_tx, "TSO");
1639                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
1640                     CTLFLAG_RD, &txr->total_packets,
1641                     "Queue Packets Transmitted");
1642         }
1643
1644         for (i = 0, rx_que = sc->rx_queues; i < sc->num_rx_queues; i++, rx_que++) {
1645                 struct rx_ring *rxr = &rx_que->rxr;
1646                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1647                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1648                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name");
1649                 queue_list = SYSCTL_CHILDREN(queue_node);
1650
1651                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
1652                     CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
1653                     &sc->rx_queues[i], 0,
1654                     ixgbe_sysctl_interrupt_rate_handler, "IU",
1655                     "Interrupt Rate");
1656                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
1657                     CTLFLAG_RD, &(sc->rx_queues[i].irqs),
1658                     "irqs on this queue");
1659                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
1660                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, rxr, 0,
1661                     ixgbe_sysctl_rdh_handler, "IU", "Receive Descriptor Head");
1662                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
1663                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, rxr, 0,
1664                     ixgbe_sysctl_rdt_handler, "IU", "Receive Descriptor Tail");
1665                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
1666                     CTLFLAG_RD, &rxr->rx_packets, "Queue Packets Received");
1667                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
1668                     CTLFLAG_RD, &rxr->rx_bytes, "Queue Bytes Received");
1669                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
1670                     CTLFLAG_RD, &rxr->rx_copies, "Copied RX Frames");
1671                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
1672                     CTLFLAG_RD, &rxr->rx_discarded, "Discarded RX packets");
1673         }
1674
1675         /* MAC stats get their own sub node */
1676
1677         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
1678             CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "MAC Statistics");
1679         stat_list = SYSCTL_CHILDREN(stat_node);
1680
1681         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_errs",
1682             CTLFLAG_RD, &sc->ierrors, IXGBE_SYSCTL_DESC_RX_ERRS);
1683         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
1684             CTLFLAG_RD, &stats->crcerrs, "CRC Errors");
1685         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
1686             CTLFLAG_RD, &stats->illerrc, "Illegal Byte Errors");
1687         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
1688             CTLFLAG_RD, &stats->errbc, "Byte Errors");
1689         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
1690             CTLFLAG_RD, &stats->mspdc, "MAC Short Packets Discarded");
1691         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
1692             CTLFLAG_RD, &stats->mlfc, "MAC Local Faults");
1693         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
1694             CTLFLAG_RD, &stats->mrfc, "MAC Remote Faults");
1695         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
1696             CTLFLAG_RD, &stats->rlec, "Receive Length Errors");
1697         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_missed_packets",
1698             CTLFLAG_RD, &stats->mpc[0], "RX Missed Packet Count");
1699
1700         /* Flow Control stats */
1701         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
1702             CTLFLAG_RD, &stats->lxontxc, "Link XON Transmitted");
1703         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
1704             CTLFLAG_RD, &stats->lxonrxc, "Link XON Received");
1705         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
1706             CTLFLAG_RD, &stats->lxofftxc, "Link XOFF Transmitted");
1707         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
1708             CTLFLAG_RD, &stats->lxoffrxc, "Link XOFF Received");
1709
1710         /* Packet Reception Stats */
1711         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
1712             CTLFLAG_RD, &stats->tor, "Total Octets Received");
1713         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
1714             CTLFLAG_RD, &stats->gorc, "Good Octets Received");
1715         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
1716             CTLFLAG_RD, &stats->tpr, "Total Packets Received");
1717         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
1718             CTLFLAG_RD, &stats->gprc, "Good Packets Received");
1719         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
1720             CTLFLAG_RD, &stats->mprc, "Multicast Packets Received");
1721         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
1722             CTLFLAG_RD, &stats->bprc, "Broadcast Packets Received");
1723         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
1724             CTLFLAG_RD, &stats->prc64, "64 byte frames received ");
1725         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
1726             CTLFLAG_RD, &stats->prc127, "65-127 byte frames received");
1727         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
1728             CTLFLAG_RD, &stats->prc255, "128-255 byte frames received");
1729         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
1730             CTLFLAG_RD, &stats->prc511, "256-511 byte frames received");
1731         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
1732             CTLFLAG_RD, &stats->prc1023, "512-1023 byte frames received");
1733         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
1734             CTLFLAG_RD, &stats->prc1522, "1023-1522 byte frames received");
1735         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
1736             CTLFLAG_RD, &stats->ruc, "Receive Undersized");
1737         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
1738             CTLFLAG_RD, &stats->rfc, "Fragmented Packets Received ");
1739         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
1740             CTLFLAG_RD, &stats->roc, "Oversized Packets Received");
1741         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
1742             CTLFLAG_RD, &stats->rjc, "Received Jabber");
1743         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
1744             CTLFLAG_RD, &stats->mngprc, "Management Packets Received");
1745         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
1746             CTLFLAG_RD, &stats->mngptc, "Management Packets Dropped");
1747         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
1748             CTLFLAG_RD, &stats->xec, "Checksum Errors");
1749
1750         /* Packet Transmission Stats */
1751         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
1752             CTLFLAG_RD, &stats->gotc, "Good Octets Transmitted");
1753         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
1754             CTLFLAG_RD, &stats->tpt, "Total Packets Transmitted");
1755         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
1756             CTLFLAG_RD, &stats->gptc, "Good Packets Transmitted");
1757         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
1758             CTLFLAG_RD, &stats->bptc, "Broadcast Packets Transmitted");
1759         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
1760             CTLFLAG_RD, &stats->mptc, "Multicast Packets Transmitted");
1761         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
1762             CTLFLAG_RD, &stats->mngptc, "Management Packets Transmitted");
1763         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
1764             CTLFLAG_RD, &stats->ptc64, "64 byte frames transmitted ");
1765         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
1766             CTLFLAG_RD, &stats->ptc127, "65-127 byte frames transmitted");
1767         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
1768             CTLFLAG_RD, &stats->ptc255, "128-255 byte frames transmitted");
1769         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
1770             CTLFLAG_RD, &stats->ptc511, "256-511 byte frames transmitted");
1771         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
1772             CTLFLAG_RD, &stats->ptc1023, "512-1023 byte frames transmitted");
1773         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
1774             CTLFLAG_RD, &stats->ptc1522, "1024-1522 byte frames transmitted");
1775 } /* ixgbe_add_hw_stats */
1776
1777 /************************************************************************
1778  * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function
1779  *
1780  *   Retrieves the TDH value from the hardware
1781  ************************************************************************/
1782 static int
1783 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
1784 {
1785         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
1786         int            error;
1787         unsigned int   val;
1788
1789         if (!txr)
1790                 return (0);
1791
1792         val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_TDH(txr->me));
1793         error = sysctl_handle_int(oidp, &val, 0, req);
1794         if (error || !req->newptr)
1795                 return error;
1796
1797         return (0);
1798 } /* ixgbe_sysctl_tdh_handler */
1799
1800 /************************************************************************
1801  * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function
1802  *
1803  *   Retrieves the TDT value from the hardware
1804  ************************************************************************/
1805 static int
1806 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
1807 {
1808         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
1809         int            error;
1810         unsigned int   val;
1811
1812         if (!txr)
1813                 return (0);
1814
1815         val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_TDT(txr->me));
1816         error = sysctl_handle_int(oidp, &val, 0, req);
1817         if (error || !req->newptr)
1818                 return error;
1819
1820         return (0);
1821 } /* ixgbe_sysctl_tdt_handler */
1822
1823 /************************************************************************
1824  * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function
1825  *
1826  *   Retrieves the RDH value from the hardware
1827  ************************************************************************/
1828 static int
1829 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
1830 {
1831         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
1832         int            error;
1833         unsigned int   val;
1834
1835         if (!rxr)
1836                 return (0);
1837
1838         val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_RDH(rxr->me));
1839         error = sysctl_handle_int(oidp, &val, 0, req);
1840         if (error || !req->newptr)
1841                 return error;
1842
1843         return (0);
1844 } /* ixgbe_sysctl_rdh_handler */
1845
1846 /************************************************************************
1847  * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function
1848  *
1849  *   Retrieves the RDT value from the hardware
1850  ************************************************************************/
1851 static int
1852 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
1853 {
1854         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
1855         int            error;
1856         unsigned int   val;
1857
1858         if (!rxr)
1859                 return (0);
1860
1861         val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_RDT(rxr->me));
1862         error = sysctl_handle_int(oidp, &val, 0, req);
1863         if (error || !req->newptr)
1864                 return error;
1865
1866         return (0);
1867 } /* ixgbe_sysctl_rdt_handler */
1868
1869 /************************************************************************
1870  * ixgbe_if_vlan_register
1871  *
1872  *   Run via vlan config EVENT, it enables us to use the
1873  *   HW Filter table since we can get the vlan id. This
1874  *   just creates the entry in the soft version of the
1875  *   VFTA, init will repopulate the real table.
1876  ************************************************************************/
1877 static void
1878 ixgbe_if_vlan_register(if_ctx_t ctx, u16 vtag)
1879 {
1880         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1881         u16            index, bit;
1882
1883         index = (vtag >> 5) & 0x7F;
1884         bit = vtag & 0x1F;
1885         sc->shadow_vfta[index] |= (1 << bit);
1886         ++sc->num_vlans;
1887         ixgbe_setup_vlan_hw_support(ctx);
1888 } /* ixgbe_if_vlan_register */
1889
1890 /************************************************************************
1891  * ixgbe_if_vlan_unregister
1892  *
1893  *   Run via vlan unconfig EVENT, remove our entry in the soft vfta.
1894  ************************************************************************/
1895 static void
1896 ixgbe_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
1897 {
1898         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1899         u16            index, bit;
1900
1901         index = (vtag >> 5) & 0x7F;
1902         bit = vtag & 0x1F;
1903         sc->shadow_vfta[index] &= ~(1 << bit);
1904         --sc->num_vlans;
1905         /* Re-init to load the changes */
1906         ixgbe_setup_vlan_hw_support(ctx);
1907 } /* ixgbe_if_vlan_unregister */
1908
1909 /************************************************************************
1910  * ixgbe_setup_vlan_hw_support
1911  ************************************************************************/
1912 static void
1913 ixgbe_setup_vlan_hw_support(if_ctx_t ctx)
1914 {
1915         struct ifnet    *ifp = iflib_get_ifp(ctx);
1916         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
1917         struct ixgbe_hw *hw = &sc->hw;
1918         struct rx_ring  *rxr;
1919         int             i;
1920         u32             ctrl;
1921
1922
1923         /*
1924          * We get here thru init_locked, meaning
1925          * a soft reset, this has already cleared
1926          * the VFTA and other state, so if there
1927          * have been no vlan's registered do nothing.
1928          */
1929         if (sc->num_vlans == 0)
1930                 return;
1931
1932         /* Setup the queues for vlans */
1933         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1934                 for (i = 0; i < sc->num_rx_queues; i++) {
1935                         rxr = &sc->rx_queues[i].rxr;
1936                         /* On 82599 the VLAN enable is per/queue in RXDCTL */
1937                         if (hw->mac.type != ixgbe_mac_82598EB) {
1938                                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
1939                                 ctrl |= IXGBE_RXDCTL_VME;
1940                                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl);
1941                         }
1942                         rxr->vtag_strip = true;
1943                 }
1944         }
1945
1946         if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
1947                 return;
1948         /*
1949          * A soft reset zero's out the VFTA, so
1950          * we need to repopulate it now.
1951          */
1952         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1953                 if (sc->shadow_vfta[i] != 0)
1954                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
1955                             sc->shadow_vfta[i]);
1956
1957         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1958         /* Enable the Filter Table if enabled */
1959         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
1960                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
1961                 ctrl |= IXGBE_VLNCTRL_VFE;
1962         }
1963         if (hw->mac.type == ixgbe_mac_82598EB)
1964                 ctrl |= IXGBE_VLNCTRL_VME;
1965         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1966 } /* ixgbe_setup_vlan_hw_support */
1967
1968 /************************************************************************
1969  * ixgbe_get_slot_info
1970  *
1971  *   Get the width and transaction speed of
1972  *   the slot this adapter is plugged into.
1973  ************************************************************************/
1974 static void
1975 ixgbe_get_slot_info(struct ixgbe_softc *sc)
1976 {
1977         device_t        dev = iflib_get_dev(sc->ctx);
1978         struct ixgbe_hw *hw = &sc->hw;
1979         int             bus_info_valid = true;
1980         u32             offset;
1981         u16             link;
1982
1983         /* Some devices are behind an internal bridge */
1984         switch (hw->device_id) {
1985         case IXGBE_DEV_ID_82599_SFP_SF_QP:
1986         case IXGBE_DEV_ID_82599_QSFP_SF_QP:
1987                 goto get_parent_info;
1988         default:
1989                 break;
1990         }
1991
1992         ixgbe_get_bus_info(hw);
1993
1994         /*
1995          * Some devices don't use PCI-E, but there is no need
1996          * to display "Unknown" for bus speed and width.
1997          */
1998         switch (hw->mac.type) {
1999         case ixgbe_mac_X550EM_x:
2000         case ixgbe_mac_X550EM_a:
2001                 return;
2002         default:
2003                 goto display;
2004         }
2005
2006 get_parent_info:
2007         /*
2008          * For the Quad port adapter we need to parse back
2009          * up the PCI tree to find the speed of the expansion
2010          * slot into which this adapter is plugged. A bit more work.
2011          */
2012         dev = device_get_parent(device_get_parent(dev));
2013 #ifdef IXGBE_DEBUG
2014         device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev),
2015             pci_get_slot(dev), pci_get_function(dev));
2016 #endif
2017         dev = device_get_parent(device_get_parent(dev));
2018 #ifdef IXGBE_DEBUG
2019         device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev),
2020             pci_get_slot(dev), pci_get_function(dev));
2021 #endif
2022         /* Now get the PCI Express Capabilities offset */
2023         if (pci_find_cap(dev, PCIY_EXPRESS, &offset)) {
2024                 /*
2025                  * Hmm...can't get PCI-Express capabilities.
2026                  * Falling back to default method.
2027                  */
2028                 bus_info_valid = false;
2029                 ixgbe_get_bus_info(hw);
2030                 goto display;
2031         }
2032         /* ...and read the Link Status Register */
2033         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
2034         ixgbe_set_pci_config_data_generic(hw, link);
2035
2036 display:
2037         device_printf(dev, "PCI Express Bus: Speed %s %s\n",
2038             ((hw->bus.speed == ixgbe_bus_speed_8000)    ? "8.0GT/s"  :
2039              (hw->bus.speed == ixgbe_bus_speed_5000)    ? "5.0GT/s"  :
2040              (hw->bus.speed == ixgbe_bus_speed_2500)    ? "2.5GT/s"  :
2041              "Unknown"),
2042             ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
2043              (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
2044              (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
2045              "Unknown"));
2046
2047         if (bus_info_valid) {
2048                 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
2049                     ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
2050                     (hw->bus.speed == ixgbe_bus_speed_2500))) {
2051                         device_printf(dev, "PCI-Express bandwidth available for this card\n     is not sufficient for optimal performance.\n");
2052                         device_printf(dev, "For optimal performance a x8 PCIE, or x4 PCIE Gen2 slot is required.\n");
2053                 }
2054                 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
2055                     ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
2056                     (hw->bus.speed < ixgbe_bus_speed_8000))) {
2057                         device_printf(dev, "PCI-Express bandwidth available for this card\n     is not sufficient for optimal performance.\n");
2058                         device_printf(dev, "For optimal performance a x8 PCIE Gen3 slot is required.\n");
2059                 }
2060         } else
2061                 device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n");
2062
2063         return;
2064 } /* ixgbe_get_slot_info */
2065
2066 /************************************************************************
2067  * ixgbe_if_msix_intr_assign
2068  *
2069  *   Setup MSI-X Interrupt resources and handlers
2070  ************************************************************************/
2071 static int
2072 ixgbe_if_msix_intr_assign(if_ctx_t ctx, int msix)
2073 {
2074         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
2075         struct ix_rx_queue *rx_que = sc->rx_queues;
2076         struct ix_tx_queue *tx_que;
2077         int                error, rid, vector = 0;
2078         char               buf[16];
2079
2080         /* Admin Que is vector 0*/
2081         rid = vector + 1;
2082         for (int i = 0; i < sc->num_rx_queues; i++, vector++, rx_que++) {
2083                 rid = vector + 1;
2084
2085                 snprintf(buf, sizeof(buf), "rxq%d", i);
2086                 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
2087                     IFLIB_INTR_RXTX, ixgbe_msix_que, rx_que, rx_que->rxr.me, buf);
2088
2089                 if (error) {
2090                         device_printf(iflib_get_dev(ctx),
2091                             "Failed to allocate que int %d err: %d", i, error);
2092                         sc->num_rx_queues = i + 1;
2093                         goto fail;
2094                 }
2095
2096                 rx_que->msix = vector;
2097         }
2098         for (int i = 0; i < sc->num_tx_queues; i++) {
2099                 snprintf(buf, sizeof(buf), "txq%d", i);
2100                 tx_que = &sc->tx_queues[i];
2101                 tx_que->msix = i % sc->num_rx_queues;
2102                 iflib_softirq_alloc_generic(ctx,
2103                     &sc->rx_queues[tx_que->msix].que_irq,
2104                     IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
2105         }
2106         rid = vector + 1;
2107         error = iflib_irq_alloc_generic(ctx, &sc->irq, rid,
2108             IFLIB_INTR_ADMIN, ixgbe_msix_link, sc, 0, "aq");
2109         if (error) {
2110                 device_printf(iflib_get_dev(ctx),
2111                     "Failed to register admin handler");
2112                 return (error);
2113         }
2114
2115         sc->vector = vector;
2116
2117         return (0);
2118 fail:
2119         iflib_irq_free(ctx, &sc->irq);
2120         rx_que = sc->rx_queues;
2121         for (int i = 0; i < sc->num_rx_queues; i++, rx_que++)
2122                 iflib_irq_free(ctx, &rx_que->que_irq);
2123
2124         return (error);
2125 } /* ixgbe_if_msix_intr_assign */
2126
2127 static inline void
2128 ixgbe_perform_aim(struct ixgbe_softc *sc, struct ix_rx_queue *que)
2129 {
2130         uint32_t newitr = 0;
2131         struct rx_ring *rxr = &que->rxr;
2132
2133         /*
2134          * Do Adaptive Interrupt Moderation:
2135          *  - Write out last calculated setting
2136          *  - Calculate based on average size over
2137          *    the last interval.
2138          */
2139         if (que->eitr_setting) {
2140                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(que->msix),
2141                     que->eitr_setting);
2142         }
2143
2144         que->eitr_setting = 0;
2145         /* Idle, do nothing */
2146         if (rxr->bytes == 0) {
2147                 return;
2148         }
2149
2150         if ((rxr->bytes) && (rxr->packets)) {
2151                 newitr = (rxr->bytes / rxr->packets);
2152         }
2153
2154         newitr += 24; /* account for hardware frame, crc */
2155         /* set an upper boundary */
2156         newitr = min(newitr, 3000);
2157
2158         /* Be nice to the mid range */
2159         if ((newitr > 300) && (newitr < 1200)) {
2160                 newitr = (newitr / 3);
2161         } else {
2162                 newitr = (newitr / 2);
2163         }
2164
2165         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
2166                 newitr |= newitr << 16;
2167         } else {
2168                 newitr |= IXGBE_EITR_CNT_WDIS;
2169         }
2170
2171         /* save for next interrupt */
2172         que->eitr_setting = newitr;
2173
2174         /* Reset state */
2175         rxr->bytes = 0;
2176         rxr->packets = 0;
2177
2178         return;
2179 }
2180
2181 /*********************************************************************
2182  * ixgbe_msix_que - MSI-X Queue Interrupt Service routine
2183  **********************************************************************/
2184 static int
2185 ixgbe_msix_que(void *arg)
2186 {
2187         struct ix_rx_queue *que = arg;
2188         struct ixgbe_softc     *sc = que->sc;
2189         struct ifnet       *ifp = iflib_get_ifp(que->sc->ctx);
2190
2191         /* Protect against spurious interrupts */
2192         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2193                 return (FILTER_HANDLED);
2194
2195         ixgbe_disable_queue(sc, que->msix);
2196         ++que->irqs;
2197
2198         /* Check for AIM */
2199         if (sc->enable_aim) {
2200                 ixgbe_perform_aim(sc, que);
2201         }
2202
2203         return (FILTER_SCHEDULE_THREAD);
2204 } /* ixgbe_msix_que */
2205
2206 /************************************************************************
2207  * ixgbe_media_status - Media Ioctl callback
2208  *
2209  *   Called whenever the user queries the status of
2210  *   the interface using ifconfig.
2211  ************************************************************************/
2212 static void
2213 ixgbe_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
2214 {
2215         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
2216         struct ixgbe_hw *hw = &sc->hw;
2217         int             layer;
2218
2219         INIT_DEBUGOUT("ixgbe_if_media_status: begin");
2220
2221         ifmr->ifm_status = IFM_AVALID;
2222         ifmr->ifm_active = IFM_ETHER;
2223
2224         if (!sc->link_active)
2225                 return;
2226
2227         ifmr->ifm_status |= IFM_ACTIVE;
2228         layer = sc->phy_layer;
2229
2230         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T ||
2231             layer & IXGBE_PHYSICAL_LAYER_1000BASE_T ||
2232             layer & IXGBE_PHYSICAL_LAYER_100BASE_TX ||
2233             layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
2234                 switch (sc->link_speed) {
2235                 case IXGBE_LINK_SPEED_10GB_FULL:
2236                         ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
2237                         break;
2238                 case IXGBE_LINK_SPEED_1GB_FULL:
2239                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
2240                         break;
2241                 case IXGBE_LINK_SPEED_100_FULL:
2242                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
2243                         break;
2244                 case IXGBE_LINK_SPEED_10_FULL:
2245                         ifmr->ifm_active |= IFM_10_T | IFM_FDX;
2246                         break;
2247                 }
2248         if (hw->mac.type == ixgbe_mac_X550)
2249                 switch (sc->link_speed) {
2250                 case IXGBE_LINK_SPEED_5GB_FULL:
2251                         ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
2252                         break;
2253                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2254                         ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
2255                         break;
2256                 }
2257         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
2258             layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
2259                 switch (sc->link_speed) {
2260                 case IXGBE_LINK_SPEED_10GB_FULL:
2261                         ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;
2262                         break;
2263                 }
2264         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
2265                 switch (sc->link_speed) {
2266                 case IXGBE_LINK_SPEED_10GB_FULL:
2267                         ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
2268                         break;
2269                 case IXGBE_LINK_SPEED_1GB_FULL:
2270                         ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
2271                         break;
2272                 }
2273         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM)
2274                 switch (sc->link_speed) {
2275                 case IXGBE_LINK_SPEED_10GB_FULL:
2276                         ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;
2277                         break;
2278                 case IXGBE_LINK_SPEED_1GB_FULL:
2279                         ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
2280                         break;
2281                 }
2282         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR ||
2283             layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
2284                 switch (sc->link_speed) {
2285                 case IXGBE_LINK_SPEED_10GB_FULL:
2286                         ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
2287                         break;
2288                 case IXGBE_LINK_SPEED_1GB_FULL:
2289                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
2290                         break;
2291                 }
2292         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
2293                 switch (sc->link_speed) {
2294                 case IXGBE_LINK_SPEED_10GB_FULL:
2295                         ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
2296                         break;
2297                 }
2298         /*
2299          * XXX: These need to use the proper media types once
2300          * they're added.
2301          */
2302 #ifndef IFM_ETH_XTYPE
2303         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
2304                 switch (sc->link_speed) {
2305                 case IXGBE_LINK_SPEED_10GB_FULL:
2306                         ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
2307                         break;
2308                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2309                         ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
2310                         break;
2311                 case IXGBE_LINK_SPEED_1GB_FULL:
2312                         ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
2313                         break;
2314                 }
2315         else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
2316             layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
2317             layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
2318                 switch (sc->link_speed) {
2319                 case IXGBE_LINK_SPEED_10GB_FULL:
2320                         ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
2321                         break;
2322                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2323                         ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
2324                         break;
2325                 case IXGBE_LINK_SPEED_1GB_FULL:
2326                         ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
2327                         break;
2328                 }
2329 #else
2330         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
2331                 switch (sc->link_speed) {
2332                 case IXGBE_LINK_SPEED_10GB_FULL:
2333                         ifmr->ifm_active |= IFM_10G_KR | IFM_FDX;
2334                         break;
2335                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2336                         ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
2337                         break;
2338                 case IXGBE_LINK_SPEED_1GB_FULL:
2339                         ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
2340                         break;
2341                 }
2342         else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
2343             layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
2344             layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
2345                 switch (sc->link_speed) {
2346                 case IXGBE_LINK_SPEED_10GB_FULL:
2347                         ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX;
2348                         break;
2349                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2350                         ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
2351                         break;
2352                 case IXGBE_LINK_SPEED_1GB_FULL:
2353                         ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
2354                         break;
2355                 }
2356 #endif
2357
2358         /* If nothing is recognized... */
2359         if (IFM_SUBTYPE(ifmr->ifm_active) == 0)
2360                 ifmr->ifm_active |= IFM_UNKNOWN;
2361
2362         /* Display current flow control setting used on link */
2363         if (hw->fc.current_mode == ixgbe_fc_rx_pause ||
2364             hw->fc.current_mode == ixgbe_fc_full)
2365                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
2366         if (hw->fc.current_mode == ixgbe_fc_tx_pause ||
2367             hw->fc.current_mode == ixgbe_fc_full)
2368                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
2369 } /* ixgbe_media_status */
2370
2371 /************************************************************************
2372  * ixgbe_media_change - Media Ioctl callback
2373  *
2374  *   Called when the user changes speed/duplex using
2375  *   media/mediopt option with ifconfig.
2376  ************************************************************************/
2377 static int
2378 ixgbe_if_media_change(if_ctx_t ctx)
2379 {
2380         struct ixgbe_softc   *sc = iflib_get_softc(ctx);
2381         struct ifmedia   *ifm = iflib_get_media(ctx);
2382         struct ixgbe_hw  *hw = &sc->hw;
2383         ixgbe_link_speed speed = 0;
2384
2385         INIT_DEBUGOUT("ixgbe_if_media_change: begin");
2386
2387         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2388                 return (EINVAL);
2389
2390         if (hw->phy.media_type == ixgbe_media_type_backplane)
2391                 return (EPERM);
2392
2393         /*
2394          * We don't actually need to check against the supported
2395          * media types of the adapter; ifmedia will take care of
2396          * that for us.
2397          */
2398         switch (IFM_SUBTYPE(ifm->ifm_media)) {
2399         case IFM_AUTO:
2400         case IFM_10G_T:
2401                 speed |= IXGBE_LINK_SPEED_100_FULL;
2402                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2403                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
2404                 break;
2405         case IFM_10G_LRM:
2406         case IFM_10G_LR:
2407 #ifndef IFM_ETH_XTYPE
2408         case IFM_10G_SR: /* KR, too */
2409         case IFM_10G_CX4: /* KX4 */
2410 #else
2411         case IFM_10G_KR:
2412         case IFM_10G_KX4:
2413 #endif
2414                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2415                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
2416                 break;
2417 #ifndef IFM_ETH_XTYPE
2418         case IFM_1000_CX: /* KX */
2419 #else
2420         case IFM_1000_KX:
2421 #endif
2422         case IFM_1000_LX:
2423         case IFM_1000_SX:
2424                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2425                 break;
2426         case IFM_1000_T:
2427                 speed |= IXGBE_LINK_SPEED_100_FULL;
2428                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2429                 break;
2430         case IFM_10G_TWINAX:
2431                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
2432                 break;
2433         case IFM_5000_T:
2434                 speed |= IXGBE_LINK_SPEED_5GB_FULL;
2435                 break;
2436         case IFM_2500_T:
2437                 speed |= IXGBE_LINK_SPEED_2_5GB_FULL;
2438                 break;
2439         case IFM_100_TX:
2440                 speed |= IXGBE_LINK_SPEED_100_FULL;
2441                 break;
2442         case IFM_10_T:
2443                 speed |= IXGBE_LINK_SPEED_10_FULL;
2444                 break;
2445         default:
2446                 goto invalid;
2447         }
2448
2449         hw->mac.autotry_restart = true;
2450         hw->mac.ops.setup_link(hw, speed, true);
2451         sc->advertise =
2452             ((speed & IXGBE_LINK_SPEED_10GB_FULL)  ? 0x4  : 0) |
2453             ((speed & IXGBE_LINK_SPEED_5GB_FULL)   ? 0x20 : 0) |
2454             ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) |
2455             ((speed & IXGBE_LINK_SPEED_1GB_FULL)   ? 0x2  : 0) |
2456             ((speed & IXGBE_LINK_SPEED_100_FULL)   ? 0x1  : 0) |
2457             ((speed & IXGBE_LINK_SPEED_10_FULL)    ? 0x8  : 0);
2458
2459         return (0);
2460
2461 invalid:
2462         device_printf(iflib_get_dev(ctx), "Invalid media type!\n");
2463
2464         return (EINVAL);
2465 } /* ixgbe_if_media_change */
2466
2467 /************************************************************************
2468  * ixgbe_set_promisc
2469  ************************************************************************/
2470 static int
2471 ixgbe_if_promisc_set(if_ctx_t ctx, int flags)
2472 {
2473         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2474         struct ifnet   *ifp = iflib_get_ifp(ctx);
2475         u32            rctl;
2476         int            mcnt = 0;
2477
2478         rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
2479         rctl &= (~IXGBE_FCTRL_UPE);
2480         if (ifp->if_flags & IFF_ALLMULTI)
2481                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2482         else {
2483                 mcnt = min(if_llmaddr_count(ifp), MAX_NUM_MULTICAST_ADDRESSES);
2484         }
2485         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2486                 rctl &= (~IXGBE_FCTRL_MPE);
2487         IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);
2488
2489         if (ifp->if_flags & IFF_PROMISC) {
2490                 rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2491                 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);
2492         } else if (ifp->if_flags & IFF_ALLMULTI) {
2493                 rctl |= IXGBE_FCTRL_MPE;
2494                 rctl &= ~IXGBE_FCTRL_UPE;
2495                 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);
2496         }
2497         return (0);
2498 } /* ixgbe_if_promisc_set */
2499
2500 /************************************************************************
2501  * ixgbe_msix_link - Link status change ISR (MSI/MSI-X)
2502  ************************************************************************/
2503 static int
2504 ixgbe_msix_link(void *arg)
2505 {
2506         struct ixgbe_softc  *sc = arg;
2507         struct ixgbe_hw *hw = &sc->hw;
2508         u32             eicr, eicr_mask;
2509         s32             retval;
2510
2511         ++sc->link_irq;
2512
2513         /* Pause other interrupts */
2514         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER);
2515
2516         /* First get the cause */
2517         eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
2518         /* Be sure the queue bits are not cleared */
2519         eicr &= ~IXGBE_EICR_RTX_QUEUE;
2520         /* Clear interrupt with write */
2521         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
2522
2523         /* Link status change */
2524         if (eicr & IXGBE_EICR_LSC) {
2525                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
2526                 sc->task_requests |= IXGBE_REQUEST_TASK_LSC;
2527         }
2528
2529         if (sc->hw.mac.type != ixgbe_mac_82598EB) {
2530                 if ((sc->feat_en & IXGBE_FEATURE_FDIR) &&
2531                     (eicr & IXGBE_EICR_FLOW_DIR)) {
2532                         /* This is probably overkill :) */
2533                         if (!atomic_cmpset_int(&sc->fdir_reinit, 0, 1))
2534                                 return (FILTER_HANDLED);
2535                         /* Disable the interrupt */
2536                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
2537                         sc->task_requests |= IXGBE_REQUEST_TASK_FDIR;
2538                 } else
2539                         if (eicr & IXGBE_EICR_ECC) {
2540                                 device_printf(iflib_get_dev(sc->ctx),
2541                                    "\nCRITICAL: ECC ERROR!! Please Reboot!!\n");
2542                                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
2543                         }
2544
2545                 /* Check for over temp condition */
2546                 if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR) {
2547                         switch (sc->hw.mac.type) {
2548                         case ixgbe_mac_X550EM_a:
2549                                 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a))
2550                                         break;
2551                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC,
2552                                     IXGBE_EICR_GPI_SDP0_X550EM_a);
2553                                 IXGBE_WRITE_REG(hw, IXGBE_EICR,
2554                                     IXGBE_EICR_GPI_SDP0_X550EM_a);
2555                                 retval = hw->phy.ops.check_overtemp(hw);
2556                                 if (retval != IXGBE_ERR_OVERTEMP)
2557                                         break;
2558                                 device_printf(iflib_get_dev(sc->ctx),
2559                                     "\nCRITICAL: OVER TEMP!! PHY IS SHUT DOWN!!\n");
2560                                 device_printf(iflib_get_dev(sc->ctx),
2561                                     "System shutdown required!\n");
2562                                 break;
2563                         default:
2564                                 if (!(eicr & IXGBE_EICR_TS))
2565                                         break;
2566                                 retval = hw->phy.ops.check_overtemp(hw);
2567                                 if (retval != IXGBE_ERR_OVERTEMP)
2568                                         break;
2569                                 device_printf(iflib_get_dev(sc->ctx),
2570                                     "\nCRITICAL: OVER TEMP!! PHY IS SHUT DOWN!!\n");
2571                                 device_printf(iflib_get_dev(sc->ctx),
2572                                     "System shutdown required!\n");
2573                                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
2574                                 break;
2575                         }
2576                 }
2577
2578                 /* Check for VF message */
2579                 if ((sc->feat_en & IXGBE_FEATURE_SRIOV) &&
2580                     (eicr & IXGBE_EICR_MAILBOX))
2581                         sc->task_requests |= IXGBE_REQUEST_TASK_MBX;
2582         }
2583
2584         if (ixgbe_is_sfp(hw)) {
2585                 /* Pluggable optics-related interrupt */
2586                 if (hw->mac.type >= ixgbe_mac_X540)
2587                         eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
2588                 else
2589                         eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
2590
2591                 if (eicr & eicr_mask) {
2592                         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
2593                         sc->task_requests |= IXGBE_REQUEST_TASK_MOD;
2594                 }
2595
2596                 if ((hw->mac.type == ixgbe_mac_82599EB) &&
2597                     (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
2598                         IXGBE_WRITE_REG(hw, IXGBE_EICR,
2599                             IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
2600                         sc->task_requests |= IXGBE_REQUEST_TASK_MSF;
2601                 }
2602         }
2603
2604         /* Check for fan failure */
2605         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL) {
2606                 ixgbe_check_fan_failure(sc, eicr, true);
2607                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
2608         }
2609
2610         /* External PHY interrupt */
2611         if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&
2612             (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
2613                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540);
2614                 sc->task_requests |= IXGBE_REQUEST_TASK_PHY;
2615         }
2616
2617         return (sc->task_requests != 0) ? FILTER_SCHEDULE_THREAD : FILTER_HANDLED;
2618 } /* ixgbe_msix_link */
2619
2620 /************************************************************************
2621  * ixgbe_sysctl_interrupt_rate_handler
2622  ************************************************************************/
2623 static int
2624 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
2625 {
2626         struct ix_rx_queue *que = ((struct ix_rx_queue *)oidp->oid_arg1);
2627         int                error;
2628         unsigned int       reg, usec, rate;
2629
2630         reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_EITR(que->msix));
2631         usec = ((reg & 0x0FF8) >> 3);
2632         if (usec > 0)
2633                 rate = 500000 / usec;
2634         else
2635                 rate = 0;
2636         error = sysctl_handle_int(oidp, &rate, 0, req);
2637         if (error || !req->newptr)
2638                 return error;
2639         reg &= ~0xfff; /* default, no limitation */
2640         ixgbe_max_interrupt_rate = 0;
2641         if (rate > 0 && rate < 500000) {
2642                 if (rate < 1000)
2643                         rate = 1000;
2644                 ixgbe_max_interrupt_rate = rate;
2645                 reg |= ((4000000/rate) & 0xff8);
2646         }
2647         IXGBE_WRITE_REG(&que->sc->hw, IXGBE_EITR(que->msix), reg);
2648
2649         return (0);
2650 } /* ixgbe_sysctl_interrupt_rate_handler */
2651
2652 /************************************************************************
2653  * ixgbe_add_device_sysctls
2654  ************************************************************************/
2655 static void
2656 ixgbe_add_device_sysctls(if_ctx_t ctx)
2657 {
2658         struct ixgbe_softc         *sc = iflib_get_softc(ctx);
2659         device_t               dev = iflib_get_dev(ctx);
2660         struct ixgbe_hw        *hw = &sc->hw;
2661         struct sysctl_oid_list *child;
2662         struct sysctl_ctx_list *ctx_list;
2663
2664         ctx_list = device_get_sysctl_ctx(dev);
2665         child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
2666
2667         /* Sysctls for all devices */
2668         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc",
2669             CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2670             sc, 0, ixgbe_sysctl_flowcntl, "I",
2671             IXGBE_SYSCTL_DESC_SET_FC);
2672
2673         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "advertise_speed",
2674             CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2675             sc, 0, ixgbe_sysctl_advertise, "I",
2676             IXGBE_SYSCTL_DESC_ADV_SPEED);
2677
2678         sc->enable_aim = ixgbe_enable_aim;
2679         SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim", CTLFLAG_RW,
2680             &sc->enable_aim, 0, "Interrupt Moderation");
2681
2682         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version",
2683             CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
2684             ixgbe_sysctl_print_fw_version, "A", "Prints FW/NVM Versions");
2685
2686 #ifdef IXGBE_DEBUG
2687         /* testing sysctls (for all devices) */
2688         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "power_state",
2689             CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2690             sc, 0, ixgbe_sysctl_power_state,
2691             "I", "PCI Power State");
2692
2693         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "print_rss_config",
2694             CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
2695             ixgbe_sysctl_print_rss_config, "A", "Prints RSS Configuration");
2696 #endif
2697         /* for X550 series devices */
2698         if (hw->mac.type >= ixgbe_mac_X550)
2699                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac",
2700                     CTLTYPE_U16 | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2701                     sc, 0, ixgbe_sysctl_dmac,
2702                     "I", "DMA Coalesce");
2703
2704         /* for WoL-capable devices */
2705         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
2706                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "wol_enable",
2707                     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
2708                     ixgbe_sysctl_wol_enable, "I", "Enable/Disable Wake on LAN");
2709
2710                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "wufc",
2711                     CTLTYPE_U32 | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2712                     sc, 0, ixgbe_sysctl_wufc,
2713                     "I", "Enable/Disable Wake Up Filters");
2714         }
2715
2716         /* for X552/X557-AT devices */
2717         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
2718                 struct sysctl_oid *phy_node;
2719                 struct sysctl_oid_list *phy_list;
2720
2721                 phy_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "phy",
2722                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "External PHY sysctls");
2723                 phy_list = SYSCTL_CHILDREN(phy_node);
2724
2725                 SYSCTL_ADD_PROC(ctx_list, phy_list, OID_AUTO, "temp",
2726                     CTLTYPE_U16 | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
2727                     sc, 0, ixgbe_sysctl_phy_temp,
2728                     "I", "Current External PHY Temperature (Celsius)");
2729
2730                 SYSCTL_ADD_PROC(ctx_list, phy_list, OID_AUTO,
2731                     "overtemp_occurred",
2732                     CTLTYPE_U16 | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
2733                     ixgbe_sysctl_phy_overtemp_occurred, "I",
2734                     "External PHY High Temperature Event Occurred");
2735         }
2736
2737         if (sc->feat_cap & IXGBE_FEATURE_EEE) {
2738                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_state",
2739                     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
2740                     ixgbe_sysctl_eee_state, "I", "EEE Power Save State");
2741         }
2742 } /* ixgbe_add_device_sysctls */
2743
2744 /************************************************************************
2745  * ixgbe_allocate_pci_resources
2746  ************************************************************************/
2747 static int
2748 ixgbe_allocate_pci_resources(if_ctx_t ctx)
2749 {
2750         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2751         device_t        dev = iflib_get_dev(ctx);
2752         int             rid;
2753
2754         rid = PCIR_BAR(0);
2755         sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
2756             RF_ACTIVE);
2757
2758         if (!(sc->pci_mem)) {
2759                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2760                 return (ENXIO);
2761         }
2762
2763         /* Save bus_space values for READ/WRITE_REG macros */
2764         sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem);
2765         sc->osdep.mem_bus_space_handle =
2766             rman_get_bushandle(sc->pci_mem);
2767         /* Set hw values for shared code */
2768         sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle;
2769
2770         return (0);
2771 } /* ixgbe_allocate_pci_resources */
2772
2773 /************************************************************************
2774  * ixgbe_detach - Device removal routine
2775  *
2776  *   Called when the driver is being removed.
2777  *   Stops the adapter and deallocates all the resources
2778  *   that were allocated for driver operation.
2779  *
2780  *   return 0 on success, positive on failure
2781  ************************************************************************/
2782 static int
2783 ixgbe_if_detach(if_ctx_t ctx)
2784 {
2785         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2786         device_t       dev = iflib_get_dev(ctx);
2787         u32            ctrl_ext;
2788
2789         INIT_DEBUGOUT("ixgbe_detach: begin");
2790
2791         if (ixgbe_pci_iov_detach(dev) != 0) {
2792                 device_printf(dev, "SR-IOV in use; detach first.\n");
2793                 return (EBUSY);
2794         }
2795
2796         ixgbe_setup_low_power_mode(ctx);
2797
2798         /* let hardware know driver is unloading */
2799         ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
2800         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
2801         IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
2802
2803         ixgbe_free_pci_resources(ctx);
2804         free(sc->mta, M_IXGBE);
2805
2806         return (0);
2807 } /* ixgbe_if_detach */
2808
2809 /************************************************************************
2810  * ixgbe_setup_low_power_mode - LPLU/WoL preparation
2811  *
2812  *   Prepare the adapter/port for LPLU and/or WoL
2813  ************************************************************************/
2814 static int
2815 ixgbe_setup_low_power_mode(if_ctx_t ctx)
2816 {
2817         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
2818         struct ixgbe_hw *hw = &sc->hw;
2819         device_t        dev = iflib_get_dev(ctx);
2820         s32             error = 0;
2821
2822         if (!hw->wol_enabled)
2823                 ixgbe_set_phy_power(hw, false);
2824
2825         /* Limit power management flow to X550EM baseT */
2826         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
2827             hw->phy.ops.enter_lplu) {
2828                 /* Turn off support for APM wakeup. (Using ACPI instead) */
2829                 IXGBE_WRITE_REG(hw, IXGBE_GRC,
2830                     IXGBE_READ_REG(hw, IXGBE_GRC) & ~(u32)2);
2831
2832                 /*
2833                  * Clear Wake Up Status register to prevent any previous wakeup
2834                  * events from waking us up immediately after we suspend.
2835                  */
2836                 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
2837
2838                 /*
2839                  * Program the Wakeup Filter Control register with user filter
2840                  * settings
2841                  */
2842                 IXGBE_WRITE_REG(hw, IXGBE_WUFC, sc->wufc);
2843
2844                 /* Enable wakeups and power management in Wakeup Control */
2845                 IXGBE_WRITE_REG(hw, IXGBE_WUC,
2846                     IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN);
2847
2848                 /* X550EM baseT adapters need a special LPLU flow */
2849                 hw->phy.reset_disable = true;
2850                 ixgbe_if_stop(ctx);
2851                 error = hw->phy.ops.enter_lplu(hw);
2852                 if (error)
2853                         device_printf(dev, "Error entering LPLU: %d\n", error);
2854                 hw->phy.reset_disable = false;
2855         } else {
2856                 /* Just stop for other adapters */
2857                 ixgbe_if_stop(ctx);
2858         }
2859
2860         return error;
2861 } /* ixgbe_setup_low_power_mode */
2862
2863 /************************************************************************
2864  * ixgbe_shutdown - Shutdown entry point
2865  ************************************************************************/
2866 static int
2867 ixgbe_if_shutdown(if_ctx_t ctx)
2868 {
2869         int error = 0;
2870
2871         INIT_DEBUGOUT("ixgbe_shutdown: begin");
2872
2873         error = ixgbe_setup_low_power_mode(ctx);
2874
2875         return (error);
2876 } /* ixgbe_if_shutdown */
2877
2878 /************************************************************************
2879  * ixgbe_suspend
2880  *
2881  *   From D0 to D3
2882  ************************************************************************/
2883 static int
2884 ixgbe_if_suspend(if_ctx_t ctx)
2885 {
2886         int error = 0;
2887
2888         INIT_DEBUGOUT("ixgbe_suspend: begin");
2889
2890         error = ixgbe_setup_low_power_mode(ctx);
2891
2892         return (error);
2893 } /* ixgbe_if_suspend */
2894
2895 /************************************************************************
2896  * ixgbe_resume
2897  *
2898  *   From D3 to D0
2899  ************************************************************************/
2900 static int
2901 ixgbe_if_resume(if_ctx_t ctx)
2902 {
2903         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
2904         device_t        dev = iflib_get_dev(ctx);
2905         struct ifnet    *ifp = iflib_get_ifp(ctx);
2906         struct ixgbe_hw *hw = &sc->hw;
2907         u32             wus;
2908
2909         INIT_DEBUGOUT("ixgbe_resume: begin");
2910
2911         /* Read & clear WUS register */
2912         wus = IXGBE_READ_REG(hw, IXGBE_WUS);
2913         if (wus)
2914                 device_printf(dev, "Woken up by (WUS): %#010x\n",
2915                     IXGBE_READ_REG(hw, IXGBE_WUS));
2916         IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
2917         /* And clear WUFC until next low-power transition */
2918         IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
2919
2920         /*
2921          * Required after D3->D0 transition;
2922          * will re-advertise all previous advertised speeds
2923          */
2924         if (ifp->if_flags & IFF_UP)
2925                 ixgbe_if_init(ctx);
2926
2927         return (0);
2928 } /* ixgbe_if_resume */
2929
2930 /************************************************************************
2931  * ixgbe_if_mtu_set - Ioctl mtu entry point
2932  *
2933  *   Return 0 on success, EINVAL on failure
2934  ************************************************************************/
2935 static int
2936 ixgbe_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
2937 {
2938         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2939         int error = 0;
2940
2941         IOCTL_DEBUGOUT("ioctl: SIOCIFMTU (Set Interface MTU)");
2942
2943         if (mtu > IXGBE_MAX_MTU) {
2944                 error = EINVAL;
2945         } else {
2946                 sc->max_frame_size = mtu + IXGBE_MTU_HDR;
2947         }
2948
2949         return error;
2950 } /* ixgbe_if_mtu_set */
2951
2952 /************************************************************************
2953  * ixgbe_if_crcstrip_set
2954  ************************************************************************/
2955 static void
2956 ixgbe_if_crcstrip_set(if_ctx_t ctx, int onoff, int crcstrip)
2957 {
2958         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2959         struct ixgbe_hw *hw = &sc->hw;
2960         /* crc stripping is set in two places:
2961          * IXGBE_HLREG0 (modified on init_locked and hw reset)
2962          * IXGBE_RDRXCTL (set by the original driver in
2963          *      ixgbe_setup_hw_rsc() called in init_locked.
2964          *      We disable the setting when netmap is compiled in).
2965          * We update the values here, but also in ixgbe.c because
2966          * init_locked sometimes is called outside our control.
2967          */
2968         uint32_t hl, rxc;
2969
2970         hl = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2971         rxc = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2972 #ifdef NETMAP
2973         if (netmap_verbose)
2974                 D("%s read  HLREG 0x%x rxc 0x%x",
2975                         onoff ? "enter" : "exit", hl, rxc);
2976 #endif
2977         /* hw requirements ... */
2978         rxc &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2979         rxc |= IXGBE_RDRXCTL_RSCACKC;
2980         if (onoff && !crcstrip) {
2981                 /* keep the crc. Fast rx */
2982                 hl &= ~IXGBE_HLREG0_RXCRCSTRP;
2983                 rxc &= ~IXGBE_RDRXCTL_CRCSTRIP;
2984         } else {
2985                 /* reset default mode */
2986                 hl |= IXGBE_HLREG0_RXCRCSTRP;
2987                 rxc |= IXGBE_RDRXCTL_CRCSTRIP;
2988         }
2989 #ifdef NETMAP
2990         if (netmap_verbose)
2991                 D("%s write HLREG 0x%x rxc 0x%x",
2992                         onoff ? "enter" : "exit", hl, rxc);
2993 #endif
2994         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hl);
2995         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rxc);
2996 } /* ixgbe_if_crcstrip_set */
2997
2998 /*********************************************************************
2999  * ixgbe_if_init - Init entry point
3000  *
3001  *   Used in two ways: It is used by the stack as an init
3002  *   entry point in network interface structure. It is also
3003  *   used by the driver as a hw/sw initialization routine to
3004  *   get to a consistent state.
3005  *
3006  *   Return 0 on success, positive on failure
3007  **********************************************************************/
3008 void
3009 ixgbe_if_init(if_ctx_t ctx)
3010 {
3011         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
3012         struct ifnet       *ifp = iflib_get_ifp(ctx);
3013         device_t           dev = iflib_get_dev(ctx);
3014         struct ixgbe_hw *hw = &sc->hw;
3015         struct ix_rx_queue *rx_que;
3016         struct ix_tx_queue *tx_que;
3017         u32             txdctl, mhadd;
3018         u32             rxdctl, rxctrl;
3019         u32             ctrl_ext;
3020
3021         int             i, j, err;
3022
3023         INIT_DEBUGOUT("ixgbe_if_init: begin");
3024
3025         /* Queue indices may change with IOV mode */
3026         ixgbe_align_all_queue_indices(sc);
3027
3028         /* reprogram the RAR[0] in case user changed it. */
3029         ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, IXGBE_RAH_AV);
3030
3031         /* Get the latest mac address, User can use a LAA */
3032         bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
3033         ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, 1);
3034         hw->addr_ctrl.rar_used_count = 1;
3035
3036         ixgbe_init_hw(hw);
3037
3038         ixgbe_initialize_iov(sc);
3039
3040         ixgbe_initialize_transmit_units(ctx);
3041
3042         /* Setup Multicast table */
3043         ixgbe_if_multi_set(ctx);
3044
3045         /* Determine the correct mbuf pool, based on frame size */
3046         sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
3047
3048         /* Configure RX settings */
3049         ixgbe_initialize_receive_units(ctx);
3050
3051         /*
3052          * Initialize variable holding task enqueue requests
3053          * from MSI-X interrupts
3054          */
3055         sc->task_requests = 0;
3056
3057         /* Enable SDP & MSI-X interrupts based on adapter */
3058         ixgbe_config_gpie(sc);
3059
3060         /* Set MTU size */
3061         if (ifp->if_mtu > ETHERMTU) {
3062                 /* aka IXGBE_MAXFRS on 82599 and newer */
3063                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3064                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
3065                 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
3066                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3067         }
3068
3069         /* Now enable all the queues */
3070         for (i = 0, tx_que = sc->tx_queues; i < sc->num_tx_queues; i++, tx_que++) {
3071                 struct tx_ring *txr = &tx_que->txr;
3072
3073                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me));
3074                 txdctl |= IXGBE_TXDCTL_ENABLE;
3075                 /* Set WTHRESH to 8, burst writeback */
3076                 txdctl |= (8 << 16);
3077                 /*
3078                  * When the internal queue falls below PTHRESH (32),
3079                  * start prefetching as long as there are at least
3080                  * HTHRESH (1) buffers ready. The values are taken
3081                  * from the Intel linux driver 3.8.21.
3082                  * Prefetching enables tx line rate even with 1 queue.
3083                  */
3084                 txdctl |= (32 << 0) | (1 << 8);
3085                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl);
3086         }
3087
3088         for (i = 0, rx_que = sc->rx_queues; i < sc->num_rx_queues; i++, rx_que++) {
3089                 struct rx_ring *rxr = &rx_que->rxr;
3090
3091                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
3092                 if (hw->mac.type == ixgbe_mac_82598EB) {
3093                         /*
3094                          * PTHRESH = 21
3095                          * HTHRESH = 4
3096                          * WTHRESH = 8
3097                          */
3098                         rxdctl &= ~0x3FFFFF;
3099                         rxdctl |= 0x080420;
3100                 }
3101                 rxdctl |= IXGBE_RXDCTL_ENABLE;
3102                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl);
3103                 for (j = 0; j < 10; j++) {
3104                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) &
3105                             IXGBE_RXDCTL_ENABLE)
3106                                 break;
3107                         else
3108                                 msec_delay(1);
3109                 }
3110                 wmb();
3111         }
3112
3113         /* Enable Receive engine */
3114         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3115         if (hw->mac.type == ixgbe_mac_82598EB)
3116                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3117         rxctrl |= IXGBE_RXCTRL_RXEN;
3118         ixgbe_enable_rx_dma(hw, rxctrl);
3119
3120         /* Set up MSI/MSI-X routing */
3121         if (ixgbe_enable_msix)  {
3122                 ixgbe_configure_ivars(sc);
3123                 /* Set up auto-mask */
3124                 if (hw->mac.type == ixgbe_mac_82598EB)
3125                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3126                 else {
3127                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
3128                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
3129                 }
3130         } else {  /* Simple settings for Legacy/MSI */
3131                 ixgbe_set_ivar(sc, 0, 0, 0);
3132                 ixgbe_set_ivar(sc, 0, 0, 1);
3133                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3134         }
3135
3136         ixgbe_init_fdir(sc);
3137
3138         /*
3139          * Check on any SFP devices that
3140          * need to be kick-started
3141          */
3142         if (hw->phy.type == ixgbe_phy_none) {
3143                 err = hw->phy.ops.identify(hw);
3144                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3145                         device_printf(dev,
3146                             "Unsupported SFP+ module type was detected.\n");
3147                         return;
3148                 }
3149         }
3150
3151         /* Set moderation on the Link interrupt */
3152         IXGBE_WRITE_REG(hw, IXGBE_EITR(sc->vector), IXGBE_LINK_ITR);
3153
3154         /* Enable power to the phy. */
3155         ixgbe_set_phy_power(hw, true);
3156
3157         /* Config/Enable Link */
3158         ixgbe_config_link(ctx);
3159
3160         /* Hardware Packet Buffer & Flow Control setup */
3161         ixgbe_config_delay_values(sc);
3162
3163         /* Initialize the FC settings */
3164         ixgbe_start_hw(hw);
3165
3166         /* Set up VLAN support and filter */
3167         ixgbe_setup_vlan_hw_support(ctx);
3168
3169         /* Setup DMA Coalescing */
3170         ixgbe_config_dmac(sc);
3171
3172         /* And now turn on interrupts */
3173         ixgbe_if_enable_intr(ctx);
3174
3175         /* Enable the use of the MBX by the VF's */
3176         if (sc->feat_en & IXGBE_FEATURE_SRIOV) {
3177                 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
3178                 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
3179                 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
3180         }
3181
3182 } /* ixgbe_init_locked */
3183
3184 /************************************************************************
3185  * ixgbe_set_ivar
3186  *
3187  *   Setup the correct IVAR register for a particular MSI-X interrupt
3188  *     (yes this is all very magic and confusing :)
3189  *    - entry is the register array entry
3190  *    - vector is the MSI-X vector for this queue
3191  *    - type is RX/TX/MISC
3192  ************************************************************************/
3193 static void
3194 ixgbe_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type)
3195 {
3196         struct ixgbe_hw *hw = &sc->hw;
3197         u32 ivar, index;
3198
3199         vector |= IXGBE_IVAR_ALLOC_VAL;
3200
3201         switch (hw->mac.type) {
3202         case ixgbe_mac_82598EB:
3203                 if (type == -1)
3204                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
3205                 else
3206                         entry += (type * 64);
3207                 index = (entry >> 2) & 0x1F;
3208                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3209                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
3210                 ivar |= (vector << (8 * (entry & 0x3)));
3211                 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar);
3212                 break;
3213         case ixgbe_mac_82599EB:
3214         case ixgbe_mac_X540:
3215         case ixgbe_mac_X550:
3216         case ixgbe_mac_X550EM_x:
3217         case ixgbe_mac_X550EM_a:
3218                 if (type == -1) { /* MISC IVAR */
3219                         index = (entry & 1) * 8;
3220                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
3221                         ivar &= ~(0xFF << index);
3222                         ivar |= (vector << index);
3223                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
3224                 } else {          /* RX/TX IVARS */
3225                         index = (16 * (entry & 1)) + (8 * type);
3226                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
3227                         ivar &= ~(0xFF << index);
3228                         ivar |= (vector << index);
3229                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
3230                 }
3231         default:
3232                 break;
3233         }
3234 } /* ixgbe_set_ivar */
3235
3236 /************************************************************************
3237  * ixgbe_configure_ivars
3238  ************************************************************************/
3239 static void
3240 ixgbe_configure_ivars(struct ixgbe_softc *sc)
3241 {
3242         struct ix_rx_queue *rx_que = sc->rx_queues;
3243         struct ix_tx_queue *tx_que = sc->tx_queues;
3244         u32                newitr;
3245
3246         if (ixgbe_max_interrupt_rate > 0)
3247                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
3248         else {
3249                 /*
3250                  * Disable DMA coalescing if interrupt moderation is
3251                  * disabled.
3252                  */
3253                 sc->dmac = 0;
3254                 newitr = 0;
3255         }
3256
3257         for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) {
3258                 struct rx_ring *rxr = &rx_que->rxr;
3259
3260                 /* First the RX queue entry */
3261                 ixgbe_set_ivar(sc, rxr->me, rx_que->msix, 0);
3262
3263                 /* Set an Initial EITR value */
3264                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(rx_que->msix), newitr);
3265         }
3266         for (int i = 0; i < sc->num_tx_queues; i++, tx_que++) {
3267                 struct tx_ring *txr = &tx_que->txr;
3268
3269                 /* ... and the TX */
3270                 ixgbe_set_ivar(sc, txr->me, tx_que->msix, 1);
3271         }
3272         /* For the Link interrupt */
3273         ixgbe_set_ivar(sc, 1, sc->vector, -1);
3274 } /* ixgbe_configure_ivars */
3275
3276 /************************************************************************
3277  * ixgbe_config_gpie
3278  ************************************************************************/
3279 static void
3280 ixgbe_config_gpie(struct ixgbe_softc *sc)
3281 {
3282         struct ixgbe_hw *hw = &sc->hw;
3283         u32             gpie;
3284
3285         gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3286
3287         if (sc->intr_type == IFLIB_INTR_MSIX) {
3288                 /* Enable Enhanced MSI-X mode */
3289                 gpie |= IXGBE_GPIE_MSIX_MODE
3290                      |  IXGBE_GPIE_EIAME
3291                      |  IXGBE_GPIE_PBA_SUPPORT
3292                      |  IXGBE_GPIE_OCD;
3293         }
3294
3295         /* Fan Failure Interrupt */
3296         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)
3297                 gpie |= IXGBE_SDP1_GPIEN;
3298
3299         /* Thermal Sensor Interrupt */
3300         if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR)
3301                 gpie |= IXGBE_SDP0_GPIEN_X540;
3302
3303         /* Link detection */
3304         switch (hw->mac.type) {
3305         case ixgbe_mac_82599EB:
3306                 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN;
3307                 break;
3308         case ixgbe_mac_X550EM_x:
3309         case ixgbe_mac_X550EM_a:
3310                 gpie |= IXGBE_SDP0_GPIEN_X540;
3311                 break;
3312         default:
3313                 break;
3314         }
3315
3316         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3317
3318 } /* ixgbe_config_gpie */
3319
3320 /************************************************************************
3321  * ixgbe_config_delay_values
3322  *
3323  *   Requires sc->max_frame_size to be set.
3324  ************************************************************************/
3325 static void
3326 ixgbe_config_delay_values(struct ixgbe_softc *sc)
3327 {
3328         struct ixgbe_hw *hw = &sc->hw;
3329         u32             rxpb, frame, size, tmp;
3330
3331         frame = sc->max_frame_size;
3332
3333         /* Calculate High Water */
3334         switch (hw->mac.type) {
3335         case ixgbe_mac_X540:
3336         case ixgbe_mac_X550:
3337         case ixgbe_mac_X550EM_x:
3338         case ixgbe_mac_X550EM_a:
3339                 tmp = IXGBE_DV_X540(frame, frame);
3340                 break;
3341         default:
3342                 tmp = IXGBE_DV(frame, frame);
3343                 break;
3344         }
3345         size = IXGBE_BT2KB(tmp);
3346         rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
3347         hw->fc.high_water[0] = rxpb - size;
3348
3349         /* Now calculate Low Water */
3350         switch (hw->mac.type) {
3351         case ixgbe_mac_X540:
3352         case ixgbe_mac_X550:
3353         case ixgbe_mac_X550EM_x:
3354         case ixgbe_mac_X550EM_a:
3355                 tmp = IXGBE_LOW_DV_X540(frame);
3356                 break;
3357         default:
3358                 tmp = IXGBE_LOW_DV(frame);
3359                 break;
3360         }
3361         hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
3362
3363         hw->fc.pause_time = IXGBE_FC_PAUSE;
3364         hw->fc.send_xon = true;
3365 } /* ixgbe_config_delay_values */
3366
3367 /************************************************************************
3368  * ixgbe_set_multi - Multicast Update
3369  *
3370  *   Called whenever multicast address list is updated.
3371  ************************************************************************/
3372 static u_int
3373 ixgbe_mc_filter_apply(void *arg, struct sockaddr_dl *sdl, u_int idx)
3374 {
3375         struct ixgbe_softc *sc = arg;
3376         struct ixgbe_mc_addr *mta = sc->mta;
3377
3378         if (idx == MAX_NUM_MULTICAST_ADDRESSES)
3379                 return (0);
3380         bcopy(LLADDR(sdl), mta[idx].addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
3381         mta[idx].vmdq = sc->pool;
3382
3383         return (1);
3384 } /* ixgbe_mc_filter_apply */
3385
3386 static void
3387 ixgbe_if_multi_set(if_ctx_t ctx)
3388 {
3389         struct ixgbe_softc       *sc = iflib_get_softc(ctx);
3390         struct ixgbe_mc_addr *mta;
3391         struct ifnet         *ifp = iflib_get_ifp(ctx);
3392         u8                   *update_ptr;
3393         u32                  fctrl;
3394         u_int                mcnt;
3395
3396         IOCTL_DEBUGOUT("ixgbe_if_multi_set: begin");
3397
3398         mta = sc->mta;
3399         bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES);
3400
3401         mcnt = if_foreach_llmaddr(iflib_get_ifp(ctx), ixgbe_mc_filter_apply, sc);
3402
3403         fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
3404
3405         if (ifp->if_flags & IFF_PROMISC)
3406                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3407         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
3408             ifp->if_flags & IFF_ALLMULTI) {
3409                 fctrl |= IXGBE_FCTRL_MPE;
3410                 fctrl &= ~IXGBE_FCTRL_UPE;
3411         } else
3412                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3413
3414         IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl);
3415
3416         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
3417                 update_ptr = (u8 *)mta;
3418                 ixgbe_update_mc_addr_list(&sc->hw, update_ptr, mcnt,
3419                     ixgbe_mc_array_itr, true);
3420         }
3421
3422 } /* ixgbe_if_multi_set */
3423
3424 /************************************************************************
3425  * ixgbe_mc_array_itr
3426  *
3427  *   An iterator function needed by the multicast shared code.
3428  *   It feeds the shared code routine the addresses in the
3429  *   array of ixgbe_set_multi() one by one.
3430  ************************************************************************/
3431 static u8 *
3432 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
3433 {
3434         struct ixgbe_mc_addr *mta;
3435
3436         mta = (struct ixgbe_mc_addr *)*update_ptr;
3437         *vmdq = mta->vmdq;
3438
3439         *update_ptr = (u8*)(mta + 1);
3440
3441         return (mta->addr);
3442 } /* ixgbe_mc_array_itr */
3443
3444 /************************************************************************
3445  * ixgbe_local_timer - Timer routine
3446  *
3447  *   Checks for link status, updates statistics,
3448  *   and runs the watchdog check.
3449  ************************************************************************/
3450 static void
3451 ixgbe_if_timer(if_ctx_t ctx, uint16_t qid)
3452 {
3453         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3454
3455         if (qid != 0)
3456                 return;
3457
3458         /* Check for pluggable optics */
3459         if (sc->sfp_probe)
3460                 if (!ixgbe_sfp_probe(ctx))
3461                         return; /* Nothing to do */
3462
3463         ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0);
3464
3465         /* Fire off the adminq task */
3466         iflib_admin_intr_deferred(ctx);
3467
3468 } /* ixgbe_if_timer */
3469
3470 /************************************************************************
3471  * ixgbe_sfp_probe
3472  *
3473  *   Determine if a port had optics inserted.
3474  ************************************************************************/
3475 static bool
3476 ixgbe_sfp_probe(if_ctx_t ctx)
3477 {
3478         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3479         struct ixgbe_hw *hw = &sc->hw;
3480         device_t        dev = iflib_get_dev(ctx);
3481         bool            result = false;
3482
3483         if ((hw->phy.type == ixgbe_phy_nl) &&
3484             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
3485                 s32 ret = hw->phy.ops.identify_sfp(hw);
3486                 if (ret)
3487                         goto out;
3488                 ret = hw->phy.ops.reset(hw);
3489                 sc->sfp_probe = false;
3490                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3491                         device_printf(dev, "Unsupported SFP+ module detected!");
3492                         device_printf(dev,
3493                             "Reload driver with supported module.\n");
3494                         goto out;
3495                 } else
3496                         device_printf(dev, "SFP+ module detected!\n");
3497                 /* We now have supported optics */
3498                 result = true;
3499         }
3500 out:
3501
3502         return (result);
3503 } /* ixgbe_sfp_probe */
3504
3505 /************************************************************************
3506  * ixgbe_handle_mod - Tasklet for SFP module interrupts
3507  ************************************************************************/
3508 static void
3509 ixgbe_handle_mod(void *context)
3510 {
3511         if_ctx_t        ctx = context;
3512         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3513         struct ixgbe_hw *hw = &sc->hw;
3514         device_t        dev = iflib_get_dev(ctx);
3515         u32             err, cage_full = 0;
3516
3517         if (sc->hw.need_crosstalk_fix) {
3518                 switch (hw->mac.type) {
3519                 case ixgbe_mac_82599EB:
3520                         cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
3521                             IXGBE_ESDP_SDP2;
3522                         break;
3523                 case ixgbe_mac_X550EM_x:
3524                 case ixgbe_mac_X550EM_a:
3525                         cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
3526                             IXGBE_ESDP_SDP0;
3527                         break;
3528                 default:
3529                         break;
3530                 }
3531
3532                 if (!cage_full)
3533                         goto handle_mod_out;
3534         }
3535
3536         err = hw->phy.ops.identify_sfp(hw);
3537         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3538                 device_printf(dev,
3539                     "Unsupported SFP+ module type was detected.\n");
3540                 goto handle_mod_out;
3541         }
3542
3543         if (hw->mac.type == ixgbe_mac_82598EB)
3544                 err = hw->phy.ops.reset(hw);
3545         else
3546                 err = hw->mac.ops.setup_sfp(hw);
3547
3548         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3549                 device_printf(dev,
3550                     "Setup failure - unsupported SFP+ module type.\n");
3551                 goto handle_mod_out;
3552         }
3553         sc->task_requests |= IXGBE_REQUEST_TASK_MSF;
3554         return;
3555
3556 handle_mod_out:
3557         sc->task_requests &= ~(IXGBE_REQUEST_TASK_MSF);
3558 } /* ixgbe_handle_mod */
3559
3560
3561 /************************************************************************
3562  * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts
3563  ************************************************************************/
3564 static void
3565 ixgbe_handle_msf(void *context)
3566 {
3567         if_ctx_t        ctx = context;
3568         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3569         struct ixgbe_hw *hw = &sc->hw;
3570         u32             autoneg;
3571         bool            negotiate;
3572
3573         /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */
3574         sc->phy_layer = ixgbe_get_supported_physical_layer(hw);
3575
3576         autoneg = hw->phy.autoneg_advertised;
3577         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3578                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
3579         if (hw->mac.ops.setup_link)
3580                 hw->mac.ops.setup_link(hw, autoneg, true);
3581
3582         /* Adjust media types shown in ifconfig */
3583         ifmedia_removeall(sc->media);
3584         ixgbe_add_media_types(sc->ctx);
3585         ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO);
3586 } /* ixgbe_handle_msf */
3587
3588 /************************************************************************
3589  * ixgbe_handle_phy - Tasklet for external PHY interrupts
3590  ************************************************************************/
3591 static void
3592 ixgbe_handle_phy(void *context)
3593 {
3594         if_ctx_t        ctx = context;
3595         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3596         struct ixgbe_hw *hw = &sc->hw;
3597         int             error;
3598
3599         error = hw->phy.ops.handle_lasi(hw);
3600         if (error == IXGBE_ERR_OVERTEMP)
3601                 device_printf(sc->dev, "CRITICAL: EXTERNAL PHY OVER TEMP!!  PHY will downshift to lower power state!\n");
3602         else if (error)
3603                 device_printf(sc->dev,
3604                     "Error handling LASI interrupt: %d\n", error);
3605 } /* ixgbe_handle_phy */
3606
3607 /************************************************************************
3608  * ixgbe_if_stop - Stop the hardware
3609  *
3610  *   Disables all traffic on the adapter by issuing a
3611  *   global reset on the MAC and deallocates TX/RX buffers.
3612  ************************************************************************/
3613 static void
3614 ixgbe_if_stop(if_ctx_t ctx)
3615 {
3616         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3617         struct ixgbe_hw *hw = &sc->hw;
3618
3619         INIT_DEBUGOUT("ixgbe_if_stop: begin\n");
3620
3621         ixgbe_reset_hw(hw);
3622         hw->adapter_stopped = false;
3623         ixgbe_stop_adapter(hw);
3624         if (hw->mac.type == ixgbe_mac_82599EB)
3625                 ixgbe_stop_mac_link_on_d3_82599(hw);
3626         /* Turn off the laser - noop with no optics */
3627         ixgbe_disable_tx_laser(hw);
3628
3629         /* Update the stack */
3630         sc->link_up = false;
3631         ixgbe_if_update_admin_status(ctx);
3632
3633         /* reprogram the RAR[0] in case user changed it. */
3634         ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);
3635
3636         return;
3637 } /* ixgbe_if_stop */
3638
3639 /************************************************************************
3640  * ixgbe_update_link_status - Update OS on link state
3641  *
3642  * Note: Only updates the OS on the cached link state.
3643  *       The real check of the hardware only happens with
3644  *       a link interrupt.
3645  ************************************************************************/
3646 static void
3647 ixgbe_if_update_admin_status(if_ctx_t ctx)
3648 {
3649         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3650         device_t       dev = iflib_get_dev(ctx);
3651
3652         if (sc->link_up) {
3653                 if (sc->link_active == false) {
3654                         if (bootverbose)
3655                                 device_printf(dev, "Link is up %d Gbps %s \n",
3656                                     ((sc->link_speed == 128) ? 10 : 1),
3657                                     "Full Duplex");
3658                         sc->link_active = true;
3659                         /* Update any Flow Control changes */
3660                         ixgbe_fc_enable(&sc->hw);
3661                         /* Update DMA coalescing config */
3662                         ixgbe_config_dmac(sc);
3663                         /* should actually be negotiated value */
3664                         iflib_link_state_change(ctx, LINK_STATE_UP, IF_Gbps(10));
3665
3666                         if (sc->feat_en & IXGBE_FEATURE_SRIOV)
3667                                 ixgbe_ping_all_vfs(sc);
3668                 }
3669         } else { /* Link down */
3670                 if (sc->link_active == true) {
3671                         if (bootverbose)
3672                                 device_printf(dev, "Link is Down\n");
3673                         iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
3674                         sc->link_active = false;
3675                         if (sc->feat_en & IXGBE_FEATURE_SRIOV)
3676                                 ixgbe_ping_all_vfs(sc);
3677                 }
3678         }
3679
3680         /* Handle task requests from msix_link() */
3681         if (sc->task_requests & IXGBE_REQUEST_TASK_MOD)
3682                 ixgbe_handle_mod(ctx);
3683         if (sc->task_requests & IXGBE_REQUEST_TASK_MSF)
3684                 ixgbe_handle_msf(ctx);
3685         if (sc->task_requests & IXGBE_REQUEST_TASK_MBX)
3686                 ixgbe_handle_mbx(ctx);
3687         if (sc->task_requests & IXGBE_REQUEST_TASK_FDIR)
3688                 ixgbe_reinit_fdir(ctx);
3689         if (sc->task_requests & IXGBE_REQUEST_TASK_PHY)
3690                 ixgbe_handle_phy(ctx);
3691         sc->task_requests = 0;
3692
3693         ixgbe_update_stats_counters(sc);
3694 } /* ixgbe_if_update_admin_status */
3695
3696 /************************************************************************
3697  * ixgbe_config_dmac - Configure DMA Coalescing
3698  ************************************************************************/
3699 static void
3700 ixgbe_config_dmac(struct ixgbe_softc *sc)
3701 {
3702         struct ixgbe_hw          *hw = &sc->hw;
3703         struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config;
3704
3705         if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config)
3706                 return;
3707
3708         if (dcfg->watchdog_timer ^ sc->dmac ||
3709             dcfg->link_speed ^ sc->link_speed) {
3710                 dcfg->watchdog_timer = sc->dmac;
3711                 dcfg->fcoe_en = false;
3712                 dcfg->link_speed = sc->link_speed;
3713                 dcfg->num_tcs = 1;
3714
3715                 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n",
3716                     dcfg->watchdog_timer, dcfg->link_speed);
3717
3718                 hw->mac.ops.dmac_config(hw);
3719         }
3720 } /* ixgbe_config_dmac */
3721
3722 /************************************************************************
3723  * ixgbe_if_enable_intr
3724  ************************************************************************/
3725 void
3726 ixgbe_if_enable_intr(if_ctx_t ctx)
3727 {
3728         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
3729         struct ixgbe_hw    *hw = &sc->hw;
3730         struct ix_rx_queue *que = sc->rx_queues;
3731         u32                mask, fwsm;
3732
3733         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3734
3735         switch (sc->hw.mac.type) {
3736         case ixgbe_mac_82599EB:
3737                 mask |= IXGBE_EIMS_ECC;
3738                 /* Temperature sensor on some scs */
3739                 mask |= IXGBE_EIMS_GPI_SDP0;
3740                 /* SFP+ (RX_LOS_N & MOD_ABS_N) */
3741                 mask |= IXGBE_EIMS_GPI_SDP1;
3742                 mask |= IXGBE_EIMS_GPI_SDP2;
3743                 break;
3744         case ixgbe_mac_X540:
3745                 /* Detect if Thermal Sensor is enabled */
3746                 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
3747                 if (fwsm & IXGBE_FWSM_TS_ENABLED)
3748                         mask |= IXGBE_EIMS_TS;
3749                 mask |= IXGBE_EIMS_ECC;
3750                 break;
3751         case ixgbe_mac_X550:
3752                 /* MAC thermal sensor is automatically enabled */
3753                 mask |= IXGBE_EIMS_TS;
3754                 mask |= IXGBE_EIMS_ECC;
3755                 break;
3756         case ixgbe_mac_X550EM_x:
3757         case ixgbe_mac_X550EM_a:
3758                 /* Some devices use SDP0 for important information */
3759                 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
3760                     hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
3761                     hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
3762                     hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T)
3763                         mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
3764                 if (hw->phy.type == ixgbe_phy_x550em_ext_t)
3765                         mask |= IXGBE_EICR_GPI_SDP0_X540;
3766                 mask |= IXGBE_EIMS_ECC;
3767                 break;
3768         default:
3769                 break;
3770         }
3771
3772         /* Enable Fan Failure detection */
3773         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)
3774                 mask |= IXGBE_EIMS_GPI_SDP1;
3775         /* Enable SR-IOV */
3776         if (sc->feat_en & IXGBE_FEATURE_SRIOV)
3777                 mask |= IXGBE_EIMS_MAILBOX;
3778         /* Enable Flow Director */
3779         if (sc->feat_en & IXGBE_FEATURE_FDIR)
3780                 mask |= IXGBE_EIMS_FLOW_DIR;
3781
3782         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3783
3784         /* With MSI-X we use auto clear */
3785         if (sc->intr_type == IFLIB_INTR_MSIX) {
3786                 mask = IXGBE_EIMS_ENABLE_MASK;
3787                 /* Don't autoclear Link */
3788                 mask &= ~IXGBE_EIMS_OTHER;
3789                 mask &= ~IXGBE_EIMS_LSC;
3790                 if (sc->feat_cap & IXGBE_FEATURE_SRIOV)
3791                         mask &= ~IXGBE_EIMS_MAILBOX;
3792                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
3793         }
3794
3795         /*
3796          * Now enable all queues, this is done separately to
3797          * allow for handling the extended (beyond 32) MSI-X
3798          * vectors that can be used by 82599
3799          */
3800         for (int i = 0; i < sc->num_rx_queues; i++, que++)
3801                 ixgbe_enable_queue(sc, que->msix);
3802
3803         IXGBE_WRITE_FLUSH(hw);
3804
3805 } /* ixgbe_if_enable_intr */
3806
3807 /************************************************************************
3808  * ixgbe_disable_intr
3809  ************************************************************************/
3810 static void
3811 ixgbe_if_disable_intr(if_ctx_t ctx)
3812 {
3813         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3814
3815         if (sc->intr_type == IFLIB_INTR_MSIX)
3816                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0);
3817         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
3818                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0);
3819         } else {
3820                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000);
3821                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0);
3822                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0);
3823         }
3824         IXGBE_WRITE_FLUSH(&sc->hw);
3825
3826 } /* ixgbe_if_disable_intr */
3827
3828 /************************************************************************
3829  * ixgbe_link_intr_enable
3830  ************************************************************************/
3831 static void
3832 ixgbe_link_intr_enable(if_ctx_t ctx)
3833 {
3834         struct ixgbe_hw *hw = &((struct ixgbe_softc *)iflib_get_softc(ctx))->hw;
3835
3836         /* Re-enable other interrupts */
3837         IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
3838 } /* ixgbe_link_intr_enable */
3839
3840 /************************************************************************
3841  * ixgbe_if_rx_queue_intr_enable
3842  ************************************************************************/
3843 static int
3844 ixgbe_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
3845 {
3846         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
3847         struct ix_rx_queue *que = &sc->rx_queues[rxqid];
3848
3849         ixgbe_enable_queue(sc, que->msix);
3850
3851         return (0);
3852 } /* ixgbe_if_rx_queue_intr_enable */
3853
3854 /************************************************************************
3855  * ixgbe_enable_queue
3856  ************************************************************************/
3857 static void
3858 ixgbe_enable_queue(struct ixgbe_softc *sc, u32 vector)
3859 {
3860         struct ixgbe_hw *hw = &sc->hw;
3861         u64             queue = 1ULL << vector;
3862         u32             mask;
3863
3864         if (hw->mac.type == ixgbe_mac_82598EB) {
3865                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
3866                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3867         } else {
3868                 mask = (queue & 0xFFFFFFFF);
3869                 if (mask)
3870                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
3871                 mask = (queue >> 32);
3872                 if (mask)
3873                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
3874         }
3875 } /* ixgbe_enable_queue */
3876
3877 /************************************************************************
3878  * ixgbe_disable_queue
3879  ************************************************************************/
3880 static void
3881 ixgbe_disable_queue(struct ixgbe_softc *sc, u32 vector)
3882 {
3883         struct ixgbe_hw *hw = &sc->hw;
3884         u64             queue = 1ULL << vector;
3885         u32             mask;
3886
3887         if (hw->mac.type == ixgbe_mac_82598EB) {
3888                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
3889                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
3890         } else {
3891                 mask = (queue & 0xFFFFFFFF);
3892                 if (mask)
3893                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
3894                 mask = (queue >> 32);
3895                 if (mask)
3896                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
3897         }
3898 } /* ixgbe_disable_queue */
3899
3900 /************************************************************************
3901  * ixgbe_intr - Legacy Interrupt Service Routine
3902  ************************************************************************/
3903 int
3904 ixgbe_intr(void *arg)
3905 {
3906         struct ixgbe_softc     *sc = arg;
3907         struct ix_rx_queue *que = sc->rx_queues;
3908         struct ixgbe_hw    *hw = &sc->hw;
3909         if_ctx_t           ctx = sc->ctx;
3910         u32                eicr, eicr_mask;
3911
3912         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3913
3914         ++que->irqs;
3915         if (eicr == 0) {
3916                 ixgbe_if_enable_intr(ctx);
3917                 return (FILTER_HANDLED);
3918         }
3919
3920         /* Check for fan failure */
3921         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
3922             (eicr & IXGBE_EICR_GPI_SDP1)) {
3923                 device_printf(sc->dev,
3924                     "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
3925                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3926         }
3927
3928         /* Link status change */
3929         if (eicr & IXGBE_EICR_LSC) {
3930                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
3931                 iflib_admin_intr_deferred(ctx);
3932         }
3933
3934         if (ixgbe_is_sfp(hw)) {
3935                 /* Pluggable optics-related interrupt */
3936                 if (hw->mac.type >= ixgbe_mac_X540)
3937                         eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
3938                 else
3939                         eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
3940
3941                 if (eicr & eicr_mask) {
3942                         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
3943                         sc->task_requests |= IXGBE_REQUEST_TASK_MOD;
3944                 }
3945
3946                 if ((hw->mac.type == ixgbe_mac_82599EB) &&
3947                     (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
3948                         IXGBE_WRITE_REG(hw, IXGBE_EICR,
3949                             IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3950                         sc->task_requests |= IXGBE_REQUEST_TASK_MSF;
3951                 }
3952         }
3953
3954         /* External PHY interrupt */
3955         if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&
3956             (eicr & IXGBE_EICR_GPI_SDP0_X540))
3957                 sc->task_requests |= IXGBE_REQUEST_TASK_PHY;
3958
3959         return (FILTER_SCHEDULE_THREAD);
3960 } /* ixgbe_intr */
3961
3962 /************************************************************************
3963  * ixgbe_free_pci_resources
3964  ************************************************************************/
3965 static void
3966 ixgbe_free_pci_resources(if_ctx_t ctx)
3967 {
3968         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3969         struct         ix_rx_queue *que = sc->rx_queues;
3970         device_t       dev = iflib_get_dev(ctx);
3971
3972         /* Release all MSI-X queue resources */
3973         if (sc->intr_type == IFLIB_INTR_MSIX)
3974                 iflib_irq_free(ctx, &sc->irq);
3975
3976         if (que != NULL) {
3977                 for (int i = 0; i < sc->num_rx_queues; i++, que++) {
3978                         iflib_irq_free(ctx, &que->que_irq);
3979                 }
3980         }
3981
3982         if (sc->pci_mem != NULL)
3983                 bus_release_resource(dev, SYS_RES_MEMORY,
3984                     rman_get_rid(sc->pci_mem), sc->pci_mem);
3985 } /* ixgbe_free_pci_resources */
3986
3987 /************************************************************************
3988  * ixgbe_sysctl_flowcntl
3989  *
3990  *   SYSCTL wrapper around setting Flow Control
3991  ************************************************************************/
3992 static int
3993 ixgbe_sysctl_flowcntl(SYSCTL_HANDLER_ARGS)
3994 {
3995         struct ixgbe_softc *sc;
3996         int            error, fc;
3997
3998         sc = (struct ixgbe_softc *)arg1;
3999         fc = sc->hw.fc.current_mode;
4000
4001         error = sysctl_handle_int(oidp, &fc, 0, req);
4002         if ((error) || (req->newptr == NULL))
4003                 return (error);
4004
4005         /* Don't bother if it's not changed */
4006         if (fc == sc->hw.fc.current_mode)
4007                 return (0);
4008
4009         return ixgbe_set_flowcntl(sc, fc);
4010 } /* ixgbe_sysctl_flowcntl */
4011
4012 /************************************************************************
4013  * ixgbe_set_flowcntl - Set flow control
4014  *
4015  *   Flow control values:
4016  *     0 - off
4017  *     1 - rx pause
4018  *     2 - tx pause
4019  *     3 - full
4020  ************************************************************************/
4021 static int
4022 ixgbe_set_flowcntl(struct ixgbe_softc *sc, int fc)
4023 {
4024         switch (fc) {
4025         case ixgbe_fc_rx_pause:
4026         case ixgbe_fc_tx_pause:
4027         case ixgbe_fc_full:
4028                 sc->hw.fc.requested_mode = fc;
4029                 if (sc->num_rx_queues > 1)
4030                         ixgbe_disable_rx_drop(sc);
4031                 break;
4032         case ixgbe_fc_none:
4033                 sc->hw.fc.requested_mode = ixgbe_fc_none;
4034                 if (sc->num_rx_queues > 1)
4035                         ixgbe_enable_rx_drop(sc);
4036                 break;
4037         default:
4038                 return (EINVAL);
4039         }
4040
4041         /* Don't autoneg if forcing a value */
4042         sc->hw.fc.disable_fc_autoneg = true;
4043         ixgbe_fc_enable(&sc->hw);
4044
4045         return (0);
4046 } /* ixgbe_set_flowcntl */
4047
4048 /************************************************************************
4049  * ixgbe_enable_rx_drop
4050  *
4051  *   Enable the hardware to drop packets when the buffer is
4052  *   full. This is useful with multiqueue, so that no single
4053  *   queue being full stalls the entire RX engine. We only
4054  *   enable this when Multiqueue is enabled AND Flow Control
4055  *   is disabled.
4056  ************************************************************************/
4057 static void
4058 ixgbe_enable_rx_drop(struct ixgbe_softc *sc)
4059 {
4060         struct ixgbe_hw *hw = &sc->hw;
4061         struct rx_ring  *rxr;
4062         u32             srrctl;
4063
4064         for (int i = 0; i < sc->num_rx_queues; i++) {
4065                 rxr = &sc->rx_queues[i].rxr;
4066                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
4067                 srrctl |= IXGBE_SRRCTL_DROP_EN;
4068                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
4069         }
4070
4071         /* enable drop for each vf */
4072         for (int i = 0; i < sc->num_vfs; i++) {
4073                 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4074                                 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) |
4075                                 IXGBE_QDE_ENABLE));
4076         }
4077 } /* ixgbe_enable_rx_drop */
4078
4079 /************************************************************************
4080  * ixgbe_disable_rx_drop
4081  ************************************************************************/
4082 static void
4083 ixgbe_disable_rx_drop(struct ixgbe_softc *sc)
4084 {
4085         struct ixgbe_hw *hw = &sc->hw;
4086         struct rx_ring  *rxr;
4087         u32             srrctl;
4088
4089         for (int i = 0; i < sc->num_rx_queues; i++) {
4090                 rxr = &sc->rx_queues[i].rxr;
4091                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
4092                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
4093                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
4094         }
4095
4096         /* disable drop for each vf */
4097         for (int i = 0; i < sc->num_vfs; i++) {
4098                 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4099                     (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT)));
4100         }
4101 } /* ixgbe_disable_rx_drop */
4102
4103 /************************************************************************
4104  * ixgbe_sysctl_advertise
4105  *
4106  *   SYSCTL wrapper around setting advertised speed
4107  ************************************************************************/
4108 static int
4109 ixgbe_sysctl_advertise(SYSCTL_HANDLER_ARGS)
4110 {
4111         struct ixgbe_softc *sc;
4112         int            error, advertise;
4113
4114         sc = (struct ixgbe_softc *)arg1;
4115         advertise = sc->advertise;
4116
4117         error = sysctl_handle_int(oidp, &advertise, 0, req);
4118         if ((error) || (req->newptr == NULL))
4119                 return (error);
4120
4121         return ixgbe_set_advertise(sc, advertise);
4122 } /* ixgbe_sysctl_advertise */
4123
4124 /************************************************************************
4125  * ixgbe_set_advertise - Control advertised link speed
4126  *
4127  *   Flags:
4128  *     0x1  - advertise 100 Mb
4129  *     0x2  - advertise 1G
4130  *     0x4  - advertise 10G
4131  *     0x8  - advertise 10 Mb (yes, Mb)
4132  *     0x10 - advertise 2.5G (disabled by default)
4133  *     0x20 - advertise 5G (disabled by default)
4134  *
4135  ************************************************************************/
4136 static int
4137 ixgbe_set_advertise(struct ixgbe_softc *sc, int advertise)
4138 {
4139         device_t         dev = iflib_get_dev(sc->ctx);
4140         struct ixgbe_hw  *hw;
4141         ixgbe_link_speed speed = 0;
4142         ixgbe_link_speed link_caps = 0;
4143         s32              err = IXGBE_NOT_IMPLEMENTED;
4144         bool             negotiate = false;
4145
4146         /* Checks to validate new value */
4147         if (sc->advertise == advertise) /* no change */
4148                 return (0);
4149
4150         hw = &sc->hw;
4151
4152         /* No speed changes for backplane media */
4153         if (hw->phy.media_type == ixgbe_media_type_backplane)
4154                 return (ENODEV);
4155
4156         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
4157               (hw->phy.multispeed_fiber))) {
4158                 device_printf(dev, "Advertised speed can only be set on copper or multispeed fiber media types.\n");
4159                 return (EINVAL);
4160         }
4161
4162         if (advertise < 0x1 || advertise > 0x3F) {
4163                 device_printf(dev, "Invalid advertised speed; valid modes are 0x1 through 0x3F\n");
4164                 return (EINVAL);
4165         }
4166
4167         if (hw->mac.ops.get_link_capabilities) {
4168                 err = hw->mac.ops.get_link_capabilities(hw, &link_caps,
4169                     &negotiate);
4170                 if (err != IXGBE_SUCCESS) {
4171                         device_printf(dev, "Unable to determine supported advertise speeds\n");
4172                         return (ENODEV);
4173                 }
4174         }
4175
4176         /* Set new value and report new advertised mode */
4177         if (advertise & 0x1) {
4178                 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) {
4179                         device_printf(dev, "Interface does not support 100Mb advertised speed\n");
4180                         return (EINVAL);
4181                 }
4182                 speed |= IXGBE_LINK_SPEED_100_FULL;
4183         }
4184         if (advertise & 0x2) {
4185                 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) {
4186                         device_printf(dev, "Interface does not support 1Gb advertised speed\n");
4187                         return (EINVAL);
4188                 }
4189                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
4190         }
4191         if (advertise & 0x4) {
4192                 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) {
4193                         device_printf(dev, "Interface does not support 10Gb advertised speed\n");
4194                         return (EINVAL);
4195                 }
4196                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
4197         }
4198         if (advertise & 0x8) {
4199                 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) {
4200                         device_printf(dev, "Interface does not support 10Mb advertised speed\n");
4201                         return (EINVAL);
4202                 }
4203                 speed |= IXGBE_LINK_SPEED_10_FULL;
4204         }
4205         if (advertise & 0x10) {
4206                 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) {
4207                         device_printf(dev, "Interface does not support 2.5G advertised speed\n");
4208                         return (EINVAL);
4209                 }
4210                 speed |= IXGBE_LINK_SPEED_2_5GB_FULL;
4211         }
4212         if (advertise & 0x20) {
4213                 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) {
4214                         device_printf(dev, "Interface does not support 5G advertised speed\n");
4215                         return (EINVAL);
4216                 }
4217                 speed |= IXGBE_LINK_SPEED_5GB_FULL;
4218         }
4219
4220         hw->mac.autotry_restart = true;
4221         hw->mac.ops.setup_link(hw, speed, true);
4222         sc->advertise = advertise;
4223
4224         return (0);
4225 } /* ixgbe_set_advertise */
4226
4227 /************************************************************************
4228  * ixgbe_get_default_advertise - Get default advertised speed settings
4229  *
4230  *   Formatted for sysctl usage.
4231  *   Flags:
4232  *     0x1 - advertise 100 Mb
4233  *     0x2 - advertise 1G
4234  *     0x4 - advertise 10G
4235  *     0x8 - advertise 10 Mb (yes, Mb)
4236  *     0x10 - advertise 2.5G (disabled by default)
4237  *     0x20 - advertise 5G (disabled by default)
4238  ************************************************************************/
4239 static int
4240 ixgbe_get_default_advertise(struct ixgbe_softc *sc)
4241 {
4242         struct ixgbe_hw  *hw = &sc->hw;
4243         int              speed;
4244         ixgbe_link_speed link_caps = 0;
4245         s32              err;
4246         bool             negotiate = false;
4247
4248         /*
4249          * Advertised speed means nothing unless it's copper or
4250          * multi-speed fiber
4251          */
4252         if (!(hw->phy.media_type == ixgbe_media_type_copper) &&
4253             !(hw->phy.multispeed_fiber))
4254                 return (0);
4255
4256         err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate);
4257         if (err != IXGBE_SUCCESS)
4258                 return (0);
4259
4260         if (hw->mac.type == ixgbe_mac_X550) {
4261                 /*
4262                  * 2.5G and 5G autonegotiation speeds on X550
4263                  * are disabled by default due to reported
4264                  * interoperability issues with some switches.
4265                  */
4266                 link_caps &= ~(IXGBE_LINK_SPEED_2_5GB_FULL |
4267                     IXGBE_LINK_SPEED_5GB_FULL);
4268         }
4269
4270         speed =
4271             ((link_caps & IXGBE_LINK_SPEED_10GB_FULL)  ? 0x4  : 0) |
4272             ((link_caps & IXGBE_LINK_SPEED_5GB_FULL)   ? 0x20 : 0) |
4273             ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) |
4274             ((link_caps & IXGBE_LINK_SPEED_1GB_FULL)   ? 0x2  : 0) |
4275             ((link_caps & IXGBE_LINK_SPEED_100_FULL)   ? 0x1  : 0) |
4276             ((link_caps & IXGBE_LINK_SPEED_10_FULL)    ? 0x8  : 0);
4277
4278         return speed;
4279 } /* ixgbe_get_default_advertise */
4280
4281 /************************************************************************
4282  * ixgbe_sysctl_dmac - Manage DMA Coalescing
4283  *
4284  *   Control values:
4285  *     0/1 - off / on (use default value of 1000)
4286  *
4287  *     Legal timer values are:
4288  *     50,100,250,500,1000,2000,5000,10000
4289  *
4290  *     Turning off interrupt moderation will also turn this off.
4291  ************************************************************************/
4292 static int
4293 ixgbe_sysctl_dmac(SYSCTL_HANDLER_ARGS)
4294 {
4295         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4296         struct ifnet   *ifp = iflib_get_ifp(sc->ctx);
4297         int            error;
4298         u16            newval;
4299
4300         newval = sc->dmac;
4301         error = sysctl_handle_16(oidp, &newval, 0, req);
4302         if ((error) || (req->newptr == NULL))
4303                 return (error);
4304
4305         switch (newval) {
4306         case 0:
4307                 /* Disabled */
4308                 sc->dmac = 0;
4309                 break;
4310         case 1:
4311                 /* Enable and use default */
4312                 sc->dmac = 1000;
4313                 break;
4314         case 50:
4315         case 100:
4316         case 250:
4317         case 500:
4318         case 1000:
4319         case 2000:
4320         case 5000:
4321         case 10000:
4322                 /* Legal values - allow */
4323                 sc->dmac = newval;
4324                 break;
4325         default:
4326                 /* Do nothing, illegal value */
4327                 return (EINVAL);
4328         }
4329
4330         /* Re-initialize hardware if it's already running */
4331         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
4332                 ifp->if_init(ifp);
4333
4334         return (0);
4335 } /* ixgbe_sysctl_dmac */
4336
4337 #ifdef IXGBE_DEBUG
4338 /************************************************************************
4339  * ixgbe_sysctl_power_state
4340  *
4341  *   Sysctl to test power states
4342  *   Values:
4343  *     0      - set device to D0
4344  *     3      - set device to D3
4345  *     (none) - get current device power state
4346  ************************************************************************/
4347 static int
4348 ixgbe_sysctl_power_state(SYSCTL_HANDLER_ARGS)
4349 {
4350         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4351         device_t       dev = sc->dev;
4352         int            curr_ps, new_ps, error = 0;
4353
4354         curr_ps = new_ps = pci_get_powerstate(dev);
4355
4356         error = sysctl_handle_int(oidp, &new_ps, 0, req);
4357         if ((error) || (req->newptr == NULL))
4358                 return (error);
4359
4360         if (new_ps == curr_ps)
4361                 return (0);
4362
4363         if (new_ps == 3 && curr_ps == 0)
4364                 error = DEVICE_SUSPEND(dev);
4365         else if (new_ps == 0 && curr_ps == 3)
4366                 error = DEVICE_RESUME(dev);
4367         else
4368                 return (EINVAL);
4369
4370         device_printf(dev, "New state: %d\n", pci_get_powerstate(dev));
4371
4372         return (error);
4373 } /* ixgbe_sysctl_power_state */
4374 #endif
4375
4376 /************************************************************************
4377  * ixgbe_sysctl_wol_enable
4378  *
4379  *   Sysctl to enable/disable the WoL capability,
4380  *   if supported by the adapter.
4381  *
4382  *   Values:
4383  *     0 - disabled
4384  *     1 - enabled
4385  ************************************************************************/
4386 static int
4387 ixgbe_sysctl_wol_enable(SYSCTL_HANDLER_ARGS)
4388 {
4389         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4390         struct ixgbe_hw *hw = &sc->hw;
4391         int             new_wol_enabled;
4392         int             error = 0;
4393
4394         new_wol_enabled = hw->wol_enabled;
4395         error = sysctl_handle_int(oidp, &new_wol_enabled, 0, req);
4396         if ((error) || (req->newptr == NULL))
4397                 return (error);
4398         new_wol_enabled = !!(new_wol_enabled);
4399         if (new_wol_enabled == hw->wol_enabled)
4400                 return (0);
4401
4402         if (new_wol_enabled > 0 && !sc->wol_support)
4403                 return (ENODEV);
4404         else
4405                 hw->wol_enabled = new_wol_enabled;
4406
4407         return (0);
4408 } /* ixgbe_sysctl_wol_enable */
4409
4410 /************************************************************************
4411  * ixgbe_sysctl_wufc - Wake Up Filter Control
4412  *
4413  *   Sysctl to enable/disable the types of packets that the
4414  *   adapter will wake up on upon receipt.
4415  *   Flags:
4416  *     0x1  - Link Status Change
4417  *     0x2  - Magic Packet
4418  *     0x4  - Direct Exact
4419  *     0x8  - Directed Multicast
4420  *     0x10 - Broadcast
4421  *     0x20 - ARP/IPv4 Request Packet
4422  *     0x40 - Direct IPv4 Packet
4423  *     0x80 - Direct IPv6 Packet
4424  *
4425  *   Settings not listed above will cause the sysctl to return an error.
4426  ************************************************************************/
4427 static int
4428 ixgbe_sysctl_wufc(SYSCTL_HANDLER_ARGS)
4429 {
4430         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4431         int            error = 0;
4432         u32            new_wufc;
4433
4434         new_wufc = sc->wufc;
4435
4436         error = sysctl_handle_32(oidp, &new_wufc, 0, req);
4437         if ((error) || (req->newptr == NULL))
4438                 return (error);
4439         if (new_wufc == sc->wufc)
4440                 return (0);
4441
4442         if (new_wufc & 0xffffff00)
4443                 return (EINVAL);
4444
4445         new_wufc &= 0xff;
4446         new_wufc |= (0xffffff & sc->wufc);
4447         sc->wufc = new_wufc;
4448
4449         return (0);
4450 } /* ixgbe_sysctl_wufc */
4451
4452 #ifdef IXGBE_DEBUG
4453 /************************************************************************
4454  * ixgbe_sysctl_print_rss_config
4455  ************************************************************************/
4456 static int
4457 ixgbe_sysctl_print_rss_config(SYSCTL_HANDLER_ARGS)
4458 {
4459         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4460         struct ixgbe_hw *hw = &sc->hw;
4461         device_t        dev = sc->dev;
4462         struct sbuf     *buf;
4463         int             error = 0, reta_size;
4464         u32             reg;
4465
4466         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4467         if (!buf) {
4468                 device_printf(dev, "Could not allocate sbuf for output.\n");
4469                 return (ENOMEM);
4470         }
4471
4472         // TODO: use sbufs to make a string to print out
4473         /* Set multiplier for RETA setup and table size based on MAC */
4474         switch (sc->hw.mac.type) {
4475         case ixgbe_mac_X550:
4476         case ixgbe_mac_X550EM_x:
4477         case ixgbe_mac_X550EM_a:
4478                 reta_size = 128;
4479                 break;
4480         default:
4481                 reta_size = 32;
4482                 break;
4483         }
4484
4485         /* Print out the redirection table */
4486         sbuf_cat(buf, "\n");
4487         for (int i = 0; i < reta_size; i++) {
4488                 if (i < 32) {
4489                         reg = IXGBE_READ_REG(hw, IXGBE_RETA(i));
4490                         sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg);
4491                 } else {
4492                         reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32));
4493                         sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg);
4494                 }
4495         }
4496
4497         // TODO: print more config
4498
4499         error = sbuf_finish(buf);
4500         if (error)
4501                 device_printf(dev, "Error finishing sbuf: %d\n", error);
4502
4503         sbuf_delete(buf);
4504
4505         return (0);
4506 } /* ixgbe_sysctl_print_rss_config */
4507 #endif /* IXGBE_DEBUG */
4508
4509 /************************************************************************
4510  * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY
4511  *
4512  *   For X552/X557-AT devices using an external PHY
4513  ************************************************************************/
4514 static int
4515 ixgbe_sysctl_phy_temp(SYSCTL_HANDLER_ARGS)
4516 {
4517         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4518         struct ixgbe_hw *hw = &sc->hw;
4519         u16             reg;
4520
4521         if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {
4522                 device_printf(iflib_get_dev(sc->ctx),
4523                     "Device has no supported external thermal sensor.\n");
4524                 return (ENODEV);
4525         }
4526
4527         if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP,
4528             IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
4529                 device_printf(iflib_get_dev(sc->ctx),
4530                     "Error reading from PHY's current temperature register\n");
4531                 return (EAGAIN);
4532         }
4533
4534         /* Shift temp for output */
4535         reg = reg >> 8;
4536
4537         return (sysctl_handle_16(oidp, NULL, reg, req));
4538 } /* ixgbe_sysctl_phy_temp */
4539
4540 /************************************************************************
4541  * ixgbe_sysctl_phy_overtemp_occurred
4542  *
4543  *   Reports (directly from the PHY) whether the current PHY
4544  *   temperature is over the overtemp threshold.
4545  ************************************************************************/
4546 static int
4547 ixgbe_sysctl_phy_overtemp_occurred(SYSCTL_HANDLER_ARGS)
4548 {
4549         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4550         struct ixgbe_hw *hw = &sc->hw;
4551         u16             reg;
4552
4553         if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {
4554                 device_printf(iflib_get_dev(sc->ctx),
4555                     "Device has no supported external thermal sensor.\n");
4556                 return (ENODEV);
4557         }
4558
4559         if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS,
4560             IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
4561                 device_printf(iflib_get_dev(sc->ctx),
4562                     "Error reading from PHY's temperature status register\n");
4563                 return (EAGAIN);
4564         }
4565
4566         /* Get occurrence bit */
4567         reg = !!(reg & 0x4000);
4568
4569         return (sysctl_handle_16(oidp, 0, reg, req));
4570 } /* ixgbe_sysctl_phy_overtemp_occurred */
4571
4572 /************************************************************************
4573  * ixgbe_sysctl_eee_state
4574  *
4575  *   Sysctl to set EEE power saving feature
4576  *   Values:
4577  *     0      - disable EEE
4578  *     1      - enable EEE
4579  *     (none) - get current device EEE state
4580  ************************************************************************/
4581 static int
4582 ixgbe_sysctl_eee_state(SYSCTL_HANDLER_ARGS)
4583 {
4584         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4585         device_t       dev = sc->dev;
4586         struct ifnet   *ifp = iflib_get_ifp(sc->ctx);
4587         int            curr_eee, new_eee, error = 0;
4588         s32            retval;
4589
4590         curr_eee = new_eee = !!(sc->feat_en & IXGBE_FEATURE_EEE);
4591
4592         error = sysctl_handle_int(oidp, &new_eee, 0, req);
4593         if ((error) || (req->newptr == NULL))
4594                 return (error);
4595
4596         /* Nothing to do */
4597         if (new_eee == curr_eee)
4598                 return (0);
4599
4600         /* Not supported */
4601         if (!(sc->feat_cap & IXGBE_FEATURE_EEE))
4602                 return (EINVAL);
4603
4604         /* Bounds checking */
4605         if ((new_eee < 0) || (new_eee > 1))
4606                 return (EINVAL);
4607
4608         retval = ixgbe_setup_eee(&sc->hw, new_eee);
4609         if (retval) {
4610                 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval);
4611                 return (EINVAL);
4612         }
4613
4614         /* Restart auto-neg */
4615         ifp->if_init(ifp);
4616
4617         device_printf(dev, "New EEE state: %d\n", new_eee);
4618
4619         /* Cache new value */
4620         if (new_eee)
4621                 sc->feat_en |= IXGBE_FEATURE_EEE;
4622         else
4623                 sc->feat_en &= ~IXGBE_FEATURE_EEE;
4624
4625         return (error);
4626 } /* ixgbe_sysctl_eee_state */
4627
4628 /************************************************************************
4629  * ixgbe_init_device_features
4630  ************************************************************************/
4631 static void
4632 ixgbe_init_device_features(struct ixgbe_softc *sc)
4633 {
4634         sc->feat_cap = IXGBE_FEATURE_NETMAP
4635                           | IXGBE_FEATURE_RSS
4636                           | IXGBE_FEATURE_MSI
4637                           | IXGBE_FEATURE_MSIX
4638                           | IXGBE_FEATURE_LEGACY_IRQ;
4639
4640         /* Set capabilities first... */
4641         switch (sc->hw.mac.type) {
4642         case ixgbe_mac_82598EB:
4643                 if (sc->hw.device_id == IXGBE_DEV_ID_82598AT)
4644                         sc->feat_cap |= IXGBE_FEATURE_FAN_FAIL;
4645                 break;
4646         case ixgbe_mac_X540:
4647                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4648                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4649                 if ((sc->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) &&
4650                     (sc->hw.bus.func == 0))
4651                         sc->feat_cap |= IXGBE_FEATURE_BYPASS;
4652                 break;
4653         case ixgbe_mac_X550:
4654                 sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
4655                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4656                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4657                 break;
4658         case ixgbe_mac_X550EM_x:
4659                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4660                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4661                 break;
4662         case ixgbe_mac_X550EM_a:
4663                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4664                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4665                 sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
4666                 if ((sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) ||
4667                     (sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) {
4668                         sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
4669                         sc->feat_cap |= IXGBE_FEATURE_EEE;
4670                 }
4671                 break;
4672         case ixgbe_mac_82599EB:
4673                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4674                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4675                 if ((sc->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) &&
4676                     (sc->hw.bus.func == 0))
4677                         sc->feat_cap |= IXGBE_FEATURE_BYPASS;
4678                 if (sc->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP)
4679                         sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
4680                 break;
4681         default:
4682                 break;
4683         }
4684
4685         /* Enabled by default... */
4686         /* Fan failure detection */
4687         if (sc->feat_cap & IXGBE_FEATURE_FAN_FAIL)
4688                 sc->feat_en |= IXGBE_FEATURE_FAN_FAIL;
4689         /* Netmap */
4690         if (sc->feat_cap & IXGBE_FEATURE_NETMAP)
4691                 sc->feat_en |= IXGBE_FEATURE_NETMAP;
4692         /* EEE */
4693         if (sc->feat_cap & IXGBE_FEATURE_EEE)
4694                 sc->feat_en |= IXGBE_FEATURE_EEE;
4695         /* Thermal Sensor */
4696         if (sc->feat_cap & IXGBE_FEATURE_TEMP_SENSOR)
4697                 sc->feat_en |= IXGBE_FEATURE_TEMP_SENSOR;
4698
4699         /* Enabled via global sysctl... */
4700         /* Flow Director */
4701         if (ixgbe_enable_fdir) {
4702                 if (sc->feat_cap & IXGBE_FEATURE_FDIR)
4703                         sc->feat_en |= IXGBE_FEATURE_FDIR;
4704                 else
4705                         device_printf(sc->dev, "Device does not support Flow Director. Leaving disabled.");
4706         }
4707         /*
4708          * Message Signal Interrupts - Extended (MSI-X)
4709          * Normal MSI is only enabled if MSI-X calls fail.
4710          */
4711         if (!ixgbe_enable_msix)
4712                 sc->feat_cap &= ~IXGBE_FEATURE_MSIX;
4713         /* Receive-Side Scaling (RSS) */
4714         if ((sc->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss)
4715                 sc->feat_en |= IXGBE_FEATURE_RSS;
4716
4717         /* Disable features with unmet dependencies... */
4718         /* No MSI-X */
4719         if (!(sc->feat_cap & IXGBE_FEATURE_MSIX)) {
4720                 sc->feat_cap &= ~IXGBE_FEATURE_RSS;
4721                 sc->feat_cap &= ~IXGBE_FEATURE_SRIOV;
4722                 sc->feat_en &= ~IXGBE_FEATURE_RSS;
4723                 sc->feat_en &= ~IXGBE_FEATURE_SRIOV;
4724         }
4725 } /* ixgbe_init_device_features */
4726
4727 /************************************************************************
4728  * ixgbe_check_fan_failure
4729  ************************************************************************/
4730 static void
4731 ixgbe_check_fan_failure(struct ixgbe_softc *sc, u32 reg, bool in_interrupt)
4732 {
4733         u32 mask;
4734
4735         mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&sc->hw) :
4736             IXGBE_ESDP_SDP1;
4737
4738         if (reg & mask)
4739                 device_printf(sc->dev, "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
4740 } /* ixgbe_check_fan_failure */
4741
4742 /************************************************************************
4743  * ixgbe_sbuf_fw_version
4744  ************************************************************************/
4745 static void
4746 ixgbe_sbuf_fw_version(struct ixgbe_hw *hw, struct sbuf *buf)
4747 {
4748         struct ixgbe_nvm_version nvm_ver = {0};
4749         uint16_t phyfw = 0;
4750         int status;
4751         const char *space = "";
4752
4753         ixgbe_get_oem_prod_version(hw, &nvm_ver); /* OEM's NVM version */
4754         ixgbe_get_orom_version(hw, &nvm_ver); /* Option ROM */
4755         ixgbe_get_etk_id(hw, &nvm_ver); /* eTrack identifies a build in Intel's SCM */
4756         status = ixgbe_get_phy_firmware_version(hw, &phyfw);
4757
4758         if (nvm_ver.oem_valid) {
4759                 sbuf_printf(buf, "NVM OEM V%d.%d R%d", nvm_ver.oem_major,
4760                     nvm_ver.oem_minor, nvm_ver.oem_release);
4761                 space = " ";
4762         }
4763
4764         if (nvm_ver.or_valid) {
4765                 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d",
4766                     space, nvm_ver.or_major, nvm_ver.or_build, nvm_ver.or_patch);
4767                 space = " ";
4768         }
4769
4770         if (nvm_ver.etk_id != ((NVM_VER_INVALID << NVM_ETK_SHIFT) |
4771             NVM_VER_INVALID)) {
4772                 sbuf_printf(buf, "%seTrack 0x%08x", space, nvm_ver.etk_id);
4773                 space = " ";
4774         }
4775
4776         if (phyfw != 0 && status == IXGBE_SUCCESS)
4777                 sbuf_printf(buf, "%sPHY FW V%d", space, phyfw);
4778 } /* ixgbe_sbuf_fw_version */
4779
4780 /************************************************************************
4781  * ixgbe_print_fw_version
4782  ************************************************************************/
4783 static void
4784 ixgbe_print_fw_version(if_ctx_t ctx)
4785 {
4786         struct ixgbe_softc *sc = iflib_get_softc(ctx);
4787         struct ixgbe_hw *hw = &sc->hw;
4788         device_t dev = sc->dev;
4789         struct sbuf *buf;
4790         int error = 0;
4791
4792         buf = sbuf_new_auto();
4793         if (!buf) {
4794                 device_printf(dev, "Could not allocate sbuf for output.\n");
4795                 return;
4796         }
4797
4798         ixgbe_sbuf_fw_version(hw, buf);
4799
4800         error = sbuf_finish(buf);
4801         if (error)
4802                 device_printf(dev, "Error finishing sbuf: %d\n", error);
4803         else if (sbuf_len(buf))
4804                 device_printf(dev, "%s\n", sbuf_data(buf));
4805
4806         sbuf_delete(buf);
4807 } /* ixgbe_print_fw_version */
4808
4809 /************************************************************************
4810  * ixgbe_sysctl_print_fw_version
4811  ************************************************************************/
4812 static int
4813 ixgbe_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS)
4814 {
4815         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4816         struct ixgbe_hw *hw = &sc->hw;
4817         device_t dev = sc->dev;
4818         struct sbuf *buf;
4819         int error = 0;
4820
4821         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4822         if (!buf) {
4823                 device_printf(dev, "Could not allocate sbuf for output.\n");
4824                 return (ENOMEM);
4825         }
4826
4827         ixgbe_sbuf_fw_version(hw, buf);
4828
4829         error = sbuf_finish(buf);
4830         if (error)
4831                 device_printf(dev, "Error finishing sbuf: %d\n", error);
4832
4833         sbuf_delete(buf);
4834
4835         return (0);
4836 } /* ixgbe_sysctl_print_fw_version */