sys/dev/ixgbe/if_ix.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2017, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37 #include "opt_rss.h"
  38
  39 #include "ixgbe.h"
  40 #include "ixgbe_sriov.h"
  41 #include "ifdi_if.h"
  42
  43 #include <net/netmap.h>
  44 #include <dev/netmap/netmap_kern.h>
  45
  46 /************************************************************************
  47  * Driver version
  48  ************************************************************************/
  49 char ixgbe_driver_version[] = "4.0.1-k";
  50
  51 /************************************************************************
  52  * PCI Device ID Table
  53  *
  54  *   Used by probe to select devices to load on
  55  *   Last field stores an index into ixgbe_strings
  56  *   Last entry must be all 0s
  57  *
  58  *   { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  59  ************************************************************************/
  60 static pci_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT,  "Intel(R) 82598EB AF (Dual Fiber)"),
  63   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT,  "Intel(R) 82598EB AF (Fiber)"),
  64   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4,  "Intel(R) 82598EB AT (CX4)"),
  65   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT,  "Intel(R) 82598EB AT"),
  66   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2,  "Intel(R) 82598EB AT2"),
  67   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598,  "Intel(R) 82598"),
  68   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT,  "Intel(R) 82598EB AF DA (Dual Fiber)"),
  69   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT,  "Intel(R) 82598EB AT (Dual CX4)"),
  70   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR,  "Intel(R) 82598EB AF (Dual Fiber LR)"),
  71   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM,  "Intel(R) 82598EB AF (Dual Fiber SR)"),
  72   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM,  "Intel(R) 82598EB LOM"),
  73   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4,  "Intel(R) X520 82599 (KX4)"),
  74   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ,  "Intel(R) X520 82599 (KX4 Mezzanine)"),
  75   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP,  "Intel(R) X520 82599ES (SFI/SFP+)"),
  76   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM,  "Intel(R) X520 82599 (XAUI/BX4)"),
  77   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4,  "Intel(R) X520 82599 (Dual CX4)"),
  78   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM,  "Intel(R) X520-T 82599 LOM"),
  79   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE,  "Intel(R) X520 82599 (Combined Backplane)"),
  80   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE,  "Intel(R) X520 82599 (Backplane w/FCoE)"),
  81   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2,  "Intel(R) X520 82599 (Dual SFP+)"),
  82   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE,  "Intel(R) X520 82599 (Dual SFP+ w/FCoE)"),
  83   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP,  "Intel(R) X520-1 82599EN (SFP+)"),
  84   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP,  "Intel(R) X520-4 82599 (Quad SFP+)"),
  85   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP,  "Intel(R) X520-Q1 82599 (QSFP+)"),
  86   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T,  "Intel(R) X540-AT2"),
  87   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1,  "Intel(R) X540-T1"),
  88   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T,  "Intel(R) X550-T2"),
  89   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, "Intel(R) X550-T1"),
  90   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR,  "Intel(R) X552 (KR Backplane)"),
  91   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4,  "Intel(R) X552 (KX4 Backplane)"),
  92   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T,  "Intel(R) X552/X557-AT (10GBASE-T)"),
  93   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T,  "Intel(R) X552 (1000BASE-T)"),
  94   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, "Intel(R) X552 (SFP+)"),
  95   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, "Intel(R) X553 (KR Backplane)"),
  96   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, "Intel(R) X553 L (KR Backplane)"),
  97   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, "Intel(R) X553 (SFP+)"),
  98   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, "Intel(R) X553 N (SFP+)"),
  99   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, "Intel(R) X553 (1GbE SGMII)"),
 100   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, "Intel(R) X553 L (1GbE SGMII)"),
 101   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, "Intel(R) X553/X557-AT (10GBASE-T)"),
 102   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, "Intel(R) X553 (1GbE)"),
 103   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, "Intel(R) X553 L (1GbE)"),
 104   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, "Intel(R) X540-T2 (Bypass)"),
 105   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, "Intel(R) X520 82599 (Bypass)"),
 106         /* required last entry */
 107   PVID_END
 108 };
 109
 110 static void *ixgbe_register(device_t);
 111 static int  ixgbe_if_attach_pre(if_ctx_t);
 112 static int  ixgbe_if_attach_post(if_ctx_t);
 113 static int  ixgbe_if_detach(if_ctx_t);
 114 static int  ixgbe_if_shutdown(if_ctx_t);
 115 static int  ixgbe_if_suspend(if_ctx_t);
 116 static int  ixgbe_if_resume(if_ctx_t);
 117
 118 static void ixgbe_if_stop(if_ctx_t);
 119 void ixgbe_if_enable_intr(if_ctx_t);
 120 static void ixgbe_if_disable_intr(if_ctx_t);
 121 static void ixgbe_link_intr_enable(if_ctx_t);
 122 static int  ixgbe_if_rx_queue_intr_enable(if_ctx_t, uint16_t);
 123 static void ixgbe_if_media_status(if_ctx_t, struct ifmediareq *);
 124 static int  ixgbe_if_media_change(if_ctx_t);
 125 static int  ixgbe_if_msix_intr_assign(if_ctx_t, int);
 126 static int  ixgbe_if_mtu_set(if_ctx_t, uint32_t);
 127 static void ixgbe_if_crcstrip_set(if_ctx_t, int, int);
 128 static void ixgbe_if_multi_set(if_ctx_t);
 129 static int  ixgbe_if_promisc_set(if_ctx_t, int);
 130 static int  ixgbe_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
 131 static int  ixgbe_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
 132 static void ixgbe_if_queues_free(if_ctx_t);
 133 static void ixgbe_if_timer(if_ctx_t, uint16_t);
 134 static void ixgbe_if_update_admin_status(if_ctx_t);
 135 static void ixgbe_if_vlan_register(if_ctx_t, u16);
 136 static void ixgbe_if_vlan_unregister(if_ctx_t, u16);
 137 static int  ixgbe_if_i2c_req(if_ctx_t, struct ifi2creq *);
 138 static bool ixgbe_if_needs_restart(if_ctx_t, enum iflib_restart_event);
 139 int ixgbe_intr(void *);
 140
 141 /************************************************************************
 142  * Function prototypes
 143  ************************************************************************/
 144 static uint64_t ixgbe_if_get_counter(if_ctx_t, ift_counter);
 145
 146 static void ixgbe_enable_queue(struct ixgbe_softc *, u32);
 147 static void ixgbe_disable_queue(struct ixgbe_softc *, u32);
 148 static void ixgbe_add_device_sysctls(if_ctx_t);
 149 static int  ixgbe_allocate_pci_resources(if_ctx_t);
 150 static int  ixgbe_setup_low_power_mode(if_ctx_t);
 151
 152 static void ixgbe_config_dmac(struct ixgbe_softc *);
 153 static void ixgbe_configure_ivars(struct ixgbe_softc *);
 154 static void ixgbe_set_ivar(struct ixgbe_softc *, u8, u8, s8);
 155 static u8   *ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 156 static bool ixgbe_sfp_probe(if_ctx_t);
 157
 158 static void ixgbe_free_pci_resources(if_ctx_t);
 159
 160 static int  ixgbe_msix_link(void *);
 161 static int  ixgbe_msix_que(void *);
 162 static void ixgbe_initialize_rss_mapping(struct ixgbe_softc *);
 163 static void ixgbe_initialize_receive_units(if_ctx_t);
 164 static void ixgbe_initialize_transmit_units(if_ctx_t);
 165
 166 static int  ixgbe_setup_interface(if_ctx_t);
 167 static void ixgbe_init_device_features(struct ixgbe_softc *);
 168 static void ixgbe_check_fan_failure(struct ixgbe_softc *, u32, bool);
 169 static void ixgbe_sbuf_fw_version(struct ixgbe_hw *, struct sbuf *);
 170 static void ixgbe_print_fw_version(if_ctx_t);
 171 static void ixgbe_add_media_types(if_ctx_t);
 172 static void ixgbe_update_stats_counters(struct ixgbe_softc *);
 173 static void ixgbe_config_link(if_ctx_t);
 174 static void ixgbe_get_slot_info(struct ixgbe_softc *);
 175 static void ixgbe_check_wol_support(struct ixgbe_softc *);
 176 static void ixgbe_enable_rx_drop(struct ixgbe_softc *);
 177 static void ixgbe_disable_rx_drop(struct ixgbe_softc *);
 178
 179 static void ixgbe_add_hw_stats(struct ixgbe_softc *);
 180 static int  ixgbe_set_flowcntl(struct ixgbe_softc *, int);
 181 static int  ixgbe_set_advertise(struct ixgbe_softc *, int);
 182 static int  ixgbe_get_default_advertise(struct ixgbe_softc *);
 183 static void ixgbe_setup_vlan_hw_support(if_ctx_t);
 184 static void ixgbe_config_gpie(struct ixgbe_softc *);
 185 static void ixgbe_config_delay_values(struct ixgbe_softc *);
 186
 187 /* Sysctl handlers */
 188 static int  ixgbe_sysctl_flowcntl(SYSCTL_HANDLER_ARGS);
 189 static int  ixgbe_sysctl_advertise(SYSCTL_HANDLER_ARGS);
 190 static int  ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS);
 191 static int  ixgbe_sysctl_dmac(SYSCTL_HANDLER_ARGS);
 192 static int  ixgbe_sysctl_phy_temp(SYSCTL_HANDLER_ARGS);
 193 static int  ixgbe_sysctl_phy_overtemp_occurred(SYSCTL_HANDLER_ARGS);
 194 static int  ixgbe_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS);
 195 #ifdef IXGBE_DEBUG
 196 static int  ixgbe_sysctl_power_state(SYSCTL_HANDLER_ARGS);
 197 static int  ixgbe_sysctl_print_rss_config(SYSCTL_HANDLER_ARGS);
 198 #endif
 199 static int  ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS);
 200 static int  ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS);
 201 static int  ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS);
 202 static int  ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS);
 203 static int  ixgbe_sysctl_eee_state(SYSCTL_HANDLER_ARGS);
 204 static int  ixgbe_sysctl_wol_enable(SYSCTL_HANDLER_ARGS);
 205 static int  ixgbe_sysctl_wufc(SYSCTL_HANDLER_ARGS);
 206
 207 /* Deferred interrupt tasklets */
 208 static void ixgbe_handle_msf(void *);
 209 static void ixgbe_handle_mod(void *);
 210 static void ixgbe_handle_phy(void *);
 211
 212 /************************************************************************
 213  *  FreeBSD Device Interface Entry Points
 214  ************************************************************************/
 215 static device_method_t ix_methods[] = {
 216         /* Device interface */
 217         DEVMETHOD(device_register, ixgbe_register),
 218         DEVMETHOD(device_probe, iflib_device_probe),
 219         DEVMETHOD(device_attach, iflib_device_attach),
 220         DEVMETHOD(device_detach, iflib_device_detach),
 221         DEVMETHOD(device_shutdown, iflib_device_shutdown),
 222         DEVMETHOD(device_suspend, iflib_device_suspend),
 223         DEVMETHOD(device_resume, iflib_device_resume),
 224 #ifdef PCI_IOV
 225         DEVMETHOD(pci_iov_init, iflib_device_iov_init),
 226         DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit),
 227         DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf),
 228 #endif /* PCI_IOV */
 229         DEVMETHOD_END
 230 };
 231
 232 static driver_t ix_driver = {
 233         "ix", ix_methods, sizeof(struct ixgbe_softc),
 234 };
 235
 236 DRIVER_MODULE(ix, pci, ix_driver, 0, 0);
 237 IFLIB_PNP_INFO(pci, ix_driver, ixgbe_vendor_info_array);
 238 MODULE_DEPEND(ix, pci, 1, 1, 1);
 239 MODULE_DEPEND(ix, ether, 1, 1, 1);
 240 MODULE_DEPEND(ix, iflib, 1, 1, 1);
 241
 242 static device_method_t ixgbe_if_methods[] = {
 243         DEVMETHOD(ifdi_attach_pre, ixgbe_if_attach_pre),
 244         DEVMETHOD(ifdi_attach_post, ixgbe_if_attach_post),
 245         DEVMETHOD(ifdi_detach, ixgbe_if_detach),
 246         DEVMETHOD(ifdi_shutdown, ixgbe_if_shutdown),
 247         DEVMETHOD(ifdi_suspend, ixgbe_if_suspend),
 248         DEVMETHOD(ifdi_resume, ixgbe_if_resume),
 249         DEVMETHOD(ifdi_init, ixgbe_if_init),
 250         DEVMETHOD(ifdi_stop, ixgbe_if_stop),
 251         DEVMETHOD(ifdi_msix_intr_assign, ixgbe_if_msix_intr_assign),
 252         DEVMETHOD(ifdi_intr_enable, ixgbe_if_enable_intr),
 253         DEVMETHOD(ifdi_intr_disable, ixgbe_if_disable_intr),
 254         DEVMETHOD(ifdi_link_intr_enable, ixgbe_link_intr_enable),
 255         DEVMETHOD(ifdi_tx_queue_intr_enable, ixgbe_if_rx_queue_intr_enable),
 256         DEVMETHOD(ifdi_rx_queue_intr_enable, ixgbe_if_rx_queue_intr_enable),
 257         DEVMETHOD(ifdi_tx_queues_alloc, ixgbe_if_tx_queues_alloc),
 258         DEVMETHOD(ifdi_rx_queues_alloc, ixgbe_if_rx_queues_alloc),
 259         DEVMETHOD(ifdi_queues_free, ixgbe_if_queues_free),
 260         DEVMETHOD(ifdi_update_admin_status, ixgbe_if_update_admin_status),
 261         DEVMETHOD(ifdi_multi_set, ixgbe_if_multi_set),
 262         DEVMETHOD(ifdi_mtu_set, ixgbe_if_mtu_set),
 263         DEVMETHOD(ifdi_crcstrip_set, ixgbe_if_crcstrip_set),
 264         DEVMETHOD(ifdi_media_status, ixgbe_if_media_status),
 265         DEVMETHOD(ifdi_media_change, ixgbe_if_media_change),
 266         DEVMETHOD(ifdi_promisc_set, ixgbe_if_promisc_set),
 267         DEVMETHOD(ifdi_timer, ixgbe_if_timer),
 268         DEVMETHOD(ifdi_vlan_register, ixgbe_if_vlan_register),
 269         DEVMETHOD(ifdi_vlan_unregister, ixgbe_if_vlan_unregister),
 270         DEVMETHOD(ifdi_get_counter, ixgbe_if_get_counter),
 271         DEVMETHOD(ifdi_i2c_req, ixgbe_if_i2c_req),
 272         DEVMETHOD(ifdi_needs_restart, ixgbe_if_needs_restart),
 273 #ifdef PCI_IOV
 274         DEVMETHOD(ifdi_iov_init, ixgbe_if_iov_init),
 275         DEVMETHOD(ifdi_iov_uninit, ixgbe_if_iov_uninit),
 276         DEVMETHOD(ifdi_iov_vf_add, ixgbe_if_iov_vf_add),
 277 #endif /* PCI_IOV */
 278         DEVMETHOD_END
 279 };
 280
 281 /*
 282  * TUNEABLE PARAMETERS:
 283  */
 284
 285 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 286     "IXGBE driver parameters");
 287 static driver_t ixgbe_if_driver = {
 288   "ixgbe_if", ixgbe_if_methods, sizeof(struct ixgbe_softc)
 289 };
 290
 291 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 292 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
 293     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
 294
 295 /* Flow control setting, default to full */
 296 static int ixgbe_flow_control = ixgbe_fc_full;
 297 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN,
 298     &ixgbe_flow_control, 0, "Default flow control used for all adapters");
 299
 300 /* Advertise Speed, default to 0 (auto) */
 301 static int ixgbe_advertise_speed = 0;
 302 SYSCTL_INT(_hw_ix, OID_AUTO, advertise_speed, CTLFLAG_RDTUN,
 303     &ixgbe_advertise_speed, 0, "Default advertised speed for all adapters");
 304
 305 /*
 306  * Smart speed setting, default to on
 307  * this only works as a compile option
 308  * right now as its during attach, set
 309  * this to 'ixgbe_smart_speed_off' to
 310  * disable.
 311  */
 312 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 313
 314 /*
 315  * MSI-X should be the default for best performance,
 316  * but this allows it to be forced off for testing.
 317  */
 318 static int ixgbe_enable_msix = 1;
 319 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
 320     "Enable MSI-X interrupts");
 321
 322 /*
 323  * Defining this on will allow the use
 324  * of unsupported SFP+ modules, note that
 325  * doing so you are on your own :)
 326  */
 327 static int allow_unsupported_sfp = false;
 328 SYSCTL_INT(_hw_ix, OID_AUTO, unsupported_sfp, CTLFLAG_RDTUN,
 329     &allow_unsupported_sfp, 0,
 330     "Allow unsupported SFP modules...use at your own risk");
 331
 332 /*
 333  * Not sure if Flow Director is fully baked,
 334  * so we'll default to turning it off.
 335  */
 336 static int ixgbe_enable_fdir = 0;
 337 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0,
 338     "Enable Flow Director");
 339
 340 /* Receive-Side Scaling */
 341 static int ixgbe_enable_rss = 1;
 342 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0,
 343     "Enable Receive-Side Scaling (RSS)");
 344
 345 /*
 346  * AIM: Adaptive Interrupt Moderation
 347  * which means that the interrupt rate
 348  * is varied over time based on the
 349  * traffic for that interrupt vector
 350  */
 351 static int ixgbe_enable_aim = false;
 352 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &ixgbe_enable_aim, 0,
 353     "Enable adaptive interrupt moderation");
 354
 355 #if 0
 356 /* Keep running tab on them for sanity check */
 357 static int ixgbe_total_ports;
 358 #endif
 359
 360 MALLOC_DEFINE(M_IXGBE, "ix", "ix driver allocations");
 361
 362 /*
 363  * For Flow Director: this is the number of TX packets we sample
 364  * for the filter pool, this means every 20th packet will be probed.
 365  *
 366  * This feature can be disabled by setting this to 0.
 367  */
 368 static int atr_sample_rate = 20;
 369
 370 extern struct if_txrx ixgbe_txrx;
 371
 372 static struct if_shared_ctx ixgbe_sctx_init = {
 373         .isc_magic = IFLIB_MAGIC,
 374         .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
 375         .isc_tx_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 376         .isc_tx_maxsegsize = PAGE_SIZE,
 377         .isc_tso_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 378         .isc_tso_maxsegsize = PAGE_SIZE,
 379         .isc_rx_maxsize = PAGE_SIZE*4,
 380         .isc_rx_nsegments = 1,
 381         .isc_rx_maxsegsize = PAGE_SIZE*4,
 382         .isc_nfl = 1,
 383         .isc_ntxqs = 1,
 384         .isc_nrxqs = 1,
 385
 386         .isc_admin_intrcnt = 1,
 387         .isc_vendor_info = ixgbe_vendor_info_array,
 388         .isc_driver_version = ixgbe_driver_version,
 389         .isc_driver = &ixgbe_if_driver,
 390         .isc_flags = IFLIB_TSO_INIT_IP,
 391
 392         .isc_nrxd_min = {MIN_RXD},
 393         .isc_ntxd_min = {MIN_TXD},
 394         .isc_nrxd_max = {MAX_RXD},
 395         .isc_ntxd_max = {MAX_TXD},
 396         .isc_nrxd_default = {DEFAULT_RXD},
 397         .isc_ntxd_default = {DEFAULT_TXD},
 398 };
 399
 400 /************************************************************************
 401  * ixgbe_if_tx_queues_alloc
 402  ************************************************************************/
 403 static int
 404 ixgbe_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
 405     int ntxqs, int ntxqsets)
 406 {
 407         struct ixgbe_softc *sc = iflib_get_softc(ctx);
 408         if_softc_ctx_t     scctx = sc->shared;
 409         struct ix_tx_queue *que;
 410         int                i, j, error;
 411
 412         MPASS(sc->num_tx_queues > 0);
 413         MPASS(sc->num_tx_queues == ntxqsets);
 414         MPASS(ntxqs == 1);
 415
 416         /* Allocate queue structure memory */
 417         sc->tx_queues =
 418             (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * ntxqsets,
 419                                          M_IXGBE, M_NOWAIT | M_ZERO);
 420         if (!sc->tx_queues) {
 421                 device_printf(iflib_get_dev(ctx),
 422                     "Unable to allocate TX ring memory\n");
 423                 return (ENOMEM);
 424         }
 425
 426         for (i = 0, que = sc->tx_queues; i < ntxqsets; i++, que++) {
 427                 struct tx_ring *txr = &que->txr;
 428
 429                 /* In case SR-IOV is enabled, align the index properly */
 430                 txr->me = ixgbe_vf_que_index(sc->iov_mode, sc->pool,
 431                     i);
 432
 433                 txr->sc = que->sc = sc;
 434
 435                 /* Allocate report status array */
 436                 txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_IXGBE, M_NOWAIT | M_ZERO);
 437                 if (txr->tx_rsq == NULL) {
 438                         error = ENOMEM;
 439                         goto fail;
 440                 }
 441                 for (j = 0; j < scctx->isc_ntxd[0]; j++)
 442                         txr->tx_rsq[j] = QIDX_INVALID;
 443                 /* get the virtual and physical address of the hardware queues */
 444                 txr->tail = IXGBE_TDT(txr->me);
 445                 txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i];
 446                 txr->tx_paddr = paddrs[i];
 447
 448                 txr->bytes = 0;
 449                 txr->total_packets = 0;
 450
 451                 /* Set the rate at which we sample packets */
 452                 if (sc->feat_en & IXGBE_FEATURE_FDIR)
 453                         txr->atr_sample = atr_sample_rate;
 454
 455         }
 456
 457         device_printf(iflib_get_dev(ctx), "allocated for %d queues\n",
 458             sc->num_tx_queues);
 459
 460         return (0);
 461
 462 fail:
 463         ixgbe_if_queues_free(ctx);
 464
 465         return (error);
 466 } /* ixgbe_if_tx_queues_alloc */
 467
 468 /************************************************************************
 469  * ixgbe_if_rx_queues_alloc
 470  ************************************************************************/
 471 static int
 472 ixgbe_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
 473     int nrxqs, int nrxqsets)
 474 {
 475         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 476         struct ix_rx_queue *que;
 477         int                i;
 478
 479         MPASS(sc->num_rx_queues > 0);
 480         MPASS(sc->num_rx_queues == nrxqsets);
 481         MPASS(nrxqs == 1);
 482
 483         /* Allocate queue structure memory */
 484         sc->rx_queues =
 485             (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue)*nrxqsets,
 486                                          M_IXGBE, M_NOWAIT | M_ZERO);
 487         if (!sc->rx_queues) {
 488                 device_printf(iflib_get_dev(ctx),
 489                     "Unable to allocate TX ring memory\n");
 490                 return (ENOMEM);
 491         }
 492
 493         for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) {
 494                 struct rx_ring *rxr = &que->rxr;
 495
 496                 /* In case SR-IOV is enabled, align the index properly */
 497                 rxr->me = ixgbe_vf_que_index(sc->iov_mode, sc->pool,
 498                     i);
 499
 500                 rxr->sc = que->sc = sc;
 501
 502                 /* get the virtual and physical address of the hw queues */
 503                 rxr->tail = IXGBE_RDT(rxr->me);
 504                 rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];
 505                 rxr->rx_paddr = paddrs[i];
 506                 rxr->bytes = 0;
 507                 rxr->que = que;
 508         }
 509
 510         device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n",
 511             sc->num_rx_queues);
 512
 513         return (0);
 514 } /* ixgbe_if_rx_queues_alloc */
 515
 516 /************************************************************************
 517  * ixgbe_if_queues_free
 518  ************************************************************************/
 519 static void
 520 ixgbe_if_queues_free(if_ctx_t ctx)
 521 {
 522         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 523         struct ix_tx_queue *tx_que = sc->tx_queues;
 524         struct ix_rx_queue *rx_que = sc->rx_queues;
 525         int                i;
 526
 527         if (tx_que != NULL) {
 528                 for (i = 0; i < sc->num_tx_queues; i++, tx_que++) {
 529                         struct tx_ring *txr = &tx_que->txr;
 530                         if (txr->tx_rsq == NULL)
 531                                 break;
 532
 533                         free(txr->tx_rsq, M_IXGBE);
 534                         txr->tx_rsq = NULL;
 535                 }
 536
 537                 free(sc->tx_queues, M_IXGBE);
 538                 sc->tx_queues = NULL;
 539         }
 540         if (rx_que != NULL) {
 541                 free(sc->rx_queues, M_IXGBE);
 542                 sc->rx_queues = NULL;
 543         }
 544 } /* ixgbe_if_queues_free */
 545
 546 /************************************************************************
 547  * ixgbe_initialize_rss_mapping
 548  ************************************************************************/
 549 static void
 550 ixgbe_initialize_rss_mapping(struct ixgbe_softc *sc)
 551 {
 552         struct ixgbe_hw *hw = &sc->hw;
 553         u32             reta = 0, mrqc, rss_key[10];
 554         int             queue_id, table_size, index_mult;
 555         int             i, j;
 556         u32             rss_hash_config;
 557
 558         if (sc->feat_en & IXGBE_FEATURE_RSS) {
 559                 /* Fetch the configured RSS key */
 560                 rss_getkey((uint8_t *)&rss_key);
 561         } else {
 562                 /* set up random bits */
 563                 arc4rand(&rss_key, sizeof(rss_key), 0);
 564         }
 565
 566         /* Set multiplier for RETA setup and table size based on MAC */
 567         index_mult = 0x1;
 568         table_size = 128;
 569         switch (sc->hw.mac.type) {
 570         case ixgbe_mac_82598EB:
 571                 index_mult = 0x11;
 572                 break;
 573         case ixgbe_mac_X550:
 574         case ixgbe_mac_X550EM_x:
 575         case ixgbe_mac_X550EM_a:
 576                 table_size = 512;
 577                 break;
 578         default:
 579                 break;
 580         }
 581
 582         /* Set up the redirection table */
 583         for (i = 0, j = 0; i < table_size; i++, j++) {
 584                 if (j == sc->num_rx_queues)
 585                         j = 0;
 586
 587                 if (sc->feat_en & IXGBE_FEATURE_RSS) {
 588                         /*
 589                          * Fetch the RSS bucket id for the given indirection
 590                          * entry. Cap it at the number of configured buckets
 591                          * (which is num_rx_queues.)
 592                          */
 593                         queue_id = rss_get_indirection_to_bucket(i);
 594                         queue_id = queue_id % sc->num_rx_queues;
 595                 } else
 596                         queue_id = (j * index_mult);
 597
 598                 /*
 599                  * The low 8 bits are for hash value (n+0);
 600                  * The next 8 bits are for hash value (n+1), etc.
 601                  */
 602                 reta = reta >> 8;
 603                 reta = reta | (((uint32_t)queue_id) << 24);
 604                 if ((i & 3) == 3) {
 605                         if (i < 128)
 606                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
 607                         else
 608                                 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
 609                                     reta);
 610                         reta = 0;
 611                 }
 612         }
 613
 614         /* Now fill our hash function seeds */
 615         for (i = 0; i < 10; i++)
 616                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
 617
 618         /* Perform hash on these packet types */
 619         if (sc->feat_en & IXGBE_FEATURE_RSS)
 620                 rss_hash_config = rss_gethashconfig();
 621         else {
 622                 /*
 623                  * Disable UDP - IP fragments aren't currently being handled
 624                  * and so we end up with a mix of 2-tuple and 4-tuple
 625                  * traffic.
 626                  */
 627                 rss_hash_config = RSS_HASHTYPE_RSS_IPV4
 628                                 | RSS_HASHTYPE_RSS_TCP_IPV4
 629                                 | RSS_HASHTYPE_RSS_IPV6
 630                                 | RSS_HASHTYPE_RSS_TCP_IPV6
 631                                 | RSS_HASHTYPE_RSS_IPV6_EX
 632                                 | RSS_HASHTYPE_RSS_TCP_IPV6_EX;
 633         }
 634
 635         mrqc = IXGBE_MRQC_RSSEN;
 636         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
 637                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
 638         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
 639                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
 640         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
 641                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
 642         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
 643                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
 644         if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
 645                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
 646         if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
 647                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
 648         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
 649                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
 650         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
 651                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
 652         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
 653                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
 654         mrqc |= ixgbe_get_mrqc(sc->iov_mode);
 655         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
 656 } /* ixgbe_initialize_rss_mapping */
 657
 658 /************************************************************************
 659  * ixgbe_initialize_receive_units - Setup receive registers and features.
 660  ************************************************************************/
 661 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
 662
 663 static void
 664 ixgbe_initialize_receive_units(if_ctx_t ctx)
 665 {
 666         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 667         if_softc_ctx_t     scctx = sc->shared;
 668         struct ixgbe_hw    *hw = &sc->hw;
 669         struct ifnet       *ifp = iflib_get_ifp(ctx);
 670         struct ix_rx_queue *que;
 671         int                i, j;
 672         u32                bufsz, fctrl, srrctl, rxcsum;
 673         u32                hlreg;
 674
 675         /*
 676          * Make sure receives are disabled while
 677          * setting up the descriptor ring
 678          */
 679         ixgbe_disable_rx(hw);
 680
 681         /* Enable broadcasts */
 682         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
 683         fctrl |= IXGBE_FCTRL_BAM;
 684         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
 685                 fctrl |= IXGBE_FCTRL_DPF;
 686                 fctrl |= IXGBE_FCTRL_PMCF;
 687         }
 688         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
 689
 690         /* Set for Jumbo Frames? */
 691         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
 692         if (ifp->if_mtu > ETHERMTU)
 693                 hlreg |= IXGBE_HLREG0_JUMBOEN;
 694         else
 695                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
 696         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
 697
 698         bufsz = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
 699             IXGBE_SRRCTL_BSIZEPKT_SHIFT;
 700
 701         /* Setup the Base and Length of the Rx Descriptor Ring */
 702         for (i = 0, que = sc->rx_queues; i < sc->num_rx_queues; i++, que++) {
 703                 struct rx_ring *rxr = &que->rxr;
 704                 u64            rdba = rxr->rx_paddr;
 705
 706                 j = rxr->me;
 707
 708                 /* Setup the Base and Length of the Rx Descriptor Ring */
 709                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j),
 710                     (rdba & 0x00000000ffffffffULL));
 711                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32));
 712                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j),
 713                      scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));
 714
 715                 /* Set up the SRRCTL register */
 716                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j));
 717                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
 718                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
 719                 srrctl |= bufsz;
 720                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
 721
 722                 /*
 723                  * Set DROP_EN iff we have no flow control and >1 queue.
 724                  * Note that srrctl was cleared shortly before during reset,
 725                  * so we do not need to clear the bit, but do it just in case
 726                  * this code is moved elsewhere.
 727                  */
 728                 if (sc->num_rx_queues > 1 &&
 729                     sc->hw.fc.requested_mode == ixgbe_fc_none) {
 730                         srrctl |= IXGBE_SRRCTL_DROP_EN;
 731                 } else {
 732                         srrctl &= ~IXGBE_SRRCTL_DROP_EN;
 733                 }
 734
 735                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl);
 736
 737                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
 738                 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0);
 739                 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0);
 740
 741                 /* Set the driver rx tail address */
 742                 rxr->tail =  IXGBE_RDT(rxr->me);
 743         }
 744
 745         if (sc->hw.mac.type != ixgbe_mac_82598EB) {
 746                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR
 747                             | IXGBE_PSRTYPE_UDPHDR
 748                             | IXGBE_PSRTYPE_IPV4HDR
 749                             | IXGBE_PSRTYPE_IPV6HDR;
 750                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
 751         }
 752
 753         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
 754
 755         ixgbe_initialize_rss_mapping(sc);
 756
 757         if (sc->num_rx_queues > 1) {
 758                 /* RSS and RX IPP Checksum are mutually exclusive */
 759                 rxcsum |= IXGBE_RXCSUM_PCSD;
 760         }
 761
 762         if (ifp->if_capenable & IFCAP_RXCSUM)
 763                 rxcsum |= IXGBE_RXCSUM_PCSD;
 764
 765         /* This is useful for calculating UDP/IP fragment checksums */
 766         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
 767                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
 768
 769         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
 770
 771 } /* ixgbe_initialize_receive_units */
 772
 773 /************************************************************************
 774  * ixgbe_initialize_transmit_units - Enable transmit units.
 775  ************************************************************************/
 776 static void
 777 ixgbe_initialize_transmit_units(if_ctx_t ctx)
 778 {
 779         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
 780         struct ixgbe_hw    *hw = &sc->hw;
 781         if_softc_ctx_t     scctx = sc->shared;
 782         struct ix_tx_queue *que;
 783         int i;
 784
 785         /* Setup the Base and Length of the Tx Descriptor Ring */
 786         for (i = 0, que = sc->tx_queues; i < sc->num_tx_queues;
 787             i++, que++) {
 788                 struct tx_ring     *txr = &que->txr;
 789                 u64 tdba = txr->tx_paddr;
 790                 u32 txctrl = 0;
 791                 int j = txr->me;
 792
 793                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j),
 794                     (tdba & 0x00000000ffffffffULL));
 795                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32));
 796                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j),
 797                     scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc));
 798
 799                 /* Setup the HW Tx Head and Tail descriptor pointers */
 800                 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0);
 801                 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0);
 802
 803                 /* Cache the tail address */
 804                 txr->tail = IXGBE_TDT(txr->me);
 805
 806                 txr->tx_rs_cidx = txr->tx_rs_pidx;
 807                 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
 808                 for (int k = 0; k < scctx->isc_ntxd[0]; k++)
 809                         txr->tx_rsq[k] = QIDX_INVALID;
 810
 811                 /* Disable Head Writeback */
 812                 /*
 813                  * Note: for X550 series devices, these registers are actually
 814                  * prefixed with TPH_ isntead of DCA_, but the addresses and
 815                  * fields remain the same.
 816                  */
 817                 switch (hw->mac.type) {
 818                 case ixgbe_mac_82598EB:
 819                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j));
 820                         break;
 821                 default:
 822                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j));
 823                         break;
 824                 }
 825                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
 826                 switch (hw->mac.type) {
 827                 case ixgbe_mac_82598EB:
 828                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl);
 829                         break;
 830                 default:
 831                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl);
 832                         break;
 833                 }
 834
 835         }
 836
 837         if (hw->mac.type != ixgbe_mac_82598EB) {
 838                 u32 dmatxctl, rttdcs;
 839
 840                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
 841                 dmatxctl |= IXGBE_DMATXCTL_TE;
 842                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
 843                 /* Disable arbiter to set MTQC */
 844                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
 845                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
 846                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
 847                 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
 848                     ixgbe_get_mtqc(sc->iov_mode));
 849                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
 850                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
 851         }
 852
 853 } /* ixgbe_initialize_transmit_units */
 854
 855 /************************************************************************
 856  * ixgbe_register
 857  ************************************************************************/
 858 static void *
 859 ixgbe_register(device_t dev)
 860 {
 861         return (&ixgbe_sctx_init);
 862 } /* ixgbe_register */
 863
 864 /************************************************************************
 865  * ixgbe_if_attach_pre - Device initialization routine, part 1
 866  *
 867  *   Called when the driver is being loaded.
 868  *   Identifies the type of hardware, initializes the hardware,
 869  *   and initializes iflib structures.
 870  *
 871  *   return 0 on success, positive on failure
 872  ************************************************************************/
 873 static int
 874 ixgbe_if_attach_pre(if_ctx_t ctx)
 875 {
 876         struct ixgbe_softc  *sc;
 877         device_t        dev;
 878         if_softc_ctx_t  scctx;
 879         struct ixgbe_hw *hw;
 880         int             error = 0;
 881         u32             ctrl_ext;
 882
 883         INIT_DEBUGOUT("ixgbe_attach: begin");
 884
 885         /* Allocate, clear, and link in our adapter structure */
 886         dev = iflib_get_dev(ctx);
 887         sc = iflib_get_softc(ctx);
 888         sc->hw.back = sc;
 889         sc->ctx = ctx;
 890         sc->dev = dev;
 891         scctx = sc->shared = iflib_get_softc_ctx(ctx);
 892         sc->media = iflib_get_media(ctx);
 893         hw = &sc->hw;
 894
 895         /* Determine hardware revision */
 896         hw->vendor_id = pci_get_vendor(dev);
 897         hw->device_id = pci_get_device(dev);
 898         hw->revision_id = pci_get_revid(dev);
 899         hw->subsystem_vendor_id = pci_get_subvendor(dev);
 900         hw->subsystem_device_id = pci_get_subdevice(dev);
 901
 902         /* Do base PCI setup - map BAR0 */
 903         if (ixgbe_allocate_pci_resources(ctx)) {
 904                 device_printf(dev, "Allocation of PCI resources failed\n");
 905                 return (ENXIO);
 906         }
 907
 908         /* let hardware know driver is loaded */
 909         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 910         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 911         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 912
 913         /*
 914          * Initialize the shared code
 915          */
 916         if (ixgbe_init_shared_code(hw) != 0) {
 917                 device_printf(dev, "Unable to initialize the shared code\n");
 918                 error = ENXIO;
 919                 goto err_pci;
 920         }
 921
 922         if (hw->mbx.ops.init_params)
 923                 hw->mbx.ops.init_params(hw);
 924
 925         hw->allow_unsupported_sfp = allow_unsupported_sfp;
 926
 927         if (hw->mac.type != ixgbe_mac_82598EB)
 928                 hw->phy.smart_speed = ixgbe_smart_speed;
 929
 930         ixgbe_init_device_features(sc);
 931
 932         /* Enable WoL (if supported) */
 933         ixgbe_check_wol_support(sc);
 934
 935         /* Verify adapter fan is still functional (if applicable) */
 936         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL) {
 937                 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
 938                 ixgbe_check_fan_failure(sc, esdp, false);
 939         }
 940
 941         /* Ensure SW/FW semaphore is free */
 942         ixgbe_init_swfw_semaphore(hw);
 943
 944         /* Set an initial default flow control value */
 945         hw->fc.requested_mode = ixgbe_flow_control;
 946
 947         hw->phy.reset_if_overtemp = true;
 948         error = ixgbe_reset_hw(hw);
 949         hw->phy.reset_if_overtemp = false;
 950         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 951                 /*
 952                  * No optics in this port, set up
 953                  * so the timer routine will probe
 954                  * for later insertion.
 955                  */
 956                 sc->sfp_probe = true;
 957                 error = 0;
 958         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 959                 device_printf(dev, "Unsupported SFP+ module detected!\n");
 960                 error = EIO;
 961                 goto err_pci;
 962         } else if (error) {
 963                 device_printf(dev, "Hardware initialization failed\n");
 964                 error = EIO;
 965                 goto err_pci;
 966         }
 967
 968         /* Make sure we have a good EEPROM before we read from it */
 969         if (ixgbe_validate_eeprom_checksum(&sc->hw, NULL) < 0) {
 970                 device_printf(dev, "The EEPROM Checksum Is Not Valid\n");
 971                 error = EIO;
 972                 goto err_pci;
 973         }
 974
 975         error = ixgbe_start_hw(hw);
 976         switch (error) {
 977         case IXGBE_ERR_EEPROM_VERSION:
 978                 device_printf(dev, "This device is a pre-production adapter/LOM.  Please be aware there may be issues associated with your hardware.\nIf you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
 979                 break;
 980         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 981                 device_printf(dev, "Unsupported SFP+ Module\n");
 982                 error = EIO;
 983                 goto err_pci;
 984         case IXGBE_ERR_SFP_NOT_PRESENT:
 985                 device_printf(dev, "No SFP+ Module found\n");
 986                 /* falls thru */
 987         default:
 988                 break;
 989         }
 990
 991         /* Most of the iflib initialization... */
 992
 993         iflib_set_mac(ctx, hw->mac.addr);
 994         switch (sc->hw.mac.type) {
 995         case ixgbe_mac_X550:
 996         case ixgbe_mac_X550EM_x:
 997         case ixgbe_mac_X550EM_a:
 998                 scctx->isc_rss_table_size = 512;
 999                 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 64;
1000                 break;
1001         default:
1002                 scctx->isc_rss_table_size = 128;
1003                 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 16;
1004         }
1005
1006         /* Allow legacy interrupts */
1007         ixgbe_txrx.ift_legacy_intr = ixgbe_intr;
1008
1009         scctx->isc_txqsizes[0] =
1010             roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +
1011             sizeof(u32), DBA_ALIGN),
1012         scctx->isc_rxqsizes[0] =
1013             roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),
1014             DBA_ALIGN);
1015
1016         /* XXX */
1017         scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
1018             CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO;
1019         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
1020                 scctx->isc_tx_nsegments = IXGBE_82598_SCATTER;
1021         } else {
1022                 scctx->isc_tx_csum_flags |= CSUM_SCTP |CSUM_IP6_SCTP;
1023                 scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;
1024         }
1025
1026         scctx->isc_msix_bar = pci_msix_table_bar(dev);
1027
1028         scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
1029         scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;
1030         scctx->isc_tx_tso_segsize_max = PAGE_SIZE;
1031
1032         scctx->isc_txrx = &ixgbe_txrx;
1033
1034         scctx->isc_capabilities = scctx->isc_capenable = IXGBE_CAPS;
1035
1036         return (0);
1037
1038 err_pci:
1039         ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
1040         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
1041         IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
1042         ixgbe_free_pci_resources(ctx);
1043
1044         return (error);
1045 } /* ixgbe_if_attach_pre */
1046
1047  /*********************************************************************
1048  * ixgbe_if_attach_post - Device initialization routine, part 2
1049  *
1050  *   Called during driver load, but after interrupts and
1051  *   resources have been allocated and configured.
1052  *   Sets up some data structures not relevant to iflib.
1053  *
1054  *   return 0 on success, positive on failure
1055  *********************************************************************/
1056 static int
1057 ixgbe_if_attach_post(if_ctx_t ctx)
1058 {
1059         device_t dev;
1060         struct ixgbe_softc  *sc;
1061         struct ixgbe_hw *hw;
1062         int             error = 0;
1063
1064         dev = iflib_get_dev(ctx);
1065         sc = iflib_get_softc(ctx);
1066         hw = &sc->hw;
1067
1068
1069         if (sc->intr_type == IFLIB_INTR_LEGACY &&
1070                 (sc->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) == 0) {
1071                 device_printf(dev, "Device does not support legacy interrupts");
1072                 error = ENXIO;
1073                 goto err;
1074         }
1075
1076         /* Allocate multicast array memory. */
1077         sc->mta = malloc(sizeof(*sc->mta) *
1078                               MAX_NUM_MULTICAST_ADDRESSES, M_IXGBE, M_NOWAIT);
1079         if (sc->mta == NULL) {
1080                 device_printf(dev, "Can not allocate multicast setup array\n");
1081                 error = ENOMEM;
1082                 goto err;
1083         }
1084
1085         /* hw.ix defaults init */
1086         ixgbe_set_advertise(sc, ixgbe_advertise_speed);
1087
1088         /* Enable the optics for 82599 SFP+ fiber */
1089         ixgbe_enable_tx_laser(hw);
1090
1091         /* Enable power to the phy. */
1092         ixgbe_set_phy_power(hw, true);
1093
1094         ixgbe_initialize_iov(sc);
1095
1096         error = ixgbe_setup_interface(ctx);
1097         if (error) {
1098                 device_printf(dev, "Interface setup failed: %d\n", error);
1099                 goto err;
1100         }
1101
1102         ixgbe_if_update_admin_status(ctx);
1103
1104         /* Initialize statistics */
1105         ixgbe_update_stats_counters(sc);
1106         ixgbe_add_hw_stats(sc);
1107
1108         /* Check PCIE slot type/speed/width */
1109         ixgbe_get_slot_info(sc);
1110
1111         /*
1112          * Do time init and sysctl init here, but
1113          * only on the first port of a bypass sc.
1114          */
1115         ixgbe_bypass_init(sc);
1116
1117         /* Display NVM and Option ROM versions */
1118         ixgbe_print_fw_version(ctx);
1119
1120         /* Set an initial dmac value */
1121         sc->dmac = 0;
1122         /* Set initial advertised speeds (if applicable) */
1123         sc->advertise = ixgbe_get_default_advertise(sc);
1124
1125         if (sc->feat_cap & IXGBE_FEATURE_SRIOV)
1126                 ixgbe_define_iov_schemas(dev, &error);
1127
1128         /* Add sysctls */
1129         ixgbe_add_device_sysctls(ctx);
1130
1131         return (0);
1132 err:
1133         return (error);
1134 } /* ixgbe_if_attach_post */
1135
1136 /************************************************************************
1137  * ixgbe_check_wol_support
1138  *
1139  *   Checks whether the adapter's ports are capable of
1140  *   Wake On LAN by reading the adapter's NVM.
1141  *
1142  *   Sets each port's hw->wol_enabled value depending
1143  *   on the value read here.
1144  ************************************************************************/
1145 static void
1146 ixgbe_check_wol_support(struct ixgbe_softc *sc)
1147 {
1148         struct ixgbe_hw *hw = &sc->hw;
1149         u16             dev_caps = 0;
1150
1151         /* Find out WoL support for port */
1152         sc->wol_support = hw->wol_enabled = 0;
1153         ixgbe_get_device_caps(hw, &dev_caps);
1154         if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
1155             ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) &&
1156              hw->bus.func == 0))
1157                 sc->wol_support = hw->wol_enabled = 1;
1158
1159         /* Save initial wake up filter configuration */
1160         sc->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC);
1161
1162         return;
1163 } /* ixgbe_check_wol_support */
1164
1165 /************************************************************************
1166  * ixgbe_setup_interface
1167  *
1168  *   Setup networking device structure and register an interface.
1169  ************************************************************************/
1170 static int
1171 ixgbe_setup_interface(if_ctx_t ctx)
1172 {
1173         struct ifnet   *ifp = iflib_get_ifp(ctx);
1174         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1175
1176         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
1177
1178         if_setbaudrate(ifp, IF_Gbps(10));
1179
1180         sc->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1181
1182         sc->phy_layer = ixgbe_get_supported_physical_layer(&sc->hw);
1183
1184         ixgbe_add_media_types(ctx);
1185
1186         /* Autoselect media by default */
1187         ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO);
1188
1189         return (0);
1190 } /* ixgbe_setup_interface */
1191
1192 /************************************************************************
1193  * ixgbe_if_get_counter
1194  ************************************************************************/
1195 static uint64_t
1196 ixgbe_if_get_counter(if_ctx_t ctx, ift_counter cnt)
1197 {
1198         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1199         if_t           ifp = iflib_get_ifp(ctx);
1200
1201         switch (cnt) {
1202         case IFCOUNTER_IPACKETS:
1203                 return (sc->ipackets);
1204         case IFCOUNTER_OPACKETS:
1205                 return (sc->opackets);
1206         case IFCOUNTER_IBYTES:
1207                 return (sc->ibytes);
1208         case IFCOUNTER_OBYTES:
1209                 return (sc->obytes);
1210         case IFCOUNTER_IMCASTS:
1211                 return (sc->imcasts);
1212         case IFCOUNTER_OMCASTS:
1213                 return (sc->omcasts);
1214         case IFCOUNTER_COLLISIONS:
1215                 return (0);
1216         case IFCOUNTER_IQDROPS:
1217                 return (sc->iqdrops);
1218         case IFCOUNTER_OQDROPS:
1219                 return (0);
1220         case IFCOUNTER_IERRORS:
1221                 return (sc->ierrors);
1222         default:
1223                 return (if_get_counter_default(ifp, cnt));
1224         }
1225 } /* ixgbe_if_get_counter */
1226
1227 /************************************************************************
1228  * ixgbe_if_i2c_req
1229  ************************************************************************/
1230 static int
1231 ixgbe_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req)
1232 {
1233         struct ixgbe_softc              *sc = iflib_get_softc(ctx);
1234         struct ixgbe_hw         *hw = &sc->hw;
1235         int                     i;
1236
1237
1238         if (hw->phy.ops.read_i2c_byte == NULL)
1239                 return (ENXIO);
1240         for (i = 0; i < req->len; i++)
1241                 hw->phy.ops.read_i2c_byte(hw, req->offset + i,
1242                     req->dev_addr, &req->data[i]);
1243         return (0);
1244 } /* ixgbe_if_i2c_req */
1245
1246 /* ixgbe_if_needs_restart - Tell iflib when the driver needs to be reinitialized
1247  * @ctx: iflib context
1248  * @event: event code to check
1249  *
1250  * Defaults to returning true for unknown events.
1251  *
1252  * @returns true if iflib needs to reinit the interface
1253  */
1254 static bool
1255 ixgbe_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event)
1256 {
1257         switch (event) {
1258         case IFLIB_RESTART_VLAN_CONFIG:
1259                 return (false);
1260         default:
1261                 return (true);
1262         }
1263 }
1264
1265 /************************************************************************
1266  * ixgbe_add_media_types
1267  ************************************************************************/
1268 static void
1269 ixgbe_add_media_types(if_ctx_t ctx)
1270 {
1271         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
1272         struct ixgbe_hw *hw = &sc->hw;
1273         device_t        dev = iflib_get_dev(ctx);
1274         u64             layer;
1275
1276         layer = sc->phy_layer = ixgbe_get_supported_physical_layer(hw);
1277
1278         /* Media types with matching FreeBSD media defines */
1279         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T)
1280                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_T, 0, NULL);
1281         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T)
1282                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1283         if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
1284                 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL);
1285         if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
1286                 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL);
1287
1288         if (hw->mac.type == ixgbe_mac_X550) {
1289                 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_T, 0, NULL);
1290                 ifmedia_add(sc->media, IFM_ETHER | IFM_5000_T, 0, NULL);
1291         }
1292
1293         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
1294             layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
1295                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_TWINAX, 0,
1296                     NULL);
1297
1298         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
1299                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
1300                 if (hw->phy.multispeed_fiber)
1301                         ifmedia_add(sc->media, IFM_ETHER | IFM_1000_LX, 0,
1302                             NULL);
1303         }
1304         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
1305                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
1306                 if (hw->phy.multispeed_fiber)
1307                         ifmedia_add(sc->media, IFM_ETHER | IFM_1000_SX, 0,
1308                             NULL);
1309         } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
1310                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
1311         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
1312                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
1313
1314 #ifdef IFM_ETH_XTYPE
1315         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
1316                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
1317         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4)
1318                 ifmedia_add( sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
1319         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
1320                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
1321         if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX)
1322                 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL);
1323 #else
1324         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
1325                 device_printf(dev, "Media supported: 10GbaseKR\n");
1326                 device_printf(dev, "10GbaseKR mapped to 10GbaseSR\n");
1327                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
1328         }
1329         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) {
1330                 device_printf(dev, "Media supported: 10GbaseKX4\n");
1331                 device_printf(dev, "10GbaseKX4 mapped to 10GbaseCX4\n");
1332                 ifmedia_add(sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
1333         }
1334         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) {
1335                 device_printf(dev, "Media supported: 1000baseKX\n");
1336                 device_printf(dev, "1000baseKX mapped to 1000baseCX\n");
1337                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
1338         }
1339         if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) {
1340                 device_printf(dev, "Media supported: 2500baseKX\n");
1341                 device_printf(dev, "2500baseKX mapped to 2500baseSX\n");
1342                 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_SX, 0, NULL);
1343         }
1344 #endif
1345         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX)
1346                 device_printf(dev, "Media supported: 1000baseBX\n");
1347
1348         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
1349                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
1350                     0, NULL);
1351                 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1352         }
1353
1354         ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1355 } /* ixgbe_add_media_types */
1356
1357 /************************************************************************
1358  * ixgbe_is_sfp
1359  ************************************************************************/
1360 static inline bool
1361 ixgbe_is_sfp(struct ixgbe_hw *hw)
1362 {
1363         switch (hw->mac.type) {
1364         case ixgbe_mac_82598EB:
1365                 if (hw->phy.type == ixgbe_phy_nl)
1366                         return (true);
1367                 return (false);
1368         case ixgbe_mac_82599EB:
1369                 switch (hw->mac.ops.get_media_type(hw)) {
1370                 case ixgbe_media_type_fiber:
1371                 case ixgbe_media_type_fiber_qsfp:
1372                         return (true);
1373                 default:
1374                         return (false);
1375                 }
1376         case ixgbe_mac_X550EM_x:
1377         case ixgbe_mac_X550EM_a:
1378                 if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber)
1379                         return (true);
1380                 return (false);
1381         default:
1382                 return (false);
1383         }
1384 } /* ixgbe_is_sfp */
1385
1386 /************************************************************************
1387  * ixgbe_config_link
1388  ************************************************************************/
1389 static void
1390 ixgbe_config_link(if_ctx_t ctx)
1391 {
1392         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
1393         struct ixgbe_hw *hw = &sc->hw;
1394         u32             autoneg, err = 0;
1395         bool            sfp, negotiate;
1396
1397         sfp = ixgbe_is_sfp(hw);
1398
1399         if (sfp) {
1400                 sc->task_requests |= IXGBE_REQUEST_TASK_MOD;
1401                 iflib_admin_intr_deferred(ctx);
1402         } else {
1403                 if (hw->mac.ops.check_link)
1404                         err = ixgbe_check_link(hw, &sc->link_speed,
1405                             &sc->link_up, false);
1406                 if (err)
1407                         return;
1408                 autoneg = hw->phy.autoneg_advertised;
1409                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
1410                         err = hw->mac.ops.get_link_capabilities(hw, &autoneg,
1411                             &negotiate);
1412                 if (err)
1413                         return;
1414
1415                 if (hw->mac.type == ixgbe_mac_X550 &&
1416                     hw->phy.autoneg_advertised == 0) {
1417                         /*
1418                          * 2.5G and 5G autonegotiation speeds on X550
1419                          * are disabled by default due to reported
1420                          * interoperability issues with some switches.
1421                          *
1422                          * The second condition checks if any operations
1423                          * involving setting autonegotiation speeds have
1424                          * been performed prior to this ixgbe_config_link()
1425                          * call.
1426                          *
1427                          * If hw->phy.autoneg_advertised does not
1428                          * equal 0, this means that the user might have
1429                          * set autonegotiation speeds via the sysctl
1430                          * before bringing the interface up. In this
1431                          * case, we should not disable 2.5G and 5G
1432                          * since that speeds might be selected by the
1433                          * user.
1434                          *
1435                          * Otherwise (i.e. if hw->phy.autoneg_advertised
1436                          * is set to 0), it is the first time we set
1437                          * autonegotiation preferences and the default
1438                          * set of speeds should exclude 2.5G and 5G.
1439                          */
1440                         autoneg &= ~(IXGBE_LINK_SPEED_2_5GB_FULL |
1441                             IXGBE_LINK_SPEED_5GB_FULL);
1442                 }
1443
1444                 if (hw->mac.ops.setup_link)
1445                         err = hw->mac.ops.setup_link(hw, autoneg,
1446                             sc->link_up);
1447         }
1448 } /* ixgbe_config_link */
1449
1450 /************************************************************************
1451  * ixgbe_update_stats_counters - Update board statistics counters.
1452  ************************************************************************/
1453 static void
1454 ixgbe_update_stats_counters(struct ixgbe_softc *sc)
1455 {
1456         struct ixgbe_hw       *hw = &sc->hw;
1457         struct ixgbe_hw_stats *stats = &sc->stats.pf;
1458         u32                   missed_rx = 0, bprc, lxon, lxoff, total;
1459         u32                   lxoffrxc;
1460         u64                   total_missed_rx = 0;
1461
1462         stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
1463         stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
1464         stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
1465         stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
1466         stats->mpc[0] += IXGBE_READ_REG(hw, IXGBE_MPC(0));
1467
1468         for (int i = 0; i < 16; i++) {
1469                 stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
1470                 stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
1471                 stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
1472         }
1473         stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
1474         stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
1475         stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
1476
1477         /* Hardware workaround, gprc counts missed packets */
1478         stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
1479         stats->gprc -= missed_rx;
1480
1481         if (hw->mac.type != ixgbe_mac_82598EB) {
1482                 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
1483                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
1484                 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
1485                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
1486                 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
1487                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
1488                 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
1489                 lxoffrxc = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
1490                 stats->lxoffrxc += lxoffrxc;
1491         } else {
1492                 stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
1493                 lxoffrxc = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
1494                 stats->lxoffrxc += lxoffrxc;
1495                 /* 82598 only has a counter in the high register */
1496                 stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
1497                 stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
1498                 stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
1499         }
1500
1501         /*
1502          * For watchdog management we need to know if we have been paused
1503          * during the last interval, so capture that here.
1504         */
1505         if (lxoffrxc)
1506                 sc->shared->isc_pause_frames = 1;
1507
1508         /*
1509          * Workaround: mprc hardware is incorrectly counting
1510          * broadcasts, so for now we subtract those.
1511          */
1512         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
1513         stats->bprc += bprc;
1514         stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
1515         if (hw->mac.type == ixgbe_mac_82598EB)
1516                 stats->mprc -= bprc;
1517
1518         stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
1519         stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
1520         stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
1521         stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
1522         stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
1523         stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
1524
1525         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
1526         stats->lxontxc += lxon;
1527         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
1528         stats->lxofftxc += lxoff;
1529         total = lxon + lxoff;
1530
1531         stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
1532         stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
1533         stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
1534         stats->gptc -= total;
1535         stats->mptc -= total;
1536         stats->ptc64 -= total;
1537         stats->gotc -= total * ETHER_MIN_LEN;
1538
1539         stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
1540         stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
1541         stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
1542         stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
1543         stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
1544         stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
1545         stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
1546         stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
1547         stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
1548         stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
1549         stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
1550         stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
1551         stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
1552         stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
1553         stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
1554         stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
1555         stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
1556         stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
1557         /* Only read FCOE on 82599 */
1558         if (hw->mac.type != ixgbe_mac_82598EB) {
1559                 stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
1560                 stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
1561                 stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
1562                 stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
1563                 stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
1564         }
1565
1566         /* Fill out the OS statistics structure */
1567         IXGBE_SET_IPACKETS(sc, stats->gprc);
1568         IXGBE_SET_OPACKETS(sc, stats->gptc);
1569         IXGBE_SET_IBYTES(sc, stats->gorc);
1570         IXGBE_SET_OBYTES(sc, stats->gotc);
1571         IXGBE_SET_IMCASTS(sc, stats->mprc);
1572         IXGBE_SET_OMCASTS(sc, stats->mptc);
1573         IXGBE_SET_COLLISIONS(sc, 0);
1574         IXGBE_SET_IQDROPS(sc, total_missed_rx);
1575
1576         /*
1577          * Aggregate following types of errors as RX errors:
1578          * - CRC error count,
1579          * - illegal byte error count,
1580          * - checksum error count,
1581          * - missed packets count,
1582          * - length error count,
1583          * - undersized packets count,
1584          * - fragmented packets count,
1585          * - oversized packets count,
1586          * - jabber count.
1587          */
1588         IXGBE_SET_IERRORS(sc, stats->crcerrs + stats->illerrc + stats->xec +
1589             stats->mpc[0] + stats->rlec + stats->ruc + stats->rfc + stats->roc +
1590             stats->rjc);
1591 } /* ixgbe_update_stats_counters */
1592
1593 /************************************************************************
1594  * ixgbe_add_hw_stats
1595  *
1596  *   Add sysctl variables, one per statistic, to the system.
1597  ************************************************************************/
1598 static void
1599 ixgbe_add_hw_stats(struct ixgbe_softc *sc)
1600 {
1601         device_t               dev = iflib_get_dev(sc->ctx);
1602         struct ix_rx_queue     *rx_que;
1603         struct ix_tx_queue     *tx_que;
1604         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
1605         struct sysctl_oid      *tree = device_get_sysctl_tree(dev);
1606         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
1607         struct ixgbe_hw_stats  *stats = &sc->stats.pf;
1608         struct sysctl_oid      *stat_node, *queue_node;
1609         struct sysctl_oid_list *stat_list, *queue_list;
1610         int                    i;
1611
1612 #define QUEUE_NAME_LEN 32
1613         char                   namebuf[QUEUE_NAME_LEN];
1614
1615         /* Driver Statistics */
1616         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
1617             CTLFLAG_RD, &sc->dropped_pkts, "Driver dropped packets");
1618         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
1619             CTLFLAG_RD, &sc->watchdog_events, "Watchdog timeouts");
1620         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
1621             CTLFLAG_RD, &sc->link_irq, "Link MSI-X IRQ Handled");
1622
1623         for (i = 0, tx_que = sc->tx_queues; i < sc->num_tx_queues; i++, tx_que++) {
1624                 struct tx_ring *txr = &tx_que->txr;
1625                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1626                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1627                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name");
1628                 queue_list = SYSCTL_CHILDREN(queue_node);
1629
1630                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
1631                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, txr, 0,
1632                     ixgbe_sysctl_tdh_handler, "IU", "Transmit Descriptor Head");
1633                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
1634                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, txr, 0,
1635                     ixgbe_sysctl_tdt_handler, "IU", "Transmit Descriptor Tail");
1636                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
1637                     CTLFLAG_RD, &txr->tso_tx, "TSO");
1638                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
1639                     CTLFLAG_RD, &txr->total_packets,
1640                     "Queue Packets Transmitted");
1641         }
1642
1643         for (i = 0, rx_que = sc->rx_queues; i < sc->num_rx_queues; i++, rx_que++) {
1644                 struct rx_ring *rxr = &rx_que->rxr;
1645                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
1646                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
1647                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name");
1648                 queue_list = SYSCTL_CHILDREN(queue_node);
1649
1650                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
1651                     CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
1652                     &sc->rx_queues[i], 0,
1653                     ixgbe_sysctl_interrupt_rate_handler, "IU",
1654                     "Interrupt Rate");
1655                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
1656                     CTLFLAG_RD, &(sc->rx_queues[i].irqs),
1657                     "irqs on this queue");
1658                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
1659                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, rxr, 0,
1660                     ixgbe_sysctl_rdh_handler, "IU", "Receive Descriptor Head");
1661                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
1662                     CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, rxr, 0,
1663                     ixgbe_sysctl_rdt_handler, "IU", "Receive Descriptor Tail");
1664                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
1665                     CTLFLAG_RD, &rxr->rx_packets, "Queue Packets Received");
1666                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
1667                     CTLFLAG_RD, &rxr->rx_bytes, "Queue Bytes Received");
1668                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
1669                     CTLFLAG_RD, &rxr->rx_copies, "Copied RX Frames");
1670                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
1671                     CTLFLAG_RD, &rxr->rx_discarded, "Discarded RX packets");
1672         }
1673
1674         /* MAC stats get their own sub node */
1675
1676         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
1677             CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "MAC Statistics");
1678         stat_list = SYSCTL_CHILDREN(stat_node);
1679
1680         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_errs",
1681             CTLFLAG_RD, &sc->ierrors, IXGBE_SYSCTL_DESC_RX_ERRS);
1682         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
1683             CTLFLAG_RD, &stats->crcerrs, "CRC Errors");
1684         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
1685             CTLFLAG_RD, &stats->illerrc, "Illegal Byte Errors");
1686         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
1687             CTLFLAG_RD, &stats->errbc, "Byte Errors");
1688         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
1689             CTLFLAG_RD, &stats->mspdc, "MAC Short Packets Discarded");
1690         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
1691             CTLFLAG_RD, &stats->mlfc, "MAC Local Faults");
1692         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
1693             CTLFLAG_RD, &stats->mrfc, "MAC Remote Faults");
1694         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
1695             CTLFLAG_RD, &stats->rlec, "Receive Length Errors");
1696         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_missed_packets",
1697             CTLFLAG_RD, &stats->mpc[0], "RX Missed Packet Count");
1698
1699         /* Flow Control stats */
1700         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
1701             CTLFLAG_RD, &stats->lxontxc, "Link XON Transmitted");
1702         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
1703             CTLFLAG_RD, &stats->lxonrxc, "Link XON Received");
1704         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
1705             CTLFLAG_RD, &stats->lxofftxc, "Link XOFF Transmitted");
1706         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
1707             CTLFLAG_RD, &stats->lxoffrxc, "Link XOFF Received");
1708
1709         /* Packet Reception Stats */
1710         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
1711             CTLFLAG_RD, &stats->tor, "Total Octets Received");
1712         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
1713             CTLFLAG_RD, &stats->gorc, "Good Octets Received");
1714         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
1715             CTLFLAG_RD, &stats->tpr, "Total Packets Received");
1716         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
1717             CTLFLAG_RD, &stats->gprc, "Good Packets Received");
1718         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
1719             CTLFLAG_RD, &stats->mprc, "Multicast Packets Received");
1720         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
1721             CTLFLAG_RD, &stats->bprc, "Broadcast Packets Received");
1722         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
1723             CTLFLAG_RD, &stats->prc64, "64 byte frames received ");
1724         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
1725             CTLFLAG_RD, &stats->prc127, "65-127 byte frames received");
1726         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
1727             CTLFLAG_RD, &stats->prc255, "128-255 byte frames received");
1728         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
1729             CTLFLAG_RD, &stats->prc511, "256-511 byte frames received");
1730         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
1731             CTLFLAG_RD, &stats->prc1023, "512-1023 byte frames received");
1732         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
1733             CTLFLAG_RD, &stats->prc1522, "1023-1522 byte frames received");
1734         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
1735             CTLFLAG_RD, &stats->ruc, "Receive Undersized");
1736         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
1737             CTLFLAG_RD, &stats->rfc, "Fragmented Packets Received ");
1738         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
1739             CTLFLAG_RD, &stats->roc, "Oversized Packets Received");
1740         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
1741             CTLFLAG_RD, &stats->rjc, "Received Jabber");
1742         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
1743             CTLFLAG_RD, &stats->mngprc, "Management Packets Received");
1744         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
1745             CTLFLAG_RD, &stats->mngptc, "Management Packets Dropped");
1746         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
1747             CTLFLAG_RD, &stats->xec, "Checksum Errors");
1748
1749         /* Packet Transmission Stats */
1750         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
1751             CTLFLAG_RD, &stats->gotc, "Good Octets Transmitted");
1752         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
1753             CTLFLAG_RD, &stats->tpt, "Total Packets Transmitted");
1754         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
1755             CTLFLAG_RD, &stats->gptc, "Good Packets Transmitted");
1756         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
1757             CTLFLAG_RD, &stats->bptc, "Broadcast Packets Transmitted");
1758         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
1759             CTLFLAG_RD, &stats->mptc, "Multicast Packets Transmitted");
1760         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
1761             CTLFLAG_RD, &stats->mngptc, "Management Packets Transmitted");
1762         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
1763             CTLFLAG_RD, &stats->ptc64, "64 byte frames transmitted ");
1764         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
1765             CTLFLAG_RD, &stats->ptc127, "65-127 byte frames transmitted");
1766         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
1767             CTLFLAG_RD, &stats->ptc255, "128-255 byte frames transmitted");
1768         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
1769             CTLFLAG_RD, &stats->ptc511, "256-511 byte frames transmitted");
1770         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
1771             CTLFLAG_RD, &stats->ptc1023, "512-1023 byte frames transmitted");
1772         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
1773             CTLFLAG_RD, &stats->ptc1522, "1024-1522 byte frames transmitted");
1774 } /* ixgbe_add_hw_stats */
1775
1776 /************************************************************************
1777  * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function
1778  *
1779  *   Retrieves the TDH value from the hardware
1780  ************************************************************************/
1781 static int
1782 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
1783 {
1784         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
1785         int            error;
1786         unsigned int   val;
1787
1788         if (!txr)
1789                 return (0);
1790
1791         val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_TDH(txr->me));
1792         error = sysctl_handle_int(oidp, &val, 0, req);
1793         if (error || !req->newptr)
1794                 return error;
1795
1796         return (0);
1797 } /* ixgbe_sysctl_tdh_handler */
1798
1799 /************************************************************************
1800  * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function
1801  *
1802  *   Retrieves the TDT value from the hardware
1803  ************************************************************************/
1804 static int
1805 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
1806 {
1807         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
1808         int            error;
1809         unsigned int   val;
1810
1811         if (!txr)
1812                 return (0);
1813
1814         val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_TDT(txr->me));
1815         error = sysctl_handle_int(oidp, &val, 0, req);
1816         if (error || !req->newptr)
1817                 return error;
1818
1819         return (0);
1820 } /* ixgbe_sysctl_tdt_handler */
1821
1822 /************************************************************************
1823  * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function
1824  *
1825  *   Retrieves the RDH value from the hardware
1826  ************************************************************************/
1827 static int
1828 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
1829 {
1830         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
1831         int            error;
1832         unsigned int   val;
1833
1834         if (!rxr)
1835                 return (0);
1836
1837         val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_RDH(rxr->me));
1838         error = sysctl_handle_int(oidp, &val, 0, req);
1839         if (error || !req->newptr)
1840                 return error;
1841
1842         return (0);
1843 } /* ixgbe_sysctl_rdh_handler */
1844
1845 /************************************************************************
1846  * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function
1847  *
1848  *   Retrieves the RDT value from the hardware
1849  ************************************************************************/
1850 static int
1851 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
1852 {
1853         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
1854         int            error;
1855         unsigned int   val;
1856
1857         if (!rxr)
1858                 return (0);
1859
1860         val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_RDT(rxr->me));
1861         error = sysctl_handle_int(oidp, &val, 0, req);
1862         if (error || !req->newptr)
1863                 return error;
1864
1865         return (0);
1866 } /* ixgbe_sysctl_rdt_handler */
1867
1868 /************************************************************************
1869  * ixgbe_if_vlan_register
1870  *
1871  *   Run via vlan config EVENT, it enables us to use the
1872  *   HW Filter table since we can get the vlan id. This
1873  *   just creates the entry in the soft version of the
1874  *   VFTA, init will repopulate the real table.
1875  ************************************************************************/
1876 static void
1877 ixgbe_if_vlan_register(if_ctx_t ctx, u16 vtag)
1878 {
1879         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1880         u16            index, bit;
1881
1882         index = (vtag >> 5) & 0x7F;
1883         bit = vtag & 0x1F;
1884         sc->shadow_vfta[index] |= (1 << bit);
1885         ++sc->num_vlans;
1886         ixgbe_setup_vlan_hw_support(ctx);
1887 } /* ixgbe_if_vlan_register */
1888
1889 /************************************************************************
1890  * ixgbe_if_vlan_unregister
1891  *
1892  *   Run via vlan unconfig EVENT, remove our entry in the soft vfta.
1893  ************************************************************************/
1894 static void
1895 ixgbe_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
1896 {
1897         struct ixgbe_softc *sc = iflib_get_softc(ctx);
1898         u16            index, bit;
1899
1900         index = (vtag >> 5) & 0x7F;
1901         bit = vtag & 0x1F;
1902         sc->shadow_vfta[index] &= ~(1 << bit);
1903         --sc->num_vlans;
1904         /* Re-init to load the changes */
1905         ixgbe_setup_vlan_hw_support(ctx);
1906 } /* ixgbe_if_vlan_unregister */
1907
1908 /************************************************************************
1909  * ixgbe_setup_vlan_hw_support
1910  ************************************************************************/
1911 static void
1912 ixgbe_setup_vlan_hw_support(if_ctx_t ctx)
1913 {
1914         struct ifnet    *ifp = iflib_get_ifp(ctx);
1915         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
1916         struct ixgbe_hw *hw = &sc->hw;
1917         struct rx_ring  *rxr;
1918         int             i;
1919         u32             ctrl;
1920
1921
1922         /*
1923          * We get here thru init_locked, meaning
1924          * a soft reset, this has already cleared
1925          * the VFTA and other state, so if there
1926          * have been no vlan's registered do nothing.
1927          */
1928         if (sc->num_vlans == 0)
1929                 return;
1930
1931         /* Setup the queues for vlans */
1932         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1933                 for (i = 0; i < sc->num_rx_queues; i++) {
1934                         rxr = &sc->rx_queues[i].rxr;
1935                         /* On 82599 the VLAN enable is per/queue in RXDCTL */
1936                         if (hw->mac.type != ixgbe_mac_82598EB) {
1937                                 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
1938                                 ctrl |= IXGBE_RXDCTL_VME;
1939                                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl);
1940                         }
1941                         rxr->vtag_strip = true;
1942                 }
1943         }
1944
1945         if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
1946                 return;
1947         /*
1948          * A soft reset zero's out the VFTA, so
1949          * we need to repopulate it now.
1950          */
1951         for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1952                 if (sc->shadow_vfta[i] != 0)
1953                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
1954                             sc->shadow_vfta[i]);
1955
1956         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1957         /* Enable the Filter Table if enabled */
1958         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
1959                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
1960                 ctrl |= IXGBE_VLNCTRL_VFE;
1961         }
1962         if (hw->mac.type == ixgbe_mac_82598EB)
1963                 ctrl |= IXGBE_VLNCTRL_VME;
1964         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
1965 } /* ixgbe_setup_vlan_hw_support */
1966
1967 /************************************************************************
1968  * ixgbe_get_slot_info
1969  *
1970  *   Get the width and transaction speed of
1971  *   the slot this adapter is plugged into.
1972  ************************************************************************/
1973 static void
1974 ixgbe_get_slot_info(struct ixgbe_softc *sc)
1975 {
1976         device_t        dev = iflib_get_dev(sc->ctx);
1977         struct ixgbe_hw *hw = &sc->hw;
1978         int             bus_info_valid = true;
1979         u32             offset;
1980         u16             link;
1981
1982         /* Some devices are behind an internal bridge */
1983         switch (hw->device_id) {
1984         case IXGBE_DEV_ID_82599_SFP_SF_QP:
1985         case IXGBE_DEV_ID_82599_QSFP_SF_QP:
1986                 goto get_parent_info;
1987         default:
1988                 break;
1989         }
1990
1991         ixgbe_get_bus_info(hw);
1992
1993         /*
1994          * Some devices don't use PCI-E, but there is no need
1995          * to display "Unknown" for bus speed and width.
1996          */
1997         switch (hw->mac.type) {
1998         case ixgbe_mac_X550EM_x:
1999         case ixgbe_mac_X550EM_a:
2000                 return;
2001         default:
2002                 goto display;
2003         }
2004
2005 get_parent_info:
2006         /*
2007          * For the Quad port adapter we need to parse back
2008          * up the PCI tree to find the speed of the expansion
2009          * slot into which this adapter is plugged. A bit more work.
2010          */
2011         dev = device_get_parent(device_get_parent(dev));
2012 #ifdef IXGBE_DEBUG
2013         device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev),
2014             pci_get_slot(dev), pci_get_function(dev));
2015 #endif
2016         dev = device_get_parent(device_get_parent(dev));
2017 #ifdef IXGBE_DEBUG
2018         device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev),
2019             pci_get_slot(dev), pci_get_function(dev));
2020 #endif
2021         /* Now get the PCI Express Capabilities offset */
2022         if (pci_find_cap(dev, PCIY_EXPRESS, &offset)) {
2023                 /*
2024                  * Hmm...can't get PCI-Express capabilities.
2025                  * Falling back to default method.
2026                  */
2027                 bus_info_valid = false;
2028                 ixgbe_get_bus_info(hw);
2029                 goto display;
2030         }
2031         /* ...and read the Link Status Register */
2032         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
2033         ixgbe_set_pci_config_data_generic(hw, link);
2034
2035 display:
2036         device_printf(dev, "PCI Express Bus: Speed %s %s\n",
2037             ((hw->bus.speed == ixgbe_bus_speed_8000)    ? "8.0GT/s"  :
2038              (hw->bus.speed == ixgbe_bus_speed_5000)    ? "5.0GT/s"  :
2039              (hw->bus.speed == ixgbe_bus_speed_2500)    ? "2.5GT/s"  :
2040              "Unknown"),
2041             ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
2042              (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
2043              (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
2044              "Unknown"));
2045
2046         if (bus_info_valid) {
2047                 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
2048                     ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
2049                     (hw->bus.speed == ixgbe_bus_speed_2500))) {
2050                         device_printf(dev, "PCI-Express bandwidth available for this card\n     is not sufficient for optimal performance.\n");
2051                         device_printf(dev, "For optimal performance a x8 PCIE, or x4 PCIE Gen2 slot is required.\n");
2052                 }
2053                 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
2054                     ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
2055                     (hw->bus.speed < ixgbe_bus_speed_8000))) {
2056                         device_printf(dev, "PCI-Express bandwidth available for this card\n     is not sufficient for optimal performance.\n");
2057                         device_printf(dev, "For optimal performance a x8 PCIE Gen3 slot is required.\n");
2058                 }
2059         } else
2060                 device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n");
2061
2062         return;
2063 } /* ixgbe_get_slot_info */
2064
2065 /************************************************************************
2066  * ixgbe_if_msix_intr_assign
2067  *
2068  *   Setup MSI-X Interrupt resources and handlers
2069  ************************************************************************/
2070 static int
2071 ixgbe_if_msix_intr_assign(if_ctx_t ctx, int msix)
2072 {
2073         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
2074         struct ix_rx_queue *rx_que = sc->rx_queues;
2075         struct ix_tx_queue *tx_que;
2076         int                error, rid, vector = 0;
2077         char               buf[16];
2078
2079         /* Admin Que is vector 0*/
2080         rid = vector + 1;
2081         for (int i = 0; i < sc->num_rx_queues; i++, vector++, rx_que++) {
2082                 rid = vector + 1;
2083
2084                 snprintf(buf, sizeof(buf), "rxq%d", i);
2085                 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
2086                     IFLIB_INTR_RXTX, ixgbe_msix_que, rx_que, rx_que->rxr.me, buf);
2087
2088                 if (error) {
2089                         device_printf(iflib_get_dev(ctx),
2090                             "Failed to allocate que int %d err: %d", i, error);
2091                         sc->num_rx_queues = i + 1;
2092                         goto fail;
2093                 }
2094
2095                 rx_que->msix = vector;
2096         }
2097         for (int i = 0; i < sc->num_tx_queues; i++) {
2098                 snprintf(buf, sizeof(buf), "txq%d", i);
2099                 tx_que = &sc->tx_queues[i];
2100                 tx_que->msix = i % sc->num_rx_queues;
2101                 iflib_softirq_alloc_generic(ctx,
2102                     &sc->rx_queues[tx_que->msix].que_irq,
2103                     IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
2104         }
2105         rid = vector + 1;
2106         error = iflib_irq_alloc_generic(ctx, &sc->irq, rid,
2107             IFLIB_INTR_ADMIN, ixgbe_msix_link, sc, 0, "aq");
2108         if (error) {
2109                 device_printf(iflib_get_dev(ctx),
2110                     "Failed to register admin handler");
2111                 return (error);
2112         }
2113
2114         sc->vector = vector;
2115
2116         return (0);
2117 fail:
2118         iflib_irq_free(ctx, &sc->irq);
2119         rx_que = sc->rx_queues;
2120         for (int i = 0; i < sc->num_rx_queues; i++, rx_que++)
2121                 iflib_irq_free(ctx, &rx_que->que_irq);
2122
2123         return (error);
2124 } /* ixgbe_if_msix_intr_assign */
2125
2126 static inline void
2127 ixgbe_perform_aim(struct ixgbe_softc *sc, struct ix_rx_queue *que)
2128 {
2129         uint32_t newitr = 0;
2130         struct rx_ring *rxr = &que->rxr;
2131
2132         /*
2133          * Do Adaptive Interrupt Moderation:
2134          *  - Write out last calculated setting
2135          *  - Calculate based on average size over
2136          *    the last interval.
2137          */
2138         if (que->eitr_setting) {
2139                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(que->msix),
2140                     que->eitr_setting);
2141         }
2142
2143         que->eitr_setting = 0;
2144         /* Idle, do nothing */
2145         if (rxr->bytes == 0) {
2146                 return;
2147         }
2148
2149         if ((rxr->bytes) && (rxr->packets)) {
2150                 newitr = (rxr->bytes / rxr->packets);
2151         }
2152
2153         newitr += 24; /* account for hardware frame, crc */
2154         /* set an upper boundary */
2155         newitr = min(newitr, 3000);
2156
2157         /* Be nice to the mid range */
2158         if ((newitr > 300) && (newitr < 1200)) {
2159                 newitr = (newitr / 3);
2160         } else {
2161                 newitr = (newitr / 2);
2162         }
2163
2164         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
2165                 newitr |= newitr << 16;
2166         } else {
2167                 newitr |= IXGBE_EITR_CNT_WDIS;
2168         }
2169
2170         /* save for next interrupt */
2171         que->eitr_setting = newitr;
2172
2173         /* Reset state */
2174         rxr->bytes = 0;
2175         rxr->packets = 0;
2176
2177         return;
2178 }
2179
2180 /*********************************************************************
2181  * ixgbe_msix_que - MSI-X Queue Interrupt Service routine
2182  **********************************************************************/
2183 static int
2184 ixgbe_msix_que(void *arg)
2185 {
2186         struct ix_rx_queue *que = arg;
2187         struct ixgbe_softc     *sc = que->sc;
2188         struct ifnet       *ifp = iflib_get_ifp(que->sc->ctx);
2189
2190         /* Protect against spurious interrupts */
2191         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2192                 return (FILTER_HANDLED);
2193
2194         ixgbe_disable_queue(sc, que->msix);
2195         ++que->irqs;
2196
2197         /* Check for AIM */
2198         if (sc->enable_aim) {
2199                 ixgbe_perform_aim(sc, que);
2200         }
2201
2202         return (FILTER_SCHEDULE_THREAD);
2203 } /* ixgbe_msix_que */
2204
2205 /************************************************************************
2206  * ixgbe_media_status - Media Ioctl callback
2207  *
2208  *   Called whenever the user queries the status of
2209  *   the interface using ifconfig.
2210  ************************************************************************/
2211 static void
2212 ixgbe_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
2213 {
2214         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
2215         struct ixgbe_hw *hw = &sc->hw;
2216         int             layer;
2217
2218         INIT_DEBUGOUT("ixgbe_if_media_status: begin");
2219
2220         ifmr->ifm_status = IFM_AVALID;
2221         ifmr->ifm_active = IFM_ETHER;
2222
2223         if (!sc->link_active)
2224                 return;
2225
2226         ifmr->ifm_status |= IFM_ACTIVE;
2227         layer = sc->phy_layer;
2228
2229         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T ||
2230             layer & IXGBE_PHYSICAL_LAYER_1000BASE_T ||
2231             layer & IXGBE_PHYSICAL_LAYER_100BASE_TX ||
2232             layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
2233                 switch (sc->link_speed) {
2234                 case IXGBE_LINK_SPEED_10GB_FULL:
2235                         ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
2236                         break;
2237                 case IXGBE_LINK_SPEED_1GB_FULL:
2238                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
2239                         break;
2240                 case IXGBE_LINK_SPEED_100_FULL:
2241                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
2242                         break;
2243                 case IXGBE_LINK_SPEED_10_FULL:
2244                         ifmr->ifm_active |= IFM_10_T | IFM_FDX;
2245                         break;
2246                 }
2247         if (hw->mac.type == ixgbe_mac_X550)
2248                 switch (sc->link_speed) {
2249                 case IXGBE_LINK_SPEED_5GB_FULL:
2250                         ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
2251                         break;
2252                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2253                         ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
2254                         break;
2255                 }
2256         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
2257             layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
2258                 switch (sc->link_speed) {
2259                 case IXGBE_LINK_SPEED_10GB_FULL:
2260                         ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;
2261                         break;
2262                 }
2263         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
2264                 switch (sc->link_speed) {
2265                 case IXGBE_LINK_SPEED_10GB_FULL:
2266                         ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
2267                         break;
2268                 case IXGBE_LINK_SPEED_1GB_FULL:
2269                         ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
2270                         break;
2271                 }
2272         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM)
2273                 switch (sc->link_speed) {
2274                 case IXGBE_LINK_SPEED_10GB_FULL:
2275                         ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;
2276                         break;
2277                 case IXGBE_LINK_SPEED_1GB_FULL:
2278                         ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
2279                         break;
2280                 }
2281         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR ||
2282             layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
2283                 switch (sc->link_speed) {
2284                 case IXGBE_LINK_SPEED_10GB_FULL:
2285                         ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
2286                         break;
2287                 case IXGBE_LINK_SPEED_1GB_FULL:
2288                         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
2289                         break;
2290                 }
2291         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
2292                 switch (sc->link_speed) {
2293                 case IXGBE_LINK_SPEED_10GB_FULL:
2294                         ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
2295                         break;
2296                 }
2297         /*
2298          * XXX: These need to use the proper media types once
2299          * they're added.
2300          */
2301 #ifndef IFM_ETH_XTYPE
2302         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
2303                 switch (sc->link_speed) {
2304                 case IXGBE_LINK_SPEED_10GB_FULL:
2305                         ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
2306                         break;
2307                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2308                         ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
2309                         break;
2310                 case IXGBE_LINK_SPEED_1GB_FULL:
2311                         ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
2312                         break;
2313                 }
2314         else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
2315             layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
2316             layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
2317                 switch (sc->link_speed) {
2318                 case IXGBE_LINK_SPEED_10GB_FULL:
2319                         ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
2320                         break;
2321                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2322                         ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
2323                         break;
2324                 case IXGBE_LINK_SPEED_1GB_FULL:
2325                         ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
2326                         break;
2327                 }
2328 #else
2329         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
2330                 switch (sc->link_speed) {
2331                 case IXGBE_LINK_SPEED_10GB_FULL:
2332                         ifmr->ifm_active |= IFM_10G_KR | IFM_FDX;
2333                         break;
2334                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2335                         ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
2336                         break;
2337                 case IXGBE_LINK_SPEED_1GB_FULL:
2338                         ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
2339                         break;
2340                 }
2341         else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
2342             layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
2343             layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
2344                 switch (sc->link_speed) {
2345                 case IXGBE_LINK_SPEED_10GB_FULL:
2346                         ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX;
2347                         break;
2348                 case IXGBE_LINK_SPEED_2_5GB_FULL:
2349                         ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
2350                         break;
2351                 case IXGBE_LINK_SPEED_1GB_FULL:
2352                         ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
2353                         break;
2354                 }
2355 #endif
2356
2357         /* If nothing is recognized... */
2358         if (IFM_SUBTYPE(ifmr->ifm_active) == 0)
2359                 ifmr->ifm_active |= IFM_UNKNOWN;
2360
2361         /* Display current flow control setting used on link */
2362         if (hw->fc.current_mode == ixgbe_fc_rx_pause ||
2363             hw->fc.current_mode == ixgbe_fc_full)
2364                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
2365         if (hw->fc.current_mode == ixgbe_fc_tx_pause ||
2366             hw->fc.current_mode == ixgbe_fc_full)
2367                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
2368 } /* ixgbe_media_status */
2369
2370 /************************************************************************
2371  * ixgbe_media_change - Media Ioctl callback
2372  *
2373  *   Called when the user changes speed/duplex using
2374  *   media/mediopt option with ifconfig.
2375  ************************************************************************/
2376 static int
2377 ixgbe_if_media_change(if_ctx_t ctx)
2378 {
2379         struct ixgbe_softc   *sc = iflib_get_softc(ctx);
2380         struct ifmedia   *ifm = iflib_get_media(ctx);
2381         struct ixgbe_hw  *hw = &sc->hw;
2382         ixgbe_link_speed speed = 0;
2383
2384         INIT_DEBUGOUT("ixgbe_if_media_change: begin");
2385
2386         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2387                 return (EINVAL);
2388
2389         if (hw->phy.media_type == ixgbe_media_type_backplane)
2390                 return (EPERM);
2391
2392         /*
2393          * We don't actually need to check against the supported
2394          * media types of the adapter; ifmedia will take care of
2395          * that for us.
2396          */
2397         switch (IFM_SUBTYPE(ifm->ifm_media)) {
2398         case IFM_AUTO:
2399         case IFM_10G_T:
2400                 speed |= IXGBE_LINK_SPEED_100_FULL;
2401                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2402                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
2403                 break;
2404         case IFM_10G_LRM:
2405         case IFM_10G_LR:
2406 #ifndef IFM_ETH_XTYPE
2407         case IFM_10G_SR: /* KR, too */
2408         case IFM_10G_CX4: /* KX4 */
2409 #else
2410         case IFM_10G_KR:
2411         case IFM_10G_KX4:
2412 #endif
2413                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2414                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
2415                 break;
2416 #ifndef IFM_ETH_XTYPE
2417         case IFM_1000_CX: /* KX */
2418 #else
2419         case IFM_1000_KX:
2420 #endif
2421         case IFM_1000_LX:
2422         case IFM_1000_SX:
2423                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2424                 break;
2425         case IFM_1000_T:
2426                 speed |= IXGBE_LINK_SPEED_100_FULL;
2427                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
2428                 break;
2429         case IFM_10G_TWINAX:
2430                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
2431                 break;
2432         case IFM_5000_T:
2433                 speed |= IXGBE_LINK_SPEED_5GB_FULL;
2434                 break;
2435         case IFM_2500_T:
2436                 speed |= IXGBE_LINK_SPEED_2_5GB_FULL;
2437                 break;
2438         case IFM_100_TX:
2439                 speed |= IXGBE_LINK_SPEED_100_FULL;
2440                 break;
2441         case IFM_10_T:
2442                 speed |= IXGBE_LINK_SPEED_10_FULL;
2443                 break;
2444         default:
2445                 goto invalid;
2446         }
2447
2448         hw->mac.autotry_restart = true;
2449         hw->mac.ops.setup_link(hw, speed, true);
2450         sc->advertise =
2451             ((speed & IXGBE_LINK_SPEED_10GB_FULL)  ? 0x4  : 0) |
2452             ((speed & IXGBE_LINK_SPEED_5GB_FULL)   ? 0x20 : 0) |
2453             ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) |
2454             ((speed & IXGBE_LINK_SPEED_1GB_FULL)   ? 0x2  : 0) |
2455             ((speed & IXGBE_LINK_SPEED_100_FULL)   ? 0x1  : 0) |
2456             ((speed & IXGBE_LINK_SPEED_10_FULL)    ? 0x8  : 0);
2457
2458         return (0);
2459
2460 invalid:
2461         device_printf(iflib_get_dev(ctx), "Invalid media type!\n");
2462
2463         return (EINVAL);
2464 } /* ixgbe_if_media_change */
2465
2466 /************************************************************************
2467  * ixgbe_set_promisc
2468  ************************************************************************/
2469 static int
2470 ixgbe_if_promisc_set(if_ctx_t ctx, int flags)
2471 {
2472         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2473         struct ifnet   *ifp = iflib_get_ifp(ctx);
2474         u32            rctl;
2475         int            mcnt = 0;
2476
2477         rctl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
2478         rctl &= (~IXGBE_FCTRL_UPE);
2479         if (ifp->if_flags & IFF_ALLMULTI)
2480                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2481         else {
2482                 mcnt = min(if_llmaddr_count(ifp), MAX_NUM_MULTICAST_ADDRESSES);
2483         }
2484         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2485                 rctl &= (~IXGBE_FCTRL_MPE);
2486         IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);
2487
2488         if (ifp->if_flags & IFF_PROMISC) {
2489                 rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
2490                 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);
2491         } else if (ifp->if_flags & IFF_ALLMULTI) {
2492                 rctl |= IXGBE_FCTRL_MPE;
2493                 rctl &= ~IXGBE_FCTRL_UPE;
2494                 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, rctl);
2495         }
2496         return (0);
2497 } /* ixgbe_if_promisc_set */
2498
2499 /************************************************************************
2500  * ixgbe_msix_link - Link status change ISR (MSI/MSI-X)
2501  ************************************************************************/
2502 static int
2503 ixgbe_msix_link(void *arg)
2504 {
2505         struct ixgbe_softc  *sc = arg;
2506         struct ixgbe_hw *hw = &sc->hw;
2507         u32             eicr, eicr_mask;
2508         s32             retval;
2509
2510         ++sc->link_irq;
2511
2512         /* Pause other interrupts */
2513         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER);
2514
2515         /* First get the cause */
2516         eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
2517         /* Be sure the queue bits are not cleared */
2518         eicr &= ~IXGBE_EICR_RTX_QUEUE;
2519         /* Clear interrupt with write */
2520         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
2521
2522         /* Link status change */
2523         if (eicr & IXGBE_EICR_LSC) {
2524                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
2525                 sc->task_requests |= IXGBE_REQUEST_TASK_LSC;
2526         }
2527
2528         if (sc->hw.mac.type != ixgbe_mac_82598EB) {
2529                 if ((sc->feat_en & IXGBE_FEATURE_FDIR) &&
2530                     (eicr & IXGBE_EICR_FLOW_DIR)) {
2531                         /* This is probably overkill :) */
2532                         if (!atomic_cmpset_int(&sc->fdir_reinit, 0, 1))
2533                                 return (FILTER_HANDLED);
2534                         /* Disable the interrupt */
2535                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
2536                         sc->task_requests |= IXGBE_REQUEST_TASK_FDIR;
2537                 } else
2538                         if (eicr & IXGBE_EICR_ECC) {
2539                                 device_printf(iflib_get_dev(sc->ctx),
2540                                    "\nCRITICAL: ECC ERROR!! Please Reboot!!\n");
2541                                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
2542                         }
2543
2544                 /* Check for over temp condition */
2545                 if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR) {
2546                         switch (sc->hw.mac.type) {
2547                         case ixgbe_mac_X550EM_a:
2548                                 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a))
2549                                         break;
2550                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC,
2551                                     IXGBE_EICR_GPI_SDP0_X550EM_a);
2552                                 IXGBE_WRITE_REG(hw, IXGBE_EICR,
2553                                     IXGBE_EICR_GPI_SDP0_X550EM_a);
2554                                 retval = hw->phy.ops.check_overtemp(hw);
2555                                 if (retval != IXGBE_ERR_OVERTEMP)
2556                                         break;
2557                                 device_printf(iflib_get_dev(sc->ctx),
2558                                     "\nCRITICAL: OVER TEMP!! PHY IS SHUT DOWN!!\n");
2559                                 device_printf(iflib_get_dev(sc->ctx),
2560                                     "System shutdown required!\n");
2561                                 break;
2562                         default:
2563                                 if (!(eicr & IXGBE_EICR_TS))
2564                                         break;
2565                                 retval = hw->phy.ops.check_overtemp(hw);
2566                                 if (retval != IXGBE_ERR_OVERTEMP)
2567                                         break;
2568                                 device_printf(iflib_get_dev(sc->ctx),
2569                                     "\nCRITICAL: OVER TEMP!! PHY IS SHUT DOWN!!\n");
2570                                 device_printf(iflib_get_dev(sc->ctx),
2571                                     "System shutdown required!\n");
2572                                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
2573                                 break;
2574                         }
2575                 }
2576
2577                 /* Check for VF message */
2578                 if ((sc->feat_en & IXGBE_FEATURE_SRIOV) &&
2579                     (eicr & IXGBE_EICR_MAILBOX))
2580                         sc->task_requests |= IXGBE_REQUEST_TASK_MBX;
2581         }
2582
2583         if (ixgbe_is_sfp(hw)) {
2584                 /* Pluggable optics-related interrupt */
2585                 if (hw->mac.type >= ixgbe_mac_X540)
2586                         eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
2587                 else
2588                         eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
2589
2590                 if (eicr & eicr_mask) {
2591                         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
2592                         sc->task_requests |= IXGBE_REQUEST_TASK_MOD;
2593                 }
2594
2595                 if ((hw->mac.type == ixgbe_mac_82599EB) &&
2596                     (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
2597                         IXGBE_WRITE_REG(hw, IXGBE_EICR,
2598                             IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
2599                         sc->task_requests |= IXGBE_REQUEST_TASK_MSF;
2600                 }
2601         }
2602
2603         /* Check for fan failure */
2604         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL) {
2605                 ixgbe_check_fan_failure(sc, eicr, true);
2606                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
2607         }
2608
2609         /* External PHY interrupt */
2610         if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&
2611             (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
2612                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540);
2613                 sc->task_requests |= IXGBE_REQUEST_TASK_PHY;
2614         }
2615
2616         return (sc->task_requests != 0) ? FILTER_SCHEDULE_THREAD : FILTER_HANDLED;
2617 } /* ixgbe_msix_link */
2618
2619 /************************************************************************
2620  * ixgbe_sysctl_interrupt_rate_handler
2621  ************************************************************************/
2622 static int
2623 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
2624 {
2625         struct ix_rx_queue *que = ((struct ix_rx_queue *)oidp->oid_arg1);
2626         int                error;
2627         unsigned int       reg, usec, rate;
2628
2629         reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_EITR(que->msix));
2630         usec = ((reg & 0x0FF8) >> 3);
2631         if (usec > 0)
2632                 rate = 500000 / usec;
2633         else
2634                 rate = 0;
2635         error = sysctl_handle_int(oidp, &rate, 0, req);
2636         if (error || !req->newptr)
2637                 return error;
2638         reg &= ~0xfff; /* default, no limitation */
2639         ixgbe_max_interrupt_rate = 0;
2640         if (rate > 0 && rate < 500000) {
2641                 if (rate < 1000)
2642                         rate = 1000;
2643                 ixgbe_max_interrupt_rate = rate;
2644                 reg |= ((4000000/rate) & 0xff8);
2645         }
2646         IXGBE_WRITE_REG(&que->sc->hw, IXGBE_EITR(que->msix), reg);
2647
2648         return (0);
2649 } /* ixgbe_sysctl_interrupt_rate_handler */
2650
2651 /************************************************************************
2652  * ixgbe_add_device_sysctls
2653  ************************************************************************/
2654 static void
2655 ixgbe_add_device_sysctls(if_ctx_t ctx)
2656 {
2657         struct ixgbe_softc         *sc = iflib_get_softc(ctx);
2658         device_t               dev = iflib_get_dev(ctx);
2659         struct ixgbe_hw        *hw = &sc->hw;
2660         struct sysctl_oid_list *child;
2661         struct sysctl_ctx_list *ctx_list;
2662
2663         ctx_list = device_get_sysctl_ctx(dev);
2664         child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
2665
2666         /* Sysctls for all devices */
2667         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc",
2668             CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2669             sc, 0, ixgbe_sysctl_flowcntl, "I",
2670             IXGBE_SYSCTL_DESC_SET_FC);
2671
2672         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "advertise_speed",
2673             CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2674             sc, 0, ixgbe_sysctl_advertise, "I",
2675             IXGBE_SYSCTL_DESC_ADV_SPEED);
2676
2677         sc->enable_aim = ixgbe_enable_aim;
2678         SYSCTL_ADD_INT(ctx_list, child, OID_AUTO, "enable_aim", CTLFLAG_RW,
2679             &sc->enable_aim, 0, "Interrupt Moderation");
2680
2681         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fw_version",
2682             CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
2683             ixgbe_sysctl_print_fw_version, "A", "Prints FW/NVM Versions");
2684
2685 #ifdef IXGBE_DEBUG
2686         /* testing sysctls (for all devices) */
2687         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "power_state",
2688             CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2689             sc, 0, ixgbe_sysctl_power_state,
2690             "I", "PCI Power State");
2691
2692         SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "print_rss_config",
2693             CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
2694             ixgbe_sysctl_print_rss_config, "A", "Prints RSS Configuration");
2695 #endif
2696         /* for X550 series devices */
2697         if (hw->mac.type >= ixgbe_mac_X550)
2698                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac",
2699                     CTLTYPE_U16 | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2700                     sc, 0, ixgbe_sysctl_dmac,
2701                     "I", "DMA Coalesce");
2702
2703         /* for WoL-capable devices */
2704         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
2705                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "wol_enable",
2706                     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
2707                     ixgbe_sysctl_wol_enable, "I", "Enable/Disable Wake on LAN");
2708
2709                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "wufc",
2710                     CTLTYPE_U32 | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
2711                     sc, 0, ixgbe_sysctl_wufc,
2712                     "I", "Enable/Disable Wake Up Filters");
2713         }
2714
2715         /* for X552/X557-AT devices */
2716         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
2717                 struct sysctl_oid *phy_node;
2718                 struct sysctl_oid_list *phy_list;
2719
2720                 phy_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "phy",
2721                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "External PHY sysctls");
2722                 phy_list = SYSCTL_CHILDREN(phy_node);
2723
2724                 SYSCTL_ADD_PROC(ctx_list, phy_list, OID_AUTO, "temp",
2725                     CTLTYPE_U16 | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
2726                     sc, 0, ixgbe_sysctl_phy_temp,
2727                     "I", "Current External PHY Temperature (Celsius)");
2728
2729                 SYSCTL_ADD_PROC(ctx_list, phy_list, OID_AUTO,
2730                     "overtemp_occurred",
2731                     CTLTYPE_U16 | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0,
2732                     ixgbe_sysctl_phy_overtemp_occurred, "I",
2733                     "External PHY High Temperature Event Occurred");
2734         }
2735
2736         if (sc->feat_cap & IXGBE_FEATURE_EEE) {
2737                 SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_state",
2738                     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0,
2739                     ixgbe_sysctl_eee_state, "I", "EEE Power Save State");
2740         }
2741 } /* ixgbe_add_device_sysctls */
2742
2743 /************************************************************************
2744  * ixgbe_allocate_pci_resources
2745  ************************************************************************/
2746 static int
2747 ixgbe_allocate_pci_resources(if_ctx_t ctx)
2748 {
2749         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2750         device_t        dev = iflib_get_dev(ctx);
2751         int             rid;
2752
2753         rid = PCIR_BAR(0);
2754         sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
2755             RF_ACTIVE);
2756
2757         if (!(sc->pci_mem)) {
2758                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2759                 return (ENXIO);
2760         }
2761
2762         /* Save bus_space values for READ/WRITE_REG macros */
2763         sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem);
2764         sc->osdep.mem_bus_space_handle =
2765             rman_get_bushandle(sc->pci_mem);
2766         /* Set hw values for shared code */
2767         sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle;
2768
2769         return (0);
2770 } /* ixgbe_allocate_pci_resources */
2771
2772 /************************************************************************
2773  * ixgbe_detach - Device removal routine
2774  *
2775  *   Called when the driver is being removed.
2776  *   Stops the adapter and deallocates all the resources
2777  *   that were allocated for driver operation.
2778  *
2779  *   return 0 on success, positive on failure
2780  ************************************************************************/
2781 static int
2782 ixgbe_if_detach(if_ctx_t ctx)
2783 {
2784         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2785         device_t       dev = iflib_get_dev(ctx);
2786         u32            ctrl_ext;
2787
2788         INIT_DEBUGOUT("ixgbe_detach: begin");
2789
2790         if (ixgbe_pci_iov_detach(dev) != 0) {
2791                 device_printf(dev, "SR-IOV in use; detach first.\n");
2792                 return (EBUSY);
2793         }
2794
2795         ixgbe_setup_low_power_mode(ctx);
2796
2797         /* let hardware know driver is unloading */
2798         ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
2799         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
2800         IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
2801
2802         ixgbe_free_pci_resources(ctx);
2803         free(sc->mta, M_IXGBE);
2804
2805         return (0);
2806 } /* ixgbe_if_detach */
2807
2808 /************************************************************************
2809  * ixgbe_setup_low_power_mode - LPLU/WoL preparation
2810  *
2811  *   Prepare the adapter/port for LPLU and/or WoL
2812  ************************************************************************/
2813 static int
2814 ixgbe_setup_low_power_mode(if_ctx_t ctx)
2815 {
2816         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
2817         struct ixgbe_hw *hw = &sc->hw;
2818         device_t        dev = iflib_get_dev(ctx);
2819         s32             error = 0;
2820
2821         if (!hw->wol_enabled)
2822                 ixgbe_set_phy_power(hw, false);
2823
2824         /* Limit power management flow to X550EM baseT */
2825         if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
2826             hw->phy.ops.enter_lplu) {
2827                 /* Turn off support for APM wakeup. (Using ACPI instead) */
2828                 IXGBE_WRITE_REG(hw, IXGBE_GRC,
2829                     IXGBE_READ_REG(hw, IXGBE_GRC) & ~(u32)2);
2830
2831                 /*
2832                  * Clear Wake Up Status register to prevent any previous wakeup
2833                  * events from waking us up immediately after we suspend.
2834                  */
2835                 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
2836
2837                 /*
2838                  * Program the Wakeup Filter Control register with user filter
2839                  * settings
2840                  */
2841                 IXGBE_WRITE_REG(hw, IXGBE_WUFC, sc->wufc);
2842
2843                 /* Enable wakeups and power management in Wakeup Control */
2844                 IXGBE_WRITE_REG(hw, IXGBE_WUC,
2845                     IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN);
2846
2847                 /* X550EM baseT adapters need a special LPLU flow */
2848                 hw->phy.reset_disable = true;
2849                 ixgbe_if_stop(ctx);
2850                 error = hw->phy.ops.enter_lplu(hw);
2851                 if (error)
2852                         device_printf(dev, "Error entering LPLU: %d\n", error);
2853                 hw->phy.reset_disable = false;
2854         } else {
2855                 /* Just stop for other adapters */
2856                 ixgbe_if_stop(ctx);
2857         }
2858
2859         return error;
2860 } /* ixgbe_setup_low_power_mode */
2861
2862 /************************************************************************
2863  * ixgbe_shutdown - Shutdown entry point
2864  ************************************************************************/
2865 static int
2866 ixgbe_if_shutdown(if_ctx_t ctx)
2867 {
2868         int error = 0;
2869
2870         INIT_DEBUGOUT("ixgbe_shutdown: begin");
2871
2872         error = ixgbe_setup_low_power_mode(ctx);
2873
2874         return (error);
2875 } /* ixgbe_if_shutdown */
2876
2877 /************************************************************************
2878  * ixgbe_suspend
2879  *
2880  *   From D0 to D3
2881  ************************************************************************/
2882 static int
2883 ixgbe_if_suspend(if_ctx_t ctx)
2884 {
2885         int error = 0;
2886
2887         INIT_DEBUGOUT("ixgbe_suspend: begin");
2888
2889         error = ixgbe_setup_low_power_mode(ctx);
2890
2891         return (error);
2892 } /* ixgbe_if_suspend */
2893
2894 /************************************************************************
2895  * ixgbe_resume
2896  *
2897  *   From D3 to D0
2898  ************************************************************************/
2899 static int
2900 ixgbe_if_resume(if_ctx_t ctx)
2901 {
2902         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
2903         device_t        dev = iflib_get_dev(ctx);
2904         struct ifnet    *ifp = iflib_get_ifp(ctx);
2905         struct ixgbe_hw *hw = &sc->hw;
2906         u32             wus;
2907
2908         INIT_DEBUGOUT("ixgbe_resume: begin");
2909
2910         /* Read & clear WUS register */
2911         wus = IXGBE_READ_REG(hw, IXGBE_WUS);
2912         if (wus)
2913                 device_printf(dev, "Woken up by (WUS): %#010x\n",
2914                     IXGBE_READ_REG(hw, IXGBE_WUS));
2915         IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
2916         /* And clear WUFC until next low-power transition */
2917         IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
2918
2919         /*
2920          * Required after D3->D0 transition;
2921          * will re-advertise all previous advertised speeds
2922          */
2923         if (ifp->if_flags & IFF_UP)
2924                 ixgbe_if_init(ctx);
2925
2926         return (0);
2927 } /* ixgbe_if_resume */
2928
2929 /************************************************************************
2930  * ixgbe_if_mtu_set - Ioctl mtu entry point
2931  *
2932  *   Return 0 on success, EINVAL on failure
2933  ************************************************************************/
2934 static int
2935 ixgbe_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
2936 {
2937         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2938         int error = 0;
2939
2940         IOCTL_DEBUGOUT("ioctl: SIOCIFMTU (Set Interface MTU)");
2941
2942         if (mtu > IXGBE_MAX_MTU) {
2943                 error = EINVAL;
2944         } else {
2945                 sc->max_frame_size = mtu + IXGBE_MTU_HDR;
2946         }
2947
2948         return error;
2949 } /* ixgbe_if_mtu_set */
2950
2951 /************************************************************************
2952  * ixgbe_if_crcstrip_set
2953  ************************************************************************/
2954 static void
2955 ixgbe_if_crcstrip_set(if_ctx_t ctx, int onoff, int crcstrip)
2956 {
2957         struct ixgbe_softc *sc = iflib_get_softc(ctx);
2958         struct ixgbe_hw *hw = &sc->hw;
2959         /* crc stripping is set in two places:
2960          * IXGBE_HLREG0 (modified on init_locked and hw reset)
2961          * IXGBE_RDRXCTL (set by the original driver in
2962          *      ixgbe_setup_hw_rsc() called in init_locked.
2963          *      We disable the setting when netmap is compiled in).
2964          * We update the values here, but also in ixgbe.c because
2965          * init_locked sometimes is called outside our control.
2966          */
2967         uint32_t hl, rxc;
2968
2969         hl = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2970         rxc = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2971 #ifdef NETMAP
2972         if (netmap_verbose)
2973                 D("%s read  HLREG 0x%x rxc 0x%x",
2974                         onoff ? "enter" : "exit", hl, rxc);
2975 #endif
2976         /* hw requirements ... */
2977         rxc &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2978         rxc |= IXGBE_RDRXCTL_RSCACKC;
2979         if (onoff && !crcstrip) {
2980                 /* keep the crc. Fast rx */
2981                 hl &= ~IXGBE_HLREG0_RXCRCSTRP;
2982                 rxc &= ~IXGBE_RDRXCTL_CRCSTRIP;
2983         } else {
2984                 /* reset default mode */
2985                 hl |= IXGBE_HLREG0_RXCRCSTRP;
2986                 rxc |= IXGBE_RDRXCTL_CRCSTRIP;
2987         }
2988 #ifdef NETMAP
2989         if (netmap_verbose)
2990                 D("%s write HLREG 0x%x rxc 0x%x",
2991                         onoff ? "enter" : "exit", hl, rxc);
2992 #endif
2993         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hl);
2994         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rxc);
2995 } /* ixgbe_if_crcstrip_set */
2996
2997 /*********************************************************************
2998  * ixgbe_if_init - Init entry point
2999  *
3000  *   Used in two ways: It is used by the stack as an init
3001  *   entry point in network interface structure. It is also
3002  *   used by the driver as a hw/sw initialization routine to
3003  *   get to a consistent state.
3004  *
3005  *   Return 0 on success, positive on failure
3006  **********************************************************************/
3007 void
3008 ixgbe_if_init(if_ctx_t ctx)
3009 {
3010         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
3011         struct ifnet       *ifp = iflib_get_ifp(ctx);
3012         device_t           dev = iflib_get_dev(ctx);
3013         struct ixgbe_hw *hw = &sc->hw;
3014         struct ix_rx_queue *rx_que;
3015         struct ix_tx_queue *tx_que;
3016         u32             txdctl, mhadd;
3017         u32             rxdctl, rxctrl;
3018         u32             ctrl_ext;
3019
3020         int             i, j, err;
3021
3022         INIT_DEBUGOUT("ixgbe_if_init: begin");
3023
3024         /* Queue indices may change with IOV mode */
3025         ixgbe_align_all_queue_indices(sc);
3026
3027         /* reprogram the RAR[0] in case user changed it. */
3028         ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, IXGBE_RAH_AV);
3029
3030         /* Get the latest mac address, User can use a LAA */
3031         bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
3032         ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, 1);
3033         hw->addr_ctrl.rar_used_count = 1;
3034
3035         ixgbe_init_hw(hw);
3036
3037         ixgbe_initialize_iov(sc);
3038
3039         ixgbe_initialize_transmit_units(ctx);
3040
3041         /* Setup Multicast table */
3042         ixgbe_if_multi_set(ctx);
3043
3044         /* Determine the correct mbuf pool, based on frame size */
3045         sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
3046
3047         /* Configure RX settings */
3048         ixgbe_initialize_receive_units(ctx);
3049
3050         /*
3051          * Initialize variable holding task enqueue requests
3052          * from MSI-X interrupts
3053          */
3054         sc->task_requests = 0;
3055
3056         /* Enable SDP & MSI-X interrupts based on adapter */
3057         ixgbe_config_gpie(sc);
3058
3059         /* Set MTU size */
3060         if (ifp->if_mtu > ETHERMTU) {
3061                 /* aka IXGBE_MAXFRS on 82599 and newer */
3062                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3063                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
3064                 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
3065                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3066         }
3067
3068         /* Now enable all the queues */
3069         for (i = 0, tx_que = sc->tx_queues; i < sc->num_tx_queues; i++, tx_que++) {
3070                 struct tx_ring *txr = &tx_que->txr;
3071
3072                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me));
3073                 txdctl |= IXGBE_TXDCTL_ENABLE;
3074                 /* Set WTHRESH to 8, burst writeback */
3075                 txdctl |= (8 << 16);
3076                 /*
3077                  * When the internal queue falls below PTHRESH (32),
3078                  * start prefetching as long as there are at least
3079                  * HTHRESH (1) buffers ready. The values are taken
3080                  * from the Intel linux driver 3.8.21.
3081                  * Prefetching enables tx line rate even with 1 queue.
3082                  */
3083                 txdctl |= (32 << 0) | (1 << 8);
3084                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl);
3085         }
3086
3087         for (i = 0, rx_que = sc->rx_queues; i < sc->num_rx_queues; i++, rx_que++) {
3088                 struct rx_ring *rxr = &rx_que->rxr;
3089
3090                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
3091                 if (hw->mac.type == ixgbe_mac_82598EB) {
3092                         /*
3093                          * PTHRESH = 21
3094                          * HTHRESH = 4
3095                          * WTHRESH = 8
3096                          */
3097                         rxdctl &= ~0x3FFFFF;
3098                         rxdctl |= 0x080420;
3099                 }
3100                 rxdctl |= IXGBE_RXDCTL_ENABLE;
3101                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl);
3102                 for (j = 0; j < 10; j++) {
3103                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) &
3104                             IXGBE_RXDCTL_ENABLE)
3105                                 break;
3106                         else
3107                                 msec_delay(1);
3108                 }
3109                 wmb();
3110         }
3111
3112         /* Enable Receive engine */
3113         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3114         if (hw->mac.type == ixgbe_mac_82598EB)
3115                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3116         rxctrl |= IXGBE_RXCTRL_RXEN;
3117         ixgbe_enable_rx_dma(hw, rxctrl);
3118
3119         /* Set up MSI/MSI-X routing */
3120         if (ixgbe_enable_msix)  {
3121                 ixgbe_configure_ivars(sc);
3122                 /* Set up auto-mask */
3123                 if (hw->mac.type == ixgbe_mac_82598EB)
3124                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3125                 else {
3126                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
3127                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
3128                 }
3129         } else {  /* Simple settings for Legacy/MSI */
3130                 ixgbe_set_ivar(sc, 0, 0, 0);
3131                 ixgbe_set_ivar(sc, 0, 0, 1);
3132                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3133         }
3134
3135         ixgbe_init_fdir(sc);
3136
3137         /*
3138          * Check on any SFP devices that
3139          * need to be kick-started
3140          */
3141         if (hw->phy.type == ixgbe_phy_none) {
3142                 err = hw->phy.ops.identify(hw);
3143                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3144                         device_printf(dev,
3145                             "Unsupported SFP+ module type was detected.\n");
3146                         return;
3147                 }
3148         }
3149
3150         /* Set moderation on the Link interrupt */
3151         IXGBE_WRITE_REG(hw, IXGBE_EITR(sc->vector), IXGBE_LINK_ITR);
3152
3153         /* Enable power to the phy. */
3154         ixgbe_set_phy_power(hw, true);
3155
3156         /* Config/Enable Link */
3157         ixgbe_config_link(ctx);
3158
3159         /* Hardware Packet Buffer & Flow Control setup */
3160         ixgbe_config_delay_values(sc);
3161
3162         /* Initialize the FC settings */
3163         ixgbe_start_hw(hw);
3164
3165         /* Set up VLAN support and filter */
3166         ixgbe_setup_vlan_hw_support(ctx);
3167
3168         /* Setup DMA Coalescing */
3169         ixgbe_config_dmac(sc);
3170
3171         /* And now turn on interrupts */
3172         ixgbe_if_enable_intr(ctx);
3173
3174         /* Enable the use of the MBX by the VF's */
3175         if (sc->feat_en & IXGBE_FEATURE_SRIOV) {
3176                 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
3177                 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
3178                 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
3179         }
3180
3181 } /* ixgbe_init_locked */
3182
3183 /************************************************************************
3184  * ixgbe_set_ivar
3185  *
3186  *   Setup the correct IVAR register for a particular MSI-X interrupt
3187  *     (yes this is all very magic and confusing :)
3188  *    - entry is the register array entry
3189  *    - vector is the MSI-X vector for this queue
3190  *    - type is RX/TX/MISC
3191  ************************************************************************/
3192 static void
3193 ixgbe_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type)
3194 {
3195         struct ixgbe_hw *hw = &sc->hw;
3196         u32 ivar, index;
3197
3198         vector |= IXGBE_IVAR_ALLOC_VAL;
3199
3200         switch (hw->mac.type) {
3201         case ixgbe_mac_82598EB:
3202                 if (type == -1)
3203                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
3204                 else
3205                         entry += (type * 64);
3206                 index = (entry >> 2) & 0x1F;
3207                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3208                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
3209                 ivar |= (vector << (8 * (entry & 0x3)));
3210                 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar);
3211                 break;
3212         case ixgbe_mac_82599EB:
3213         case ixgbe_mac_X540:
3214         case ixgbe_mac_X550:
3215         case ixgbe_mac_X550EM_x:
3216         case ixgbe_mac_X550EM_a:
3217                 if (type == -1) { /* MISC IVAR */
3218                         index = (entry & 1) * 8;
3219                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
3220                         ivar &= ~(0xFF << index);
3221                         ivar |= (vector << index);
3222                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
3223                 } else {          /* RX/TX IVARS */
3224                         index = (16 * (entry & 1)) + (8 * type);
3225                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
3226                         ivar &= ~(0xFF << index);
3227                         ivar |= (vector << index);
3228                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
3229                 }
3230         default:
3231                 break;
3232         }
3233 } /* ixgbe_set_ivar */
3234
3235 /************************************************************************
3236  * ixgbe_configure_ivars
3237  ************************************************************************/
3238 static void
3239 ixgbe_configure_ivars(struct ixgbe_softc *sc)
3240 {
3241         struct ix_rx_queue *rx_que = sc->rx_queues;
3242         struct ix_tx_queue *tx_que = sc->tx_queues;
3243         u32                newitr;
3244
3245         if (ixgbe_max_interrupt_rate > 0)
3246                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
3247         else {
3248                 /*
3249                  * Disable DMA coalescing if interrupt moderation is
3250                  * disabled.
3251                  */
3252                 sc->dmac = 0;
3253                 newitr = 0;
3254         }
3255
3256         for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) {
3257                 struct rx_ring *rxr = &rx_que->rxr;
3258
3259                 /* First the RX queue entry */
3260                 ixgbe_set_ivar(sc, rxr->me, rx_que->msix, 0);
3261
3262                 /* Set an Initial EITR value */
3263                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(rx_que->msix), newitr);
3264         }
3265         for (int i = 0; i < sc->num_tx_queues; i++, tx_que++) {
3266                 struct tx_ring *txr = &tx_que->txr;
3267
3268                 /* ... and the TX */
3269                 ixgbe_set_ivar(sc, txr->me, tx_que->msix, 1);
3270         }
3271         /* For the Link interrupt */
3272         ixgbe_set_ivar(sc, 1, sc->vector, -1);
3273 } /* ixgbe_configure_ivars */
3274
3275 /************************************************************************
3276  * ixgbe_config_gpie
3277  ************************************************************************/
3278 static void
3279 ixgbe_config_gpie(struct ixgbe_softc *sc)
3280 {
3281         struct ixgbe_hw *hw = &sc->hw;
3282         u32             gpie;
3283
3284         gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3285
3286         if (sc->intr_type == IFLIB_INTR_MSIX) {
3287                 /* Enable Enhanced MSI-X mode */
3288                 gpie |= IXGBE_GPIE_MSIX_MODE
3289                      |  IXGBE_GPIE_EIAME
3290                      |  IXGBE_GPIE_PBA_SUPPORT
3291                      |  IXGBE_GPIE_OCD;
3292         }
3293
3294         /* Fan Failure Interrupt */
3295         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)
3296                 gpie |= IXGBE_SDP1_GPIEN;
3297
3298         /* Thermal Sensor Interrupt */
3299         if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR)
3300                 gpie |= IXGBE_SDP0_GPIEN_X540;
3301
3302         /* Link detection */
3303         switch (hw->mac.type) {
3304         case ixgbe_mac_82599EB:
3305                 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN;
3306                 break;
3307         case ixgbe_mac_X550EM_x:
3308         case ixgbe_mac_X550EM_a:
3309                 gpie |= IXGBE_SDP0_GPIEN_X540;
3310                 break;
3311         default:
3312                 break;
3313         }
3314
3315         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3316
3317 } /* ixgbe_config_gpie */
3318
3319 /************************************************************************
3320  * ixgbe_config_delay_values
3321  *
3322  *   Requires sc->max_frame_size to be set.
3323  ************************************************************************/
3324 static void
3325 ixgbe_config_delay_values(struct ixgbe_softc *sc)
3326 {
3327         struct ixgbe_hw *hw = &sc->hw;
3328         u32             rxpb, frame, size, tmp;
3329
3330         frame = sc->max_frame_size;
3331
3332         /* Calculate High Water */
3333         switch (hw->mac.type) {
3334         case ixgbe_mac_X540:
3335         case ixgbe_mac_X550:
3336         case ixgbe_mac_X550EM_x:
3337         case ixgbe_mac_X550EM_a:
3338                 tmp = IXGBE_DV_X540(frame, frame);
3339                 break;
3340         default:
3341                 tmp = IXGBE_DV(frame, frame);
3342                 break;
3343         }
3344         size = IXGBE_BT2KB(tmp);
3345         rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
3346         hw->fc.high_water[0] = rxpb - size;
3347
3348         /* Now calculate Low Water */
3349         switch (hw->mac.type) {
3350         case ixgbe_mac_X540:
3351         case ixgbe_mac_X550:
3352         case ixgbe_mac_X550EM_x:
3353         case ixgbe_mac_X550EM_a:
3354                 tmp = IXGBE_LOW_DV_X540(frame);
3355                 break;
3356         default:
3357                 tmp = IXGBE_LOW_DV(frame);
3358                 break;
3359         }
3360         hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
3361
3362         hw->fc.pause_time = IXGBE_FC_PAUSE;
3363         hw->fc.send_xon = true;
3364 } /* ixgbe_config_delay_values */
3365
3366 /************************************************************************
3367  * ixgbe_set_multi - Multicast Update
3368  *
3369  *   Called whenever multicast address list is updated.
3370  ************************************************************************/
3371 static u_int
3372 ixgbe_mc_filter_apply(void *arg, struct sockaddr_dl *sdl, u_int idx)
3373 {
3374         struct ixgbe_softc *sc = arg;
3375         struct ixgbe_mc_addr *mta = sc->mta;
3376
3377         if (idx == MAX_NUM_MULTICAST_ADDRESSES)
3378                 return (0);
3379         bcopy(LLADDR(sdl), mta[idx].addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
3380         mta[idx].vmdq = sc->pool;
3381
3382         return (1);
3383 } /* ixgbe_mc_filter_apply */
3384
3385 static void
3386 ixgbe_if_multi_set(if_ctx_t ctx)
3387 {
3388         struct ixgbe_softc       *sc = iflib_get_softc(ctx);
3389         struct ixgbe_mc_addr *mta;
3390         struct ifnet         *ifp = iflib_get_ifp(ctx);
3391         u8                   *update_ptr;
3392         u32                  fctrl;
3393         u_int                mcnt;
3394
3395         IOCTL_DEBUGOUT("ixgbe_if_multi_set: begin");
3396
3397         mta = sc->mta;
3398         bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES);
3399
3400         mcnt = if_foreach_llmaddr(iflib_get_ifp(ctx), ixgbe_mc_filter_apply, sc);
3401
3402         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
3403                 update_ptr = (u8 *)mta;
3404                 ixgbe_update_mc_addr_list(&sc->hw, update_ptr, mcnt,
3405                     ixgbe_mc_array_itr, true);
3406         }
3407
3408         fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
3409
3410         if (ifp->if_flags & IFF_PROMISC)
3411                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3412         else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
3413             ifp->if_flags & IFF_ALLMULTI) {
3414                 fctrl |= IXGBE_FCTRL_MPE;
3415                 fctrl &= ~IXGBE_FCTRL_UPE;
3416         } else
3417                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3418
3419         IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl);
3420 } /* ixgbe_if_multi_set */
3421
3422 /************************************************************************
3423  * ixgbe_mc_array_itr
3424  *
3425  *   An iterator function needed by the multicast shared code.
3426  *   It feeds the shared code routine the addresses in the
3427  *   array of ixgbe_set_multi() one by one.
3428  ************************************************************************/
3429 static u8 *
3430 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
3431 {
3432         struct ixgbe_mc_addr *mta;
3433
3434         mta = (struct ixgbe_mc_addr *)*update_ptr;
3435         *vmdq = mta->vmdq;
3436
3437         *update_ptr = (u8*)(mta + 1);
3438
3439         return (mta->addr);
3440 } /* ixgbe_mc_array_itr */
3441
3442 /************************************************************************
3443  * ixgbe_local_timer - Timer routine
3444  *
3445  *   Checks for link status, updates statistics,
3446  *   and runs the watchdog check.
3447  ************************************************************************/
3448 static void
3449 ixgbe_if_timer(if_ctx_t ctx, uint16_t qid)
3450 {
3451         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3452
3453         if (qid != 0)
3454                 return;
3455
3456         /* Check for pluggable optics */
3457         if (sc->sfp_probe)
3458                 if (!ixgbe_sfp_probe(ctx))
3459                         return; /* Nothing to do */
3460
3461         ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0);
3462
3463         /* Fire off the adminq task */
3464         iflib_admin_intr_deferred(ctx);
3465
3466 } /* ixgbe_if_timer */
3467
3468 /************************************************************************
3469  * ixgbe_sfp_probe
3470  *
3471  *   Determine if a port had optics inserted.
3472  ************************************************************************/
3473 static bool
3474 ixgbe_sfp_probe(if_ctx_t ctx)
3475 {
3476         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3477         struct ixgbe_hw *hw = &sc->hw;
3478         device_t        dev = iflib_get_dev(ctx);
3479         bool            result = false;
3480
3481         if ((hw->phy.type == ixgbe_phy_nl) &&
3482             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
3483                 s32 ret = hw->phy.ops.identify_sfp(hw);
3484                 if (ret)
3485                         goto out;
3486                 ret = hw->phy.ops.reset(hw);
3487                 sc->sfp_probe = false;
3488                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3489                         device_printf(dev, "Unsupported SFP+ module detected!");
3490                         device_printf(dev,
3491                             "Reload driver with supported module.\n");
3492                         goto out;
3493                 } else
3494                         device_printf(dev, "SFP+ module detected!\n");
3495                 /* We now have supported optics */
3496                 result = true;
3497         }
3498 out:
3499
3500         return (result);
3501 } /* ixgbe_sfp_probe */
3502
3503 /************************************************************************
3504  * ixgbe_handle_mod - Tasklet for SFP module interrupts
3505  ************************************************************************/
3506 static void
3507 ixgbe_handle_mod(void *context)
3508 {
3509         if_ctx_t        ctx = context;
3510         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3511         struct ixgbe_hw *hw = &sc->hw;
3512         device_t        dev = iflib_get_dev(ctx);
3513         u32             err, cage_full = 0;
3514
3515         if (sc->hw.need_crosstalk_fix) {
3516                 switch (hw->mac.type) {
3517                 case ixgbe_mac_82599EB:
3518                         cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
3519                             IXGBE_ESDP_SDP2;
3520                         break;
3521                 case ixgbe_mac_X550EM_x:
3522                 case ixgbe_mac_X550EM_a:
3523                         cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
3524                             IXGBE_ESDP_SDP0;
3525                         break;
3526                 default:
3527                         break;
3528                 }
3529
3530                 if (!cage_full)
3531                         goto handle_mod_out;
3532         }
3533
3534         err = hw->phy.ops.identify_sfp(hw);
3535         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3536                 device_printf(dev,
3537                     "Unsupported SFP+ module type was detected.\n");
3538                 goto handle_mod_out;
3539         }
3540
3541         if (hw->mac.type == ixgbe_mac_82598EB)
3542                 err = hw->phy.ops.reset(hw);
3543         else
3544                 err = hw->mac.ops.setup_sfp(hw);
3545
3546         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3547                 device_printf(dev,
3548                     "Setup failure - unsupported SFP+ module type.\n");
3549                 goto handle_mod_out;
3550         }
3551         sc->task_requests |= IXGBE_REQUEST_TASK_MSF;
3552         return;
3553
3554 handle_mod_out:
3555         sc->task_requests &= ~(IXGBE_REQUEST_TASK_MSF);
3556 } /* ixgbe_handle_mod */
3557
3558
3559 /************************************************************************
3560  * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts
3561  ************************************************************************/
3562 static void
3563 ixgbe_handle_msf(void *context)
3564 {
3565         if_ctx_t        ctx = context;
3566         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3567         struct ixgbe_hw *hw = &sc->hw;
3568         u32             autoneg;
3569         bool            negotiate;
3570
3571         /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */
3572         sc->phy_layer = ixgbe_get_supported_physical_layer(hw);
3573
3574         autoneg = hw->phy.autoneg_advertised;
3575         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3576                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
3577         if (hw->mac.ops.setup_link)
3578                 hw->mac.ops.setup_link(hw, autoneg, true);
3579
3580         /* Adjust media types shown in ifconfig */
3581         ifmedia_removeall(sc->media);
3582         ixgbe_add_media_types(sc->ctx);
3583         ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO);
3584 } /* ixgbe_handle_msf */
3585
3586 /************************************************************************
3587  * ixgbe_handle_phy - Tasklet for external PHY interrupts
3588  ************************************************************************/
3589 static void
3590 ixgbe_handle_phy(void *context)
3591 {
3592         if_ctx_t        ctx = context;
3593         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3594         struct ixgbe_hw *hw = &sc->hw;
3595         int             error;
3596
3597         error = hw->phy.ops.handle_lasi(hw);
3598         if (error == IXGBE_ERR_OVERTEMP)
3599                 device_printf(sc->dev, "CRITICAL: EXTERNAL PHY OVER TEMP!!  PHY will downshift to lower power state!\n");
3600         else if (error)
3601                 device_printf(sc->dev,
3602                     "Error handling LASI interrupt: %d\n", error);
3603 } /* ixgbe_handle_phy */
3604
3605 /************************************************************************
3606  * ixgbe_if_stop - Stop the hardware
3607  *
3608  *   Disables all traffic on the adapter by issuing a
3609  *   global reset on the MAC and deallocates TX/RX buffers.
3610  ************************************************************************/
3611 static void
3612 ixgbe_if_stop(if_ctx_t ctx)
3613 {
3614         struct ixgbe_softc  *sc = iflib_get_softc(ctx);
3615         struct ixgbe_hw *hw = &sc->hw;
3616
3617         INIT_DEBUGOUT("ixgbe_if_stop: begin\n");
3618
3619         ixgbe_reset_hw(hw);
3620         hw->adapter_stopped = false;
3621         ixgbe_stop_adapter(hw);
3622         if (hw->mac.type == ixgbe_mac_82599EB)
3623                 ixgbe_stop_mac_link_on_d3_82599(hw);
3624         /* Turn off the laser - noop with no optics */
3625         ixgbe_disable_tx_laser(hw);
3626
3627         /* Update the stack */
3628         sc->link_up = false;
3629         ixgbe_if_update_admin_status(ctx);
3630
3631         /* reprogram the RAR[0] in case user changed it. */
3632         ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);
3633
3634         return;
3635 } /* ixgbe_if_stop */
3636
3637 /************************************************************************
3638  * ixgbe_update_link_status - Update OS on link state
3639  *
3640  * Note: Only updates the OS on the cached link state.
3641  *       The real check of the hardware only happens with
3642  *       a link interrupt.
3643  ************************************************************************/
3644 static void
3645 ixgbe_if_update_admin_status(if_ctx_t ctx)
3646 {
3647         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3648         device_t       dev = iflib_get_dev(ctx);
3649
3650         if (sc->link_up) {
3651                 if (sc->link_active == false) {
3652                         if (bootverbose)
3653                                 device_printf(dev, "Link is up %d Gbps %s \n",
3654                                     ((sc->link_speed == 128) ? 10 : 1),
3655                                     "Full Duplex");
3656                         sc->link_active = true;
3657                         /* Update any Flow Control changes */
3658                         ixgbe_fc_enable(&sc->hw);
3659                         /* Update DMA coalescing config */
3660                         ixgbe_config_dmac(sc);
3661                         /* should actually be negotiated value */
3662                         iflib_link_state_change(ctx, LINK_STATE_UP, IF_Gbps(10));
3663
3664                         if (sc->feat_en & IXGBE_FEATURE_SRIOV)
3665                                 ixgbe_ping_all_vfs(sc);
3666                 }
3667         } else { /* Link down */
3668                 if (sc->link_active == true) {
3669                         if (bootverbose)
3670                                 device_printf(dev, "Link is Down\n");
3671                         iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
3672                         sc->link_active = false;
3673                         if (sc->feat_en & IXGBE_FEATURE_SRIOV)
3674                                 ixgbe_ping_all_vfs(sc);
3675                 }
3676         }
3677
3678         /* Handle task requests from msix_link() */
3679         if (sc->task_requests & IXGBE_REQUEST_TASK_MOD)
3680                 ixgbe_handle_mod(ctx);
3681         if (sc->task_requests & IXGBE_REQUEST_TASK_MSF)
3682                 ixgbe_handle_msf(ctx);
3683         if (sc->task_requests & IXGBE_REQUEST_TASK_MBX)
3684                 ixgbe_handle_mbx(ctx);
3685         if (sc->task_requests & IXGBE_REQUEST_TASK_FDIR)
3686                 ixgbe_reinit_fdir(ctx);
3687         if (sc->task_requests & IXGBE_REQUEST_TASK_PHY)
3688                 ixgbe_handle_phy(ctx);
3689         sc->task_requests = 0;
3690
3691         ixgbe_update_stats_counters(sc);
3692 } /* ixgbe_if_update_admin_status */
3693
3694 /************************************************************************
3695  * ixgbe_config_dmac - Configure DMA Coalescing
3696  ************************************************************************/
3697 static void
3698 ixgbe_config_dmac(struct ixgbe_softc *sc)
3699 {
3700         struct ixgbe_hw          *hw = &sc->hw;
3701         struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config;
3702
3703         if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config)
3704                 return;
3705
3706         if (dcfg->watchdog_timer ^ sc->dmac ||
3707             dcfg->link_speed ^ sc->link_speed) {
3708                 dcfg->watchdog_timer = sc->dmac;
3709                 dcfg->fcoe_en = false;
3710                 dcfg->link_speed = sc->link_speed;
3711                 dcfg->num_tcs = 1;
3712
3713                 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n",
3714                     dcfg->watchdog_timer, dcfg->link_speed);
3715
3716                 hw->mac.ops.dmac_config(hw);
3717         }
3718 } /* ixgbe_config_dmac */
3719
3720 /************************************************************************
3721  * ixgbe_if_enable_intr
3722  ************************************************************************/
3723 void
3724 ixgbe_if_enable_intr(if_ctx_t ctx)
3725 {
3726         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
3727         struct ixgbe_hw    *hw = &sc->hw;
3728         struct ix_rx_queue *que = sc->rx_queues;
3729         u32                mask, fwsm;
3730
3731         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3732
3733         switch (sc->hw.mac.type) {
3734         case ixgbe_mac_82599EB:
3735                 mask |= IXGBE_EIMS_ECC;
3736                 /* Temperature sensor on some scs */
3737                 mask |= IXGBE_EIMS_GPI_SDP0;
3738                 /* SFP+ (RX_LOS_N & MOD_ABS_N) */
3739                 mask |= IXGBE_EIMS_GPI_SDP1;
3740                 mask |= IXGBE_EIMS_GPI_SDP2;
3741                 break;
3742         case ixgbe_mac_X540:
3743                 /* Detect if Thermal Sensor is enabled */
3744                 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
3745                 if (fwsm & IXGBE_FWSM_TS_ENABLED)
3746                         mask |= IXGBE_EIMS_TS;
3747                 mask |= IXGBE_EIMS_ECC;
3748                 break;
3749         case ixgbe_mac_X550:
3750                 /* MAC thermal sensor is automatically enabled */
3751                 mask |= IXGBE_EIMS_TS;
3752                 mask |= IXGBE_EIMS_ECC;
3753                 break;
3754         case ixgbe_mac_X550EM_x:
3755         case ixgbe_mac_X550EM_a:
3756                 /* Some devices use SDP0 for important information */
3757                 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
3758                     hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
3759                     hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
3760                     hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T)
3761                         mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
3762                 if (hw->phy.type == ixgbe_phy_x550em_ext_t)
3763                         mask |= IXGBE_EICR_GPI_SDP0_X540;
3764                 mask |= IXGBE_EIMS_ECC;
3765                 break;
3766         default:
3767                 break;
3768         }
3769
3770         /* Enable Fan Failure detection */
3771         if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)
3772                 mask |= IXGBE_EIMS_GPI_SDP1;
3773         /* Enable SR-IOV */
3774         if (sc->feat_en & IXGBE_FEATURE_SRIOV)
3775                 mask |= IXGBE_EIMS_MAILBOX;
3776         /* Enable Flow Director */
3777         if (sc->feat_en & IXGBE_FEATURE_FDIR)
3778                 mask |= IXGBE_EIMS_FLOW_DIR;
3779
3780         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3781
3782         /* With MSI-X we use auto clear */
3783         if (sc->intr_type == IFLIB_INTR_MSIX) {
3784                 mask = IXGBE_EIMS_ENABLE_MASK;
3785                 /* Don't autoclear Link */
3786                 mask &= ~IXGBE_EIMS_OTHER;
3787                 mask &= ~IXGBE_EIMS_LSC;
3788                 if (sc->feat_cap & IXGBE_FEATURE_SRIOV)
3789                         mask &= ~IXGBE_EIMS_MAILBOX;
3790                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
3791         }
3792
3793         /*
3794          * Now enable all queues, this is done separately to
3795          * allow for handling the extended (beyond 32) MSI-X
3796          * vectors that can be used by 82599
3797          */
3798         for (int i = 0; i < sc->num_rx_queues; i++, que++)
3799                 ixgbe_enable_queue(sc, que->msix);
3800
3801         IXGBE_WRITE_FLUSH(hw);
3802
3803 } /* ixgbe_if_enable_intr */
3804
3805 /************************************************************************
3806  * ixgbe_disable_intr
3807  ************************************************************************/
3808 static void
3809 ixgbe_if_disable_intr(if_ctx_t ctx)
3810 {
3811         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3812
3813         if (sc->intr_type == IFLIB_INTR_MSIX)
3814                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0);
3815         if (sc->hw.mac.type == ixgbe_mac_82598EB) {
3816                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0);
3817         } else {
3818                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000);
3819                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0);
3820                 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0);
3821         }
3822         IXGBE_WRITE_FLUSH(&sc->hw);
3823
3824 } /* ixgbe_if_disable_intr */
3825
3826 /************************************************************************
3827  * ixgbe_link_intr_enable
3828  ************************************************************************/
3829 static void
3830 ixgbe_link_intr_enable(if_ctx_t ctx)
3831 {
3832         struct ixgbe_hw *hw = &((struct ixgbe_softc *)iflib_get_softc(ctx))->hw;
3833
3834         /* Re-enable other interrupts */
3835         IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
3836 } /* ixgbe_link_intr_enable */
3837
3838 /************************************************************************
3839  * ixgbe_if_rx_queue_intr_enable
3840  ************************************************************************/
3841 static int
3842 ixgbe_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
3843 {
3844         struct ixgbe_softc     *sc = iflib_get_softc(ctx);
3845         struct ix_rx_queue *que = &sc->rx_queues[rxqid];
3846
3847         ixgbe_enable_queue(sc, que->msix);
3848
3849         return (0);
3850 } /* ixgbe_if_rx_queue_intr_enable */
3851
3852 /************************************************************************
3853  * ixgbe_enable_queue
3854  ************************************************************************/
3855 static void
3856 ixgbe_enable_queue(struct ixgbe_softc *sc, u32 vector)
3857 {
3858         struct ixgbe_hw *hw = &sc->hw;
3859         u64             queue = 1ULL << vector;
3860         u32             mask;
3861
3862         if (hw->mac.type == ixgbe_mac_82598EB) {
3863                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
3864                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3865         } else {
3866                 mask = (queue & 0xFFFFFFFF);
3867                 if (mask)
3868                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
3869                 mask = (queue >> 32);
3870                 if (mask)
3871                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
3872         }
3873 } /* ixgbe_enable_queue */
3874
3875 /************************************************************************
3876  * ixgbe_disable_queue
3877  ************************************************************************/
3878 static void
3879 ixgbe_disable_queue(struct ixgbe_softc *sc, u32 vector)
3880 {
3881         struct ixgbe_hw *hw = &sc->hw;
3882         u64             queue = 1ULL << vector;
3883         u32             mask;
3884
3885         if (hw->mac.type == ixgbe_mac_82598EB) {
3886                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
3887                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
3888         } else {
3889                 mask = (queue & 0xFFFFFFFF);
3890                 if (mask)
3891                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
3892                 mask = (queue >> 32);
3893                 if (mask)
3894                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
3895         }
3896 } /* ixgbe_disable_queue */
3897
3898 /************************************************************************
3899  * ixgbe_intr - Legacy Interrupt Service Routine
3900  ************************************************************************/
3901 int
3902 ixgbe_intr(void *arg)
3903 {
3904         struct ixgbe_softc     *sc = arg;
3905         struct ix_rx_queue *que = sc->rx_queues;
3906         struct ixgbe_hw    *hw = &sc->hw;
3907         if_ctx_t           ctx = sc->ctx;
3908         u32                eicr, eicr_mask;
3909
3910         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3911
3912         ++que->irqs;
3913         if (eicr == 0) {
3914                 ixgbe_if_enable_intr(ctx);
3915                 return (FILTER_HANDLED);
3916         }
3917
3918         /* Check for fan failure */
3919         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
3920             (eicr & IXGBE_EICR_GPI_SDP1)) {
3921                 device_printf(sc->dev,
3922                     "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
3923                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3924         }
3925
3926         /* Link status change */
3927         if (eicr & IXGBE_EICR_LSC) {
3928                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
3929                 iflib_admin_intr_deferred(ctx);
3930         }
3931
3932         if (ixgbe_is_sfp(hw)) {
3933                 /* Pluggable optics-related interrupt */
3934                 if (hw->mac.type >= ixgbe_mac_X540)
3935                         eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
3936                 else
3937                         eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
3938
3939                 if (eicr & eicr_mask) {
3940                         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
3941                         sc->task_requests |= IXGBE_REQUEST_TASK_MOD;
3942                 }
3943
3944                 if ((hw->mac.type == ixgbe_mac_82599EB) &&
3945                     (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
3946                         IXGBE_WRITE_REG(hw, IXGBE_EICR,
3947                             IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3948                         sc->task_requests |= IXGBE_REQUEST_TASK_MSF;
3949                 }
3950         }
3951
3952         /* External PHY interrupt */
3953         if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&
3954             (eicr & IXGBE_EICR_GPI_SDP0_X540))
3955                 sc->task_requests |= IXGBE_REQUEST_TASK_PHY;
3956
3957         return (FILTER_SCHEDULE_THREAD);
3958 } /* ixgbe_intr */
3959
3960 /************************************************************************
3961  * ixgbe_free_pci_resources
3962  ************************************************************************/
3963 static void
3964 ixgbe_free_pci_resources(if_ctx_t ctx)
3965 {
3966         struct ixgbe_softc *sc = iflib_get_softc(ctx);
3967         struct         ix_rx_queue *que = sc->rx_queues;
3968         device_t       dev = iflib_get_dev(ctx);
3969
3970         /* Release all MSI-X queue resources */
3971         if (sc->intr_type == IFLIB_INTR_MSIX)
3972                 iflib_irq_free(ctx, &sc->irq);
3973
3974         if (que != NULL) {
3975                 for (int i = 0; i < sc->num_rx_queues; i++, que++) {
3976                         iflib_irq_free(ctx, &que->que_irq);
3977                 }
3978         }
3979
3980         if (sc->pci_mem != NULL)
3981                 bus_release_resource(dev, SYS_RES_MEMORY,
3982                     rman_get_rid(sc->pci_mem), sc->pci_mem);
3983 } /* ixgbe_free_pci_resources */
3984
3985 /************************************************************************
3986  * ixgbe_sysctl_flowcntl
3987  *
3988  *   SYSCTL wrapper around setting Flow Control
3989  ************************************************************************/
3990 static int
3991 ixgbe_sysctl_flowcntl(SYSCTL_HANDLER_ARGS)
3992 {
3993         struct ixgbe_softc *sc;
3994         int            error, fc;
3995
3996         sc = (struct ixgbe_softc *)arg1;
3997         fc = sc->hw.fc.current_mode;
3998
3999         error = sysctl_handle_int(oidp, &fc, 0, req);
4000         if ((error) || (req->newptr == NULL))
4001                 return (error);
4002
4003         /* Don't bother if it's not changed */
4004         if (fc == sc->hw.fc.current_mode)
4005                 return (0);
4006
4007         return ixgbe_set_flowcntl(sc, fc);
4008 } /* ixgbe_sysctl_flowcntl */
4009
4010 /************************************************************************
4011  * ixgbe_set_flowcntl - Set flow control
4012  *
4013  *   Flow control values:
4014  *     0 - off
4015  *     1 - rx pause
4016  *     2 - tx pause
4017  *     3 - full
4018  ************************************************************************/
4019 static int
4020 ixgbe_set_flowcntl(struct ixgbe_softc *sc, int fc)
4021 {
4022         switch (fc) {
4023         case ixgbe_fc_rx_pause:
4024         case ixgbe_fc_tx_pause:
4025         case ixgbe_fc_full:
4026                 sc->hw.fc.requested_mode = fc;
4027                 if (sc->num_rx_queues > 1)
4028                         ixgbe_disable_rx_drop(sc);
4029                 break;
4030         case ixgbe_fc_none:
4031                 sc->hw.fc.requested_mode = ixgbe_fc_none;
4032                 if (sc->num_rx_queues > 1)
4033                         ixgbe_enable_rx_drop(sc);
4034                 break;
4035         default:
4036                 return (EINVAL);
4037         }
4038
4039         /* Don't autoneg if forcing a value */
4040         sc->hw.fc.disable_fc_autoneg = true;
4041         ixgbe_fc_enable(&sc->hw);
4042
4043         return (0);
4044 } /* ixgbe_set_flowcntl */
4045
4046 /************************************************************************
4047  * ixgbe_enable_rx_drop
4048  *
4049  *   Enable the hardware to drop packets when the buffer is
4050  *   full. This is useful with multiqueue, so that no single
4051  *   queue being full stalls the entire RX engine. We only
4052  *   enable this when Multiqueue is enabled AND Flow Control
4053  *   is disabled.
4054  ************************************************************************/
4055 static void
4056 ixgbe_enable_rx_drop(struct ixgbe_softc *sc)
4057 {
4058         struct ixgbe_hw *hw = &sc->hw;
4059         struct rx_ring  *rxr;
4060         u32             srrctl;
4061
4062         for (int i = 0; i < sc->num_rx_queues; i++) {
4063                 rxr = &sc->rx_queues[i].rxr;
4064                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
4065                 srrctl |= IXGBE_SRRCTL_DROP_EN;
4066                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
4067         }
4068
4069         /* enable drop for each vf */
4070         for (int i = 0; i < sc->num_vfs; i++) {
4071                 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4072                                 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) |
4073                                 IXGBE_QDE_ENABLE));
4074         }
4075 } /* ixgbe_enable_rx_drop */
4076
4077 /************************************************************************
4078  * ixgbe_disable_rx_drop
4079  ************************************************************************/
4080 static void
4081 ixgbe_disable_rx_drop(struct ixgbe_softc *sc)
4082 {
4083         struct ixgbe_hw *hw = &sc->hw;
4084         struct rx_ring  *rxr;
4085         u32             srrctl;
4086
4087         for (int i = 0; i < sc->num_rx_queues; i++) {
4088                 rxr = &sc->rx_queues[i].rxr;
4089                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
4090                 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
4091                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
4092         }
4093
4094         /* disable drop for each vf */
4095         for (int i = 0; i < sc->num_vfs; i++) {
4096                 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4097                     (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT)));
4098         }
4099 } /* ixgbe_disable_rx_drop */
4100
4101 /************************************************************************
4102  * ixgbe_sysctl_advertise
4103  *
4104  *   SYSCTL wrapper around setting advertised speed
4105  ************************************************************************/
4106 static int
4107 ixgbe_sysctl_advertise(SYSCTL_HANDLER_ARGS)
4108 {
4109         struct ixgbe_softc *sc;
4110         int            error, advertise;
4111
4112         sc = (struct ixgbe_softc *)arg1;
4113         advertise = sc->advertise;
4114
4115         error = sysctl_handle_int(oidp, &advertise, 0, req);
4116         if ((error) || (req->newptr == NULL))
4117                 return (error);
4118
4119         return ixgbe_set_advertise(sc, advertise);
4120 } /* ixgbe_sysctl_advertise */
4121
4122 /************************************************************************
4123  * ixgbe_set_advertise - Control advertised link speed
4124  *
4125  *   Flags:
4126  *     0x1  - advertise 100 Mb
4127  *     0x2  - advertise 1G
4128  *     0x4  - advertise 10G
4129  *     0x8  - advertise 10 Mb (yes, Mb)
4130  *     0x10 - advertise 2.5G (disabled by default)
4131  *     0x20 - advertise 5G (disabled by default)
4132  *
4133  ************************************************************************/
4134 static int
4135 ixgbe_set_advertise(struct ixgbe_softc *sc, int advertise)
4136 {
4137         device_t         dev = iflib_get_dev(sc->ctx);
4138         struct ixgbe_hw  *hw;
4139         ixgbe_link_speed speed = 0;
4140         ixgbe_link_speed link_caps = 0;
4141         s32              err = IXGBE_NOT_IMPLEMENTED;
4142         bool             negotiate = false;
4143
4144         /* Checks to validate new value */
4145         if (sc->advertise == advertise) /* no change */
4146                 return (0);
4147
4148         hw = &sc->hw;
4149
4150         /* No speed changes for backplane media */
4151         if (hw->phy.media_type == ixgbe_media_type_backplane)
4152                 return (ENODEV);
4153
4154         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
4155               (hw->phy.multispeed_fiber))) {
4156                 device_printf(dev, "Advertised speed can only be set on copper or multispeed fiber media types.\n");
4157                 return (EINVAL);
4158         }
4159
4160         if (advertise < 0x1 || advertise > 0x3F) {
4161                 device_printf(dev, "Invalid advertised speed; valid modes are 0x1 through 0x3F\n");
4162                 return (EINVAL);
4163         }
4164
4165         if (hw->mac.ops.get_link_capabilities) {
4166                 err = hw->mac.ops.get_link_capabilities(hw, &link_caps,
4167                     &negotiate);
4168                 if (err != IXGBE_SUCCESS) {
4169                         device_printf(dev, "Unable to determine supported advertise speeds\n");
4170                         return (ENODEV);
4171                 }
4172         }
4173
4174         /* Set new value and report new advertised mode */
4175         if (advertise & 0x1) {
4176                 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) {
4177                         device_printf(dev, "Interface does not support 100Mb advertised speed\n");
4178                         return (EINVAL);
4179                 }
4180                 speed |= IXGBE_LINK_SPEED_100_FULL;
4181         }
4182         if (advertise & 0x2) {
4183                 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) {
4184                         device_printf(dev, "Interface does not support 1Gb advertised speed\n");
4185                         return (EINVAL);
4186                 }
4187                 speed |= IXGBE_LINK_SPEED_1GB_FULL;
4188         }
4189         if (advertise & 0x4) {
4190                 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) {
4191                         device_printf(dev, "Interface does not support 10Gb advertised speed\n");
4192                         return (EINVAL);
4193                 }
4194                 speed |= IXGBE_LINK_SPEED_10GB_FULL;
4195         }
4196         if (advertise & 0x8) {
4197                 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) {
4198                         device_printf(dev, "Interface does not support 10Mb advertised speed\n");
4199                         return (EINVAL);
4200                 }
4201                 speed |= IXGBE_LINK_SPEED_10_FULL;
4202         }
4203         if (advertise & 0x10) {
4204                 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) {
4205                         device_printf(dev, "Interface does not support 2.5G advertised speed\n");
4206                         return (EINVAL);
4207                 }
4208                 speed |= IXGBE_LINK_SPEED_2_5GB_FULL;
4209         }
4210         if (advertise & 0x20) {
4211                 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) {
4212                         device_printf(dev, "Interface does not support 5G advertised speed\n");
4213                         return (EINVAL);
4214                 }
4215                 speed |= IXGBE_LINK_SPEED_5GB_FULL;
4216         }
4217
4218         hw->mac.autotry_restart = true;
4219         hw->mac.ops.setup_link(hw, speed, true);
4220         sc->advertise = advertise;
4221
4222         return (0);
4223 } /* ixgbe_set_advertise */
4224
4225 /************************************************************************
4226  * ixgbe_get_default_advertise - Get default advertised speed settings
4227  *
4228  *   Formatted for sysctl usage.
4229  *   Flags:
4230  *     0x1 - advertise 100 Mb
4231  *     0x2 - advertise 1G
4232  *     0x4 - advertise 10G
4233  *     0x8 - advertise 10 Mb (yes, Mb)
4234  *     0x10 - advertise 2.5G (disabled by default)
4235  *     0x20 - advertise 5G (disabled by default)
4236  ************************************************************************/
4237 static int
4238 ixgbe_get_default_advertise(struct ixgbe_softc *sc)
4239 {
4240         struct ixgbe_hw  *hw = &sc->hw;
4241         int              speed;
4242         ixgbe_link_speed link_caps = 0;
4243         s32              err;
4244         bool             negotiate = false;
4245
4246         /*
4247          * Advertised speed means nothing unless it's copper or
4248          * multi-speed fiber
4249          */
4250         if (!(hw->phy.media_type == ixgbe_media_type_copper) &&
4251             !(hw->phy.multispeed_fiber))
4252                 return (0);
4253
4254         err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate);
4255         if (err != IXGBE_SUCCESS)
4256                 return (0);
4257
4258         if (hw->mac.type == ixgbe_mac_X550) {
4259                 /*
4260                  * 2.5G and 5G autonegotiation speeds on X550
4261                  * are disabled by default due to reported
4262                  * interoperability issues with some switches.
4263                  */
4264                 link_caps &= ~(IXGBE_LINK_SPEED_2_5GB_FULL |
4265                     IXGBE_LINK_SPEED_5GB_FULL);
4266         }
4267
4268         speed =
4269             ((link_caps & IXGBE_LINK_SPEED_10GB_FULL)  ? 0x4  : 0) |
4270             ((link_caps & IXGBE_LINK_SPEED_5GB_FULL)   ? 0x20 : 0) |
4271             ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) |
4272             ((link_caps & IXGBE_LINK_SPEED_1GB_FULL)   ? 0x2  : 0) |
4273             ((link_caps & IXGBE_LINK_SPEED_100_FULL)   ? 0x1  : 0) |
4274             ((link_caps & IXGBE_LINK_SPEED_10_FULL)    ? 0x8  : 0);
4275
4276         return speed;
4277 } /* ixgbe_get_default_advertise */
4278
4279 /************************************************************************
4280  * ixgbe_sysctl_dmac - Manage DMA Coalescing
4281  *
4282  *   Control values:
4283  *     0/1 - off / on (use default value of 1000)
4284  *
4285  *     Legal timer values are:
4286  *     50,100,250,500,1000,2000,5000,10000
4287  *
4288  *     Turning off interrupt moderation will also turn this off.
4289  ************************************************************************/
4290 static int
4291 ixgbe_sysctl_dmac(SYSCTL_HANDLER_ARGS)
4292 {
4293         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4294         struct ifnet   *ifp = iflib_get_ifp(sc->ctx);
4295         int            error;
4296         u16            newval;
4297
4298         newval = sc->dmac;
4299         error = sysctl_handle_16(oidp, &newval, 0, req);
4300         if ((error) || (req->newptr == NULL))
4301                 return (error);
4302
4303         switch (newval) {
4304         case 0:
4305                 /* Disabled */
4306                 sc->dmac = 0;
4307                 break;
4308         case 1:
4309                 /* Enable and use default */
4310                 sc->dmac = 1000;
4311                 break;
4312         case 50:
4313         case 100:
4314         case 250:
4315         case 500:
4316         case 1000:
4317         case 2000:
4318         case 5000:
4319         case 10000:
4320                 /* Legal values - allow */
4321                 sc->dmac = newval;
4322                 break;
4323         default:
4324                 /* Do nothing, illegal value */
4325                 return (EINVAL);
4326         }
4327
4328         /* Re-initialize hardware if it's already running */
4329         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
4330                 ifp->if_init(ifp);
4331
4332         return (0);
4333 } /* ixgbe_sysctl_dmac */
4334
4335 #ifdef IXGBE_DEBUG
4336 /************************************************************************
4337  * ixgbe_sysctl_power_state
4338  *
4339  *   Sysctl to test power states
4340  *   Values:
4341  *     0      - set device to D0
4342  *     3      - set device to D3
4343  *     (none) - get current device power state
4344  ************************************************************************/
4345 static int
4346 ixgbe_sysctl_power_state(SYSCTL_HANDLER_ARGS)
4347 {
4348         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4349         device_t       dev = sc->dev;
4350         int            curr_ps, new_ps, error = 0;
4351
4352         curr_ps = new_ps = pci_get_powerstate(dev);
4353
4354         error = sysctl_handle_int(oidp, &new_ps, 0, req);
4355         if ((error) || (req->newptr == NULL))
4356                 return (error);
4357
4358         if (new_ps == curr_ps)
4359                 return (0);
4360
4361         if (new_ps == 3 && curr_ps == 0)
4362                 error = DEVICE_SUSPEND(dev);
4363         else if (new_ps == 0 && curr_ps == 3)
4364                 error = DEVICE_RESUME(dev);
4365         else
4366                 return (EINVAL);
4367
4368         device_printf(dev, "New state: %d\n", pci_get_powerstate(dev));
4369
4370         return (error);
4371 } /* ixgbe_sysctl_power_state */
4372 #endif
4373
4374 /************************************************************************
4375  * ixgbe_sysctl_wol_enable
4376  *
4377  *   Sysctl to enable/disable the WoL capability,
4378  *   if supported by the adapter.
4379  *
4380  *   Values:
4381  *     0 - disabled
4382  *     1 - enabled
4383  ************************************************************************/
4384 static int
4385 ixgbe_sysctl_wol_enable(SYSCTL_HANDLER_ARGS)
4386 {
4387         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4388         struct ixgbe_hw *hw = &sc->hw;
4389         int             new_wol_enabled;
4390         int             error = 0;
4391
4392         new_wol_enabled = hw->wol_enabled;
4393         error = sysctl_handle_int(oidp, &new_wol_enabled, 0, req);
4394         if ((error) || (req->newptr == NULL))
4395                 return (error);
4396         new_wol_enabled = !!(new_wol_enabled);
4397         if (new_wol_enabled == hw->wol_enabled)
4398                 return (0);
4399
4400         if (new_wol_enabled > 0 && !sc->wol_support)
4401                 return (ENODEV);
4402         else
4403                 hw->wol_enabled = new_wol_enabled;
4404
4405         return (0);
4406 } /* ixgbe_sysctl_wol_enable */
4407
4408 /************************************************************************
4409  * ixgbe_sysctl_wufc - Wake Up Filter Control
4410  *
4411  *   Sysctl to enable/disable the types of packets that the
4412  *   adapter will wake up on upon receipt.
4413  *   Flags:
4414  *     0x1  - Link Status Change
4415  *     0x2  - Magic Packet
4416  *     0x4  - Direct Exact
4417  *     0x8  - Directed Multicast
4418  *     0x10 - Broadcast
4419  *     0x20 - ARP/IPv4 Request Packet
4420  *     0x40 - Direct IPv4 Packet
4421  *     0x80 - Direct IPv6 Packet
4422  *
4423  *   Settings not listed above will cause the sysctl to return an error.
4424  ************************************************************************/
4425 static int
4426 ixgbe_sysctl_wufc(SYSCTL_HANDLER_ARGS)
4427 {
4428         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4429         int            error = 0;
4430         u32            new_wufc;
4431
4432         new_wufc = sc->wufc;
4433
4434         error = sysctl_handle_32(oidp, &new_wufc, 0, req);
4435         if ((error) || (req->newptr == NULL))
4436                 return (error);
4437         if (new_wufc == sc->wufc)
4438                 return (0);
4439
4440         if (new_wufc & 0xffffff00)
4441                 return (EINVAL);
4442
4443         new_wufc &= 0xff;
4444         new_wufc |= (0xffffff & sc->wufc);
4445         sc->wufc = new_wufc;
4446
4447         return (0);
4448 } /* ixgbe_sysctl_wufc */
4449
4450 #ifdef IXGBE_DEBUG
4451 /************************************************************************
4452  * ixgbe_sysctl_print_rss_config
4453  ************************************************************************/
4454 static int
4455 ixgbe_sysctl_print_rss_config(SYSCTL_HANDLER_ARGS)
4456 {
4457         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4458         struct ixgbe_hw *hw = &sc->hw;
4459         device_t        dev = sc->dev;
4460         struct sbuf     *buf;
4461         int             error = 0, reta_size;
4462         u32             reg;
4463
4464         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4465         if (!buf) {
4466                 device_printf(dev, "Could not allocate sbuf for output.\n");
4467                 return (ENOMEM);
4468         }
4469
4470         // TODO: use sbufs to make a string to print out
4471         /* Set multiplier for RETA setup and table size based on MAC */
4472         switch (sc->hw.mac.type) {
4473         case ixgbe_mac_X550:
4474         case ixgbe_mac_X550EM_x:
4475         case ixgbe_mac_X550EM_a:
4476                 reta_size = 128;
4477                 break;
4478         default:
4479                 reta_size = 32;
4480                 break;
4481         }
4482
4483         /* Print out the redirection table */
4484         sbuf_cat(buf, "\n");
4485         for (int i = 0; i < reta_size; i++) {
4486                 if (i < 32) {
4487                         reg = IXGBE_READ_REG(hw, IXGBE_RETA(i));
4488                         sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg);
4489                 } else {
4490                         reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32));
4491                         sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg);
4492                 }
4493         }
4494
4495         // TODO: print more config
4496
4497         error = sbuf_finish(buf);
4498         if (error)
4499                 device_printf(dev, "Error finishing sbuf: %d\n", error);
4500
4501         sbuf_delete(buf);
4502
4503         return (0);
4504 } /* ixgbe_sysctl_print_rss_config */
4505 #endif /* IXGBE_DEBUG */
4506
4507 /************************************************************************
4508  * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY
4509  *
4510  *   For X552/X557-AT devices using an external PHY
4511  ************************************************************************/
4512 static int
4513 ixgbe_sysctl_phy_temp(SYSCTL_HANDLER_ARGS)
4514 {
4515         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4516         struct ixgbe_hw *hw = &sc->hw;
4517         u16             reg;
4518
4519         if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {
4520                 device_printf(iflib_get_dev(sc->ctx),
4521                     "Device has no supported external thermal sensor.\n");
4522                 return (ENODEV);
4523         }
4524
4525         if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP,
4526             IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
4527                 device_printf(iflib_get_dev(sc->ctx),
4528                     "Error reading from PHY's current temperature register\n");
4529                 return (EAGAIN);
4530         }
4531
4532         /* Shift temp for output */
4533         reg = reg >> 8;
4534
4535         return (sysctl_handle_16(oidp, NULL, reg, req));
4536 } /* ixgbe_sysctl_phy_temp */
4537
4538 /************************************************************************
4539  * ixgbe_sysctl_phy_overtemp_occurred
4540  *
4541  *   Reports (directly from the PHY) whether the current PHY
4542  *   temperature is over the overtemp threshold.
4543  ************************************************************************/
4544 static int
4545 ixgbe_sysctl_phy_overtemp_occurred(SYSCTL_HANDLER_ARGS)
4546 {
4547         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4548         struct ixgbe_hw *hw = &sc->hw;
4549         u16             reg;
4550
4551         if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {
4552                 device_printf(iflib_get_dev(sc->ctx),
4553                     "Device has no supported external thermal sensor.\n");
4554                 return (ENODEV);
4555         }
4556
4557         if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS,
4558             IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
4559                 device_printf(iflib_get_dev(sc->ctx),
4560                     "Error reading from PHY's temperature status register\n");
4561                 return (EAGAIN);
4562         }
4563
4564         /* Get occurrence bit */
4565         reg = !!(reg & 0x4000);
4566
4567         return (sysctl_handle_16(oidp, 0, reg, req));
4568 } /* ixgbe_sysctl_phy_overtemp_occurred */
4569
4570 /************************************************************************
4571  * ixgbe_sysctl_eee_state
4572  *
4573  *   Sysctl to set EEE power saving feature
4574  *   Values:
4575  *     0      - disable EEE
4576  *     1      - enable EEE
4577  *     (none) - get current device EEE state
4578  ************************************************************************/
4579 static int
4580 ixgbe_sysctl_eee_state(SYSCTL_HANDLER_ARGS)
4581 {
4582         struct ixgbe_softc *sc = (struct ixgbe_softc *)arg1;
4583         device_t       dev = sc->dev;
4584         struct ifnet   *ifp = iflib_get_ifp(sc->ctx);
4585         int            curr_eee, new_eee, error = 0;
4586         s32            retval;
4587
4588         curr_eee = new_eee = !!(sc->feat_en & IXGBE_FEATURE_EEE);
4589
4590         error = sysctl_handle_int(oidp, &new_eee, 0, req);
4591         if ((error) || (req->newptr == NULL))
4592                 return (error);
4593
4594         /* Nothing to do */
4595         if (new_eee == curr_eee)
4596                 return (0);
4597
4598         /* Not supported */
4599         if (!(sc->feat_cap & IXGBE_FEATURE_EEE))
4600                 return (EINVAL);
4601
4602         /* Bounds checking */
4603         if ((new_eee < 0) || (new_eee > 1))
4604                 return (EINVAL);
4605
4606         retval = ixgbe_setup_eee(&sc->hw, new_eee);
4607         if (retval) {
4608                 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval);
4609                 return (EINVAL);
4610         }
4611
4612         /* Restart auto-neg */
4613         ifp->if_init(ifp);
4614
4615         device_printf(dev, "New EEE state: %d\n", new_eee);
4616
4617         /* Cache new value */
4618         if (new_eee)
4619                 sc->feat_en |= IXGBE_FEATURE_EEE;
4620         else
4621                 sc->feat_en &= ~IXGBE_FEATURE_EEE;
4622
4623         return (error);
4624 } /* ixgbe_sysctl_eee_state */
4625
4626 /************************************************************************
4627  * ixgbe_init_device_features
4628  ************************************************************************/
4629 static void
4630 ixgbe_init_device_features(struct ixgbe_softc *sc)
4631 {
4632         sc->feat_cap = IXGBE_FEATURE_NETMAP
4633                           | IXGBE_FEATURE_RSS
4634                           | IXGBE_FEATURE_MSI
4635                           | IXGBE_FEATURE_MSIX
4636                           | IXGBE_FEATURE_LEGACY_IRQ;
4637
4638         /* Set capabilities first... */
4639         switch (sc->hw.mac.type) {
4640         case ixgbe_mac_82598EB:
4641                 if (sc->hw.device_id == IXGBE_DEV_ID_82598AT)
4642                         sc->feat_cap |= IXGBE_FEATURE_FAN_FAIL;
4643                 break;
4644         case ixgbe_mac_X540:
4645                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4646                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4647                 if ((sc->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) &&
4648                     (sc->hw.bus.func == 0))
4649                         sc->feat_cap |= IXGBE_FEATURE_BYPASS;
4650                 break;
4651         case ixgbe_mac_X550:
4652                 sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
4653                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4654                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4655                 break;
4656         case ixgbe_mac_X550EM_x:
4657                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4658                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4659                 break;
4660         case ixgbe_mac_X550EM_a:
4661                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4662                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4663                 sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
4664                 if ((sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) ||
4665                     (sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) {
4666                         sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
4667                         sc->feat_cap |= IXGBE_FEATURE_EEE;
4668                 }
4669                 break;
4670         case ixgbe_mac_82599EB:
4671                 sc->feat_cap |= IXGBE_FEATURE_SRIOV;
4672                 sc->feat_cap |= IXGBE_FEATURE_FDIR;
4673                 if ((sc->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) &&
4674                     (sc->hw.bus.func == 0))
4675                         sc->feat_cap |= IXGBE_FEATURE_BYPASS;
4676                 if (sc->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP)
4677                         sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
4678                 break;
4679         default:
4680                 break;
4681         }
4682
4683         /* Enabled by default... */
4684         /* Fan failure detection */
4685         if (sc->feat_cap & IXGBE_FEATURE_FAN_FAIL)
4686                 sc->feat_en |= IXGBE_FEATURE_FAN_FAIL;
4687         /* Netmap */
4688         if (sc->feat_cap & IXGBE_FEATURE_NETMAP)
4689                 sc->feat_en |= IXGBE_FEATURE_NETMAP;
4690         /* EEE */
4691         if (sc->feat_cap & IXGBE_FEATURE_EEE)
4692                 sc->feat_en |= IXGBE_FEATURE_EEE;
4693         /* Thermal Sensor */
4694         if (sc->feat_cap & IXGBE_FEATURE_TEMP_SENSOR)
4695                 sc->feat_en |= IXGBE_FEATURE_TEMP_SENSOR;
4696
4697         /* Enabled via global sysctl... */
4698         /* Flow Director */
4699         if (ixgbe_enable_fdir) {
4700                 if (sc->feat_cap & IXGBE_FEATURE_FDIR)
4701                         sc->feat_en |= IXGBE_FEATURE_FDIR;
4702                 else
4703                         device_printf(sc->dev, "Device does not support Flow Director. Leaving disabled.");
4704         }
4705         /*
4706          * Message Signal Interrupts - Extended (MSI-X)
4707          * Normal MSI is only enabled if MSI-X calls fail.
4708          */
4709         if (!ixgbe_enable_msix)
4710                 sc->feat_cap &= ~IXGBE_FEATURE_MSIX;
4711         /* Receive-Side Scaling (RSS) */
4712         if ((sc->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss)
4713                 sc->feat_en |= IXGBE_FEATURE_RSS;
4714
4715         /* Disable features with unmet dependencies... */
4716         /* No MSI-X */
4717         if (!(sc->feat_cap & IXGBE_FEATURE_MSIX)) {
4718                 sc->feat_cap &= ~IXGBE_FEATURE_RSS;
4719                 sc->feat_cap &= ~IXGBE_FEATURE_SRIOV;
4720                 sc->feat_en &= ~IXGBE_FEATURE_RSS;
4721                 sc->feat_en &= ~IXGBE_FEATURE_SRIOV;
4722         }
4723 } /* ixgbe_init_device_features */
4724
4725 /************************************************************************
4726  * ixgbe_check_fan_failure
4727  ************************************************************************/
4728 static void
4729 ixgbe_check_fan_failure(struct ixgbe_softc *sc, u32 reg, bool in_interrupt)
4730 {
4731         u32 mask;
4732
4733         mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&sc->hw) :
4734             IXGBE_ESDP_SDP1;
4735
4736         if (reg & mask)
4737                 device_printf(sc->dev, "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
4738 } /* ixgbe_check_fan_failure */
4739
4740 /************************************************************************
4741  * ixgbe_sbuf_fw_version
4742  ************************************************************************/
4743 static void
4744 ixgbe_sbuf_fw_version(struct ixgbe_hw *hw, struct sbuf *buf)
4745 {
4746         struct ixgbe_nvm_version nvm_ver = {0};
4747         uint16_t phyfw = 0;
4748         int status;
4749         const char *space = "";
4750
4751         ixgbe_get_oem_prod_version(hw, &nvm_ver); /* OEM's NVM version */
4752         ixgbe_get_orom_version(hw, &nvm_ver); /* Option ROM */
4753         ixgbe_get_etk_id(hw, &nvm_ver); /* eTrack identifies a build in Intel's SCM */
4754         status = ixgbe_get_phy_firmware_version(hw, &phyfw);
4755
4756         if (nvm_ver.oem_valid) {
4757                 sbuf_printf(buf, "NVM OEM V%d.%d R%d", nvm_ver.oem_major,
4758                     nvm_ver.oem_minor, nvm_ver.oem_release);
4759                 space = " ";
4760         }
4761
4762         if (nvm_ver.or_valid) {
4763                 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d",
4764                     space, nvm_ver.or_major, nvm_ver.or_build, nvm_ver.or_patch);
4765                 space = " ";
4766         }
4767
4768         if (nvm_ver.etk_id != ((NVM_VER_INVALID << NVM_ETK_SHIFT) |
4769             NVM_VER_INVALID)) {
4770                 sbuf_printf(buf, "%seTrack 0x%08x", space, nvm_ver.etk_id);
4771                 space = " ";
4772         }
4773
4774         if (phyfw != 0 && status == IXGBE_SUCCESS)
4775                 sbuf_printf(buf, "%sPHY FW V%d", space, phyfw);
4776 } /* ixgbe_sbuf_fw_version */
4777
4778 /************************************************************************
4779  * ixgbe_print_fw_version
4780  ************************************************************************/
4781 static void
4782 ixgbe_print_fw_version(if_ctx_t ctx)
4783 {
4784         struct ixgbe_softc *sc = iflib_get_softc(ctx);
4785         struct ixgbe_hw *hw = &sc->hw;
4786         device_t dev = sc->dev;
4787         struct sbuf *buf;
4788         int error = 0;
4789
4790         buf = sbuf_new_auto();
4791         if (!buf) {
4792                 device_printf(dev, "Could not allocate sbuf for output.\n");
4793                 return;
4794         }
4795
4796         ixgbe_sbuf_fw_version(hw, buf);
4797
4798         error = sbuf_finish(buf);
4799         if (error)
4800                 device_printf(dev, "Error finishing sbuf: %d\n", error);
4801         else if (sbuf_len(buf))
4802                 device_printf(dev, "%s\n", sbuf_data(buf));
4803
4804         sbuf_delete(buf);
4805 } /* ixgbe_print_fw_version */
4806
4807 /************************************************************************
4808  * ixgbe_sysctl_print_fw_version
4809  ************************************************************************/
4810 static int
4811 ixgbe_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS)
4812 {
4813         struct ixgbe_softc  *sc = (struct ixgbe_softc *)arg1;
4814         struct ixgbe_hw *hw = &sc->hw;
4815         device_t dev = sc->dev;
4816         struct sbuf *buf;
4817         int error = 0;
4818
4819         buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4820         if (!buf) {
4821                 device_printf(dev, "Could not allocate sbuf for output.\n");
4822                 return (ENOMEM);
4823         }
4824
4825         ixgbe_sbuf_fw_version(hw, buf);
4826
4827         error = sbuf_finish(buf);
4828         if (error)
4829                 device_printf(dev, "Error finishing sbuf: %d\n", error);
4830
4831         sbuf_delete(buf);
4832
4833         return (0);
4834 } /* ixgbe_sysctl_print_fw_version */