sys/dev/igc/if_igc.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org>
   5  * All rights reserved.
   6  * Copyright (c) 2021 Rubicon Communications, LLC (Netgate)
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "if_igc.h"
  34 #include <sys/sbuf.h>
  35 #include <machine/_inttypes.h>
  36
  37 #ifdef RSS
  38 #include <net/rss_config.h>
  39 #include <netinet/in_rss.h>
  40 #endif
  41
  42 /*********************************************************************
  43  *  PCI Device ID Table
  44  *
  45  *  Used by probe to select devices to load on
  46  *  Last entry must be all 0s
  47  *
  48  *  { Vendor ID, Device ID, String }
  49  *********************************************************************/
  50
  51 static pci_vendor_info_t igc_vendor_info_array[] =
  52 {
  53         /* Intel(R) PRO/1000 Network Connection - igc */
  54         PVID(0x8086, IGC_DEV_ID_I225_LM, "Intel(R) Ethernet Controller I225-LM"),
  55         PVID(0x8086, IGC_DEV_ID_I225_V, "Intel(R) Ethernet Controller I225-V"),
  56         PVID(0x8086, IGC_DEV_ID_I225_K, "Intel(R) Ethernet Controller I225-K"),
  57         PVID(0x8086, IGC_DEV_ID_I225_I, "Intel(R) Ethernet Controller I225-I"),
  58         PVID(0x8086, IGC_DEV_ID_I220_V, "Intel(R) Ethernet Controller I220-V"),
  59         PVID(0x8086, IGC_DEV_ID_I225_K2, "Intel(R) Ethernet Controller I225-K(2)"),
  60         PVID(0x8086, IGC_DEV_ID_I225_LMVP, "Intel(R) Ethernet Controller I225-LMvP(2)"),
  61         PVID(0x8086, IGC_DEV_ID_I226_K, "Intel(R) Ethernet Controller I226-K"),
  62         PVID(0x8086, IGC_DEV_ID_I225_IT, "Intel(R) Ethernet Controller I225-IT(2)"),
  63         PVID(0x8086, IGC_DEV_ID_I226_LM, "Intel(R) Ethernet Controller I226-LM"),
  64         PVID(0x8086, IGC_DEV_ID_I226_V, "Intel(R) Ethernet Controller I226-V"),
  65         PVID(0x8086, IGC_DEV_ID_I226_IT, "Intel(R) Ethernet Controller I226-IT"),
  66         PVID(0x8086, IGC_DEV_ID_I221_V, "Intel(R) Ethernet Controller I221-V"),
  67         PVID(0x8086, IGC_DEV_ID_I226_BLANK_NVM, "Intel(R) Ethernet Controller I226(blankNVM)"),
  68         PVID(0x8086, IGC_DEV_ID_I225_BLANK_NVM, "Intel(R) Ethernet Controller I225(blankNVM)"),
  69         /* required last entry */
  70         PVID_END
  71 };
  72
  73 /*********************************************************************
  74  *  Function prototypes
  75  *********************************************************************/
  76 static void     *igc_register(device_t dev);
  77 static int      igc_if_attach_pre(if_ctx_t ctx);
  78 static int      igc_if_attach_post(if_ctx_t ctx);
  79 static int      igc_if_detach(if_ctx_t ctx);
  80 static int      igc_if_shutdown(if_ctx_t ctx);
  81 static int      igc_if_suspend(if_ctx_t ctx);
  82 static int      igc_if_resume(if_ctx_t ctx);
  83
  84 static int      igc_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
  85 static int      igc_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets);
  86 static void     igc_if_queues_free(if_ctx_t ctx);
  87
  88 static uint64_t igc_if_get_counter(if_ctx_t, ift_counter);
  89 static void     igc_if_init(if_ctx_t ctx);
  90 static void     igc_if_stop(if_ctx_t ctx);
  91 static void     igc_if_media_status(if_ctx_t, struct ifmediareq *);
  92 static int      igc_if_media_change(if_ctx_t ctx);
  93 static int      igc_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
  94 static void     igc_if_timer(if_ctx_t ctx, uint16_t qid);
  95 static void     igc_if_vlan_register(if_ctx_t ctx, u16 vtag);
  96 static void     igc_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
  97 static void     igc_if_watchdog_reset(if_ctx_t ctx);
  98 static bool     igc_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event);
  99
 100 static void     igc_identify_hardware(if_ctx_t ctx);
 101 static int      igc_allocate_pci_resources(if_ctx_t ctx);
 102 static void     igc_free_pci_resources(if_ctx_t ctx);
 103 static void     igc_reset(if_ctx_t ctx);
 104 static int      igc_setup_interface(if_ctx_t ctx);
 105 static int      igc_setup_msix(if_ctx_t ctx);
 106
 107 static void     igc_initialize_transmit_unit(if_ctx_t ctx);
 108 static void     igc_initialize_receive_unit(if_ctx_t ctx);
 109
 110 static void     igc_if_intr_enable(if_ctx_t ctx);
 111 static void     igc_if_intr_disable(if_ctx_t ctx);
 112 static int      igc_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
 113 static int      igc_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid);
 114 static void     igc_if_multi_set(if_ctx_t ctx);
 115 static void     igc_if_update_admin_status(if_ctx_t ctx);
 116 static void     igc_if_debug(if_ctx_t ctx);
 117 static void     igc_update_stats_counters(struct igc_adapter *);
 118 static void     igc_add_hw_stats(struct igc_adapter *adapter);
 119 static int      igc_if_set_promisc(if_ctx_t ctx, int flags);
 120 static void     igc_setup_vlan_hw_support(struct igc_adapter *);
 121 static int      igc_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
 122 static void     igc_print_nvm_info(struct igc_adapter *);
 123 static int      igc_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 124 static int      igc_get_rs(SYSCTL_HANDLER_ARGS);
 125 static void     igc_print_debug_info(struct igc_adapter *);
 126 static int      igc_is_valid_ether_addr(u8 *);
 127 static int      igc_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 128 static void     igc_add_int_delay_sysctl(struct igc_adapter *, const char *,
 129                     const char *, struct igc_int_delay_info *, int, int);
 130 /* Management and WOL Support */
 131 static void     igc_get_hw_control(struct igc_adapter *);
 132 static void     igc_release_hw_control(struct igc_adapter *);
 133 static void     igc_get_wakeup(if_ctx_t ctx);
 134 static void     igc_enable_wakeup(if_ctx_t ctx);
 135
 136 int             igc_intr(void *arg);
 137
 138 /* MSI-X handlers */
 139 static int      igc_if_msix_intr_assign(if_ctx_t, int);
 140 static int      igc_msix_link(void *);
 141 static void     igc_handle_link(void *context);
 142
 143 static int      igc_set_flowcntl(SYSCTL_HANDLER_ARGS);
 144 static int      igc_sysctl_eee(SYSCTL_HANDLER_ARGS);
 145
 146 static int      igc_get_regs(SYSCTL_HANDLER_ARGS);
 147
 148 static void     igc_configure_queues(struct igc_adapter *adapter);
 149
 150
 151 /*********************************************************************
 152  *  FreeBSD Device Interface Entry Points
 153  *********************************************************************/
 154 static device_method_t igc_methods[] = {
 155         /* Device interface */
 156         DEVMETHOD(device_register, igc_register),
 157         DEVMETHOD(device_probe, iflib_device_probe),
 158         DEVMETHOD(device_attach, iflib_device_attach),
 159         DEVMETHOD(device_detach, iflib_device_detach),
 160         DEVMETHOD(device_shutdown, iflib_device_shutdown),
 161         DEVMETHOD(device_suspend, iflib_device_suspend),
 162         DEVMETHOD(device_resume, iflib_device_resume),
 163         DEVMETHOD_END
 164 };
 165
 166 static driver_t igc_driver = {
 167         "igc", igc_methods, sizeof(struct igc_adapter),
 168 };
 169
 170 static devclass_t igc_devclass;
 171 DRIVER_MODULE(igc, pci, igc_driver, igc_devclass, 0, 0);
 172
 173 MODULE_DEPEND(igc, pci, 1, 1, 1);
 174 MODULE_DEPEND(igc, ether, 1, 1, 1);
 175 MODULE_DEPEND(igc, iflib, 1, 1, 1);
 176
 177 IFLIB_PNP_INFO(pci, igc, igc_vendor_info_array);
 178
 179 static device_method_t igc_if_methods[] = {
 180         DEVMETHOD(ifdi_attach_pre, igc_if_attach_pre),
 181         DEVMETHOD(ifdi_attach_post, igc_if_attach_post),
 182         DEVMETHOD(ifdi_detach, igc_if_detach),
 183         DEVMETHOD(ifdi_shutdown, igc_if_shutdown),
 184         DEVMETHOD(ifdi_suspend, igc_if_suspend),
 185         DEVMETHOD(ifdi_resume, igc_if_resume),
 186         DEVMETHOD(ifdi_init, igc_if_init),
 187         DEVMETHOD(ifdi_stop, igc_if_stop),
 188         DEVMETHOD(ifdi_msix_intr_assign, igc_if_msix_intr_assign),
 189         DEVMETHOD(ifdi_intr_enable, igc_if_intr_enable),
 190         DEVMETHOD(ifdi_intr_disable, igc_if_intr_disable),
 191         DEVMETHOD(ifdi_tx_queues_alloc, igc_if_tx_queues_alloc),
 192         DEVMETHOD(ifdi_rx_queues_alloc, igc_if_rx_queues_alloc),
 193         DEVMETHOD(ifdi_queues_free, igc_if_queues_free),
 194         DEVMETHOD(ifdi_update_admin_status, igc_if_update_admin_status),
 195         DEVMETHOD(ifdi_multi_set, igc_if_multi_set),
 196         DEVMETHOD(ifdi_media_status, igc_if_media_status),
 197         DEVMETHOD(ifdi_media_change, igc_if_media_change),
 198         DEVMETHOD(ifdi_mtu_set, igc_if_mtu_set),
 199         DEVMETHOD(ifdi_promisc_set, igc_if_set_promisc),
 200         DEVMETHOD(ifdi_timer, igc_if_timer),
 201         DEVMETHOD(ifdi_watchdog_reset, igc_if_watchdog_reset),
 202         DEVMETHOD(ifdi_vlan_register, igc_if_vlan_register),
 203         DEVMETHOD(ifdi_vlan_unregister, igc_if_vlan_unregister),
 204         DEVMETHOD(ifdi_get_counter, igc_if_get_counter),
 205         DEVMETHOD(ifdi_rx_queue_intr_enable, igc_if_rx_queue_intr_enable),
 206         DEVMETHOD(ifdi_tx_queue_intr_enable, igc_if_tx_queue_intr_enable),
 207         DEVMETHOD(ifdi_debug, igc_if_debug),
 208         DEVMETHOD(ifdi_needs_restart, igc_if_needs_restart),
 209         DEVMETHOD_END
 210 };
 211
 212 static driver_t igc_if_driver = {
 213         "igc_if", igc_if_methods, sizeof(struct igc_adapter)
 214 };
 215
 216 /*********************************************************************
 217  *  Tunable default values.
 218  *********************************************************************/
 219
 220 #define IGC_TICKS_TO_USECS(ticks)       ((1024 * (ticks) + 500) / 1000)
 221 #define IGC_USECS_TO_TICKS(usecs)       ((1000 * (usecs) + 512) / 1024)
 222
 223 #define MAX_INTS_PER_SEC        8000
 224 #define DEFAULT_ITR             (1000000000/(MAX_INTS_PER_SEC * 256))
 225
 226 /* Allow common code without TSO */
 227 #ifndef CSUM_TSO
 228 #define CSUM_TSO        0
 229 #endif
 230
 231 static SYSCTL_NODE(_hw, OID_AUTO, igc, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 232     "igc driver parameters");
 233
 234 static int igc_disable_crc_stripping = 0;
 235 SYSCTL_INT(_hw_igc, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN,
 236     &igc_disable_crc_stripping, 0, "Disable CRC Stripping");
 237
 238 static int igc_tx_int_delay_dflt = IGC_TICKS_TO_USECS(IGC_TIDV_VAL);
 239 static int igc_rx_int_delay_dflt = IGC_TICKS_TO_USECS(IGC_RDTR_VAL);
 240 SYSCTL_INT(_hw_igc, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &igc_tx_int_delay_dflt,
 241     0, "Default transmit interrupt delay in usecs");
 242 SYSCTL_INT(_hw_igc, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &igc_rx_int_delay_dflt,
 243     0, "Default receive interrupt delay in usecs");
 244
 245 static int igc_tx_abs_int_delay_dflt = IGC_TICKS_TO_USECS(IGC_TADV_VAL);
 246 static int igc_rx_abs_int_delay_dflt = IGC_TICKS_TO_USECS(IGC_RADV_VAL);
 247 SYSCTL_INT(_hw_igc, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN,
 248     &igc_tx_abs_int_delay_dflt, 0,
 249     "Default transmit interrupt delay limit in usecs");
 250 SYSCTL_INT(_hw_igc, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN,
 251     &igc_rx_abs_int_delay_dflt, 0,
 252     "Default receive interrupt delay limit in usecs");
 253
 254 static int igc_smart_pwr_down = false;
 255 SYSCTL_INT(_hw_igc, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &igc_smart_pwr_down,
 256     0, "Set to true to leave smart power down enabled on newer adapters");
 257
 258 /* Controls whether promiscuous also shows bad packets */
 259 static int igc_debug_sbp = true;
 260 SYSCTL_INT(_hw_igc, OID_AUTO, sbp, CTLFLAG_RDTUN, &igc_debug_sbp, 0,
 261     "Show bad packets in promiscuous mode");
 262
 263 /* How many packets rxeof tries to clean at a time */
 264 static int igc_rx_process_limit = 100;
 265 SYSCTL_INT(_hw_igc, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
 266     &igc_rx_process_limit, 0,
 267     "Maximum number of received packets to process "
 268     "at a time, -1 means unlimited");
 269
 270 /* Energy efficient ethernet - default to OFF */
 271 static int igc_eee_setting = 1;
 272 SYSCTL_INT(_hw_igc, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &igc_eee_setting, 0,
 273     "Enable Energy Efficient Ethernet");
 274
 275 /*
 276 ** Tuneable Interrupt rate
 277 */
 278 static int igc_max_interrupt_rate = 8000;
 279 SYSCTL_INT(_hw_igc, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
 280     &igc_max_interrupt_rate, 0, "Maximum interrupts per second");
 281
 282 extern struct if_txrx igc_txrx;
 283
 284 static struct if_shared_ctx igc_sctx_init = {
 285         .isc_magic = IFLIB_MAGIC,
 286         .isc_q_align = PAGE_SIZE,
 287         .isc_tx_maxsize = IGC_TSO_SIZE + sizeof(struct ether_vlan_header),
 288         .isc_tx_maxsegsize = PAGE_SIZE,
 289         .isc_tso_maxsize = IGC_TSO_SIZE + sizeof(struct ether_vlan_header),
 290         .isc_tso_maxsegsize = IGC_TSO_SEG_SIZE,
 291         .isc_rx_maxsize = MAX_JUMBO_FRAME_SIZE,
 292         .isc_rx_nsegments = 1,
 293         .isc_rx_maxsegsize = MJUM9BYTES,
 294         .isc_nfl = 1,
 295         .isc_nrxqs = 1,
 296         .isc_ntxqs = 1,
 297         .isc_admin_intrcnt = 1,
 298         .isc_vendor_info = igc_vendor_info_array,
 299         .isc_driver_version = "1",
 300         .isc_driver = &igc_if_driver,
 301         .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM,
 302
 303         .isc_nrxd_min = {IGC_MIN_RXD},
 304         .isc_ntxd_min = {IGC_MIN_TXD},
 305         .isc_nrxd_max = {IGC_MAX_RXD},
 306         .isc_ntxd_max = {IGC_MAX_TXD},
 307         .isc_nrxd_default = {IGC_DEFAULT_RXD},
 308         .isc_ntxd_default = {IGC_DEFAULT_TXD},
 309 };
 310
 311 /*****************************************************************
 312  *
 313  * Dump Registers
 314  *
 315  ****************************************************************/
 316 #define IGC_REGS_LEN 739
 317
 318 static int igc_get_regs(SYSCTL_HANDLER_ARGS)
 319 {
 320         struct igc_adapter *adapter = (struct igc_adapter *)arg1;
 321         struct igc_hw *hw = &adapter->hw;
 322         struct sbuf *sb;
 323         u32 *regs_buff;
 324         int rc;
 325
 326         regs_buff = malloc(sizeof(u32) * IGC_REGS_LEN, M_DEVBUF, M_WAITOK);
 327         memset(regs_buff, 0, IGC_REGS_LEN * sizeof(u32));
 328
 329         rc = sysctl_wire_old_buffer(req, 0);
 330         MPASS(rc == 0);
 331         if (rc != 0) {
 332                 free(regs_buff, M_DEVBUF);
 333                 return (rc);
 334         }
 335
 336         sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req);
 337         MPASS(sb != NULL);
 338         if (sb == NULL) {
 339                 free(regs_buff, M_DEVBUF);
 340                 return (ENOMEM);
 341         }
 342
 343         /* General Registers */
 344         regs_buff[0] = IGC_READ_REG(hw, IGC_CTRL);
 345         regs_buff[1] = IGC_READ_REG(hw, IGC_STATUS);
 346         regs_buff[2] = IGC_READ_REG(hw, IGC_CTRL_EXT);
 347         regs_buff[3] = IGC_READ_REG(hw, IGC_ICR);
 348         regs_buff[4] = IGC_READ_REG(hw, IGC_RCTL);
 349         regs_buff[5] = IGC_READ_REG(hw, IGC_RDLEN(0));
 350         regs_buff[6] = IGC_READ_REG(hw, IGC_RDH(0));
 351         regs_buff[7] = IGC_READ_REG(hw, IGC_RDT(0));
 352         regs_buff[8] = IGC_READ_REG(hw, IGC_RXDCTL(0));
 353         regs_buff[9] = IGC_READ_REG(hw, IGC_RDBAL(0));
 354         regs_buff[10] = IGC_READ_REG(hw, IGC_RDBAH(0));
 355         regs_buff[11] = IGC_READ_REG(hw, IGC_TCTL);
 356         regs_buff[12] = IGC_READ_REG(hw, IGC_TDBAL(0));
 357         regs_buff[13] = IGC_READ_REG(hw, IGC_TDBAH(0));
 358         regs_buff[14] = IGC_READ_REG(hw, IGC_TDLEN(0));
 359         regs_buff[15] = IGC_READ_REG(hw, IGC_TDH(0));
 360         regs_buff[16] = IGC_READ_REG(hw, IGC_TDT(0));
 361         regs_buff[17] = IGC_READ_REG(hw, IGC_TXDCTL(0));
 362
 363         sbuf_printf(sb, "General Registers\n");
 364         sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]);
 365         sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]);
 366         sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]);
 367
 368         sbuf_printf(sb, "Interrupt Registers\n");
 369         sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]);
 370
 371         sbuf_printf(sb, "RX Registers\n");
 372         sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]);
 373         sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]);
 374         sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]);
 375         sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]);
 376         sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]);
 377         sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]);
 378         sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]);
 379
 380         sbuf_printf(sb, "TX Registers\n");
 381         sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]);
 382         sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]);
 383         sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]);
 384         sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]);
 385         sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]);
 386         sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]);
 387         sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]);
 388         sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]);
 389         sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]);
 390         sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]);
 391         sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]);
 392
 393         free(regs_buff, M_DEVBUF);
 394
 395 #ifdef DUMP_DESCS
 396         {
 397                 if_softc_ctx_t scctx = adapter->shared;
 398                 struct rx_ring *rxr = &rx_que->rxr;
 399                 struct tx_ring *txr = &tx_que->txr;
 400                 int ntxd = scctx->isc_ntxd[0];
 401                 int nrxd = scctx->isc_nrxd[0];
 402                 int j;
 403
 404         for (j = 0; j < nrxd; j++) {
 405                 u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error);
 406                 u32 length =  le32toh(rxr->rx_base[j].wb.upper.length);
 407                 sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" PRIx64 "  Error:%d  Length:%d\n", j, rxr->rx_base[j].read.buffer_addr, staterr, length);
 408         }
 409
 410         for (j = 0; j < min(ntxd, 256); j++) {
 411                 unsigned int *ptr = (unsigned int *)&txr->tx_base[j];
 412
 413                 sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x  eop: %d DD=%d\n",
 414                             j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop,
 415                             buf->eop != -1 ? txr->tx_base[buf->eop].upper.fields.status & IGC_TXD_STAT_DD : 0);
 416
 417         }
 418         }
 419 #endif
 420
 421         rc = sbuf_finish(sb);
 422         sbuf_delete(sb);
 423         return(rc);
 424 }
 425
 426 static void *
 427 igc_register(device_t dev)
 428 {
 429         return (&igc_sctx_init);
 430 }
 431
 432 static int
 433 igc_set_num_queues(if_ctx_t ctx)
 434 {
 435         int maxqueues;
 436
 437         maxqueues = 4;
 438
 439         return (maxqueues);
 440 }
 441
 442 #define IGC_CAPS                                                        \
 443     IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |              \
 444     IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 |  \
 445     IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 |\
 446     IFCAP_TSO6
 447
 448 /*********************************************************************
 449  *  Device initialization routine
 450  *
 451  *  The attach entry point is called when the driver is being loaded.
 452  *  This routine identifies the type of hardware, allocates all resources
 453  *  and initializes the hardware.
 454  *
 455  *  return 0 on success, positive on failure
 456  *********************************************************************/
 457 static int
 458 igc_if_attach_pre(if_ctx_t ctx)
 459 {
 460         struct igc_adapter *adapter;
 461         if_softc_ctx_t scctx;
 462         device_t dev;
 463         struct igc_hw *hw;
 464         int error = 0;
 465
 466         INIT_DEBUGOUT("igc_if_attach_pre: begin");
 467         dev = iflib_get_dev(ctx);
 468         adapter = iflib_get_softc(ctx);
 469
 470         adapter->ctx = adapter->osdep.ctx = ctx;
 471         adapter->dev = adapter->osdep.dev = dev;
 472         scctx = adapter->shared = iflib_get_softc_ctx(ctx);
 473         adapter->media = iflib_get_media(ctx);
 474         hw = &adapter->hw;
 475
 476         adapter->tx_process_limit = scctx->isc_ntxd[0];
 477
 478         /* SYSCTL stuff */
 479         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 480             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 481             OID_AUTO, "nvm", CTLTYPE_INT | CTLFLAG_RW,
 482             adapter, 0, igc_sysctl_nvm_info, "I", "NVM Information");
 483
 484         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 485             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 486             OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW,
 487             adapter, 0, igc_sysctl_debug_info, "I", "Debug Information");
 488
 489         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 490             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 491             OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 492             adapter, 0, igc_set_flowcntl, "I", "Flow Control");
 493
 494         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 495             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 496             OID_AUTO, "reg_dump",
 497             CTLTYPE_STRING | CTLFLAG_RD, adapter, 0,
 498             igc_get_regs, "A", "Dump Registers");
 499
 500         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 501             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 502             OID_AUTO, "rs_dump",
 503             CTLTYPE_INT | CTLFLAG_RW, adapter, 0,
 504             igc_get_rs, "I", "Dump RS indexes");
 505
 506         /* Determine hardware and mac info */
 507         igc_identify_hardware(ctx);
 508
 509         scctx->isc_tx_nsegments = IGC_MAX_SCATTER;
 510         scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = igc_set_num_queues(ctx);
 511         if (bootverbose)
 512                 device_printf(dev, "attach_pre capping queues at %d\n",
 513                     scctx->isc_ntxqsets_max);
 514
 515         scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(union igc_adv_tx_desc), IGC_DBA_ALIGN);
 516         scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN);
 517         scctx->isc_txd_size[0] = sizeof(union igc_adv_tx_desc);
 518         scctx->isc_rxd_size[0] = sizeof(union igc_adv_rx_desc);
 519         scctx->isc_txrx = &igc_txrx;
 520         scctx->isc_tx_tso_segments_max = IGC_MAX_SCATTER;
 521         scctx->isc_tx_tso_size_max = IGC_TSO_SIZE;
 522         scctx->isc_tx_tso_segsize_max = IGC_TSO_SEG_SIZE;
 523         scctx->isc_capabilities = scctx->isc_capenable = IGC_CAPS;
 524         scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO |
 525                 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_SCTP | CSUM_IP6_SCTP;
 526
 527         /*
 528         ** Some new devices, as with ixgbe, now may
 529         ** use a different BAR, so we need to keep
 530         ** track of which is used.
 531         */
 532         scctx->isc_msix_bar = PCIR_BAR(IGC_MSIX_BAR);
 533         if (pci_read_config(dev, scctx->isc_msix_bar, 4) == 0)
 534                 scctx->isc_msix_bar += 4;
 535
 536         /* Setup PCI resources */
 537         if (igc_allocate_pci_resources(ctx)) {
 538                 device_printf(dev, "Allocation of PCI resources failed\n");
 539                 error = ENXIO;
 540                 goto err_pci;
 541         }
 542
 543         /* Do Shared Code initialization */
 544         error = igc_setup_init_funcs(hw, true);
 545         if (error) {
 546                 device_printf(dev, "Setup of Shared code failed, error %d\n",
 547                     error);
 548                 error = ENXIO;
 549                 goto err_pci;
 550         }
 551
 552         igc_setup_msix(ctx);
 553         igc_get_bus_info(hw);
 554
 555         /* Set up some sysctls for the tunable interrupt delays */
 556         igc_add_int_delay_sysctl(adapter, "rx_int_delay",
 557             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 558             IGC_REGISTER(hw, IGC_RDTR), igc_rx_int_delay_dflt);
 559         igc_add_int_delay_sysctl(adapter, "tx_int_delay",
 560             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 561             IGC_REGISTER(hw, IGC_TIDV), igc_tx_int_delay_dflt);
 562         igc_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 563             "receive interrupt delay limit in usecs",
 564             &adapter->rx_abs_int_delay,
 565             IGC_REGISTER(hw, IGC_RADV),
 566             igc_rx_abs_int_delay_dflt);
 567         igc_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 568             "transmit interrupt delay limit in usecs",
 569             &adapter->tx_abs_int_delay,
 570             IGC_REGISTER(hw, IGC_TADV),
 571             igc_tx_abs_int_delay_dflt);
 572         igc_add_int_delay_sysctl(adapter, "itr",
 573             "interrupt delay limit in usecs/4",
 574             &adapter->tx_itr,
 575             IGC_REGISTER(hw, IGC_ITR),
 576             DEFAULT_ITR);
 577
 578         hw->mac.autoneg = DO_AUTO_NEG;
 579         hw->phy.autoneg_wait_to_complete = false;
 580         hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 581
 582         /* Copper options */
 583         if (hw->phy.media_type == igc_media_type_copper) {
 584                 hw->phy.mdix = AUTO_ALL_MODES;
 585         }
 586
 587         /*
 588          * Set the frame limits assuming
 589          * standard ethernet sized frames.
 590          */
 591         scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size =
 592             ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 593
 594         /* Allocate multicast array memory. */
 595         adapter->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN *
 596             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 597         if (adapter->mta == NULL) {
 598                 device_printf(dev, "Can not allocate multicast setup array\n");
 599                 error = ENOMEM;
 600                 goto err_late;
 601         }
 602
 603         /* Check SOL/IDER usage */
 604         if (igc_check_reset_block(hw))
 605                 device_printf(dev, "PHY reset is blocked"
 606                               " due to SOL/IDER session.\n");
 607
 608         /* Sysctl for setting Energy Efficient Ethernet */
 609         adapter->hw.dev_spec._i225.eee_disable = igc_eee_setting;
 610         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 611             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 612             OID_AUTO, "eee_control",
 613             CTLTYPE_INT | CTLFLAG_RW,
 614             adapter, 0, igc_sysctl_eee, "I",
 615             "Disable Energy Efficient Ethernet");
 616
 617         /*
 618         ** Start from a known state, this is
 619         ** important in reading the nvm and
 620         ** mac from that.
 621         */
 622         igc_reset_hw(hw);
 623
 624         /* Make sure we have a good EEPROM before we read from it */
 625         if (igc_validate_nvm_checksum(hw) < 0) {
 626                 /*
 627                 ** Some PCI-E parts fail the first check due to
 628                 ** the link being in sleep state, call it again,
 629                 ** if it fails a second time its a real issue.
 630                 */
 631                 if (igc_validate_nvm_checksum(hw) < 0) {
 632                         device_printf(dev,
 633                             "The EEPROM Checksum Is Not Valid\n");
 634                         error = EIO;
 635                         goto err_late;
 636                 }
 637         }
 638
 639         /* Copy the permanent MAC address out of the EEPROM */
 640         if (igc_read_mac_addr(hw) < 0) {
 641                 device_printf(dev, "EEPROM read error while reading MAC"
 642                               " address\n");
 643                 error = EIO;
 644                 goto err_late;
 645         }
 646
 647         if (!igc_is_valid_ether_addr(hw->mac.addr)) {
 648                 device_printf(dev, "Invalid MAC address\n");
 649                 error = EIO;
 650                 goto err_late;
 651         }
 652
 653         /*
 654          * Get Wake-on-Lan and Management info for later use
 655          */
 656         igc_get_wakeup(ctx);
 657
 658         /* Enable only WOL MAGIC by default */
 659         scctx->isc_capenable &= ~IFCAP_WOL;
 660         if (adapter->wol != 0)
 661                 scctx->isc_capenable |= IFCAP_WOL_MAGIC;
 662
 663         iflib_set_mac(ctx, hw->mac.addr);
 664
 665         return (0);
 666
 667 err_late:
 668         igc_release_hw_control(adapter);
 669 err_pci:
 670         igc_free_pci_resources(ctx);
 671         free(adapter->mta, M_DEVBUF);
 672
 673         return (error);
 674 }
 675
 676 static int
 677 igc_if_attach_post(if_ctx_t ctx)
 678 {
 679         struct igc_adapter *adapter = iflib_get_softc(ctx);
 680         struct igc_hw *hw = &adapter->hw;
 681         int error = 0;
 682
 683         /* Setup OS specific network interface */
 684         error = igc_setup_interface(ctx);
 685         if (error != 0) {
 686                 goto err_late;
 687         }
 688
 689         igc_reset(ctx);
 690
 691         /* Initialize statistics */
 692         igc_update_stats_counters(adapter);
 693         hw->mac.get_link_status = true;
 694         igc_if_update_admin_status(ctx);
 695         igc_add_hw_stats(adapter);
 696
 697         /* the driver can now take control from firmware */
 698         igc_get_hw_control(adapter);
 699
 700         INIT_DEBUGOUT("igc_if_attach_post: end");
 701
 702         return (error);
 703
 704 err_late:
 705         igc_release_hw_control(adapter);
 706         igc_free_pci_resources(ctx);
 707         igc_if_queues_free(ctx);
 708         free(adapter->mta, M_DEVBUF);
 709
 710         return (error);
 711 }
 712
 713 /*********************************************************************
 714  *  Device removal routine
 715  *
 716  *  The detach entry point is called when the driver is being removed.
 717  *  This routine stops the adapter and deallocates all the resources
 718  *  that were allocated for driver operation.
 719  *
 720  *  return 0 on success, positive on failure
 721  *********************************************************************/
 722 static int
 723 igc_if_detach(if_ctx_t ctx)
 724 {
 725         struct igc_adapter      *adapter = iflib_get_softc(ctx);
 726
 727         INIT_DEBUGOUT("igc_if_detach: begin");
 728
 729         igc_phy_hw_reset(&adapter->hw);
 730
 731         igc_release_hw_control(adapter);
 732         igc_free_pci_resources(ctx);
 733
 734         return (0);
 735 }
 736
 737 /*********************************************************************
 738  *
 739  *  Shutdown entry point
 740  *
 741  **********************************************************************/
 742
 743 static int
 744 igc_if_shutdown(if_ctx_t ctx)
 745 {
 746         return igc_if_suspend(ctx);
 747 }
 748
 749 /*
 750  * Suspend/resume device methods.
 751  */
 752 static int
 753 igc_if_suspend(if_ctx_t ctx)
 754 {
 755         struct igc_adapter *adapter = iflib_get_softc(ctx);
 756
 757         igc_release_hw_control(adapter);
 758         igc_enable_wakeup(ctx);
 759         return (0);
 760 }
 761
 762 static int
 763 igc_if_resume(if_ctx_t ctx)
 764 {
 765         igc_if_init(ctx);
 766
 767         return(0);
 768 }
 769
 770 static int
 771 igc_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
 772 {
 773         int max_frame_size;
 774         struct igc_adapter *adapter = iflib_get_softc(ctx);
 775         if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx);
 776
 777          IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 778
 779          /* 9K Jumbo Frame size */
 780          max_frame_size = 9234;
 781
 782         if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) {
 783                 return (EINVAL);
 784         }
 785
 786         scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size =
 787             mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 788         return (0);
 789 }
 790
 791 /*********************************************************************
 792  *  Init entry point
 793  *
 794  *  This routine is used in two ways. It is used by the stack as
 795  *  init entry point in network interface structure. It is also used
 796  *  by the driver as a hw/sw initialization routine to get to a
 797  *  consistent state.
 798  *
 799  **********************************************************************/
 800 static void
 801 igc_if_init(if_ctx_t ctx)
 802 {
 803         struct igc_adapter *adapter = iflib_get_softc(ctx);
 804         if_softc_ctx_t scctx = adapter->shared;
 805         struct ifnet *ifp = iflib_get_ifp(ctx);
 806         struct igc_tx_queue *tx_que;
 807         int i;
 808
 809         INIT_DEBUGOUT("igc_if_init: begin");
 810
 811         /* Get the latest mac address, User can use a LAA */
 812         bcopy(if_getlladdr(ifp), adapter->hw.mac.addr,
 813             ETHER_ADDR_LEN);
 814
 815         /* Put the address into the Receive Address Array */
 816         igc_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 817
 818         /* Initialize the hardware */
 819         igc_reset(ctx);
 820         igc_if_update_admin_status(ctx);
 821
 822         for (i = 0, tx_que = adapter->tx_queues; i < adapter->tx_num_queues; i++, tx_que++) {
 823                 struct tx_ring *txr = &tx_que->txr;
 824
 825                 txr->tx_rs_cidx = txr->tx_rs_pidx;
 826
 827                 /* Initialize the last processed descriptor to be the end of
 828                  * the ring, rather than the start, so that we avoid an
 829                  * off-by-one error when calculating how many descriptors are
 830                  * done in the credits_update function.
 831                  */
 832                 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
 833         }
 834
 835         /* Setup VLAN support, basic and offload if available */
 836         IGC_WRITE_REG(&adapter->hw, IGC_VET, ETHERTYPE_VLAN);
 837
 838         /* Prepare transmit descriptors and buffers */
 839         igc_initialize_transmit_unit(ctx);
 840
 841         /* Setup Multicast table */
 842         igc_if_multi_set(ctx);
 843
 844         adapter->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
 845         igc_initialize_receive_unit(ctx);
 846
 847         /* Use real VLAN Filter support? */
 848         if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
 849                 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
 850                         /* Use real VLAN Filter support */
 851                         igc_setup_vlan_hw_support(adapter);
 852                 else {
 853                         u32 ctrl;
 854                         ctrl = IGC_READ_REG(&adapter->hw, IGC_CTRL);
 855                         ctrl |= IGC_CTRL_VME;
 856                         IGC_WRITE_REG(&adapter->hw, IGC_CTRL, ctrl);
 857                 }
 858         }
 859
 860         /* Don't lose promiscuous settings */
 861         igc_if_set_promisc(ctx, IFF_PROMISC);
 862         igc_clear_hw_cntrs_base_generic(&adapter->hw);
 863
 864         if (adapter->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */
 865                 igc_configure_queues(adapter);
 866
 867         /* this clears any pending interrupts */
 868         IGC_READ_REG(&adapter->hw, IGC_ICR);
 869         IGC_WRITE_REG(&adapter->hw, IGC_ICS, IGC_ICS_LSC);
 870
 871         /* the driver can now take control from firmware */
 872         igc_get_hw_control(adapter);
 873
 874         /* Set Energy Efficient Ethernet */
 875         igc_set_eee_i225(&adapter->hw, true, true, true);
 876 }
 877
 878 /*********************************************************************
 879  *
 880  *  Fast Legacy/MSI Combined Interrupt Service routine
 881  *
 882  *********************************************************************/
 883 int
 884 igc_intr(void *arg)
 885 {
 886         struct igc_adapter *adapter = arg;
 887         if_ctx_t ctx = adapter->ctx;
 888         u32 reg_icr;
 889
 890         reg_icr = IGC_READ_REG(&adapter->hw, IGC_ICR);
 891
 892         /* Hot eject? */
 893         if (reg_icr == 0xffffffff)
 894                 return FILTER_STRAY;
 895
 896         /* Definitely not our interrupt. */
 897         if (reg_icr == 0x0)
 898                 return FILTER_STRAY;
 899
 900         if ((reg_icr & IGC_ICR_INT_ASSERTED) == 0)
 901                 return FILTER_STRAY;
 902
 903         /*
 904          * Only MSI-X interrupts have one-shot behavior by taking advantage
 905          * of the EIAC register.  Thus, explicitly disable interrupts.  This
 906          * also works around the MSI message reordering errata on certain
 907          * systems.
 908          */
 909         IFDI_INTR_DISABLE(ctx);
 910
 911         /* Link status change */
 912         if (reg_icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC))
 913                 igc_handle_link(ctx);
 914
 915         if (reg_icr & IGC_ICR_RXO)
 916                 adapter->rx_overruns++;
 917
 918         return (FILTER_SCHEDULE_THREAD);
 919 }
 920
 921 static int
 922 igc_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
 923 {
 924         struct igc_adapter *adapter = iflib_get_softc(ctx);
 925         struct igc_rx_queue *rxq = &adapter->rx_queues[rxqid];
 926
 927         IGC_WRITE_REG(&adapter->hw, IGC_EIMS, rxq->eims);
 928         return (0);
 929 }
 930
 931 static int
 932 igc_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid)
 933 {
 934         struct igc_adapter *adapter = iflib_get_softc(ctx);
 935         struct igc_tx_queue *txq = &adapter->tx_queues[txqid];
 936
 937         IGC_WRITE_REG(&adapter->hw, IGC_EIMS, txq->eims);
 938         return (0);
 939 }
 940
 941 /*********************************************************************
 942  *
 943  *  MSI-X RX Interrupt Service routine
 944  *
 945  **********************************************************************/
 946 static int
 947 igc_msix_que(void *arg)
 948 {
 949         struct igc_rx_queue *que = arg;
 950
 951         ++que->irqs;
 952
 953         return (FILTER_SCHEDULE_THREAD);
 954 }
 955
 956 /*********************************************************************
 957  *
 958  *  MSI-X Link Fast Interrupt Service routine
 959  *
 960  **********************************************************************/
 961 static int
 962 igc_msix_link(void *arg)
 963 {
 964         struct igc_adapter *adapter = arg;
 965         u32 reg_icr;
 966
 967         ++adapter->link_irq;
 968         MPASS(adapter->hw.back != NULL);
 969         reg_icr = IGC_READ_REG(&adapter->hw, IGC_ICR);
 970
 971         if (reg_icr & IGC_ICR_RXO)
 972                 adapter->rx_overruns++;
 973
 974         if (reg_icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
 975                 igc_handle_link(adapter->ctx);
 976         }
 977
 978         IGC_WRITE_REG(&adapter->hw, IGC_IMS, IGC_IMS_LSC);
 979         IGC_WRITE_REG(&adapter->hw, IGC_EIMS, adapter->link_mask);
 980
 981         return (FILTER_HANDLED);
 982 }
 983
 984 static void
 985 igc_handle_link(void *context)
 986 {
 987         if_ctx_t ctx = context;
 988         struct igc_adapter *adapter = iflib_get_softc(ctx);
 989
 990         adapter->hw.mac.get_link_status = true;
 991         iflib_admin_intr_deferred(ctx);
 992 }
 993
 994 /*********************************************************************
 995  *
 996  *  Media Ioctl callback
 997  *
 998  *  This routine is called whenever the user queries the status of
 999  *  the interface using ifconfig.
1000  *
1001  **********************************************************************/
1002 static void
1003 igc_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
1004 {
1005         struct igc_adapter *adapter = iflib_get_softc(ctx);
1006
1007         INIT_DEBUGOUT("igc_if_media_status: begin");
1008
1009         iflib_admin_intr_deferred(ctx);
1010
1011         ifmr->ifm_status = IFM_AVALID;
1012         ifmr->ifm_active = IFM_ETHER;
1013
1014         if (!adapter->link_active) {
1015                 return;
1016         }
1017
1018         ifmr->ifm_status |= IFM_ACTIVE;
1019
1020         switch (adapter->link_speed) {
1021         case 10:
1022                 ifmr->ifm_active |= IFM_10_T;
1023                 break;
1024         case 100:
1025                 ifmr->ifm_active |= IFM_100_TX;
1026                 break;
1027         case 1000:
1028                 ifmr->ifm_active |= IFM_1000_T;
1029                 break;
1030         case 2500:
1031                 ifmr->ifm_active |= IFM_2500_T;
1032                 break;
1033         }
1034
1035         if (adapter->link_duplex == FULL_DUPLEX)
1036                 ifmr->ifm_active |= IFM_FDX;
1037         else
1038                 ifmr->ifm_active |= IFM_HDX;
1039 }
1040
1041 /*********************************************************************
1042  *
1043  *  Media Ioctl callback
1044  *
1045  *  This routine is called when the user changes speed/duplex using
1046  *  media/mediopt option with ifconfig.
1047  *
1048  **********************************************************************/
1049 static int
1050 igc_if_media_change(if_ctx_t ctx)
1051 {
1052         struct igc_adapter *adapter = iflib_get_softc(ctx);
1053         struct ifmedia *ifm = iflib_get_media(ctx);
1054
1055         INIT_DEBUGOUT("igc_if_media_change: begin");
1056
1057         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1058                 return (EINVAL);
1059
1060         adapter->hw.mac.autoneg = DO_AUTO_NEG;
1061
1062         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1063         case IFM_AUTO:
1064                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1065                 break;
1066         case IFM_2500_T:
1067                 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL;
1068                 break;
1069         case IFM_1000_T:
1070                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1071                 break;
1072         case IFM_100_TX:
1073                 if ((ifm->ifm_media & IFM_GMASK) == IFM_HDX)
1074                         adapter->hw.phy.autoneg_advertised = ADVERTISE_100_HALF;
1075                 else
1076                         adapter->hw.phy.autoneg_advertised = ADVERTISE_100_FULL;
1077                 break;
1078         case IFM_10_T:
1079                 if ((ifm->ifm_media & IFM_GMASK) == IFM_HDX)
1080                         adapter->hw.phy.autoneg_advertised = ADVERTISE_10_HALF;
1081                 else
1082                         adapter->hw.phy.autoneg_advertised = ADVERTISE_10_FULL;
1083                 break;
1084         default:
1085                 device_printf(adapter->dev, "Unsupported media type\n");
1086         }
1087
1088         igc_if_init(ctx);
1089
1090         return (0);
1091 }
1092
1093 static int
1094 igc_if_set_promisc(if_ctx_t ctx, int flags)
1095 {
1096         struct igc_adapter *adapter = iflib_get_softc(ctx);
1097         struct ifnet *ifp = iflib_get_ifp(ctx);
1098         u32 reg_rctl;
1099         int mcnt = 0;
1100
1101         reg_rctl = IGC_READ_REG(&adapter->hw, IGC_RCTL);
1102         reg_rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_UPE);
1103         if (flags & IFF_ALLMULTI)
1104                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
1105         else
1106                 mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES);
1107
1108         /* Don't disable if in MAX groups */
1109         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1110                 reg_rctl &=  (~IGC_RCTL_MPE);
1111         IGC_WRITE_REG(&adapter->hw, IGC_RCTL, reg_rctl);
1112
1113         if (flags & IFF_PROMISC) {
1114                 reg_rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1115                 /* Turn this on if you want to see bad packets */
1116                 if (igc_debug_sbp)
1117                         reg_rctl |= IGC_RCTL_SBP;
1118                 IGC_WRITE_REG(&adapter->hw, IGC_RCTL, reg_rctl);
1119         } else if (flags & IFF_ALLMULTI) {
1120                 reg_rctl |= IGC_RCTL_MPE;
1121                 reg_rctl &= ~IGC_RCTL_UPE;
1122                 IGC_WRITE_REG(&adapter->hw, IGC_RCTL, reg_rctl);
1123         }
1124         return (0);
1125 }
1126
1127 static u_int
1128 igc_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx)
1129 {
1130         u8 *mta = arg;
1131
1132         if (idx == MAX_NUM_MULTICAST_ADDRESSES)
1133                 return (0);
1134
1135         bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN);
1136
1137         return (1);
1138 }
1139
1140 /*********************************************************************
1141  *  Multicast Update
1142  *
1143  *  This routine is called whenever multicast address list is updated.
1144  *
1145  **********************************************************************/
1146
1147 static void
1148 igc_if_multi_set(if_ctx_t ctx)
1149 {
1150         struct igc_adapter *adapter = iflib_get_softc(ctx);
1151         struct ifnet *ifp = iflib_get_ifp(ctx);
1152         u8  *mta; /* Multicast array memory */
1153         u32 reg_rctl = 0;
1154         int mcnt = 0;
1155
1156         IOCTL_DEBUGOUT("igc_set_multi: begin");
1157
1158         mta = adapter->mta;
1159         bzero(mta, sizeof(u8) * ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
1160
1161         mcnt = if_foreach_llmaddr(ifp, igc_copy_maddr, mta);
1162
1163         reg_rctl = IGC_READ_REG(&adapter->hw, IGC_RCTL);
1164
1165         if (if_getflags(ifp) & IFF_PROMISC) {
1166                 reg_rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1167                 /* Turn this on if you want to see bad packets */
1168                 if (igc_debug_sbp)
1169                         reg_rctl |= IGC_RCTL_SBP;
1170         } else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
1171               if_getflags(ifp) & IFF_ALLMULTI) {
1172                 reg_rctl |= IGC_RCTL_MPE;
1173                 reg_rctl &= ~IGC_RCTL_UPE;
1174         } else
1175                 reg_rctl &= ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
1176
1177         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
1178                 igc_update_mc_addr_list(&adapter->hw, mta, mcnt);
1179
1180         IGC_WRITE_REG(&adapter->hw, IGC_RCTL, reg_rctl);
1181 }
1182
1183 /*********************************************************************
1184  *  Timer routine
1185  *
1186  *  This routine schedules igc_if_update_admin_status() to check for
1187  *  link status and to gather statistics as well as to perform some
1188  *  controller-specific hardware patting.
1189  *
1190  **********************************************************************/
1191 static void
1192 igc_if_timer(if_ctx_t ctx, uint16_t qid)
1193 {
1194
1195         if (qid != 0)
1196                 return;
1197
1198         iflib_admin_intr_deferred(ctx);
1199 }
1200
1201 static void
1202 igc_if_update_admin_status(if_ctx_t ctx)
1203 {
1204         struct igc_adapter *adapter = iflib_get_softc(ctx);
1205         struct igc_hw *hw = &adapter->hw;
1206         device_t dev = iflib_get_dev(ctx);
1207         u32 link_check, thstat, ctrl;
1208
1209         link_check = thstat = ctrl = 0;
1210         /* Get the cached link value or read phy for real */
1211         switch (hw->phy.media_type) {
1212         case igc_media_type_copper:
1213                 if (hw->mac.get_link_status == true) {
1214                         /* Do the work to read phy */
1215                         igc_check_for_link(hw);
1216                         link_check = !hw->mac.get_link_status;
1217                 } else
1218                         link_check = true;
1219                 break;
1220         case igc_media_type_unknown:
1221                 igc_check_for_link(hw);
1222                 link_check = !hw->mac.get_link_status;
1223                 /* FALLTHROUGH */
1224         default:
1225                 break;
1226         }
1227
1228         /* Now check for a transition */
1229         if (link_check && (adapter->link_active == 0)) {
1230                 igc_get_speed_and_duplex(hw, &adapter->link_speed,
1231                     &adapter->link_duplex);
1232                 if (bootverbose)
1233                         device_printf(dev, "Link is up %d Mbps %s\n",
1234                             adapter->link_speed,
1235                             ((adapter->link_duplex == FULL_DUPLEX) ?
1236                             "Full Duplex" : "Half Duplex"));
1237                 adapter->link_active = 1;
1238                 iflib_link_state_change(ctx, LINK_STATE_UP,
1239                     IF_Mbps(adapter->link_speed));
1240         } else if (!link_check && (adapter->link_active == 1)) {
1241                 adapter->link_speed = 0;
1242                 adapter->link_duplex = 0;
1243                 adapter->link_active = 0;
1244                 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
1245         }
1246         igc_update_stats_counters(adapter);
1247 }
1248
1249 static void
1250 igc_if_watchdog_reset(if_ctx_t ctx)
1251 {
1252         struct igc_adapter *adapter = iflib_get_softc(ctx);
1253
1254         /*
1255          * Just count the event; iflib(4) will already trigger a
1256          * sufficient reset of the controller.
1257          */
1258         adapter->watchdog_events++;
1259 }
1260
1261 /*********************************************************************
1262  *
1263  *  This routine disables all traffic on the adapter by issuing a
1264  *  global reset on the MAC.
1265  *
1266  **********************************************************************/
1267 static void
1268 igc_if_stop(if_ctx_t ctx)
1269 {
1270         struct igc_adapter *adapter = iflib_get_softc(ctx);
1271
1272         INIT_DEBUGOUT("igc_if_stop: begin");
1273
1274         igc_reset_hw(&adapter->hw);
1275         IGC_WRITE_REG(&adapter->hw, IGC_WUC, 0);
1276 }
1277
1278 /*********************************************************************
1279  *
1280  *  Determine hardware revision.
1281  *
1282  **********************************************************************/
1283 static void
1284 igc_identify_hardware(if_ctx_t ctx)
1285 {
1286         device_t dev = iflib_get_dev(ctx);
1287         struct igc_adapter *adapter = iflib_get_softc(ctx);
1288
1289         /* Make sure our PCI config space has the necessary stuff set */
1290         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1291
1292         /* Save off the information about this board */
1293         adapter->hw.vendor_id = pci_get_vendor(dev);
1294         adapter->hw.device_id = pci_get_device(dev);
1295         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
1296         adapter->hw.subsystem_vendor_id =
1297             pci_read_config(dev, PCIR_SUBVEND_0, 2);
1298         adapter->hw.subsystem_device_id =
1299             pci_read_config(dev, PCIR_SUBDEV_0, 2);
1300
1301         /* Do Shared Code Init and Setup */
1302         if (igc_set_mac_type(&adapter->hw)) {
1303                 device_printf(dev, "Setup init failure\n");
1304                 return;
1305         }
1306 }
1307
1308 static int
1309 igc_allocate_pci_resources(if_ctx_t ctx)
1310 {
1311         struct igc_adapter *adapter = iflib_get_softc(ctx);
1312         device_t dev = iflib_get_dev(ctx);
1313         int rid;
1314
1315         rid = PCIR_BAR(0);
1316         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1317             &rid, RF_ACTIVE);
1318         if (adapter->memory == NULL) {
1319                 device_printf(dev, "Unable to allocate bus resource: memory\n");
1320                 return (ENXIO);
1321         }
1322         adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->memory);
1323         adapter->osdep.mem_bus_space_handle =
1324             rman_get_bushandle(adapter->memory);
1325         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
1326
1327         adapter->hw.back = &adapter->osdep;
1328
1329         return (0);
1330 }
1331
1332 /*********************************************************************
1333  *
1334  *  Set up the MSI-X Interrupt handlers
1335  *
1336  **********************************************************************/
1337 static int
1338 igc_if_msix_intr_assign(if_ctx_t ctx, int msix)
1339 {
1340         struct igc_adapter *adapter = iflib_get_softc(ctx);
1341         struct igc_rx_queue *rx_que = adapter->rx_queues;
1342         struct igc_tx_queue *tx_que = adapter->tx_queues;
1343         int error, rid, i, vector = 0, rx_vectors;
1344         char buf[16];
1345
1346         /* First set up ring resources */
1347         for (i = 0; i < adapter->rx_num_queues; i++, rx_que++, vector++) {
1348                 rid = vector + 1;
1349                 snprintf(buf, sizeof(buf), "rxq%d", i);
1350                 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RXTX, igc_msix_que, rx_que, rx_que->me, buf);
1351                 if (error) {
1352                         device_printf(iflib_get_dev(ctx), "Failed to allocate que int %d err: %d", i, error);
1353                         adapter->rx_num_queues = i + 1;
1354                         goto fail;
1355                 }
1356
1357                 rx_que->msix =  vector;
1358
1359                 /*
1360                  * Set the bit to enable interrupt
1361                  * in IGC_IMS -- bits 20 and 21
1362                  * are for RX0 and RX1, note this has
1363                  * NOTHING to do with the MSI-X vector
1364                  */
1365                 rx_que->eims = 1 << vector;
1366         }
1367         rx_vectors = vector;
1368
1369         vector = 0;
1370         for (i = 0; i < adapter->tx_num_queues; i++, tx_que++, vector++) {
1371                 snprintf(buf, sizeof(buf), "txq%d", i);
1372                 tx_que = &adapter->tx_queues[i];
1373                 iflib_softirq_alloc_generic(ctx,
1374                     &adapter->rx_queues[i % adapter->rx_num_queues].que_irq,
1375                     IFLIB_INTR_TX, tx_que, tx_que->me, buf);
1376
1377                 tx_que->msix = (vector % adapter->rx_num_queues);
1378
1379                 /*
1380                  * Set the bit to enable interrupt
1381                  * in IGC_IMS -- bits 22 and 23
1382                  * are for TX0 and TX1, note this has
1383                  * NOTHING to do with the MSI-X vector
1384                  */
1385                 tx_que->eims = 1 << i;
1386         }
1387
1388         /* Link interrupt */
1389         rid = rx_vectors + 1;
1390         error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid, IFLIB_INTR_ADMIN, igc_msix_link, adapter, 0, "aq");
1391
1392         if (error) {
1393                 device_printf(iflib_get_dev(ctx), "Failed to register admin handler");
1394                 goto fail;
1395         }
1396         adapter->linkvec = rx_vectors;
1397         return (0);
1398 fail:
1399         iflib_irq_free(ctx, &adapter->irq);
1400         rx_que = adapter->rx_queues;
1401         for (int i = 0; i < adapter->rx_num_queues; i++, rx_que++)
1402                 iflib_irq_free(ctx, &rx_que->que_irq);
1403         return (error);
1404 }
1405
1406 static void
1407 igc_configure_queues(struct igc_adapter *adapter)
1408 {
1409         struct igc_hw *hw = &adapter->hw;
1410         struct igc_rx_queue *rx_que;
1411         struct igc_tx_queue *tx_que;
1412         u32 ivar = 0, newitr = 0;
1413
1414         /* First turn on RSS capability */
1415         IGC_WRITE_REG(hw, IGC_GPIE,
1416             IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | IGC_GPIE_PBA |
1417             IGC_GPIE_NSICR);
1418
1419         /* Turn on MSI-X */
1420         /* RX entries */
1421         for (int i = 0; i < adapter->rx_num_queues; i++) {
1422                 u32 index = i >> 1;
1423                 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index);
1424                 rx_que = &adapter->rx_queues[i];
1425                 if (i & 1) {
1426                         ivar &= 0xFF00FFFF;
1427                         ivar |= (rx_que->msix | IGC_IVAR_VALID) << 16;
1428                 } else {
1429                         ivar &= 0xFFFFFF00;
1430                         ivar |= rx_que->msix | IGC_IVAR_VALID;
1431                 }
1432                 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar);
1433         }
1434         /* TX entries */
1435         for (int i = 0; i < adapter->tx_num_queues; i++) {
1436                 u32 index = i >> 1;
1437                 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index);
1438                 tx_que = &adapter->tx_queues[i];
1439                 if (i & 1) {
1440                         ivar &= 0x00FFFFFF;
1441                         ivar |= (tx_que->msix | IGC_IVAR_VALID) << 24;
1442                 } else {
1443                         ivar &= 0xFFFF00FF;
1444                         ivar |= (tx_que->msix | IGC_IVAR_VALID) << 8;
1445                 }
1446                 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar);
1447                 adapter->que_mask |= tx_que->eims;
1448         }
1449
1450         /* And for the link interrupt */
1451         ivar = (adapter->linkvec | IGC_IVAR_VALID) << 8;
1452         adapter->link_mask = 1 << adapter->linkvec;
1453         IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar);
1454
1455         /* Set the starting interrupt rate */
1456         if (igc_max_interrupt_rate > 0)
1457                 newitr = (4000000 / igc_max_interrupt_rate) & 0x7FFC;
1458
1459         newitr |= IGC_EITR_CNT_IGNR;
1460
1461         for (int i = 0; i < adapter->rx_num_queues; i++) {
1462                 rx_que = &adapter->rx_queues[i];
1463                 IGC_WRITE_REG(hw, IGC_EITR(rx_que->msix), newitr);
1464         }
1465
1466         return;
1467 }
1468
1469 static void
1470 igc_free_pci_resources(if_ctx_t ctx)
1471 {
1472         struct igc_adapter *adapter = iflib_get_softc(ctx);
1473         struct igc_rx_queue *que = adapter->rx_queues;
1474         device_t dev = iflib_get_dev(ctx);
1475
1476         /* Release all MSI-X queue resources */
1477         if (adapter->intr_type == IFLIB_INTR_MSIX)
1478                 iflib_irq_free(ctx, &adapter->irq);
1479
1480         for (int i = 0; i < adapter->rx_num_queues; i++, que++) {
1481                 iflib_irq_free(ctx, &que->que_irq);
1482         }
1483
1484         if (adapter->memory != NULL) {
1485                 bus_release_resource(dev, SYS_RES_MEMORY,
1486                     rman_get_rid(adapter->memory), adapter->memory);
1487                 adapter->memory = NULL;
1488         }
1489
1490         if (adapter->flash != NULL) {
1491                 bus_release_resource(dev, SYS_RES_MEMORY,
1492                     rman_get_rid(adapter->flash), adapter->flash);
1493                 adapter->flash = NULL;
1494         }
1495
1496         if (adapter->ioport != NULL) {
1497                 bus_release_resource(dev, SYS_RES_IOPORT,
1498                     rman_get_rid(adapter->ioport), adapter->ioport);
1499                 adapter->ioport = NULL;
1500         }
1501 }
1502
1503 /* Set up MSI or MSI-X */
1504 static int
1505 igc_setup_msix(if_ctx_t ctx)
1506 {
1507         return (0);
1508 }
1509
1510 /*********************************************************************
1511  *
1512  *  Initialize the DMA Coalescing feature
1513  *
1514  **********************************************************************/
1515 static void
1516 igc_init_dmac(struct igc_adapter *adapter, u32 pba)
1517 {
1518         device_t        dev = adapter->dev;
1519         struct igc_hw *hw = &adapter->hw;
1520         u32             dmac, reg = ~IGC_DMACR_DMAC_EN;
1521         u16             hwm;
1522         u16             max_frame_size;
1523         int             status;
1524
1525         max_frame_size = adapter->shared->isc_max_frame_size;
1526
1527         if (adapter->dmac == 0) { /* Disabling it */
1528                 IGC_WRITE_REG(hw, IGC_DMACR, reg);
1529                 return;
1530         } else
1531                 device_printf(dev, "DMA Coalescing enabled\n");
1532
1533         /* Set starting threshold */
1534         IGC_WRITE_REG(hw, IGC_DMCTXTH, 0);
1535
1536         hwm = 64 * pba - max_frame_size / 16;
1537         if (hwm < 64 * (pba - 6))
1538                 hwm = 64 * (pba - 6);
1539         reg = IGC_READ_REG(hw, IGC_FCRTC);
1540         reg &= ~IGC_FCRTC_RTH_COAL_MASK;
1541         reg |= ((hwm << IGC_FCRTC_RTH_COAL_SHIFT)
1542                 & IGC_FCRTC_RTH_COAL_MASK);
1543         IGC_WRITE_REG(hw, IGC_FCRTC, reg);
1544
1545         dmac = pba - max_frame_size / 512;
1546         if (dmac < pba - 10)
1547                 dmac = pba - 10;
1548         reg = IGC_READ_REG(hw, IGC_DMACR);
1549         reg &= ~IGC_DMACR_DMACTHR_MASK;
1550         reg |= ((dmac << IGC_DMACR_DMACTHR_SHIFT)
1551                 & IGC_DMACR_DMACTHR_MASK);
1552
1553         /* transition to L0x or L1 if available..*/
1554         reg |= (IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK);
1555
1556         /* Check if status is 2.5Gb backplane connection
1557          * before configuration of watchdog timer, which is
1558          * in msec values in 12.8usec intervals
1559          * watchdog timer= msec values in 32usec intervals
1560          * for non 2.5Gb connection
1561          */
1562         status = IGC_READ_REG(hw, IGC_STATUS);
1563         if ((status & IGC_STATUS_2P5_SKU) &&
1564             (!(status & IGC_STATUS_2P5_SKU_OVER)))
1565                 reg |= ((adapter->dmac * 5) >> 6);
1566         else
1567                 reg |= (adapter->dmac >> 5);
1568
1569         IGC_WRITE_REG(hw, IGC_DMACR, reg);
1570
1571         IGC_WRITE_REG(hw, IGC_DMCRTRH, 0);
1572
1573         /* Set the interval before transition */
1574         reg = IGC_READ_REG(hw, IGC_DMCTLX);
1575         reg |= IGC_DMCTLX_DCFLUSH_DIS;
1576
1577         /*
1578         ** in 2.5Gb connection, TTLX unit is 0.4 usec
1579         ** which is 0x4*2 = 0xA. But delay is still 4 usec
1580         */
1581         status = IGC_READ_REG(hw, IGC_STATUS);
1582         if ((status & IGC_STATUS_2P5_SKU) &&
1583             (!(status & IGC_STATUS_2P5_SKU_OVER)))
1584                 reg |= 0xA;
1585         else
1586                 reg |= 0x4;
1587
1588         IGC_WRITE_REG(hw, IGC_DMCTLX, reg);
1589
1590         /* free space in tx packet buffer to wake from DMA coal */
1591         IGC_WRITE_REG(hw, IGC_DMCTXTH, (IGC_TXPBSIZE -
1592             (2 * max_frame_size)) >> 6);
1593
1594         /* make low power state decision controlled by DMA coal */
1595         reg = IGC_READ_REG(hw, IGC_PCIEMISC);
1596         reg &= ~IGC_PCIEMISC_LX_DECISION;
1597         IGC_WRITE_REG(hw, IGC_PCIEMISC, reg);
1598 }
1599
1600 /*********************************************************************
1601  *
1602  *  Initialize the hardware to a configuration as specified by the
1603  *  adapter structure.
1604  *
1605  **********************************************************************/
1606 static void
1607 igc_reset(if_ctx_t ctx)
1608 {
1609         device_t dev = iflib_get_dev(ctx);
1610         struct igc_adapter *adapter = iflib_get_softc(ctx);
1611         struct igc_hw *hw = &adapter->hw;
1612         u16 rx_buffer_size;
1613         u32 pba;
1614
1615         INIT_DEBUGOUT("igc_reset: begin");
1616         /* Let the firmware know the OS is in control */
1617         igc_get_hw_control(adapter);
1618
1619         /*
1620          * Packet Buffer Allocation (PBA)
1621          * Writing PBA sets the receive portion of the buffer
1622          * the remainder is used for the transmit buffer.
1623          */
1624         pba = IGC_PBA_34K;
1625
1626         INIT_DEBUGOUT1("igc_reset: pba=%dK",pba);
1627
1628         /*
1629          * These parameters control the automatic generation (Tx) and
1630          * response (Rx) to Ethernet PAUSE frames.
1631          * - High water mark should allow for at least two frames to be
1632          *   received after sending an XOFF.
1633          * - Low water mark works best when it is very near the high water mark.
1634          *   This allows the receiver to restart by sending XON when it has
1635          *   drained a bit. Here we use an arbitrary value of 1500 which will
1636          *   restart after one full frame is pulled from the buffer. There
1637          *   could be several smaller frames in the buffer and if so they will
1638          *   not trigger the XON until their total number reduces the buffer
1639          *   by 1500.
1640          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1641          */
1642         rx_buffer_size = (pba & 0xffff) << 10;
1643         hw->fc.high_water = rx_buffer_size -
1644             roundup2(adapter->hw.mac.max_frame_size, 1024);
1645         /* 16-byte granularity */
1646         hw->fc.low_water = hw->fc.high_water - 16;
1647
1648         if (adapter->fc) /* locally set flow control value? */
1649                 hw->fc.requested_mode = adapter->fc;
1650         else
1651                 hw->fc.requested_mode = igc_fc_full;
1652
1653         hw->fc.pause_time = IGC_FC_PAUSE_TIME;
1654
1655         hw->fc.send_xon = true;
1656
1657         /* Issue a global reset */
1658         igc_reset_hw(hw);
1659         IGC_WRITE_REG(hw, IGC_WUC, 0);
1660
1661         /* and a re-init */
1662         if (igc_init_hw(hw) < 0) {
1663                 device_printf(dev, "Hardware Initialization Failed\n");
1664                 return;
1665         }
1666
1667         /* Setup DMA Coalescing */
1668         igc_init_dmac(adapter, pba);
1669
1670         IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN);
1671         igc_get_phy_info(hw);
1672         igc_check_for_link(hw);
1673 }
1674
1675 /*
1676  * Initialise the RSS mapping for NICs that support multiple transmit/
1677  * receive rings.
1678  */
1679
1680 #define RSSKEYLEN 10
1681 static void
1682 igc_initialize_rss_mapping(struct igc_adapter *adapter)
1683 {
1684         struct igc_hw *hw = &adapter->hw;
1685         int i;
1686         int queue_id;
1687         u32 reta;
1688         u32 rss_key[RSSKEYLEN], mrqc, shift = 0;
1689
1690         /*
1691          * The redirection table controls which destination
1692          * queue each bucket redirects traffic to.
1693          * Each DWORD represents four queues, with the LSB
1694          * being the first queue in the DWORD.
1695          *
1696          * This just allocates buckets to queues using round-robin
1697          * allocation.
1698          *
1699          * NOTE: It Just Happens to line up with the default
1700          * RSS allocation method.
1701          */
1702
1703         /* Warning FM follows */
1704         reta = 0;
1705         for (i = 0; i < 128; i++) {
1706 #ifdef RSS
1707                 queue_id = rss_get_indirection_to_bucket(i);
1708                 /*
1709                  * If we have more queues than buckets, we'll
1710                  * end up mapping buckets to a subset of the
1711                  * queues.
1712                  *
1713                  * If we have more buckets than queues, we'll
1714                  * end up instead assigning multiple buckets
1715                  * to queues.
1716                  *
1717                  * Both are suboptimal, but we need to handle
1718                  * the case so we don't go out of bounds
1719                  * indexing arrays and such.
1720                  */
1721                 queue_id = queue_id % adapter->rx_num_queues;
1722 #else
1723                 queue_id = (i % adapter->rx_num_queues);
1724 #endif
1725                 /* Adjust if required */
1726                 queue_id = queue_id << shift;
1727
1728                 /*
1729                  * The low 8 bits are for hash value (n+0);
1730                  * The next 8 bits are for hash value (n+1), etc.
1731                  */
1732                 reta = reta >> 8;
1733                 reta = reta | ( ((uint32_t) queue_id) << 24);
1734                 if ((i & 3) == 3) {
1735                         IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta);
1736                         reta = 0;
1737                 }
1738         }
1739
1740         /* Now fill in hash table */
1741
1742         /*
1743          * MRQC: Multiple Receive Queues Command
1744          * Set queuing to RSS control, number depends on the device.
1745          */
1746         mrqc = IGC_MRQC_ENABLE_RSS_4Q;
1747
1748 #ifdef RSS
1749         /* XXX ew typecasting */
1750         rss_getkey((uint8_t *) &rss_key);
1751 #else
1752         arc4rand(&rss_key, sizeof(rss_key), 0);
1753 #endif
1754         for (i = 0; i < RSSKEYLEN; i++)
1755                 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]);
1756
1757         /*
1758          * Configure the RSS fields to hash upon.
1759          */
1760         mrqc |= (IGC_MRQC_RSS_FIELD_IPV4 |
1761             IGC_MRQC_RSS_FIELD_IPV4_TCP);
1762         mrqc |= (IGC_MRQC_RSS_FIELD_IPV6 |
1763             IGC_MRQC_RSS_FIELD_IPV6_TCP);
1764         mrqc |=( IGC_MRQC_RSS_FIELD_IPV4_UDP |
1765             IGC_MRQC_RSS_FIELD_IPV6_UDP);
1766         mrqc |=( IGC_MRQC_RSS_FIELD_IPV6_UDP_EX |
1767             IGC_MRQC_RSS_FIELD_IPV6_TCP_EX);
1768
1769         IGC_WRITE_REG(hw, IGC_MRQC, mrqc);
1770 }
1771
1772 /*********************************************************************
1773  *
1774  *  Setup networking device structure and register interface media.
1775  *
1776  **********************************************************************/
1777 static int
1778 igc_setup_interface(if_ctx_t ctx)
1779 {
1780         struct ifnet *ifp = iflib_get_ifp(ctx);
1781         struct igc_adapter *adapter = iflib_get_softc(ctx);
1782         if_softc_ctx_t scctx = adapter->shared;
1783
1784         INIT_DEBUGOUT("igc_setup_interface: begin");
1785
1786         /* Single Queue */
1787         if (adapter->tx_num_queues == 1) {
1788                 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1);
1789                 if_setsendqready(ifp);
1790         }
1791
1792         /*
1793          * Specify the media types supported by this adapter and register
1794          * callbacks to update media and link information
1795          */
1796         ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
1797         ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL);
1798         ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL);
1799         ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL);
1800         ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
1801         ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1802         ifmedia_add(adapter->media, IFM_ETHER | IFM_2500_T, 0, NULL);
1803
1804         ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1805         ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
1806         return (0);
1807 }
1808
1809 static int
1810 igc_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets)
1811 {
1812         struct igc_adapter *adapter = iflib_get_softc(ctx);
1813         if_softc_ctx_t scctx = adapter->shared;
1814         int error = IGC_SUCCESS;
1815         struct igc_tx_queue *que;
1816         int i, j;
1817
1818         MPASS(adapter->tx_num_queues > 0);
1819         MPASS(adapter->tx_num_queues == ntxqsets);
1820
1821         /* First allocate the top level queue structs */
1822         if (!(adapter->tx_queues =
1823             (struct igc_tx_queue *) malloc(sizeof(struct igc_tx_queue) *
1824             adapter->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
1825                 device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n");
1826                 return(ENOMEM);
1827         }
1828
1829         for (i = 0, que = adapter->tx_queues; i < adapter->tx_num_queues; i++, que++) {
1830                 /* Set up some basics */
1831
1832                 struct tx_ring *txr = &que->txr;
1833                 txr->adapter = que->adapter = adapter;
1834                 que->me = txr->me =  i;
1835
1836                 /* Allocate report status array */
1837                 if (!(txr->tx_rsq = (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) {
1838                         device_printf(iflib_get_dev(ctx), "failed to allocate rs_idxs memory\n");
1839                         error = ENOMEM;
1840                         goto fail;
1841                 }
1842                 for (j = 0; j < scctx->isc_ntxd[0]; j++)
1843                         txr->tx_rsq[j] = QIDX_INVALID;
1844                 /* get the virtual and physical address of the hardware queues */
1845                 txr->tx_base = (struct igc_tx_desc *)vaddrs[i*ntxqs];
1846                 txr->tx_paddr = paddrs[i*ntxqs];
1847         }
1848
1849         if (bootverbose)
1850                 device_printf(iflib_get_dev(ctx),
1851                     "allocated for %d tx_queues\n", adapter->tx_num_queues);
1852         return (0);
1853 fail:
1854         igc_if_queues_free(ctx);
1855         return (error);
1856 }
1857
1858 static int
1859 igc_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets)
1860 {
1861         struct igc_adapter *adapter = iflib_get_softc(ctx);
1862         int error = IGC_SUCCESS;
1863         struct igc_rx_queue *que;
1864         int i;
1865
1866         MPASS(adapter->rx_num_queues > 0);
1867         MPASS(adapter->rx_num_queues == nrxqsets);
1868
1869         /* First allocate the top level queue structs */
1870         if (!(adapter->rx_queues =
1871             (struct igc_rx_queue *) malloc(sizeof(struct igc_rx_queue) *
1872             adapter->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
1873                 device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n");
1874                 error = ENOMEM;
1875                 goto fail;
1876         }
1877
1878         for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
1879                 /* Set up some basics */
1880                 struct rx_ring *rxr = &que->rxr;
1881                 rxr->adapter = que->adapter = adapter;
1882                 rxr->que = que;
1883                 que->me = rxr->me =  i;
1884
1885                 /* get the virtual and physical address of the hardware queues */
1886                 rxr->rx_base = (union igc_rx_desc_extended *)vaddrs[i*nrxqs];
1887                 rxr->rx_paddr = paddrs[i*nrxqs];
1888         }
1889
1890         if (bootverbose)
1891                 device_printf(iflib_get_dev(ctx),
1892                     "allocated for %d rx_queues\n", adapter->rx_num_queues);
1893
1894         return (0);
1895 fail:
1896         igc_if_queues_free(ctx);
1897         return (error);
1898 }
1899
1900 static void
1901 igc_if_queues_free(if_ctx_t ctx)
1902 {
1903         struct igc_adapter *adapter = iflib_get_softc(ctx);
1904         struct igc_tx_queue *tx_que = adapter->tx_queues;
1905         struct igc_rx_queue *rx_que = adapter->rx_queues;
1906
1907         if (tx_que != NULL) {
1908                 for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) {
1909                         struct tx_ring *txr = &tx_que->txr;
1910                         if (txr->tx_rsq == NULL)
1911                                 break;
1912
1913                         free(txr->tx_rsq, M_DEVBUF);
1914                         txr->tx_rsq = NULL;
1915                 }
1916                 free(adapter->tx_queues, M_DEVBUF);
1917                 adapter->tx_queues = NULL;
1918         }
1919
1920         if (rx_que != NULL) {
1921                 free(adapter->rx_queues, M_DEVBUF);
1922                 adapter->rx_queues = NULL;
1923         }
1924
1925         igc_release_hw_control(adapter);
1926
1927         if (adapter->mta != NULL) {
1928                 free(adapter->mta, M_DEVBUF);
1929         }
1930 }
1931
1932 /*********************************************************************
1933  *
1934  *  Enable transmit unit.
1935  *
1936  **********************************************************************/
1937 static void
1938 igc_initialize_transmit_unit(if_ctx_t ctx)
1939 {
1940         struct igc_adapter *adapter = iflib_get_softc(ctx);
1941         if_softc_ctx_t scctx = adapter->shared;
1942         struct igc_tx_queue *que;
1943         struct tx_ring  *txr;
1944         struct igc_hw   *hw = &adapter->hw;
1945         u32 tctl, txdctl = 0;
1946
1947         INIT_DEBUGOUT("igc_initialize_transmit_unit: begin");
1948
1949         for (int i = 0; i < adapter->tx_num_queues; i++, txr++) {
1950                 u64 bus_addr;
1951                 caddr_t offp, endp;
1952
1953                 que = &adapter->tx_queues[i];
1954                 txr = &que->txr;
1955                 bus_addr = txr->tx_paddr;
1956
1957                 /* Clear checksum offload context. */
1958                 offp = (caddr_t)&txr->csum_flags;
1959                 endp = (caddr_t)(txr + 1);
1960                 bzero(offp, endp - offp);
1961
1962                 /* Base and Len of TX Ring */
1963                 IGC_WRITE_REG(hw, IGC_TDLEN(i),
1964                     scctx->isc_ntxd[0] * sizeof(struct igc_tx_desc));
1965                 IGC_WRITE_REG(hw, IGC_TDBAH(i),
1966                     (u32)(bus_addr >> 32));
1967                 IGC_WRITE_REG(hw, IGC_TDBAL(i),
1968                     (u32)bus_addr);
1969                 /* Init the HEAD/TAIL indices */
1970                 IGC_WRITE_REG(hw, IGC_TDT(i), 0);
1971                 IGC_WRITE_REG(hw, IGC_TDH(i), 0);
1972
1973                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
1974                     IGC_READ_REG(&adapter->hw, IGC_TDBAL(i)),
1975                     IGC_READ_REG(&adapter->hw, IGC_TDLEN(i)));
1976
1977                 txdctl = 0; /* clear txdctl */
1978                 txdctl |= 0x1f; /* PTHRESH */
1979                 txdctl |= 1 << 8; /* HTHRESH */
1980                 txdctl |= 1 << 16;/* WTHRESH */
1981                 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */
1982                 txdctl |= IGC_TXDCTL_GRAN;
1983                 txdctl |= 1 << 25; /* LWTHRESH */
1984
1985                 IGC_WRITE_REG(hw, IGC_TXDCTL(i), txdctl);
1986         }
1987
1988         /* Program the Transmit Control Register */
1989         tctl = IGC_READ_REG(&adapter->hw, IGC_TCTL);
1990         tctl &= ~IGC_TCTL_CT;
1991         tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN |
1992                    (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT));
1993
1994         /* This write will effectively turn on the transmit unit. */
1995         IGC_WRITE_REG(&adapter->hw, IGC_TCTL, tctl);
1996 }
1997
1998 /*********************************************************************
1999  *
2000  *  Enable receive unit.
2001  *
2002  **********************************************************************/
2003
2004 static void
2005 igc_initialize_receive_unit(if_ctx_t ctx)
2006 {
2007         struct igc_adapter *adapter = iflib_get_softc(ctx);
2008         if_softc_ctx_t scctx = adapter->shared;
2009         struct ifnet *ifp = iflib_get_ifp(ctx);
2010         struct igc_hw   *hw = &adapter->hw;
2011         struct igc_rx_queue *que;
2012         int i;
2013         u32 psize, rctl, rxcsum, srrctl = 0;
2014
2015         INIT_DEBUGOUT("igc_initialize_receive_units: begin");
2016
2017         /*
2018          * Make sure receives are disabled while setting
2019          * up the descriptor ring
2020          */
2021         rctl = IGC_READ_REG(hw, IGC_RCTL);
2022         IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN);
2023
2024         /* Setup the Receive Control Register */
2025         rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
2026         rctl |= IGC_RCTL_EN | IGC_RCTL_BAM |
2027             IGC_RCTL_LBM_NO | IGC_RCTL_RDMTS_HALF |
2028             (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
2029
2030         /* Do not store bad packets */
2031         rctl &= ~IGC_RCTL_SBP;
2032
2033         /* Enable Long Packet receive */
2034         if (if_getmtu(ifp) > ETHERMTU)
2035                 rctl |= IGC_RCTL_LPE;
2036         else
2037                 rctl &= ~IGC_RCTL_LPE;
2038
2039         /* Strip the CRC */
2040         if (!igc_disable_crc_stripping)
2041                 rctl |= IGC_RCTL_SECRC;
2042
2043         /*
2044          * Set the interrupt throttling rate. Value is calculated
2045          * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
2046          */
2047         IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR);
2048
2049         rxcsum = IGC_READ_REG(hw, IGC_RXCSUM);
2050         if (if_getcapenable(ifp) & IFCAP_RXCSUM) {
2051                 rxcsum |= IGC_RXCSUM_CRCOFL;
2052                 if (adapter->tx_num_queues > 1)
2053                         rxcsum |= IGC_RXCSUM_PCSD;
2054                 else
2055                         rxcsum |= IGC_RXCSUM_IPPCSE;
2056         } else {
2057                 if (adapter->tx_num_queues > 1)
2058                         rxcsum |= IGC_RXCSUM_PCSD;
2059                 else
2060                         rxcsum &= ~IGC_RXCSUM_TUOFL;
2061         }
2062         IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum);
2063
2064         if (adapter->rx_num_queues > 1)
2065                 igc_initialize_rss_mapping(adapter);
2066
2067         if (if_getmtu(ifp) > ETHERMTU) {
2068                 /* Set maximum packet len */
2069                 if (adapter->rx_mbuf_sz <= 4096) {
2070                         srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
2071                         rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX;
2072                 } else if (adapter->rx_mbuf_sz > 4096) {
2073                         srrctl |= 8192 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
2074                         rctl |= IGC_RCTL_SZ_8192 | IGC_RCTL_BSEX;
2075                 }
2076                 psize = scctx->isc_max_frame_size;
2077                 /* are we on a vlan? */
2078                 if (ifp->if_vlantrunk != NULL)
2079                         psize += VLAN_TAG_SIZE;
2080                 IGC_WRITE_REG(&adapter->hw, IGC_RLPML, psize);
2081         } else {
2082                 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
2083                 rctl |= IGC_RCTL_SZ_2048;
2084         }
2085
2086         /*
2087          * If TX flow control is disabled and there's >1 queue defined,
2088          * enable DROP.
2089          *
2090          * This drops frames rather than hanging the RX MAC for all queues.
2091          */
2092         if ((adapter->rx_num_queues > 1) &&
2093             (adapter->fc == igc_fc_none ||
2094              adapter->fc == igc_fc_rx_pause)) {
2095                 srrctl |= IGC_SRRCTL_DROP_EN;
2096         }
2097
2098         /* Setup the Base and Length of the Rx Descriptor Rings */
2099         for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) {
2100                 struct rx_ring *rxr = &que->rxr;
2101                 u64 bus_addr = rxr->rx_paddr;
2102                 u32 rxdctl;
2103
2104 #ifdef notyet
2105                 /* Configure for header split? -- ignore for now */
2106                 rxr->hdr_split = igc_header_split;
2107 #else
2108                 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
2109 #endif
2110
2111                 IGC_WRITE_REG(hw, IGC_RDLEN(i),
2112                               scctx->isc_nrxd[0] * sizeof(struct igc_rx_desc));
2113                 IGC_WRITE_REG(hw, IGC_RDBAH(i),
2114                               (uint32_t)(bus_addr >> 32));
2115                 IGC_WRITE_REG(hw, IGC_RDBAL(i),
2116                               (uint32_t)bus_addr);
2117                 IGC_WRITE_REG(hw, IGC_SRRCTL(i), srrctl);
2118                 /* Setup the Head and Tail Descriptor Pointers */
2119                 IGC_WRITE_REG(hw, IGC_RDH(i), 0);
2120                 IGC_WRITE_REG(hw, IGC_RDT(i), 0);
2121                 /* Enable this Queue */
2122                 rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(i));
2123                 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
2124                 rxdctl &= 0xFFF00000;
2125                 rxdctl |= IGC_RX_PTHRESH;
2126                 rxdctl |= IGC_RX_HTHRESH << 8;
2127                 rxdctl |= IGC_RX_WTHRESH << 16;
2128                 IGC_WRITE_REG(hw, IGC_RXDCTL(i), rxdctl);
2129         }
2130
2131         /* Make sure VLAN Filters are off */
2132         rctl &= ~IGC_RCTL_VFE;
2133
2134         /* Write out the settings */
2135         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
2136
2137         return;
2138 }
2139
2140 static void
2141 igc_if_vlan_register(if_ctx_t ctx, u16 vtag)
2142 {
2143         struct igc_adapter *adapter = iflib_get_softc(ctx);
2144         u32 index, bit;
2145
2146         index = (vtag >> 5) & 0x7F;
2147         bit = vtag & 0x1F;
2148         adapter->shadow_vfta[index] |= (1 << bit);
2149         ++adapter->num_vlans;
2150 }
2151
2152 static void
2153 igc_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
2154 {
2155         struct igc_adapter *adapter = iflib_get_softc(ctx);
2156         u32 index, bit;
2157
2158         index = (vtag >> 5) & 0x7F;
2159         bit = vtag & 0x1F;
2160         adapter->shadow_vfta[index] &= ~(1 << bit);
2161         --adapter->num_vlans;
2162 }
2163
2164 static void
2165 igc_setup_vlan_hw_support(struct igc_adapter *adapter)
2166 {
2167         struct igc_hw *hw = &adapter->hw;
2168         u32 reg;
2169
2170         /*
2171          * We get here thru init_locked, meaning
2172          * a soft reset, this has already cleared
2173          * the VFTA and other state, so if there
2174          * have been no vlan's registered do nothing.
2175          */
2176         if (adapter->num_vlans == 0)
2177                 return;
2178
2179         /*
2180          * A soft reset zero's out the VFTA, so
2181          * we need to repopulate it now.
2182          */
2183         for (int i = 0; i < IGC_VFTA_SIZE; i++)
2184                 if (adapter->shadow_vfta[i] != 0)
2185                         IGC_WRITE_REG_ARRAY(hw, IGC_VFTA,
2186                             i, adapter->shadow_vfta[i]);
2187
2188         reg = IGC_READ_REG(hw, IGC_CTRL);
2189         reg |= IGC_CTRL_VME;
2190         IGC_WRITE_REG(hw, IGC_CTRL, reg);
2191
2192         /* Enable the Filter Table */
2193         reg = IGC_READ_REG(hw, IGC_RCTL);
2194         reg &= ~IGC_RCTL_CFIEN;
2195         reg |= IGC_RCTL_VFE;
2196         IGC_WRITE_REG(hw, IGC_RCTL, reg);
2197 }
2198
2199 static void
2200 igc_if_intr_enable(if_ctx_t ctx)
2201 {
2202         struct igc_adapter *adapter = iflib_get_softc(ctx);
2203         struct igc_hw *hw = &adapter->hw;
2204         u32 mask;
2205
2206         if (__predict_true(adapter->intr_type == IFLIB_INTR_MSIX)) {
2207                 mask = (adapter->que_mask | adapter->link_mask);
2208                 IGC_WRITE_REG(hw, IGC_EIAC, mask);
2209                 IGC_WRITE_REG(hw, IGC_EIAM, mask);
2210                 IGC_WRITE_REG(hw, IGC_EIMS, mask);
2211                 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC);
2212         } else
2213                 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK);
2214         IGC_WRITE_FLUSH(hw);
2215 }
2216
2217 static void
2218 igc_if_intr_disable(if_ctx_t ctx)
2219 {
2220         struct igc_adapter *adapter = iflib_get_softc(ctx);
2221         struct igc_hw *hw = &adapter->hw;
2222
2223         if (__predict_true(adapter->intr_type == IFLIB_INTR_MSIX)) {
2224                 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff);
2225                 IGC_WRITE_REG(hw, IGC_EIAC, 0);
2226         }
2227         IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff);
2228         IGC_WRITE_FLUSH(hw);
2229 }
2230
2231 /*
2232  * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
2233  * For ASF and Pass Through versions of f/w this means
2234  * that the driver is loaded. For AMT version type f/w
2235  * this means that the network i/f is open.
2236  */
2237 static void
2238 igc_get_hw_control(struct igc_adapter *adapter)
2239 {
2240         u32 ctrl_ext;
2241
2242         if (adapter->vf_ifp)
2243                 return;
2244
2245         ctrl_ext = IGC_READ_REG(&adapter->hw, IGC_CTRL_EXT);
2246         IGC_WRITE_REG(&adapter->hw, IGC_CTRL_EXT,
2247             ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
2248 }
2249
2250 /*
2251  * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
2252  * For ASF and Pass Through versions of f/w this means that
2253  * the driver is no longer loaded. For AMT versions of the
2254  * f/w this means that the network i/f is closed.
2255  */
2256 static void
2257 igc_release_hw_control(struct igc_adapter *adapter)
2258 {
2259         u32 ctrl_ext;
2260
2261         ctrl_ext = IGC_READ_REG(&adapter->hw, IGC_CTRL_EXT);
2262         IGC_WRITE_REG(&adapter->hw, IGC_CTRL_EXT,
2263             ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
2264         return;
2265 }
2266
2267 static int
2268 igc_is_valid_ether_addr(u8 *addr)
2269 {
2270         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
2271
2272         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
2273                 return (false);
2274         }
2275
2276         return (true);
2277 }
2278
2279 /*
2280 ** Parse the interface capabilities with regard
2281 ** to both system management and wake-on-lan for
2282 ** later use.
2283 */
2284 static void
2285 igc_get_wakeup(if_ctx_t ctx)
2286 {
2287         struct igc_adapter *adapter = iflib_get_softc(ctx);
2288         u16 eeprom_data = 0, apme_mask;
2289
2290         apme_mask = IGC_WUC_APME;
2291         eeprom_data = IGC_READ_REG(&adapter->hw, IGC_WUC);
2292
2293         if (eeprom_data & apme_mask)
2294                 adapter->wol = IGC_WUFC_LNKC;
2295 }
2296
2297
2298 /*
2299  * Enable PCI Wake On Lan capability
2300  */
2301 static void
2302 igc_enable_wakeup(if_ctx_t ctx)
2303 {
2304         struct igc_adapter *adapter = iflib_get_softc(ctx);
2305         device_t dev = iflib_get_dev(ctx);
2306         if_t ifp = iflib_get_ifp(ctx);
2307         int error = 0;
2308         u32 pmc, ctrl, rctl;
2309         u16 status;
2310
2311         if (pci_find_cap(dev, PCIY_PMG, &pmc) != 0)
2312                 return;
2313
2314         /*
2315          * Determine type of Wakeup: note that wol
2316          * is set with all bits on by default.
2317          */
2318         if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0)
2319                 adapter->wol &= ~IGC_WUFC_MAG;
2320
2321         if ((if_getcapenable(ifp) & IFCAP_WOL_UCAST) == 0)
2322                 adapter->wol &= ~IGC_WUFC_EX;
2323
2324         if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0)
2325                 adapter->wol &= ~IGC_WUFC_MC;
2326         else {
2327                 rctl = IGC_READ_REG(&adapter->hw, IGC_RCTL);
2328                 rctl |= IGC_RCTL_MPE;
2329                 IGC_WRITE_REG(&adapter->hw, IGC_RCTL, rctl);
2330         }
2331
2332         if (!(adapter->wol & (IGC_WUFC_EX | IGC_WUFC_MAG | IGC_WUFC_MC)))
2333                 goto pme;
2334
2335         /* Advertise the wakeup capability */
2336         ctrl = IGC_READ_REG(&adapter->hw, IGC_CTRL);
2337         ctrl |= IGC_CTRL_ADVD3WUC;
2338         IGC_WRITE_REG(&adapter->hw, IGC_CTRL, ctrl);
2339
2340         /* Enable wakeup by the MAC */
2341         IGC_WRITE_REG(&adapter->hw, IGC_WUC, IGC_WUC_PME_EN);
2342         IGC_WRITE_REG(&adapter->hw, IGC_WUFC, adapter->wol);
2343
2344 pme:
2345         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
2346         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
2347         if (!error && (if_getcapenable(ifp) & IFCAP_WOL))
2348                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
2349         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
2350
2351         return;
2352 }
2353
2354 /**********************************************************************
2355  *
2356  *  Update the board statistics counters.
2357  *
2358  **********************************************************************/
2359 static void
2360 igc_update_stats_counters(struct igc_adapter *adapter)
2361 {
2362         u64 prev_xoffrxc = adapter->stats.xoffrxc;
2363
2364         adapter->stats.crcerrs += IGC_READ_REG(&adapter->hw, IGC_CRCERRS);
2365         adapter->stats.mpc += IGC_READ_REG(&adapter->hw, IGC_MPC);
2366         adapter->stats.scc += IGC_READ_REG(&adapter->hw, IGC_SCC);
2367         adapter->stats.ecol += IGC_READ_REG(&adapter->hw, IGC_ECOL);
2368
2369         adapter->stats.mcc += IGC_READ_REG(&adapter->hw, IGC_MCC);
2370         adapter->stats.latecol += IGC_READ_REG(&adapter->hw, IGC_LATECOL);
2371         adapter->stats.colc += IGC_READ_REG(&adapter->hw, IGC_COLC);
2372         adapter->stats.colc += IGC_READ_REG(&adapter->hw, IGC_RERC);
2373         adapter->stats.dc += IGC_READ_REG(&adapter->hw, IGC_DC);
2374         adapter->stats.rlec += IGC_READ_REG(&adapter->hw, IGC_RLEC);
2375         adapter->stats.xonrxc += IGC_READ_REG(&adapter->hw, IGC_XONRXC);
2376         adapter->stats.xontxc += IGC_READ_REG(&adapter->hw, IGC_XONTXC);
2377         adapter->stats.xoffrxc += IGC_READ_REG(&adapter->hw, IGC_XOFFRXC);
2378         /*
2379          * For watchdog management we need to know if we have been
2380          * paused during the last interval, so capture that here.
2381          */
2382         if (adapter->stats.xoffrxc != prev_xoffrxc)
2383                 adapter->shared->isc_pause_frames = 1;
2384         adapter->stats.xofftxc += IGC_READ_REG(&adapter->hw, IGC_XOFFTXC);
2385         adapter->stats.fcruc += IGC_READ_REG(&adapter->hw, IGC_FCRUC);
2386         adapter->stats.prc64 += IGC_READ_REG(&adapter->hw, IGC_PRC64);
2387         adapter->stats.prc127 += IGC_READ_REG(&adapter->hw, IGC_PRC127);
2388         adapter->stats.prc255 += IGC_READ_REG(&adapter->hw, IGC_PRC255);
2389         adapter->stats.prc511 += IGC_READ_REG(&adapter->hw, IGC_PRC511);
2390         adapter->stats.prc1023 += IGC_READ_REG(&adapter->hw, IGC_PRC1023);
2391         adapter->stats.prc1522 += IGC_READ_REG(&adapter->hw, IGC_PRC1522);
2392         adapter->stats.tlpic += IGC_READ_REG(&adapter->hw, IGC_TLPIC);
2393         adapter->stats.rlpic += IGC_READ_REG(&adapter->hw, IGC_RLPIC);
2394         adapter->stats.gprc += IGC_READ_REG(&adapter->hw, IGC_GPRC);
2395         adapter->stats.bprc += IGC_READ_REG(&adapter->hw, IGC_BPRC);
2396         adapter->stats.mprc += IGC_READ_REG(&adapter->hw, IGC_MPRC);
2397         adapter->stats.gptc += IGC_READ_REG(&adapter->hw, IGC_GPTC);
2398
2399         /* For the 64-bit byte counters the low dword must be read first. */
2400         /* Both registers clear on the read of the high dword */
2401
2402         adapter->stats.gorc += IGC_READ_REG(&adapter->hw, IGC_GORCL) +
2403             ((u64)IGC_READ_REG(&adapter->hw, IGC_GORCH) << 32);
2404         adapter->stats.gotc += IGC_READ_REG(&adapter->hw, IGC_GOTCL) +
2405             ((u64)IGC_READ_REG(&adapter->hw, IGC_GOTCH) << 32);
2406
2407         adapter->stats.rnbc += IGC_READ_REG(&adapter->hw, IGC_RNBC);
2408         adapter->stats.ruc += IGC_READ_REG(&adapter->hw, IGC_RUC);
2409         adapter->stats.rfc += IGC_READ_REG(&adapter->hw, IGC_RFC);
2410         adapter->stats.roc += IGC_READ_REG(&adapter->hw, IGC_ROC);
2411         adapter->stats.rjc += IGC_READ_REG(&adapter->hw, IGC_RJC);
2412
2413         adapter->stats.tor += IGC_READ_REG(&adapter->hw, IGC_TORH);
2414         adapter->stats.tot += IGC_READ_REG(&adapter->hw, IGC_TOTH);
2415
2416         adapter->stats.tpr += IGC_READ_REG(&adapter->hw, IGC_TPR);
2417         adapter->stats.tpt += IGC_READ_REG(&adapter->hw, IGC_TPT);
2418         adapter->stats.ptc64 += IGC_READ_REG(&adapter->hw, IGC_PTC64);
2419         adapter->stats.ptc127 += IGC_READ_REG(&adapter->hw, IGC_PTC127);
2420         adapter->stats.ptc255 += IGC_READ_REG(&adapter->hw, IGC_PTC255);
2421         adapter->stats.ptc511 += IGC_READ_REG(&adapter->hw, IGC_PTC511);
2422         adapter->stats.ptc1023 += IGC_READ_REG(&adapter->hw, IGC_PTC1023);
2423         adapter->stats.ptc1522 += IGC_READ_REG(&adapter->hw, IGC_PTC1522);
2424         adapter->stats.mptc += IGC_READ_REG(&adapter->hw, IGC_MPTC);
2425         adapter->stats.bptc += IGC_READ_REG(&adapter->hw, IGC_BPTC);
2426
2427         /* Interrupt Counts */
2428         adapter->stats.iac += IGC_READ_REG(&adapter->hw, IGC_IAC);
2429         adapter->stats.rxdmtc += IGC_READ_REG(&adapter->hw, IGC_RXDMTC);
2430
2431         adapter->stats.algnerrc += IGC_READ_REG(&adapter->hw, IGC_ALGNERRC);
2432         adapter->stats.tncrs += IGC_READ_REG(&adapter->hw, IGC_TNCRS);
2433         adapter->stats.htdpmc += IGC_READ_REG(&adapter->hw, IGC_HTDPMC);
2434         adapter->stats.tsctc += IGC_READ_REG(&adapter->hw, IGC_TSCTC);
2435 }
2436
2437 static uint64_t
2438 igc_if_get_counter(if_ctx_t ctx, ift_counter cnt)
2439 {
2440         struct igc_adapter *adapter = iflib_get_softc(ctx);
2441         struct ifnet *ifp = iflib_get_ifp(ctx);
2442
2443         switch (cnt) {
2444         case IFCOUNTER_COLLISIONS:
2445                 return (adapter->stats.colc);
2446         case IFCOUNTER_IERRORS:
2447                 return (adapter->dropped_pkts + adapter->stats.rxerrc +
2448                     adapter->stats.crcerrs + adapter->stats.algnerrc +
2449                     adapter->stats.ruc + adapter->stats.roc +
2450                     adapter->stats.mpc + adapter->stats.htdpmc);
2451         case IFCOUNTER_OERRORS:
2452                 return (adapter->stats.ecol + adapter->stats.latecol +
2453                     adapter->watchdog_events);
2454         default:
2455                 return (if_get_counter_default(ifp, cnt));
2456         }
2457 }
2458
2459 /* igc_if_needs_restart - Tell iflib when the driver needs to be reinitialized
2460  * @ctx: iflib context
2461  * @event: event code to check
2462  *
2463  * Defaults to returning true for unknown events.
2464  *
2465  * @returns true if iflib needs to reinit the interface
2466  */
2467 static bool
2468 igc_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event)
2469 {
2470         switch (event) {
2471         case IFLIB_RESTART_VLAN_CONFIG:
2472         default:
2473                 return (true);
2474         }
2475 }
2476
2477 /* Export a single 32-bit register via a read-only sysctl. */
2478 static int
2479 igc_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
2480 {
2481         struct igc_adapter *adapter;
2482         u_int val;
2483
2484         adapter = oidp->oid_arg1;
2485         val = IGC_READ_REG(&adapter->hw, oidp->oid_arg2);
2486         return (sysctl_handle_int(oidp, &val, 0, req));
2487 }
2488
2489 /*
2490  * Add sysctl variables, one per statistic, to the system.
2491  */
2492 static void
2493 igc_add_hw_stats(struct igc_adapter *adapter)
2494 {
2495         device_t dev = iflib_get_dev(adapter->ctx);
2496         struct igc_tx_queue *tx_que = adapter->tx_queues;
2497         struct igc_rx_queue *rx_que = adapter->rx_queues;
2498
2499         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
2500         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
2501         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
2502         struct igc_hw_stats *stats = &adapter->stats;
2503
2504         struct sysctl_oid *stat_node, *queue_node, *int_node;
2505         struct sysctl_oid_list *stat_list, *queue_list, *int_list;
2506
2507 #define QUEUE_NAME_LEN 32
2508         char namebuf[QUEUE_NAME_LEN];
2509
2510         /* Driver Statistics */
2511         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
2512                         CTLFLAG_RD, &adapter->dropped_pkts,
2513                         "Driver dropped packets");
2514         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
2515                         CTLFLAG_RD, &adapter->link_irq,
2516                         "Link MSI-X IRQ Handled");
2517         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
2518                         CTLFLAG_RD, &adapter->rx_overruns,
2519                         "RX overruns");
2520         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
2521                         CTLFLAG_RD, &adapter->watchdog_events,
2522                         "Watchdog timeouts");
2523         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
2524             CTLTYPE_UINT | CTLFLAG_RD,
2525             adapter, IGC_CTRL, igc_sysctl_reg_handler, "IU",
2526             "Device Control Register");
2527         SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
2528             CTLTYPE_UINT | CTLFLAG_RD,
2529             adapter, IGC_RCTL, igc_sysctl_reg_handler, "IU",
2530             "Receiver Control Register");
2531         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
2532                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
2533                         "Flow Control High Watermark");
2534         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
2535                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
2536                         "Flow Control Low Watermark");
2537
2538         for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) {
2539                 struct tx_ring *txr = &tx_que->txr;
2540                 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i);
2541                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2542                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Name");
2543                 queue_list = SYSCTL_CHILDREN(queue_node);
2544
2545                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
2546                     CTLTYPE_UINT | CTLFLAG_RD, adapter,
2547                     IGC_TDH(txr->me), igc_sysctl_reg_handler, "IU",
2548                     "Transmit Descriptor Head");
2549                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
2550                     CTLTYPE_UINT | CTLFLAG_RD, adapter,
2551                     IGC_TDT(txr->me), igc_sysctl_reg_handler, "IU",
2552                     "Transmit Descriptor Tail");
2553                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
2554                                 CTLFLAG_RD, &txr->tx_irq,
2555                                 "Queue MSI-X Transmit Interrupts");
2556         }
2557
2558         for (int j = 0; j < adapter->rx_num_queues; j++, rx_que++) {
2559                 struct rx_ring *rxr = &rx_que->rxr;
2560                 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j);
2561                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2562                     CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Name");
2563                 queue_list = SYSCTL_CHILDREN(queue_node);
2564
2565                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
2566                     CTLTYPE_UINT | CTLFLAG_RD, adapter,
2567                     IGC_RDH(rxr->me), igc_sysctl_reg_handler, "IU",
2568                     "Receive Descriptor Head");
2569                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
2570                     CTLTYPE_UINT | CTLFLAG_RD, adapter,
2571                     IGC_RDT(rxr->me), igc_sysctl_reg_handler, "IU",
2572                     "Receive Descriptor Tail");
2573                 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
2574                                 CTLFLAG_RD, &rxr->rx_irq,
2575                                 "Queue MSI-X Receive Interrupts");
2576         }
2577
2578         /* MAC stats get their own sub node */
2579
2580         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
2581             CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics");
2582         stat_list = SYSCTL_CHILDREN(stat_node);
2583
2584         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
2585                         CTLFLAG_RD, &stats->ecol,
2586                         "Excessive collisions");
2587         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
2588                         CTLFLAG_RD, &stats->scc,
2589                         "Single collisions");
2590         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
2591                         CTLFLAG_RD, &stats->mcc,
2592                         "Multiple collisions");
2593         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
2594                         CTLFLAG_RD, &stats->latecol,
2595                         "Late collisions");
2596         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
2597                         CTLFLAG_RD, &stats->colc,
2598                         "Collision Count");
2599         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
2600                         CTLFLAG_RD, &adapter->stats.symerrs,
2601                         "Symbol Errors");
2602         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
2603                         CTLFLAG_RD, &adapter->stats.sec,
2604                         "Sequence Errors");
2605         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
2606                         CTLFLAG_RD, &adapter->stats.dc,
2607                         "Defer Count");
2608         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
2609                         CTLFLAG_RD, &adapter->stats.mpc,
2610                         "Missed Packets");
2611         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
2612                         CTLFLAG_RD, &adapter->stats.rnbc,
2613                         "Receive No Buffers");
2614         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
2615                         CTLFLAG_RD, &adapter->stats.ruc,
2616                         "Receive Undersize");
2617         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
2618                         CTLFLAG_RD, &adapter->stats.rfc,
2619                         "Fragmented Packets Received ");
2620         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
2621                         CTLFLAG_RD, &adapter->stats.roc,
2622                         "Oversized Packets Received");
2623         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
2624                         CTLFLAG_RD, &adapter->stats.rjc,
2625                         "Recevied Jabber");
2626         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
2627                         CTLFLAG_RD, &adapter->stats.rxerrc,
2628                         "Receive Errors");
2629         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
2630                         CTLFLAG_RD, &adapter->stats.crcerrs,
2631                         "CRC errors");
2632         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
2633                         CTLFLAG_RD, &adapter->stats.algnerrc,
2634                         "Alignment Errors");
2635         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
2636                         CTLFLAG_RD, &adapter->stats.xonrxc,
2637                         "XON Received");
2638         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
2639                         CTLFLAG_RD, &adapter->stats.xontxc,
2640                         "XON Transmitted");
2641         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
2642                         CTLFLAG_RD, &adapter->stats.xoffrxc,
2643                         "XOFF Received");
2644         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
2645                         CTLFLAG_RD, &adapter->stats.xofftxc,
2646                         "XOFF Transmitted");
2647
2648         /* Packet Reception Stats */
2649         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
2650                         CTLFLAG_RD, &adapter->stats.tpr,
2651                         "Total Packets Received ");
2652         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
2653                         CTLFLAG_RD, &adapter->stats.gprc,
2654                         "Good Packets Received");
2655         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
2656                         CTLFLAG_RD, &adapter->stats.bprc,
2657                         "Broadcast Packets Received");
2658         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
2659                         CTLFLAG_RD, &adapter->stats.mprc,
2660                         "Multicast Packets Received");
2661         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
2662                         CTLFLAG_RD, &adapter->stats.prc64,
2663                         "64 byte frames received ");
2664         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
2665                         CTLFLAG_RD, &adapter->stats.prc127,
2666                         "65-127 byte frames received");
2667         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
2668                         CTLFLAG_RD, &adapter->stats.prc255,
2669                         "128-255 byte frames received");
2670         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
2671                         CTLFLAG_RD, &adapter->stats.prc511,
2672                         "256-511 byte frames received");
2673         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
2674                         CTLFLAG_RD, &adapter->stats.prc1023,
2675                         "512-1023 byte frames received");
2676         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
2677                         CTLFLAG_RD, &adapter->stats.prc1522,
2678                         "1023-1522 byte frames received");
2679         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
2680                         CTLFLAG_RD, &adapter->stats.gorc,
2681                         "Good Octets Received");
2682
2683         /* Packet Transmission Stats */
2684         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
2685                         CTLFLAG_RD, &adapter->stats.gotc,
2686                         "Good Octets Transmitted");
2687         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
2688                         CTLFLAG_RD, &adapter->stats.tpt,
2689                         "Total Packets Transmitted");
2690         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
2691                         CTLFLAG_RD, &adapter->stats.gptc,
2692                         "Good Packets Transmitted");
2693         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
2694                         CTLFLAG_RD, &adapter->stats.bptc,
2695                         "Broadcast Packets Transmitted");
2696         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
2697                         CTLFLAG_RD, &adapter->stats.mptc,
2698                         "Multicast Packets Transmitted");
2699         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
2700                         CTLFLAG_RD, &adapter->stats.ptc64,
2701                         "64 byte frames transmitted ");
2702         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
2703                         CTLFLAG_RD, &adapter->stats.ptc127,
2704                         "65-127 byte frames transmitted");
2705         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
2706                         CTLFLAG_RD, &adapter->stats.ptc255,
2707                         "128-255 byte frames transmitted");
2708         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
2709                         CTLFLAG_RD, &adapter->stats.ptc511,
2710                         "256-511 byte frames transmitted");
2711         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
2712                         CTLFLAG_RD, &adapter->stats.ptc1023,
2713                         "512-1023 byte frames transmitted");
2714         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
2715                         CTLFLAG_RD, &adapter->stats.ptc1522,
2716                         "1024-1522 byte frames transmitted");
2717         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
2718                         CTLFLAG_RD, &adapter->stats.tsctc,
2719                         "TSO Contexts Transmitted");
2720
2721         /* Interrupt Stats */
2722
2723         int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
2724             CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Interrupt Statistics");
2725         int_list = SYSCTL_CHILDREN(int_node);
2726
2727         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts",
2728                         CTLFLAG_RD, &adapter->stats.iac,
2729                         "Interrupt Assertion Count");
2730
2731         SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
2732                         CTLFLAG_RD, &adapter->stats.rxdmtc,
2733                         "Rx Desc Min Thresh Count");
2734 }
2735
2736 /**********************************************************************
2737  *
2738  *  This routine provides a way to dump out the adapter eeprom,
2739  *  often a useful debug/service tool. This only dumps the first
2740  *  32 words, stuff that matters is in that extent.
2741  *
2742  **********************************************************************/
2743 static int
2744 igc_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
2745 {
2746         struct igc_adapter *adapter = (struct igc_adapter *)arg1;
2747         int error;
2748         int result;
2749
2750         result = -1;
2751         error = sysctl_handle_int(oidp, &result, 0, req);
2752
2753         if (error || !req->newptr)
2754                 return (error);
2755
2756         /*
2757          * This value will cause a hex dump of the
2758          * first 32 16-bit words of the EEPROM to
2759          * the screen.
2760          */
2761         if (result == 1)
2762                 igc_print_nvm_info(adapter);
2763
2764         return (error);
2765 }
2766
2767 static void
2768 igc_print_nvm_info(struct igc_adapter *adapter)
2769 {
2770         u16 eeprom_data;
2771         int i, j, row = 0;
2772
2773         /* Its a bit crude, but it gets the job done */
2774         printf("\nInterface EEPROM Dump:\n");
2775         printf("Offset\n0x0000  ");
2776         for (i = 0, j = 0; i < 32; i++, j++) {
2777                 if (j == 8) { /* Make the offset block */
2778                         j = 0; ++row;
2779                         printf("\n0x00%x0  ",row);
2780                 }
2781                 igc_read_nvm(&adapter->hw, i, 1, &eeprom_data);
2782                 printf("%04x ", eeprom_data);
2783         }
2784         printf("\n");
2785 }
2786
2787 static int
2788 igc_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
2789 {
2790         struct igc_int_delay_info *info;
2791         struct igc_adapter *adapter;
2792         u32 regval;
2793         int error, usecs, ticks;
2794
2795         info = (struct igc_int_delay_info *) arg1;
2796         usecs = info->value;
2797         error = sysctl_handle_int(oidp, &usecs, 0, req);
2798         if (error != 0 || req->newptr == NULL)
2799                 return (error);
2800         if (usecs < 0 || usecs > IGC_TICKS_TO_USECS(65535))
2801                 return (EINVAL);
2802         info->value = usecs;
2803         ticks = IGC_USECS_TO_TICKS(usecs);
2804         if (info->offset == IGC_ITR)    /* units are 256ns here */
2805                 ticks *= 4;
2806
2807         adapter = info->adapter;
2808
2809         regval = IGC_READ_OFFSET(&adapter->hw, info->offset);
2810         regval = (regval & ~0xffff) | (ticks & 0xffff);
2811         /* Handle a few special cases. */
2812         switch (info->offset) {
2813         case IGC_RDTR:
2814                 break;
2815         case IGC_TIDV:
2816                 if (ticks == 0) {
2817                         adapter->txd_cmd &= ~IGC_TXD_CMD_IDE;
2818                         /* Don't write 0 into the TIDV register. */
2819                         regval++;
2820                 } else
2821                         adapter->txd_cmd |= IGC_TXD_CMD_IDE;
2822                 break;
2823         }
2824         IGC_WRITE_OFFSET(&adapter->hw, info->offset, regval);
2825         return (0);
2826 }
2827
2828 static void
2829 igc_add_int_delay_sysctl(struct igc_adapter *adapter, const char *name,
2830         const char *description, struct igc_int_delay_info *info,
2831         int offset, int value)
2832 {
2833         info->adapter = adapter;
2834         info->offset = offset;
2835         info->value = value;
2836         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
2837             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
2838             OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW,
2839             info, 0, igc_sysctl_int_delay, "I", description);
2840 }
2841
2842 /*
2843  * Set flow control using sysctl:
2844  * Flow control values:
2845  *      0 - off
2846  *      1 - rx pause
2847  *      2 - tx pause
2848  *      3 - full
2849  */
2850 static int
2851 igc_set_flowcntl(SYSCTL_HANDLER_ARGS)
2852 {
2853         int error;
2854         static int input = 3; /* default is full */
2855         struct igc_adapter      *adapter = (struct igc_adapter *) arg1;
2856
2857         error = sysctl_handle_int(oidp, &input, 0, req);
2858
2859         if ((error) || (req->newptr == NULL))
2860                 return (error);
2861
2862         if (input == adapter->fc) /* no change? */
2863                 return (error);
2864
2865         switch (input) {
2866         case igc_fc_rx_pause:
2867         case igc_fc_tx_pause:
2868         case igc_fc_full:
2869         case igc_fc_none:
2870                 adapter->hw.fc.requested_mode = input;
2871                 adapter->fc = input;
2872                 break;
2873         default:
2874                 /* Do nothing */
2875                 return (error);
2876         }
2877
2878         adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
2879         igc_force_mac_fc(&adapter->hw);
2880         return (error);
2881 }
2882
2883 /*
2884  * Manage Energy Efficient Ethernet:
2885  * Control values:
2886  *     0/1 - enabled/disabled
2887  */
2888 static int
2889 igc_sysctl_eee(SYSCTL_HANDLER_ARGS)
2890 {
2891         struct igc_adapter *adapter = (struct igc_adapter *) arg1;
2892         int error, value;
2893
2894         value = adapter->hw.dev_spec._i225.eee_disable;
2895         error = sysctl_handle_int(oidp, &value, 0, req);
2896         if (error || req->newptr == NULL)
2897                 return (error);
2898
2899         adapter->hw.dev_spec._i225.eee_disable = (value != 0);
2900         igc_if_init(adapter->ctx);
2901
2902         return (0);
2903 }
2904
2905 static int
2906 igc_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
2907 {
2908         struct igc_adapter *adapter;
2909         int error;
2910         int result;
2911
2912         result = -1;
2913         error = sysctl_handle_int(oidp, &result, 0, req);
2914
2915         if (error || !req->newptr)
2916                 return (error);
2917
2918         if (result == 1) {
2919                 adapter = (struct igc_adapter *) arg1;
2920                 igc_print_debug_info(adapter);
2921         }
2922
2923         return (error);
2924 }
2925
2926 static int
2927 igc_get_rs(SYSCTL_HANDLER_ARGS)
2928 {
2929         struct igc_adapter *adapter = (struct igc_adapter *) arg1;
2930         int error;
2931         int result;
2932
2933         result = 0;
2934         error = sysctl_handle_int(oidp, &result, 0, req);
2935
2936         if (error || !req->newptr || result != 1)
2937                 return (error);
2938         igc_dump_rs(adapter);
2939
2940         return (error);
2941 }
2942
2943 static void
2944 igc_if_debug(if_ctx_t ctx)
2945 {
2946         igc_dump_rs(iflib_get_softc(ctx));
2947 }
2948
2949 /*
2950  * This routine is meant to be fluid, add whatever is
2951  * needed for debugging a problem.  -jfv
2952  */
2953 static void
2954 igc_print_debug_info(struct igc_adapter *adapter)
2955 {
2956         device_t dev = iflib_get_dev(adapter->ctx);
2957         struct ifnet *ifp = iflib_get_ifp(adapter->ctx);
2958         struct tx_ring *txr = &adapter->tx_queues->txr;
2959         struct rx_ring *rxr = &adapter->rx_queues->rxr;
2960
2961         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
2962                 printf("Interface is RUNNING ");
2963         else
2964                 printf("Interface is NOT RUNNING\n");
2965
2966         if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE)
2967                 printf("and INACTIVE\n");
2968         else
2969                 printf("and ACTIVE\n");
2970
2971         for (int i = 0; i < adapter->tx_num_queues; i++, txr++) {
2972                 device_printf(dev, "TX Queue %d ------\n", i);
2973                 device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
2974                         IGC_READ_REG(&adapter->hw, IGC_TDH(i)),
2975                         IGC_READ_REG(&adapter->hw, IGC_TDT(i)));
2976
2977         }
2978         for (int j=0; j < adapter->rx_num_queues; j++, rxr++) {
2979                 device_printf(dev, "RX Queue %d ------\n", j);
2980                 device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
2981                         IGC_READ_REG(&adapter->hw, IGC_RDH(j)),
2982                         IGC_READ_REG(&adapter->hw, IGC_RDT(j)));
2983         }
2984 }