sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 155 static int cxgbe_media_change(struct ifnet *);
 156 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 157
 158 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 159
 160 /*
 161  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 162  * then ADAPTER_LOCK, then t4_uld_list_lock.
 163  */
 164 static struct sx t4_list_lock;
 165 SLIST_HEAD(, adapter) t4_list;
 166 #ifdef TCP_OFFLOAD
 167 static struct sx t4_uld_list_lock;
 168 SLIST_HEAD(, uld_info) t4_uld_list;
 169 #endif
 170
 171 /*
 172  * Tunables.  See tweak_tunables() too.
 173  *
 174  * Each tunable is set to a default value here if it's known at compile-time.
 175  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 176  * provide a reasonable default when the driver is loaded.
 177  *
 178  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 179  * T5 are under hw.cxl.
 180  */
 181
 182 /*
 183  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 184  */
 185 #define NTXQ_10G 16
 186 static int t4_ntxq10g = -1;
 187 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 188
 189 #define NRXQ_10G 8
 190 static int t4_nrxq10g = -1;
 191 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 192
 193 #define NTXQ_1G 4
 194 static int t4_ntxq1g = -1;
 195 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 196
 197 #define NRXQ_1G 2
 198 static int t4_nrxq1g = -1;
 199 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 200
 201 static int t4_rsrv_noflowq = 0;
 202 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 203
 204 #ifdef TCP_OFFLOAD
 205 #define NOFLDTXQ_10G 8
 206 static int t4_nofldtxq10g = -1;
 207 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 208
 209 #define NOFLDRXQ_10G 2
 210 static int t4_nofldrxq10g = -1;
 211 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 212
 213 #define NOFLDTXQ_1G 2
 214 static int t4_nofldtxq1g = -1;
 215 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 216
 217 #define NOFLDRXQ_1G 1
 218 static int t4_nofldrxq1g = -1;
 219 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 220 #endif
 221
 222 #ifdef DEV_NETMAP
 223 #define NNMTXQ_10G 2
 224 static int t4_nnmtxq10g = -1;
 225 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 226
 227 #define NNMRXQ_10G 2
 228 static int t4_nnmrxq10g = -1;
 229 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 230
 231 #define NNMTXQ_1G 1
 232 static int t4_nnmtxq1g = -1;
 233 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 234
 235 #define NNMRXQ_1G 1
 236 static int t4_nnmrxq1g = -1;
 237 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 238 #endif
 239
 240 /*
 241  * Holdoff parameters for 10G and 1G ports.
 242  */
 243 #define TMR_IDX_10G 1
 244 static int t4_tmr_idx_10g = TMR_IDX_10G;
 245 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 246
 247 #define PKTC_IDX_10G (-1)
 248 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 249 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 250
 251 #define TMR_IDX_1G 1
 252 static int t4_tmr_idx_1g = TMR_IDX_1G;
 253 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 254
 255 #define PKTC_IDX_1G (-1)
 256 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 257 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 258
 259 /*
 260  * Size (# of entries) of each tx and rx queue.
 261  */
 262 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 263 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 264
 265 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 266 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 267
 268 /*
 269  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 270  */
 271 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 272 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 273
 274 /*
 275  * Configuration file.
 276  */
 277 #define DEFAULT_CF      "default"
 278 #define FLASH_CF        "flash"
 279 #define UWIRE_CF        "uwire"
 280 #define FPGA_CF         "fpga"
 281 static char t4_cfg_file[32] = DEFAULT_CF;
 282 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 283
 284 /*
 285  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 286  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 287  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 288  *            mark or when signalled to do so, 0 to never emit PAUSE.
 289  */
 290 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 291 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 292
 293 /*
 294  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 295  * encouraged respectively).
 296  */
 297 static unsigned int t4_fw_install = 1;
 298 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 299
 300 /*
 301  * ASIC features that will be used.  Disable the ones you don't want so that the
 302  * chip resources aren't wasted on features that will not be used.
 303  */
 304 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 305 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 306
 307 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 308 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 309
 310 static int t4_toecaps_allowed = -1;
 311 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 312
 313 static int t4_rdmacaps_allowed = 0;
 314 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 315
 316 static int t4_iscsicaps_allowed = 0;
 317 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 318
 319 static int t4_fcoecaps_allowed = 0;
 320 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 321
 322 static int t5_write_combine = 0;
 323 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 324
 325 struct intrs_and_queues {
 326         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 327         uint16_t nirq;          /* Total # of vectors */
 328         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 329         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 330         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 331         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 332         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 333         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 334         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 335 #ifdef TCP_OFFLOAD
 336         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 337         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 338         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 339         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 340 #endif
 341 #ifdef DEV_NETMAP
 342         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 343         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 344         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 345         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 346 #endif
 347 };
 348
 349 struct filter_entry {
 350         uint32_t valid:1;       /* filter allocated and valid */
 351         uint32_t locked:1;      /* filter is administratively locked */
 352         uint32_t pending:1;     /* filter action is pending firmware reply */
 353         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 354         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 355
 356         struct t4_filter_specification fs;
 357 };
 358
 359 static int map_bars_0_and_4(struct adapter *);
 360 static int map_bar_2(struct adapter *);
 361 static void setup_memwin(struct adapter *);
 362 static int validate_mem_range(struct adapter *, uint32_t, int);
 363 static int fwmtype_to_hwmtype(int);
 364 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 365     uint32_t *);
 366 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 367 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 368 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 369     struct intrs_and_queues *);
 370 static int prep_firmware(struct adapter *);
 371 static int partition_resources(struct adapter *, const struct firmware *,
 372     const char *);
 373 static int get_params__pre_init(struct adapter *);
 374 static int get_params__post_init(struct adapter *);
 375 static int set_params__post_init(struct adapter *);
 376 static void t4_set_desc(struct adapter *);
 377 static void build_medialist(struct port_info *, struct ifmedia *);
 378 static int cxgbe_init_synchronized(struct port_info *);
 379 static int cxgbe_uninit_synchronized(struct port_info *);
 380 static int setup_intr_handlers(struct adapter *);
 381 static void quiesce_eq(struct adapter *, struct sge_eq *);
 382 static void quiesce_iq(struct adapter *, struct sge_iq *);
 383 static void quiesce_fl(struct adapter *, struct sge_fl *);
 384 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 385     driver_intr_t *, void *, char *);
 386 static int t4_free_irq(struct adapter *, struct irq *);
 387 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 388     unsigned int);
 389 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 390 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 391 static void cxgbe_tick(void *);
 392 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 393 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 394     struct mbuf *);
 395 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 396 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 397 static int t4_sysctls(struct adapter *);
 398 static int cxgbe_sysctls(struct port_info *);
 399 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 410 #ifdef SBUF_DRAIN
 411 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 436 #endif
 437 static inline void txq_start(struct ifnet *, struct sge_txq *);
 438 static uint32_t fconf_to_mode(uint32_t);
 439 static uint32_t mode_to_fconf(uint32_t);
 440 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 441 static int get_filter_mode(struct adapter *, uint32_t *);
 442 static int set_filter_mode(struct adapter *, uint32_t);
 443 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 444 static int get_filter(struct adapter *, struct t4_filter *);
 445 static int set_filter(struct adapter *, struct t4_filter *);
 446 static int del_filter(struct adapter *, struct t4_filter *);
 447 static void clear_filter(struct filter_entry *);
 448 static int set_filter_wr(struct adapter *, int);
 449 static int del_filter_wr(struct adapter *, int);
 450 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 451 static int load_fw(struct adapter *, struct t4_data *);
 452 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 453 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 454 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 455 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 456 #ifdef TCP_OFFLOAD
 457 static int toe_capability(struct port_info *, int);
 458 #endif
 459 static int mod_event(module_t, int, void *);
 460
 461 struct {
 462         uint16_t device;
 463         char *desc;
 464 } t4_pciids[] = {
 465         {0xa000, "Chelsio Terminator 4 FPGA"},
 466         {0x4400, "Chelsio T440-dbg"},
 467         {0x4401, "Chelsio T420-CR"},
 468         {0x4402, "Chelsio T422-CR"},
 469         {0x4403, "Chelsio T440-CR"},
 470         {0x4404, "Chelsio T420-BCH"},
 471         {0x4405, "Chelsio T440-BCH"},
 472         {0x4406, "Chelsio T440-CH"},
 473         {0x4407, "Chelsio T420-SO"},
 474         {0x4408, "Chelsio T420-CX"},
 475         {0x4409, "Chelsio T420-BT"},
 476         {0x440a, "Chelsio T404-BT"},
 477         {0x440e, "Chelsio T440-LP-CR"},
 478 }, t5_pciids[] = {
 479         {0xb000, "Chelsio Terminator 5 FPGA"},
 480         {0x5400, "Chelsio T580-dbg"},
 481         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 482         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 483         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 484         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 485         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 486         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 487         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 488         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 489         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 490         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 491         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 492         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 493         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 494 #ifdef notyet
 495         {0x5404,  "Chelsio T520-BCH"},
 496         {0x5405,  "Chelsio T540-BCH"},
 497         {0x5406,  "Chelsio T540-CH"},
 498         {0x5408,  "Chelsio T520-CX"},
 499         {0x540b,  "Chelsio B520-SR"},
 500         {0x540c,  "Chelsio B504-BT"},
 501         {0x540f,  "Chelsio Amsterdam"},
 502         {0x5413,  "Chelsio T580-CHR"},
 503 #endif
 504 };
 505
 506 #ifdef TCP_OFFLOAD
 507 /*
 508  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 509  * exactly the same for both rxq and ofld_rxq.
 510  */
 511 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 512 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 513 #endif
 514
 515 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 516 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 517 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 518
 519 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 520
 521 static int
 522 t4_probe(device_t dev)
 523 {
 524         int i;
 525         uint16_t v = pci_get_vendor(dev);
 526         uint16_t d = pci_get_device(dev);
 527         uint8_t f = pci_get_function(dev);
 528
 529         if (v != PCI_VENDOR_ID_CHELSIO)
 530                 return (ENXIO);
 531
 532         /* Attach only to PF0 of the FPGA */
 533         if (d == 0xa000 && f != 0)
 534                 return (ENXIO);
 535
 536         for (i = 0; i < nitems(t4_pciids); i++) {
 537                 if (d == t4_pciids[i].device) {
 538                         device_set_desc(dev, t4_pciids[i].desc);
 539                         return (BUS_PROBE_DEFAULT);
 540                 }
 541         }
 542
 543         return (ENXIO);
 544 }
 545
 546 static int
 547 t5_probe(device_t dev)
 548 {
 549         int i;
 550         uint16_t v = pci_get_vendor(dev);
 551         uint16_t d = pci_get_device(dev);
 552         uint8_t f = pci_get_function(dev);
 553
 554         if (v != PCI_VENDOR_ID_CHELSIO)
 555                 return (ENXIO);
 556
 557         /* Attach only to PF0 of the FPGA */
 558         if (d == 0xb000 && f != 0)
 559                 return (ENXIO);
 560
 561         for (i = 0; i < nitems(t5_pciids); i++) {
 562                 if (d == t5_pciids[i].device) {
 563                         device_set_desc(dev, t5_pciids[i].desc);
 564                         return (BUS_PROBE_DEFAULT);
 565                 }
 566         }
 567
 568         return (ENXIO);
 569 }
 570
 571 static int
 572 t4_attach(device_t dev)
 573 {
 574         struct adapter *sc;
 575         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 576         struct intrs_and_queues iaq;
 577         struct sge *s;
 578 #ifdef TCP_OFFLOAD
 579         int ofld_rqidx, ofld_tqidx;
 580 #endif
 581 #ifdef DEV_NETMAP
 582         int nm_rqidx, nm_tqidx;
 583 #endif
 584         const char *pcie_ts;
 585
 586         sc = device_get_softc(dev);
 587         sc->dev = dev;
 588
 589         pci_enable_busmaster(dev);
 590         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 591                 uint32_t v;
 592
 593                 pci_set_max_read_req(dev, 4096);
 594                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 595                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 596                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 597
 598                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 599         }
 600
 601         sc->traceq = -1;
 602         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 603         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 604             device_get_nameunit(dev));
 605
 606         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 607             device_get_nameunit(dev));
 608         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 609         sx_xlock(&t4_list_lock);
 610         SLIST_INSERT_HEAD(&t4_list, sc, link);
 611         sx_xunlock(&t4_list_lock);
 612
 613         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 614         TAILQ_INIT(&sc->sfl);
 615         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 616
 617         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 618
 619         rc = map_bars_0_and_4(sc);
 620         if (rc != 0)
 621                 goto done; /* error message displayed already */
 622
 623         /*
 624          * This is the real PF# to which we're attaching.  Works from within PCI
 625          * passthrough environments too, where pci_get_function() could return a
 626          * different PF# depending on the passthrough configuration.  We need to
 627          * use the real PF# in all our communication with the firmware.
 628          */
 629         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 630         sc->mbox = sc->pf;
 631
 632         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 633         sc->an_handler = an_not_handled;
 634         for (i = 0; i < nitems(sc->cpl_handler); i++)
 635                 sc->cpl_handler[i] = cpl_not_handled;
 636         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 637                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 638         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 639         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 641         t4_init_sge_cpl_handlers(sc);
 642
 643         /* Prepare the adapter for operation */
 644         rc = -t4_prep_adapter(sc);
 645         if (rc != 0) {
 646                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 647                 goto done;
 648         }
 649
 650         /*
 651          * Do this really early, with the memory windows set up even before the
 652          * character device.  The userland tool's register i/o and mem read
 653          * will work even in "recovery mode".
 654          */
 655         setup_memwin(sc);
 656         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 657             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 658             device_get_nameunit(dev));
 659         if (sc->cdev == NULL)
 660                 device_printf(dev, "failed to create nexus char device.\n");
 661         else
 662                 sc->cdev->si_drv1 = sc;
 663
 664         /* Go no further if recovery mode has been requested. */
 665         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 666                 device_printf(dev, "recovery mode.\n");
 667                 goto done;
 668         }
 669
 670         /* Prepare the firmware for operation */
 671         rc = prep_firmware(sc);
 672         if (rc != 0)
 673                 goto done; /* error message displayed already */
 674
 675         rc = get_params__post_init(sc);
 676         if (rc != 0)
 677                 goto done; /* error message displayed already */
 678
 679         rc = set_params__post_init(sc);
 680         if (rc != 0)
 681                 goto done; /* error message displayed already */
 682
 683         rc = map_bar_2(sc);
 684         if (rc != 0)
 685                 goto done; /* error message displayed already */
 686
 687         rc = t4_create_dma_tag(sc);
 688         if (rc != 0)
 689                 goto done; /* error message displayed already */
 690
 691         /*
 692          * First pass over all the ports - allocate VIs and initialize some
 693          * basic parameters like mac address, port type, etc.  We also figure
 694          * out whether a port is 10G or 1G and use that information when
 695          * calculating how many interrupts to attempt to allocate.
 696          */
 697         n10g = n1g = 0;
 698         for_each_port(sc, i) {
 699                 struct port_info *pi;
 700
 701                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 702                 sc->port[i] = pi;
 703
 704                 /* These must be set before t4_port_init */
 705                 pi->adapter = sc;
 706                 pi->port_id = i;
 707
 708                 /* Allocate the vi and initialize parameters like mac addr */
 709                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 710                 if (rc != 0) {
 711                         device_printf(dev, "unable to initialize port %d: %d\n",
 712                             i, rc);
 713                         free(pi, M_CXGBE);
 714                         sc->port[i] = NULL;
 715                         goto done;
 716                 }
 717
 718                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 719                 pi->link_cfg.requested_fc |= t4_pause_settings;
 720                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 721                 pi->link_cfg.fc |= t4_pause_settings;
 722
 723                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 724                 if (rc != 0) {
 725                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 726                         free(pi, M_CXGBE);
 727                         sc->port[i] = NULL;
 728                         goto done;
 729                 }
 730
 731                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 732                     device_get_nameunit(dev), i);
 733                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 734                 sc->chan_map[pi->tx_chan] = i;
 735
 736                 if (is_10G_port(pi) || is_40G_port(pi)) {
 737                         n10g++;
 738                         pi->tmr_idx = t4_tmr_idx_10g;
 739                         pi->pktc_idx = t4_pktc_idx_10g;
 740                 } else {
 741                         n1g++;
 742                         pi->tmr_idx = t4_tmr_idx_1g;
 743                         pi->pktc_idx = t4_pktc_idx_1g;
 744                 }
 745
 746                 pi->xact_addr_filt = -1;
 747                 pi->linkdnrc = -1;
 748
 749                 pi->qsize_rxq = t4_qsize_rxq;
 750                 pi->qsize_txq = t4_qsize_txq;
 751
 752                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 753                 if (pi->dev == NULL) {
 754                         device_printf(dev,
 755                             "failed to add device for port %d.\n", i);
 756                         rc = ENXIO;
 757                         goto done;
 758                 }
 759                 device_set_softc(pi->dev, pi);
 760         }
 761
 762         /*
 763          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 764          */
 765         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 766         if (rc != 0)
 767                 goto done; /* error message displayed already */
 768
 769         sc->intr_type = iaq.intr_type;
 770         sc->intr_count = iaq.nirq;
 771
 772         s = &sc->sge;
 773         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 774         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 775         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 776         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 777         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 778 #ifdef TCP_OFFLOAD
 779         if (is_offload(sc)) {
 780                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 781                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 782                 s->neq += s->nofldtxq + s->nofldrxq;
 783                 s->niq += s->nofldrxq;
 784
 785                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 786                     M_CXGBE, M_ZERO | M_WAITOK);
 787                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 788                     M_CXGBE, M_ZERO | M_WAITOK);
 789         }
 790 #endif
 791 #ifdef DEV_NETMAP
 792         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 793         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 794         s->neq += s->nnmtxq + s->nnmrxq;
 795         s->niq += s->nnmrxq;
 796
 797         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 798             M_CXGBE, M_ZERO | M_WAITOK);
 799         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 800             M_CXGBE, M_ZERO | M_WAITOK);
 801 #endif
 802
 803         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 804             M_ZERO | M_WAITOK);
 805         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 806             M_ZERO | M_WAITOK);
 807         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 808             M_ZERO | M_WAITOK);
 809         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 810             M_ZERO | M_WAITOK);
 811         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 812             M_ZERO | M_WAITOK);
 813
 814         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 815             M_ZERO | M_WAITOK);
 816
 817         t4_init_l2t(sc, M_WAITOK);
 818
 819         /*
 820          * Second pass over the ports.  This time we know the number of rx and
 821          * tx queues that each port should get.
 822          */
 823         rqidx = tqidx = 0;
 824 #ifdef TCP_OFFLOAD
 825         ofld_rqidx = ofld_tqidx = 0;
 826 #endif
 827 #ifdef DEV_NETMAP
 828         nm_rqidx = nm_tqidx = 0;
 829 #endif
 830         for_each_port(sc, i) {
 831                 struct port_info *pi = sc->port[i];
 832
 833                 if (pi == NULL)
 834                         continue;
 835
 836                 pi->first_rxq = rqidx;
 837                 pi->first_txq = tqidx;
 838                 if (is_10G_port(pi) || is_40G_port(pi)) {
 839                         pi->flags |= iaq.intr_flags_10g;
 840                         pi->nrxq = iaq.nrxq10g;
 841                         pi->ntxq = iaq.ntxq10g;
 842                 } else {
 843                         pi->flags |= iaq.intr_flags_1g;
 844                         pi->nrxq = iaq.nrxq1g;
 845                         pi->ntxq = iaq.ntxq1g;
 846                 }
 847
 848                 if (pi->ntxq > 1)
 849                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 850                 else
 851                         pi->rsrv_noflowq = 0;
 852
 853                 rqidx += pi->nrxq;
 854                 tqidx += pi->ntxq;
 855 #ifdef TCP_OFFLOAD
 856                 if (is_offload(sc)) {
 857                         pi->first_ofld_rxq = ofld_rqidx;
 858                         pi->first_ofld_txq = ofld_tqidx;
 859                         if (is_10G_port(pi) || is_40G_port(pi)) {
 860                                 pi->nofldrxq = iaq.nofldrxq10g;
 861                                 pi->nofldtxq = iaq.nofldtxq10g;
 862                         } else {
 863                                 pi->nofldrxq = iaq.nofldrxq1g;
 864                                 pi->nofldtxq = iaq.nofldtxq1g;
 865                         }
 866                         ofld_rqidx += pi->nofldrxq;
 867                         ofld_tqidx += pi->nofldtxq;
 868                 }
 869 #endif
 870 #ifdef DEV_NETMAP
 871                 pi->first_nm_rxq = nm_rqidx;
 872                 pi->first_nm_txq = nm_tqidx;
 873                 if (is_10G_port(pi) || is_40G_port(pi)) {
 874                         pi->nnmrxq = iaq.nnmrxq10g;
 875                         pi->nnmtxq = iaq.nnmtxq10g;
 876                 } else {
 877                         pi->nnmrxq = iaq.nnmrxq1g;
 878                         pi->nnmtxq = iaq.nnmtxq1g;
 879                 }
 880                 nm_rqidx += pi->nnmrxq;
 881                 nm_tqidx += pi->nnmtxq;
 882 #endif
 883         }
 884
 885         rc = setup_intr_handlers(sc);
 886         if (rc != 0) {
 887                 device_printf(dev,
 888                     "failed to setup interrupt handlers: %d\n", rc);
 889                 goto done;
 890         }
 891
 892         rc = bus_generic_attach(dev);
 893         if (rc != 0) {
 894                 device_printf(dev,
 895                     "failed to attach all child ports: %d\n", rc);
 896                 goto done;
 897         }
 898
 899         switch (sc->params.pci.speed) {
 900                 case 0x1:
 901                         pcie_ts = "2.5";
 902                         break;
 903                 case 0x2:
 904                         pcie_ts = "5.0";
 905                         break;
 906                 case 0x3:
 907                         pcie_ts = "8.0";
 908                         break;
 909                 default:
 910                         pcie_ts = "??";
 911                         break;
 912         }
 913         device_printf(dev,
 914             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 915             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 916             sc->params.nports, sc->intr_count,
 917             sc->intr_type == INTR_MSIX ? "MSI-X" :
 918             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 919             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 920
 921         t4_set_desc(sc);
 922
 923 done:
 924         if (rc != 0 && sc->cdev) {
 925                 /* cdev was created and so cxgbetool works; recover that way. */
 926                 device_printf(dev,
 927                     "error during attach, adapter is now in recovery mode.\n");
 928                 rc = 0;
 929         }
 930
 931         if (rc != 0)
 932                 t4_detach(dev);
 933         else
 934                 t4_sysctls(sc);
 935
 936         return (rc);
 937 }
 938
 939 /*
 940  * Idempotent
 941  */
 942 static int
 943 t4_detach(device_t dev)
 944 {
 945         struct adapter *sc;
 946         struct port_info *pi;
 947         int i, rc;
 948
 949         sc = device_get_softc(dev);
 950
 951         if (sc->flags & FULL_INIT_DONE)
 952                 t4_intr_disable(sc);
 953
 954         if (sc->cdev) {
 955                 destroy_dev(sc->cdev);
 956                 sc->cdev = NULL;
 957         }
 958
 959         rc = bus_generic_detach(dev);
 960         if (rc) {
 961                 device_printf(dev,
 962                     "failed to detach child devices: %d\n", rc);
 963                 return (rc);
 964         }
 965
 966         for (i = 0; i < sc->intr_count; i++)
 967                 t4_free_irq(sc, &sc->irq[i]);
 968
 969         for (i = 0; i < MAX_NPORTS; i++) {
 970                 pi = sc->port[i];
 971                 if (pi) {
 972                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 973                         if (pi->dev)
 974                                 device_delete_child(dev, pi->dev);
 975
 976                         mtx_destroy(&pi->pi_lock);
 977                         free(pi, M_CXGBE);
 978                 }
 979         }
 980
 981         if (sc->flags & FULL_INIT_DONE)
 982                 adapter_full_uninit(sc);
 983
 984         if (sc->flags & FW_OK)
 985                 t4_fw_bye(sc, sc->mbox);
 986
 987         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 988                 pci_release_msi(dev);
 989
 990         if (sc->regs_res)
 991                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 992                     sc->regs_res);
 993
 994         if (sc->udbs_res)
 995                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 996                     sc->udbs_res);
 997
 998         if (sc->msix_res)
 999                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1000                     sc->msix_res);
1001
1002         if (sc->l2t)
1003                 t4_free_l2t(sc->l2t);
1004
1005 #ifdef TCP_OFFLOAD
1006         free(sc->sge.ofld_rxq, M_CXGBE);
1007         free(sc->sge.ofld_txq, M_CXGBE);
1008 #endif
1009 #ifdef DEV_NETMAP
1010         free(sc->sge.nm_rxq, M_CXGBE);
1011         free(sc->sge.nm_txq, M_CXGBE);
1012 #endif
1013         free(sc->irq, M_CXGBE);
1014         free(sc->sge.rxq, M_CXGBE);
1015         free(sc->sge.txq, M_CXGBE);
1016         free(sc->sge.ctrlq, M_CXGBE);
1017         free(sc->sge.iqmap, M_CXGBE);
1018         free(sc->sge.eqmap, M_CXGBE);
1019         free(sc->tids.ftid_tab, M_CXGBE);
1020         t4_destroy_dma_tag(sc);
1021         if (mtx_initialized(&sc->sc_lock)) {
1022                 sx_xlock(&t4_list_lock);
1023                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1024                 sx_xunlock(&t4_list_lock);
1025                 mtx_destroy(&sc->sc_lock);
1026         }
1027
1028         if (mtx_initialized(&sc->tids.ftid_lock))
1029                 mtx_destroy(&sc->tids.ftid_lock);
1030         if (mtx_initialized(&sc->sfl_lock))
1031                 mtx_destroy(&sc->sfl_lock);
1032         if (mtx_initialized(&sc->ifp_lock))
1033                 mtx_destroy(&sc->ifp_lock);
1034         if (mtx_initialized(&sc->regwin_lock))
1035                 mtx_destroy(&sc->regwin_lock);
1036
1037         bzero(sc, sizeof(*sc));
1038
1039         return (0);
1040 }
1041
1042 static int
1043 cxgbe_probe(device_t dev)
1044 {
1045         char buf[128];
1046         struct port_info *pi = device_get_softc(dev);
1047
1048         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1049         device_set_desc_copy(dev, buf);
1050
1051         return (BUS_PROBE_DEFAULT);
1052 }
1053
1054 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1055     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1056     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1057 #define T4_CAP_ENABLE (T4_CAP)
1058
1059 static int
1060 cxgbe_attach(device_t dev)
1061 {
1062         struct port_info *pi = device_get_softc(dev);
1063         struct ifnet *ifp;
1064         char *s;
1065         int n, o;
1066
1067         /* Allocate an ifnet and set it up */
1068         ifp = if_alloc(IFT_ETHER);
1069         if (ifp == NULL) {
1070                 device_printf(dev, "Cannot allocate ifnet\n");
1071                 return (ENOMEM);
1072         }
1073         pi->ifp = ifp;
1074         ifp->if_softc = pi;
1075
1076         callout_init(&pi->tick, CALLOUT_MPSAFE);
1077
1078         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1079         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1080
1081         ifp->if_init = cxgbe_init;
1082         ifp->if_ioctl = cxgbe_ioctl;
1083         ifp->if_transmit = cxgbe_transmit;
1084         ifp->if_qflush = cxgbe_qflush;
1085         ifp->if_get_counter = cxgbe_get_counter;
1086
1087         ifp->if_capabilities = T4_CAP;
1088 #ifdef TCP_OFFLOAD
1089         if (is_offload(pi->adapter))
1090                 ifp->if_capabilities |= IFCAP_TOE;
1091 #endif
1092         ifp->if_capenable = T4_CAP_ENABLE;
1093         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1094             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1095
1096         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1097         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1098         ifp->if_hw_tsomaxsegsize = 65536;
1099
1100         /* Initialize ifmedia for this port */
1101         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1102             cxgbe_media_status);
1103         build_medialist(pi, &pi->media);
1104
1105         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1106             EVENTHANDLER_PRI_ANY);
1107
1108         ether_ifattach(ifp, pi->hw_addr);
1109
1110         n = 128;
1111         s = malloc(n, M_CXGBE, M_WAITOK);
1112         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1113         MPASS(n > o);
1114 #ifdef TCP_OFFLOAD
1115         if (is_offload(pi->adapter)) {
1116                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1117                     pi->nofldtxq, pi->nofldrxq);
1118                 MPASS(n > o);
1119         }
1120 #endif
1121 #ifdef DEV_NETMAP
1122         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1123             pi->nnmrxq);
1124         MPASS(n > o);
1125 #endif
1126         device_printf(dev, "%s\n", s);
1127         free(s, M_CXGBE);
1128
1129 #ifdef DEV_NETMAP
1130         /* nm_media handled here to keep implementation private to this file */
1131         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1132             cxgbe_media_status);
1133         build_medialist(pi, &pi->nm_media);
1134         create_netmap_ifnet(pi);        /* logs errors it something fails */
1135 #endif
1136         cxgbe_sysctls(pi);
1137
1138         return (0);
1139 }
1140
1141 static int
1142 cxgbe_detach(device_t dev)
1143 {
1144         struct port_info *pi = device_get_softc(dev);
1145         struct adapter *sc = pi->adapter;
1146         struct ifnet *ifp = pi->ifp;
1147
1148         /* Tell if_ioctl and if_init that the port is going away */
1149         ADAPTER_LOCK(sc);
1150         SET_DOOMED(pi);
1151         wakeup(&sc->flags);
1152         while (IS_BUSY(sc))
1153                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1154         SET_BUSY(sc);
1155 #ifdef INVARIANTS
1156         sc->last_op = "t4detach";
1157         sc->last_op_thr = curthread;
1158 #endif
1159         ADAPTER_UNLOCK(sc);
1160
1161         if (pi->flags & HAS_TRACEQ) {
1162                 sc->traceq = -1;        /* cloner should not create ifnet */
1163                 t4_tracer_port_detach(sc);
1164         }
1165
1166         if (pi->vlan_c)
1167                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1168
1169         PORT_LOCK(pi);
1170         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1171         callout_stop(&pi->tick);
1172         PORT_UNLOCK(pi);
1173         callout_drain(&pi->tick);
1174
1175         /* Let detach proceed even if these fail. */
1176         cxgbe_uninit_synchronized(pi);
1177         port_full_uninit(pi);
1178
1179         ifmedia_removeall(&pi->media);
1180         ether_ifdetach(pi->ifp);
1181         if_free(pi->ifp);
1182
1183 #ifdef DEV_NETMAP
1184         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1185         destroy_netmap_ifnet(pi);
1186 #endif
1187
1188         ADAPTER_LOCK(sc);
1189         CLR_BUSY(sc);
1190         wakeup(&sc->flags);
1191         ADAPTER_UNLOCK(sc);
1192
1193         return (0);
1194 }
1195
1196 static void
1197 cxgbe_init(void *arg)
1198 {
1199         struct port_info *pi = arg;
1200         struct adapter *sc = pi->adapter;
1201
1202         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1203                 return;
1204         cxgbe_init_synchronized(pi);
1205         end_synchronized_op(sc, 0);
1206 }
1207
1208 static int
1209 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1210 {
1211         int rc = 0, mtu, flags, can_sleep;
1212         struct port_info *pi = ifp->if_softc;
1213         struct adapter *sc = pi->adapter;
1214         struct ifreq *ifr = (struct ifreq *)data;
1215         uint32_t mask;
1216
1217         switch (cmd) {
1218         case SIOCSIFMTU:
1219                 mtu = ifr->ifr_mtu;
1220                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1221                         return (EINVAL);
1222
1223                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1224                 if (rc)
1225                         return (rc);
1226                 ifp->if_mtu = mtu;
1227                 if (pi->flags & PORT_INIT_DONE) {
1228                         t4_update_fl_bufsize(ifp);
1229                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1230                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1231                 }
1232                 end_synchronized_op(sc, 0);
1233                 break;
1234
1235         case SIOCSIFFLAGS:
1236                 can_sleep = 0;
1237 redo_sifflags:
1238                 rc = begin_synchronized_op(sc, pi,
1239                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1240                 if (rc)
1241                         return (rc);
1242
1243                 if (ifp->if_flags & IFF_UP) {
1244                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1245                                 flags = pi->if_flags;
1246                                 if ((ifp->if_flags ^ flags) &
1247                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1248                                         if (can_sleep == 1) {
1249                                                 end_synchronized_op(sc, 0);
1250                                                 can_sleep = 0;
1251                                                 goto redo_sifflags;
1252                                         }
1253                                         rc = update_mac_settings(ifp,
1254                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1255                                 }
1256                         } else {
1257                                 if (can_sleep == 0) {
1258                                         end_synchronized_op(sc, LOCK_HELD);
1259                                         can_sleep = 1;
1260                                         goto redo_sifflags;
1261                                 }
1262                                 rc = cxgbe_init_synchronized(pi);
1263                         }
1264                         pi->if_flags = ifp->if_flags;
1265                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1266                         if (can_sleep == 0) {
1267                                 end_synchronized_op(sc, LOCK_HELD);
1268                                 can_sleep = 1;
1269                                 goto redo_sifflags;
1270                         }
1271                         rc = cxgbe_uninit_synchronized(pi);
1272                 }
1273                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1274                 break;
1275
1276         case SIOCADDMULTI:
1277         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1278                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1279                 if (rc)
1280                         return (rc);
1281                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1282                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1283                 end_synchronized_op(sc, LOCK_HELD);
1284                 break;
1285
1286         case SIOCSIFCAP:
1287                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1288                 if (rc)
1289                         return (rc);
1290
1291                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1292                 if (mask & IFCAP_TXCSUM) {
1293                         ifp->if_capenable ^= IFCAP_TXCSUM;
1294                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1295
1296                         if (IFCAP_TSO4 & ifp->if_capenable &&
1297                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1298                                 ifp->if_capenable &= ~IFCAP_TSO4;
1299                                 if_printf(ifp,
1300                                     "tso4 disabled due to -txcsum.\n");
1301                         }
1302                 }
1303                 if (mask & IFCAP_TXCSUM_IPV6) {
1304                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1305                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1306
1307                         if (IFCAP_TSO6 & ifp->if_capenable &&
1308                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1309                                 ifp->if_capenable &= ~IFCAP_TSO6;
1310                                 if_printf(ifp,
1311                                     "tso6 disabled due to -txcsum6.\n");
1312                         }
1313                 }
1314                 if (mask & IFCAP_RXCSUM)
1315                         ifp->if_capenable ^= IFCAP_RXCSUM;
1316                 if (mask & IFCAP_RXCSUM_IPV6)
1317                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1318
1319                 /*
1320                  * Note that we leave CSUM_TSO alone (it is always set).  The
1321                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1322                  * sending a TSO request our way, so it's sufficient to toggle
1323                  * IFCAP_TSOx only.
1324                  */
1325                 if (mask & IFCAP_TSO4) {
1326                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1327                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1328                                 if_printf(ifp, "enable txcsum first.\n");
1329                                 rc = EAGAIN;
1330                                 goto fail;
1331                         }
1332                         ifp->if_capenable ^= IFCAP_TSO4;
1333                 }
1334                 if (mask & IFCAP_TSO6) {
1335                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1336                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1337                                 if_printf(ifp, "enable txcsum6 first.\n");
1338                                 rc = EAGAIN;
1339                                 goto fail;
1340                         }
1341                         ifp->if_capenable ^= IFCAP_TSO6;
1342                 }
1343                 if (mask & IFCAP_LRO) {
1344 #if defined(INET) || defined(INET6)
1345                         int i;
1346                         struct sge_rxq *rxq;
1347
1348                         ifp->if_capenable ^= IFCAP_LRO;
1349                         for_each_rxq(pi, i, rxq) {
1350                                 if (ifp->if_capenable & IFCAP_LRO)
1351                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1352                                 else
1353                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1354                         }
1355 #endif
1356                 }
1357 #ifdef TCP_OFFLOAD
1358                 if (mask & IFCAP_TOE) {
1359                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1360
1361                         rc = toe_capability(pi, enable);
1362                         if (rc != 0)
1363                                 goto fail;
1364
1365                         ifp->if_capenable ^= mask;
1366                 }
1367 #endif
1368                 if (mask & IFCAP_VLAN_HWTAGGING) {
1369                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1370                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1371                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1372                 }
1373                 if (mask & IFCAP_VLAN_MTU) {
1374                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1375
1376                         /* Need to find out how to disable auto-mtu-inflation */
1377                 }
1378                 if (mask & IFCAP_VLAN_HWTSO)
1379                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1380                 if (mask & IFCAP_VLAN_HWCSUM)
1381                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1382
1383 #ifdef VLAN_CAPABILITIES
1384                 VLAN_CAPABILITIES(ifp);
1385 #endif
1386 fail:
1387                 end_synchronized_op(sc, 0);
1388                 break;
1389
1390         case SIOCSIFMEDIA:
1391         case SIOCGIFMEDIA:
1392                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1393                 break;
1394
1395         case SIOCGI2C: {
1396                 struct ifi2creq i2c;
1397
1398                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1399                 if (rc != 0)
1400                         break;
1401                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1402                         rc = EPERM;
1403                         break;
1404                 }
1405                 if (i2c.len > sizeof(i2c.data)) {
1406                         rc = EINVAL;
1407                         break;
1408                 }
1409                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1410                 if (rc)
1411                         return (rc);
1412                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1413                     i2c.offset, i2c.len, &i2c.data[0]);
1414                 end_synchronized_op(sc, 0);
1415                 if (rc == 0)
1416                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1417                 break;
1418         }
1419
1420         default:
1421                 rc = ether_ioctl(ifp, cmd, data);
1422         }
1423
1424         return (rc);
1425 }
1426
1427 static int
1428 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1429 {
1430         struct port_info *pi = ifp->if_softc;
1431         struct adapter *sc = pi->adapter;
1432         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1433         struct buf_ring *br;
1434         int rc;
1435
1436         M_ASSERTPKTHDR(m);
1437
1438         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1439                 m_freem(m);
1440                 return (ENETDOWN);
1441         }
1442
1443         if (m->m_flags & M_FLOWID)
1444                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1445                     + pi->rsrv_noflowq);
1446         br = txq->br;
1447
1448         if (TXQ_TRYLOCK(txq) == 0) {
1449                 struct sge_eq *eq = &txq->eq;
1450
1451                 /*
1452                  * It is possible that t4_eth_tx finishes up and releases the
1453                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1454                  * need to make sure that this mbuf doesn't just sit there in
1455                  * the drbr.
1456                  */
1457
1458                 rc = drbr_enqueue(ifp, br, m);
1459                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1460                     !(eq->flags & EQ_DOOMED))
1461                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1462                 return (rc);
1463         }
1464
1465         /*
1466          * txq->m is the mbuf that is held up due to a temporary shortage of
1467          * resources and it should be put on the wire first.  Then what's in
1468          * drbr and finally the mbuf that was just passed in to us.
1469          *
1470          * Return code should indicate the fate of the mbuf that was passed in
1471          * this time.
1472          */
1473
1474         TXQ_LOCK_ASSERT_OWNED(txq);
1475         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1476
1477                 /* Queued for transmission. */
1478
1479                 rc = drbr_enqueue(ifp, br, m);
1480                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1481                 (void) t4_eth_tx(ifp, txq, m);
1482                 TXQ_UNLOCK(txq);
1483                 return (rc);
1484         }
1485
1486         /* Direct transmission. */
1487         rc = t4_eth_tx(ifp, txq, m);
1488         if (rc != 0 && txq->m)
1489                 rc = 0; /* held, will be transmitted soon (hopefully) */
1490
1491         TXQ_UNLOCK(txq);
1492         return (rc);
1493 }
1494
1495 static void
1496 cxgbe_qflush(struct ifnet *ifp)
1497 {
1498         struct port_info *pi = ifp->if_softc;
1499         struct sge_txq *txq;
1500         int i;
1501         struct mbuf *m;
1502
1503         /* queues do not exist if !PORT_INIT_DONE. */
1504         if (pi->flags & PORT_INIT_DONE) {
1505                 for_each_txq(pi, i, txq) {
1506                         TXQ_LOCK(txq);
1507                         m_freem(txq->m);
1508                         txq->m = NULL;
1509                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1510                                 m_freem(m);
1511                         TXQ_UNLOCK(txq);
1512                 }
1513         }
1514         if_qflush(ifp);
1515 }
1516
1517 static uint64_t
1518 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1519 {
1520         struct port_info *pi = ifp->if_softc;
1521         struct adapter *sc = pi->adapter;
1522         struct port_stats *s = &pi->stats;
1523
1524         cxgbe_refresh_stats(sc, pi);
1525
1526         switch (c) {
1527         case IFCOUNTER_IPACKETS:
1528                 return (s->rx_frames - s->rx_pause);
1529
1530         case IFCOUNTER_IERRORS:
1531                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1532                     s->rx_fcs_err + s->rx_len_err);
1533
1534         case IFCOUNTER_OPACKETS:
1535                 return (s->tx_frames - s->tx_pause);
1536
1537         case IFCOUNTER_OERRORS:
1538                 return (s->tx_error_frames);
1539
1540         case IFCOUNTER_IBYTES:
1541                 return (s->rx_octets - s->rx_pause * 64);
1542
1543         case IFCOUNTER_OBYTES:
1544                 return (s->tx_octets - s->tx_pause * 64);
1545
1546         case IFCOUNTER_IMCASTS:
1547                 return (s->rx_mcast_frames - s->rx_pause);
1548
1549         case IFCOUNTER_OMCASTS:
1550                 return (s->tx_mcast_frames - s->tx_pause);
1551
1552         case IFCOUNTER_IQDROPS:
1553                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1554                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1555                     s->rx_trunc3 + pi->tnl_cong_drops);
1556
1557         case IFCOUNTER_OQDROPS: {
1558                 uint64_t drops;
1559
1560                 drops = s->tx_drop;
1561                 if (pi->flags & PORT_INIT_DONE) {
1562                         int i;
1563                         struct sge_txq *txq;
1564
1565                         for_each_txq(pi, i, txq)
1566                                 drops += txq->br->br_drops;
1567                 }
1568
1569                 return (drops);
1570
1571         }
1572
1573         default:
1574                 return (if_get_counter_default(ifp, c));
1575         }
1576 }
1577
1578 static int
1579 cxgbe_media_change(struct ifnet *ifp)
1580 {
1581         struct port_info *pi = ifp->if_softc;
1582
1583         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1584
1585         return (EOPNOTSUPP);
1586 }
1587
1588 static void
1589 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1590 {
1591         struct port_info *pi = ifp->if_softc;
1592         struct ifmedia *media = NULL;
1593         struct ifmedia_entry *cur;
1594         int speed = pi->link_cfg.speed;
1595 #ifdef INVARIANTS
1596         int data = (pi->port_type << 8) | pi->mod_type;
1597 #endif
1598
1599         if (ifp == pi->ifp)
1600                 media = &pi->media;
1601 #ifdef DEV_NETMAP
1602         else if (ifp == pi->nm_ifp)
1603                 media = &pi->nm_media;
1604 #endif
1605         MPASS(media != NULL);
1606
1607         cur = media->ifm_cur;
1608         MPASS(cur->ifm_data == data);
1609
1610         ifmr->ifm_status = IFM_AVALID;
1611         if (!pi->link_cfg.link_ok)
1612                 return;
1613
1614         ifmr->ifm_status |= IFM_ACTIVE;
1615
1616         /* active and current will differ iff current media is autoselect. */
1617         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1618                 return;
1619
1620         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1621         if (speed == SPEED_10000)
1622                 ifmr->ifm_active |= IFM_10G_T;
1623         else if (speed == SPEED_1000)
1624                 ifmr->ifm_active |= IFM_1000_T;
1625         else if (speed == SPEED_100)
1626                 ifmr->ifm_active |= IFM_100_TX;
1627         else if (speed == SPEED_10)
1628                 ifmr->ifm_active |= IFM_10_T;
1629         else
1630                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1631                             speed));
1632 }
1633
1634 void
1635 t4_fatal_err(struct adapter *sc)
1636 {
1637         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1638         t4_intr_disable(sc);
1639         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1640             device_get_nameunit(sc->dev));
1641 }
1642
1643 static int
1644 map_bars_0_and_4(struct adapter *sc)
1645 {
1646         sc->regs_rid = PCIR_BAR(0);
1647         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1648             &sc->regs_rid, RF_ACTIVE);
1649         if (sc->regs_res == NULL) {
1650                 device_printf(sc->dev, "cannot map registers.\n");
1651                 return (ENXIO);
1652         }
1653         sc->bt = rman_get_bustag(sc->regs_res);
1654         sc->bh = rman_get_bushandle(sc->regs_res);
1655         sc->mmio_len = rman_get_size(sc->regs_res);
1656         setbit(&sc->doorbells, DOORBELL_KDB);
1657
1658         sc->msix_rid = PCIR_BAR(4);
1659         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1660             &sc->msix_rid, RF_ACTIVE);
1661         if (sc->msix_res == NULL) {
1662                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1663                 return (ENXIO);
1664         }
1665
1666         return (0);
1667 }
1668
1669 static int
1670 map_bar_2(struct adapter *sc)
1671 {
1672
1673         /*
1674          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1675          * to map it if RDMA is disabled.
1676          */
1677         if (is_t4(sc) && sc->rdmacaps == 0)
1678                 return (0);
1679
1680         sc->udbs_rid = PCIR_BAR(2);
1681         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1682             &sc->udbs_rid, RF_ACTIVE);
1683         if (sc->udbs_res == NULL) {
1684                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1685                 return (ENXIO);
1686         }
1687         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1688
1689         if (is_t5(sc)) {
1690                 setbit(&sc->doorbells, DOORBELL_UDB);
1691 #if defined(__i386__) || defined(__amd64__)
1692                 if (t5_write_combine) {
1693                         int rc;
1694
1695                         /*
1696                          * Enable write combining on BAR2.  This is the
1697                          * userspace doorbell BAR and is split into 128B
1698                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1699                          * with an egress queue.  The first 64B has the doorbell
1700                          * and the second 64B can be used to submit a tx work
1701                          * request with an implicit doorbell.
1702                          */
1703
1704                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1705                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1706                         if (rc == 0) {
1707                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1708                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1709                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1710                         } else {
1711                                 device_printf(sc->dev,
1712                                     "couldn't enable write combining: %d\n",
1713                                     rc);
1714                         }
1715
1716                         t4_write_reg(sc, A_SGE_STAT_CFG,
1717                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1718                 }
1719 #endif
1720         }
1721
1722         return (0);
1723 }
1724
1725 static const struct memwin t4_memwin[] = {
1726         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1727         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1728         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1729 };
1730
1731 static const struct memwin t5_memwin[] = {
1732         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1733         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1734         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1735 };
1736
1737 static void
1738 setup_memwin(struct adapter *sc)
1739 {
1740         const struct memwin *mw;
1741         int i, n;
1742         uint32_t bar0;
1743
1744         if (is_t4(sc)) {
1745                 /*
1746                  * Read low 32b of bar0 indirectly via the hardware backdoor
1747                  * mechanism.  Works from within PCI passthrough environments
1748                  * too, where rman_get_start() can return a different value.  We
1749                  * need to program the T4 memory window decoders with the actual
1750                  * addresses that will be coming across the PCIe link.
1751                  */
1752                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1753                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1754
1755                 mw = &t4_memwin[0];
1756                 n = nitems(t4_memwin);
1757         } else {
1758                 /* T5 uses the relative offset inside the PCIe BAR */
1759                 bar0 = 0;
1760
1761                 mw = &t5_memwin[0];
1762                 n = nitems(t5_memwin);
1763         }
1764
1765         for (i = 0; i < n; i++, mw++) {
1766                 t4_write_reg(sc,
1767                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1768                     (mw->base + bar0) | V_BIR(0) |
1769                     V_WINDOW(ilog2(mw->aperture) - 10));
1770         }
1771
1772         /* flush */
1773         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1774 }
1775
1776 /*
1777  * Verify that the memory range specified by the addr/len pair is valid and lies
1778  * entirely within a single region (EDCx or MCx).
1779  */
1780 static int
1781 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1782 {
1783         uint32_t em, addr_len, maddr, mlen;
1784
1785         /* Memory can only be accessed in naturally aligned 4 byte units */
1786         if (addr & 3 || len & 3 || len == 0)
1787                 return (EINVAL);
1788
1789         /* Enabled memories */
1790         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1791         if (em & F_EDRAM0_ENABLE) {
1792                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1793                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1794                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1795                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1796                     addr + len <= maddr + mlen)
1797                         return (0);
1798         }
1799         if (em & F_EDRAM1_ENABLE) {
1800                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1801                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1802                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1803                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1804                     addr + len <= maddr + mlen)
1805                         return (0);
1806         }
1807         if (em & F_EXT_MEM_ENABLE) {
1808                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1809                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1810                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1811                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1812                     addr + len <= maddr + mlen)
1813                         return (0);
1814         }
1815         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1816                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1817                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1818                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1819                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1820                     addr + len <= maddr + mlen)
1821                         return (0);
1822         }
1823
1824         return (EFAULT);
1825 }
1826
1827 static int
1828 fwmtype_to_hwmtype(int mtype)
1829 {
1830
1831         switch (mtype) {
1832         case FW_MEMTYPE_EDC0:
1833                 return (MEM_EDC0);
1834         case FW_MEMTYPE_EDC1:
1835                 return (MEM_EDC1);
1836         case FW_MEMTYPE_EXTMEM:
1837                 return (MEM_MC0);
1838         case FW_MEMTYPE_EXTMEM1:
1839                 return (MEM_MC1);
1840         default:
1841                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1842         }
1843 }
1844
1845 /*
1846  * Verify that the memory range specified by the memtype/offset/len pair is
1847  * valid and lies entirely within the memtype specified.  The global address of
1848  * the start of the range is returned in addr.
1849  */
1850 static int
1851 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1852     uint32_t *addr)
1853 {
1854         uint32_t em, addr_len, maddr, mlen;
1855
1856         /* Memory can only be accessed in naturally aligned 4 byte units */
1857         if (off & 3 || len & 3 || len == 0)
1858                 return (EINVAL);
1859
1860         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1861         switch (fwmtype_to_hwmtype(mtype)) {
1862         case MEM_EDC0:
1863                 if (!(em & F_EDRAM0_ENABLE))
1864                         return (EINVAL);
1865                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1866                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1867                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1868                 break;
1869         case MEM_EDC1:
1870                 if (!(em & F_EDRAM1_ENABLE))
1871                         return (EINVAL);
1872                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1873                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1874                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1875                 break;
1876         case MEM_MC:
1877                 if (!(em & F_EXT_MEM_ENABLE))
1878                         return (EINVAL);
1879                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1880                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1881                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1882                 break;
1883         case MEM_MC1:
1884                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1885                         return (EINVAL);
1886                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1887                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1888                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1889                 break;
1890         default:
1891                 return (EINVAL);
1892         }
1893
1894         if (mlen > 0 && off < mlen && off + len <= mlen) {
1895                 *addr = maddr + off;    /* global address */
1896                 return (0);
1897         }
1898
1899         return (EFAULT);
1900 }
1901
1902 static void
1903 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1904 {
1905         const struct memwin *mw;
1906
1907         if (is_t4(sc)) {
1908                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1909                     ("%s: incorrect memwin# (%d)", __func__, win));
1910                 mw = &t4_memwin[win];
1911         } else {
1912                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1913                     ("%s: incorrect memwin# (%d)", __func__, win));
1914                 mw = &t5_memwin[win];
1915         }
1916
1917         if (base != NULL)
1918                 *base = mw->base;
1919         if (aperture != NULL)
1920                 *aperture = mw->aperture;
1921 }
1922
1923 /*
1924  * Positions the memory window such that it can be used to access the specified
1925  * address in the chip's address space.  The return value is the offset of addr
1926  * from the start of the window.
1927  */
1928 static uint32_t
1929 position_memwin(struct adapter *sc, int n, uint32_t addr)
1930 {
1931         uint32_t start, pf;
1932         uint32_t reg;
1933
1934         KASSERT(n >= 0 && n <= 3,
1935             ("%s: invalid window %d.", __func__, n));
1936         KASSERT((addr & 3) == 0,
1937             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1938
1939         if (is_t4(sc)) {
1940                 pf = 0;
1941                 start = addr & ~0xf;    /* start must be 16B aligned */
1942         } else {
1943                 pf = V_PFNUM(sc->pf);
1944                 start = addr & ~0x7f;   /* start must be 128B aligned */
1945         }
1946         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1947
1948         t4_write_reg(sc, reg, start | pf);
1949         t4_read_reg(sc, reg);
1950
1951         return (addr - start);
1952 }
1953
1954 static int
1955 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1956     struct intrs_and_queues *iaq)
1957 {
1958         int rc, itype, navail, nrxq10g, nrxq1g, n;
1959         int nofldrxq10g = 0, nofldrxq1g = 0;
1960         int nnmrxq10g = 0, nnmrxq1g = 0;
1961
1962         bzero(iaq, sizeof(*iaq));
1963
1964         iaq->ntxq10g = t4_ntxq10g;
1965         iaq->ntxq1g = t4_ntxq1g;
1966         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1967         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1968         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1969 #ifdef TCP_OFFLOAD
1970         if (is_offload(sc)) {
1971                 iaq->nofldtxq10g = t4_nofldtxq10g;
1972                 iaq->nofldtxq1g = t4_nofldtxq1g;
1973                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1974                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1975         }
1976 #endif
1977 #ifdef DEV_NETMAP
1978         iaq->nnmtxq10g = t4_nnmtxq10g;
1979         iaq->nnmtxq1g = t4_nnmtxq1g;
1980         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1981         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1982 #endif
1983
1984         for (itype = INTR_MSIX; itype; itype >>= 1) {
1985
1986                 if ((itype & t4_intr_types) == 0)
1987                         continue;       /* not allowed */
1988
1989                 if (itype == INTR_MSIX)
1990                         navail = pci_msix_count(sc->dev);
1991                 else if (itype == INTR_MSI)
1992                         navail = pci_msi_count(sc->dev);
1993                 else
1994                         navail = 1;
1995 restart:
1996                 if (navail == 0)
1997                         continue;
1998
1999                 iaq->intr_type = itype;
2000                 iaq->intr_flags_10g = 0;
2001                 iaq->intr_flags_1g = 0;
2002
2003                 /*
2004                  * Best option: an interrupt vector for errors, one for the
2005                  * firmware event queue, and one for every rxq (NIC, TOE, and
2006                  * netmap).
2007                  */
2008                 iaq->nirq = T4_EXTRA_INTR;
2009                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
2010                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
2011                 if (iaq->nirq <= navail &&
2012                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
2013                         iaq->intr_flags_10g = INTR_ALL;
2014                         iaq->intr_flags_1g = INTR_ALL;
2015                         goto allocate;
2016                 }
2017
2018                 /*
2019                  * Second best option: a vector for errors, one for the firmware
2020                  * event queue, and vectors for either all the NIC rx queues or
2021                  * all the TOE rx queues.  The queues that don't get vectors
2022                  * will forward their interrupts to those that do.
2023                  *
2024                  * Note: netmap rx queues cannot be created early and so they
2025                  * can't be setup to receive forwarded interrupts for others.
2026                  */
2027                 iaq->nirq = T4_EXTRA_INTR;
2028                 if (nrxq10g >= nofldrxq10g) {
2029                         iaq->intr_flags_10g = INTR_RXQ;
2030                         iaq->nirq += n10g * nrxq10g;
2031 #ifdef DEV_NETMAP
2032                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2033 #endif
2034                 } else {
2035                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2036                         iaq->nirq += n10g * nofldrxq10g;
2037 #ifdef DEV_NETMAP
2038                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2039 #endif
2040                 }
2041                 if (nrxq1g >= nofldrxq1g) {
2042                         iaq->intr_flags_1g = INTR_RXQ;
2043                         iaq->nirq += n1g * nrxq1g;
2044 #ifdef DEV_NETMAP
2045                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2046 #endif
2047                 } else {
2048                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2049                         iaq->nirq += n1g * nofldrxq1g;
2050 #ifdef DEV_NETMAP
2051                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2052 #endif
2053                 }
2054                 if (iaq->nirq <= navail &&
2055                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2056                         goto allocate;
2057
2058                 /*
2059                  * Next best option: an interrupt vector for errors, one for the
2060                  * firmware event queue, and at least one per port.  At this
2061                  * point we know we'll have to downsize nrxq and/or nofldrxq
2062                  * and/or nnmrxq to fit what's available to us.
2063                  */
2064                 iaq->nirq = T4_EXTRA_INTR;
2065                 iaq->nirq += n10g + n1g;
2066                 if (iaq->nirq <= navail) {
2067                         int leftover = navail - iaq->nirq;
2068
2069                         if (n10g > 0) {
2070                                 int target = max(nrxq10g, nofldrxq10g);
2071
2072                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2073                                     INTR_RXQ : INTR_OFLD_RXQ;
2074
2075                                 n = 1;
2076                                 while (n < target && leftover >= n10g) {
2077                                         leftover -= n10g;
2078                                         iaq->nirq += n10g;
2079                                         n++;
2080                                 }
2081                                 iaq->nrxq10g = min(n, nrxq10g);
2082 #ifdef TCP_OFFLOAD
2083                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2084 #endif
2085 #ifdef DEV_NETMAP
2086                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2087 #endif
2088                         }
2089
2090                         if (n1g > 0) {
2091                                 int target = max(nrxq1g, nofldrxq1g);
2092
2093                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2094                                     INTR_RXQ : INTR_OFLD_RXQ;
2095
2096                                 n = 1;
2097                                 while (n < target && leftover >= n1g) {
2098                                         leftover -= n1g;
2099                                         iaq->nirq += n1g;
2100                                         n++;
2101                                 }
2102                                 iaq->nrxq1g = min(n, nrxq1g);
2103 #ifdef TCP_OFFLOAD
2104                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2105 #endif
2106 #ifdef DEV_NETMAP
2107                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2108 #endif
2109                         }
2110
2111                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2112                                 goto allocate;
2113                 }
2114
2115                 /*
2116                  * Least desirable option: one interrupt vector for everything.
2117                  */
2118                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2119                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2120 #ifdef TCP_OFFLOAD
2121                 if (is_offload(sc))
2122                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2123 #endif
2124 #ifdef DEV_NETMAP
2125                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2126 #endif
2127
2128 allocate:
2129                 navail = iaq->nirq;
2130                 rc = 0;
2131                 if (itype == INTR_MSIX)
2132                         rc = pci_alloc_msix(sc->dev, &navail);
2133                 else if (itype == INTR_MSI)
2134                         rc = pci_alloc_msi(sc->dev, &navail);
2135
2136                 if (rc == 0) {
2137                         if (navail == iaq->nirq)
2138                                 return (0);
2139
2140                         /*
2141                          * Didn't get the number requested.  Use whatever number
2142                          * the kernel is willing to allocate (it's in navail).
2143                          */
2144                         device_printf(sc->dev, "fewer vectors than requested, "
2145                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2146                             itype, iaq->nirq, navail);
2147                         pci_release_msi(sc->dev);
2148                         goto restart;
2149                 }
2150
2151                 device_printf(sc->dev,
2152                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2153                     itype, rc, iaq->nirq, navail);
2154         }
2155
2156         device_printf(sc->dev,
2157             "failed to find a usable interrupt type.  "
2158             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2159             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2160
2161         return (ENXIO);
2162 }
2163
2164 #define FW_VERSION(chip) ( \
2165     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2166     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2167     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2168     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2169 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2170
2171 struct fw_info {
2172         uint8_t chip;
2173         char *kld_name;
2174         char *fw_mod_name;
2175         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2176 } fw_info[] = {
2177         {
2178                 .chip = CHELSIO_T4,
2179                 .kld_name = "t4fw_cfg",
2180                 .fw_mod_name = "t4fw",
2181                 .fw_hdr = {
2182                         .chip = FW_HDR_CHIP_T4,
2183                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2184                         .intfver_nic = FW_INTFVER(T4, NIC),
2185                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2186                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2187                         .intfver_ri = FW_INTFVER(T4, RI),
2188                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2189                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2190                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2191                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2192                 },
2193         }, {
2194                 .chip = CHELSIO_T5,
2195                 .kld_name = "t5fw_cfg",
2196                 .fw_mod_name = "t5fw",
2197                 .fw_hdr = {
2198                         .chip = FW_HDR_CHIP_T5,
2199                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2200                         .intfver_nic = FW_INTFVER(T5, NIC),
2201                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2202                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2203                         .intfver_ri = FW_INTFVER(T5, RI),
2204                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2205                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2206                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2207                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2208                 },
2209         }
2210 };
2211
2212 static struct fw_info *
2213 find_fw_info(int chip)
2214 {
2215         int i;
2216
2217         for (i = 0; i < nitems(fw_info); i++) {
2218                 if (fw_info[i].chip == chip)
2219                         return (&fw_info[i]);
2220         }
2221         return (NULL);
2222 }
2223
2224 /*
2225  * Is the given firmware API compatible with the one the driver was compiled
2226  * with?
2227  */
2228 static int
2229 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2230 {
2231
2232         /* short circuit if it's the exact same firmware version */
2233         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2234                 return (1);
2235
2236         /*
2237          * XXX: Is this too conservative?  Perhaps I should limit this to the
2238          * features that are supported in the driver.
2239          */
2240 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2241         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2242             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2243             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2244                 return (1);
2245 #undef SAME_INTF
2246
2247         return (0);
2248 }
2249
2250 /*
2251  * The firmware in the KLD is usable, but should it be installed?  This routine
2252  * explains itself in detail if it indicates the KLD firmware should be
2253  * installed.
2254  */
2255 static int
2256 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2257 {
2258         const char *reason;
2259
2260         if (!card_fw_usable) {
2261                 reason = "incompatible or unusable";
2262                 goto install;
2263         }
2264
2265         if (k > c) {
2266                 reason = "older than the version bundled with this driver";
2267                 goto install;
2268         }
2269
2270         if (t4_fw_install == 2 && k != c) {
2271                 reason = "different than the version bundled with this driver";
2272                 goto install;
2273         }
2274
2275         return (0);
2276
2277 install:
2278         if (t4_fw_install == 0) {
2279                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2280                     "but the driver is prohibited from installing a different "
2281                     "firmware on the card.\n",
2282                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2283                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2284
2285                 return (0);
2286         }
2287
2288         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2289             "installing firmware %u.%u.%u.%u on card.\n",
2290             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2291             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2292             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2293             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2294
2295         return (1);
2296 }
2297 /*
2298  * Establish contact with the firmware and determine if we are the master driver
2299  * or not, and whether we are responsible for chip initialization.
2300  */
2301 static int
2302 prep_firmware(struct adapter *sc)
2303 {
2304         const struct firmware *fw = NULL, *default_cfg;
2305         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2306         enum dev_state state;
2307         struct fw_info *fw_info;
2308         struct fw_hdr *card_fw;         /* fw on the card */
2309         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2310         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2311                                            against */
2312
2313         /* Contact firmware. */
2314         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2315         if (rc < 0 || state == DEV_STATE_ERR) {
2316                 rc = -rc;
2317                 device_printf(sc->dev,
2318                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2319                 return (rc);
2320         }
2321         pf = rc;
2322         if (pf == sc->mbox)
2323                 sc->flags |= MASTER_PF;
2324         else if (state == DEV_STATE_UNINIT) {
2325                 /*
2326                  * We didn't get to be the master so we definitely won't be
2327                  * configuring the chip.  It's a bug if someone else hasn't
2328                  * configured it already.
2329                  */
2330                 device_printf(sc->dev, "couldn't be master(%d), "
2331                     "device not already initialized either(%d).\n", rc, state);
2332                 return (EDOOFUS);
2333         }
2334
2335         /* This is the firmware whose headers the driver was compiled against */
2336         fw_info = find_fw_info(chip_id(sc));
2337         if (fw_info == NULL) {
2338                 device_printf(sc->dev,
2339                     "unable to look up firmware information for chip %d.\n",
2340                     chip_id(sc));
2341                 return (EINVAL);
2342         }
2343         drv_fw = &fw_info->fw_hdr;
2344
2345         /*
2346          * The firmware KLD contains many modules.  The KLD name is also the
2347          * name of the module that contains the default config file.
2348          */
2349         default_cfg = firmware_get(fw_info->kld_name);
2350
2351         /* Read the header of the firmware on the card */
2352         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2353         rc = -t4_read_flash(sc, FLASH_FW_START,
2354             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2355         if (rc == 0)
2356                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2357         else {
2358                 device_printf(sc->dev,
2359                     "Unable to read card's firmware header: %d\n", rc);
2360                 card_fw_usable = 0;
2361         }
2362
2363         /* This is the firmware in the KLD */
2364         fw = firmware_get(fw_info->fw_mod_name);
2365         if (fw != NULL) {
2366                 kld_fw = (const void *)fw->data;
2367                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2368         } else {
2369                 kld_fw = NULL;
2370                 kld_fw_usable = 0;
2371         }
2372
2373         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2374             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2375                 /*
2376                  * Common case: the firmware on the card is an exact match and
2377                  * the KLD is an exact match too, or the KLD is
2378                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2379                  * here -- use cxgbetool loadfw if you want to reinstall the
2380                  * same firmware as the one on the card.
2381                  */
2382         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2383             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2384             be32toh(card_fw->fw_ver))) {
2385
2386                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2387                 if (rc != 0) {
2388                         device_printf(sc->dev,
2389                             "failed to install firmware: %d\n", rc);
2390                         goto done;
2391                 }
2392
2393                 /* Installed successfully, update the cached header too. */
2394                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2395                 card_fw_usable = 1;
2396                 need_fw_reset = 0;      /* already reset as part of load_fw */
2397         }
2398
2399         if (!card_fw_usable) {
2400                 uint32_t d, c, k;
2401
2402                 d = ntohl(drv_fw->fw_ver);
2403                 c = ntohl(card_fw->fw_ver);
2404                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2405
2406                 device_printf(sc->dev, "Cannot find a usable firmware: "
2407                     "fw_install %d, chip state %d, "
2408                     "driver compiled with %d.%d.%d.%d, "
2409                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2410                     t4_fw_install, state,
2411                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2412                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2413                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2414                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2415                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2416                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2417                 rc = EINVAL;
2418                 goto done;
2419         }
2420
2421         /* We're using whatever's on the card and it's known to be good. */
2422         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2423         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2424             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2425             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2426             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2427             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2428         t4_get_tp_version(sc, &sc->params.tp_vers);
2429
2430         /* Reset device */
2431         if (need_fw_reset &&
2432             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2433                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2434                 if (rc != ETIMEDOUT && rc != EIO)
2435                         t4_fw_bye(sc, sc->mbox);
2436                 goto done;
2437         }
2438         sc->flags |= FW_OK;
2439
2440         rc = get_params__pre_init(sc);
2441         if (rc != 0)
2442                 goto done; /* error message displayed already */
2443
2444         /* Partition adapter resources as specified in the config file. */
2445         if (state == DEV_STATE_UNINIT) {
2446
2447                 KASSERT(sc->flags & MASTER_PF,
2448                     ("%s: trying to change chip settings when not master.",
2449                     __func__));
2450
2451                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2452                 if (rc != 0)
2453                         goto done;      /* error message displayed already */
2454
2455                 t4_tweak_chip_settings(sc);
2456
2457                 /* get basic stuff going */
2458                 rc = -t4_fw_initialize(sc, sc->mbox);
2459                 if (rc != 0) {
2460                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2461                         goto done;
2462                 }
2463         } else {
2464                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2465                 sc->cfcsum = 0;
2466         }
2467
2468 done:
2469         free(card_fw, M_CXGBE);
2470         if (fw != NULL)
2471                 firmware_put(fw, FIRMWARE_UNLOAD);
2472         if (default_cfg != NULL)
2473                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2474
2475         return (rc);
2476 }
2477
2478 #define FW_PARAM_DEV(param) \
2479         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2480          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2481 #define FW_PARAM_PFVF(param) \
2482         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2483          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2484
2485 /*
2486  * Partition chip resources for use between various PFs, VFs, etc.
2487  */
2488 static int
2489 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2490     const char *name_prefix)
2491 {
2492         const struct firmware *cfg = NULL;
2493         int rc = 0;
2494         struct fw_caps_config_cmd caps;
2495         uint32_t mtype, moff, finicsum, cfcsum;
2496
2497         /*
2498          * Figure out what configuration file to use.  Pick the default config
2499          * file for the card if the user hasn't specified one explicitly.
2500          */
2501         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2502         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2503                 /* Card specific overrides go here. */
2504                 if (pci_get_device(sc->dev) == 0x440a)
2505                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2506                 if (is_fpga(sc))
2507                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2508         }
2509
2510         /*
2511          * We need to load another module if the profile is anything except
2512          * "default" or "flash".
2513          */
2514         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2515             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2516                 char s[32];
2517
2518                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2519                 cfg = firmware_get(s);
2520                 if (cfg == NULL) {
2521                         if (default_cfg != NULL) {
2522                                 device_printf(sc->dev,
2523                                     "unable to load module \"%s\" for "
2524                                     "configuration profile \"%s\", will use "
2525                                     "the default config file instead.\n",
2526                                     s, sc->cfg_file);
2527                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2528                                     "%s", DEFAULT_CF);
2529                         } else {
2530                                 device_printf(sc->dev,
2531                                     "unable to load module \"%s\" for "
2532                                     "configuration profile \"%s\", will use "
2533                                     "the config file on the card's flash "
2534                                     "instead.\n", s, sc->cfg_file);
2535                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2536                                     "%s", FLASH_CF);
2537                         }
2538                 }
2539         }
2540
2541         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2542             default_cfg == NULL) {
2543                 device_printf(sc->dev,
2544                     "default config file not available, will use the config "
2545                     "file on the card's flash instead.\n");
2546                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2547         }
2548
2549         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2550                 u_int cflen, i, n;
2551                 const uint32_t *cfdata;
2552                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2553
2554                 KASSERT(cfg != NULL || default_cfg != NULL,
2555                     ("%s: no config to upload", __func__));
2556
2557                 /*
2558                  * Ask the firmware where it wants us to upload the config file.
2559                  */
2560                 param = FW_PARAM_DEV(CF);
2561                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2562                 if (rc != 0) {
2563                         /* No support for config file?  Shouldn't happen. */
2564                         device_printf(sc->dev,
2565                             "failed to query config file location: %d.\n", rc);
2566                         goto done;
2567                 }
2568                 mtype = G_FW_PARAMS_PARAM_Y(val);
2569                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2570
2571                 /*
2572                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2573                  * useless stuffing/comments at the end of the config file so
2574                  * it's ok to simply throw away the last remaining bytes when
2575                  * the config file is not an exact multiple of 4.  This also
2576                  * helps with the validate_mt_off_len check.
2577                  */
2578                 if (cfg != NULL) {
2579                         cflen = cfg->datasize & ~3;
2580                         cfdata = cfg->data;
2581                 } else {
2582                         cflen = default_cfg->datasize & ~3;
2583                         cfdata = default_cfg->data;
2584                 }
2585
2586                 if (cflen > FLASH_CFG_MAX_SIZE) {
2587                         device_printf(sc->dev,
2588                             "config file too long (%d, max allowed is %d).  "
2589                             "Will try to use the config on the card, if any.\n",
2590                             cflen, FLASH_CFG_MAX_SIZE);
2591                         goto use_config_on_flash;
2592                 }
2593
2594                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2595                 if (rc != 0) {
2596                         device_printf(sc->dev,
2597                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2598                             "Will try to use the config on the card, if any.\n",
2599                             __func__, mtype, moff, cflen, rc);
2600                         goto use_config_on_flash;
2601                 }
2602
2603                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2604                 while (cflen) {
2605                         off = position_memwin(sc, 2, addr);
2606                         n = min(cflen, mw_aperture - off);
2607                         for (i = 0; i < n; i += 4)
2608                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2609                         cflen -= n;
2610                         addr += n;
2611                 }
2612         } else {
2613 use_config_on_flash:
2614                 mtype = FW_MEMTYPE_FLASH;
2615                 moff = t4_flash_cfg_addr(sc);
2616         }
2617
2618         bzero(&caps, sizeof(caps));
2619         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2620             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2621         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2622             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2623             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2624         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2625         if (rc != 0) {
2626                 device_printf(sc->dev,
2627                     "failed to pre-process config file: %d "
2628                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2629                 goto done;
2630         }
2631
2632         finicsum = be32toh(caps.finicsum);
2633         cfcsum = be32toh(caps.cfcsum);
2634         if (finicsum != cfcsum) {
2635                 device_printf(sc->dev,
2636                     "WARNING: config file checksum mismatch: %08x %08x\n",
2637                     finicsum, cfcsum);
2638         }
2639         sc->cfcsum = cfcsum;
2640
2641 #define LIMIT_CAPS(x) do { \
2642         caps.x &= htobe16(t4_##x##_allowed); \
2643 } while (0)
2644
2645         /*
2646          * Let the firmware know what features will (not) be used so it can tune
2647          * things accordingly.
2648          */
2649         LIMIT_CAPS(linkcaps);
2650         LIMIT_CAPS(niccaps);
2651         LIMIT_CAPS(toecaps);
2652         LIMIT_CAPS(rdmacaps);
2653         LIMIT_CAPS(iscsicaps);
2654         LIMIT_CAPS(fcoecaps);
2655 #undef LIMIT_CAPS
2656
2657         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2658             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2659         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2660         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2661         if (rc != 0) {
2662                 device_printf(sc->dev,
2663                     "failed to process config file: %d.\n", rc);
2664         }
2665 done:
2666         if (cfg != NULL)
2667                 firmware_put(cfg, FIRMWARE_UNLOAD);
2668         return (rc);
2669 }
2670
2671 /*
2672  * Retrieve parameters that are needed (or nice to have) very early.
2673  */
2674 static int
2675 get_params__pre_init(struct adapter *sc)
2676 {
2677         int rc;
2678         uint32_t param[2], val[2];
2679         struct fw_devlog_cmd cmd;
2680         struct devlog_params *dlog = &sc->params.devlog;
2681
2682         param[0] = FW_PARAM_DEV(PORTVEC);
2683         param[1] = FW_PARAM_DEV(CCLK);
2684         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2685         if (rc != 0) {
2686                 device_printf(sc->dev,
2687                     "failed to query parameters (pre_init): %d.\n", rc);
2688                 return (rc);
2689         }
2690
2691         sc->params.portvec = val[0];
2692         sc->params.nports = bitcount32(val[0]);
2693         sc->params.vpd.cclk = val[1];
2694
2695         /* Read device log parameters. */
2696         bzero(&cmd, sizeof(cmd));
2697         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2698             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2699         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2700         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2701         if (rc != 0) {
2702                 device_printf(sc->dev,
2703                     "failed to get devlog parameters: %d.\n", rc);
2704                 bzero(dlog, sizeof (*dlog));
2705                 rc = 0; /* devlog isn't critical for device operation */
2706         } else {
2707                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2708                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2709                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2710                 dlog->size = be32toh(cmd.memsize_devlog);
2711         }
2712
2713         return (rc);
2714 }
2715
2716 /*
2717  * Retrieve various parameters that are of interest to the driver.  The device
2718  * has been initialized by the firmware at this point.
2719  */
2720 static int
2721 get_params__post_init(struct adapter *sc)
2722 {
2723         int rc;
2724         uint32_t param[7], val[7];
2725         struct fw_caps_config_cmd caps;
2726
2727         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2728         param[1] = FW_PARAM_PFVF(EQ_START);
2729         param[2] = FW_PARAM_PFVF(FILTER_START);
2730         param[3] = FW_PARAM_PFVF(FILTER_END);
2731         param[4] = FW_PARAM_PFVF(L2T_START);
2732         param[5] = FW_PARAM_PFVF(L2T_END);
2733         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2734         if (rc != 0) {
2735                 device_printf(sc->dev,
2736                     "failed to query parameters (post_init): %d.\n", rc);
2737                 return (rc);
2738         }
2739
2740         sc->sge.iq_start = val[0];
2741         sc->sge.eq_start = val[1];
2742         sc->tids.ftid_base = val[2];
2743         sc->tids.nftids = val[3] - val[2] + 1;
2744         sc->params.ftid_min = val[2];
2745         sc->params.ftid_max = val[3];
2746         sc->vres.l2t.start = val[4];
2747         sc->vres.l2t.size = val[5] - val[4] + 1;
2748         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2749             ("%s: L2 table size (%u) larger than expected (%u)",
2750             __func__, sc->vres.l2t.size, L2T_SIZE));
2751
2752         /* get capabilites */
2753         bzero(&caps, sizeof(caps));
2754         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2755             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2756         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2757         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2758         if (rc != 0) {
2759                 device_printf(sc->dev,
2760                     "failed to get card capabilities: %d.\n", rc);
2761                 return (rc);
2762         }
2763
2764 #define READ_CAPS(x) do { \
2765         sc->x = htobe16(caps.x); \
2766 } while (0)
2767         READ_CAPS(linkcaps);
2768         READ_CAPS(niccaps);
2769         READ_CAPS(toecaps);
2770         READ_CAPS(rdmacaps);
2771         READ_CAPS(iscsicaps);
2772         READ_CAPS(fcoecaps);
2773
2774         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2775                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2776                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2777                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2778                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2779                 if (rc != 0) {
2780                         device_printf(sc->dev,
2781                             "failed to query NIC parameters: %d.\n", rc);
2782                         return (rc);
2783                 }
2784                 sc->tids.etid_base = val[0];
2785                 sc->params.etid_min = val[0];
2786                 sc->tids.netids = val[1] - val[0] + 1;
2787                 sc->params.netids = sc->tids.netids;
2788                 sc->params.eo_wr_cred = val[2];
2789                 sc->params.ethoffload = 1;
2790         }
2791
2792         if (sc->toecaps) {
2793                 /* query offload-related parameters */
2794                 param[0] = FW_PARAM_DEV(NTID);
2795                 param[1] = FW_PARAM_PFVF(SERVER_START);
2796                 param[2] = FW_PARAM_PFVF(SERVER_END);
2797                 param[3] = FW_PARAM_PFVF(TDDP_START);
2798                 param[4] = FW_PARAM_PFVF(TDDP_END);
2799                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2800                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2801                 if (rc != 0) {
2802                         device_printf(sc->dev,
2803                             "failed to query TOE parameters: %d.\n", rc);
2804                         return (rc);
2805                 }
2806                 sc->tids.ntids = val[0];
2807                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2808                 sc->tids.stid_base = val[1];
2809                 sc->tids.nstids = val[2] - val[1] + 1;
2810                 sc->vres.ddp.start = val[3];
2811                 sc->vres.ddp.size = val[4] - val[3] + 1;
2812                 sc->params.ofldq_wr_cred = val[5];
2813                 sc->params.offload = 1;
2814         }
2815         if (sc->rdmacaps) {
2816                 param[0] = FW_PARAM_PFVF(STAG_START);
2817                 param[1] = FW_PARAM_PFVF(STAG_END);
2818                 param[2] = FW_PARAM_PFVF(RQ_START);
2819                 param[3] = FW_PARAM_PFVF(RQ_END);
2820                 param[4] = FW_PARAM_PFVF(PBL_START);
2821                 param[5] = FW_PARAM_PFVF(PBL_END);
2822                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2823                 if (rc != 0) {
2824                         device_printf(sc->dev,
2825                             "failed to query RDMA parameters(1): %d.\n", rc);
2826                         return (rc);
2827                 }
2828                 sc->vres.stag.start = val[0];
2829                 sc->vres.stag.size = val[1] - val[0] + 1;
2830                 sc->vres.rq.start = val[2];
2831                 sc->vres.rq.size = val[3] - val[2] + 1;
2832                 sc->vres.pbl.start = val[4];
2833                 sc->vres.pbl.size = val[5] - val[4] + 1;
2834
2835                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2836                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2837                 param[2] = FW_PARAM_PFVF(CQ_START);
2838                 param[3] = FW_PARAM_PFVF(CQ_END);
2839                 param[4] = FW_PARAM_PFVF(OCQ_START);
2840                 param[5] = FW_PARAM_PFVF(OCQ_END);
2841                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2842                 if (rc != 0) {
2843                         device_printf(sc->dev,
2844                             "failed to query RDMA parameters(2): %d.\n", rc);
2845                         return (rc);
2846                 }
2847                 sc->vres.qp.start = val[0];
2848                 sc->vres.qp.size = val[1] - val[0] + 1;
2849                 sc->vres.cq.start = val[2];
2850                 sc->vres.cq.size = val[3] - val[2] + 1;
2851                 sc->vres.ocq.start = val[4];
2852                 sc->vres.ocq.size = val[5] - val[4] + 1;
2853         }
2854         if (sc->iscsicaps) {
2855                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2856                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2857                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2858                 if (rc != 0) {
2859                         device_printf(sc->dev,
2860                             "failed to query iSCSI parameters: %d.\n", rc);
2861                         return (rc);
2862                 }
2863                 sc->vres.iscsi.start = val[0];
2864                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2865         }
2866
2867         /*
2868          * We've got the params we wanted to query via the firmware.  Now grab
2869          * some others directly from the chip.
2870          */
2871         rc = t4_read_chip_settings(sc);
2872
2873         return (rc);
2874 }
2875
2876 static int
2877 set_params__post_init(struct adapter *sc)
2878 {
2879         uint32_t param, val;
2880
2881         /* ask for encapsulated CPLs */
2882         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2883         val = 1;
2884         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2885
2886         return (0);
2887 }
2888
2889 #undef FW_PARAM_PFVF
2890 #undef FW_PARAM_DEV
2891
2892 static void
2893 t4_set_desc(struct adapter *sc)
2894 {
2895         char buf[128];
2896         struct adapter_params *p = &sc->params;
2897
2898         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2899             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2900             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2901
2902         device_set_desc_copy(sc->dev, buf);
2903 }
2904
2905 static void
2906 build_medialist(struct port_info *pi, struct ifmedia *media)
2907 {
2908         int data, m;
2909
2910         PORT_LOCK(pi);
2911
2912         ifmedia_removeall(media);
2913
2914         m = IFM_ETHER | IFM_FDX;
2915         data = (pi->port_type << 8) | pi->mod_type;
2916
2917         switch(pi->port_type) {
2918         case FW_PORT_TYPE_BT_XFI:
2919                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2920                 break;
2921
2922         case FW_PORT_TYPE_BT_XAUI:
2923                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2924                 /* fall through */
2925
2926         case FW_PORT_TYPE_BT_SGMII:
2927                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2928                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2929                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2930                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2931                 break;
2932
2933         case FW_PORT_TYPE_CX4:
2934                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2935                 ifmedia_set(media, m | IFM_10G_CX4);
2936                 break;
2937
2938         case FW_PORT_TYPE_QSFP_10G:
2939         case FW_PORT_TYPE_SFP:
2940         case FW_PORT_TYPE_FIBER_XFI:
2941         case FW_PORT_TYPE_FIBER_XAUI:
2942                 switch (pi->mod_type) {
2943
2944                 case FW_PORT_MOD_TYPE_LR:
2945                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2946                         ifmedia_set(media, m | IFM_10G_LR);
2947                         break;
2948
2949                 case FW_PORT_MOD_TYPE_SR:
2950                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2951                         ifmedia_set(media, m | IFM_10G_SR);
2952                         break;
2953
2954                 case FW_PORT_MOD_TYPE_LRM:
2955                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2956                         ifmedia_set(media, m | IFM_10G_LRM);
2957                         break;
2958
2959                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2960                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2961                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2962                         ifmedia_set(media, m | IFM_10G_TWINAX);
2963                         break;
2964
2965                 case FW_PORT_MOD_TYPE_NONE:
2966                         m &= ~IFM_FDX;
2967                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2968                         ifmedia_set(media, m | IFM_NONE);
2969                         break;
2970
2971                 case FW_PORT_MOD_TYPE_NA:
2972                 case FW_PORT_MOD_TYPE_ER:
2973                 default:
2974                         device_printf(pi->dev,
2975                             "unknown port_type (%d), mod_type (%d)\n",
2976                             pi->port_type, pi->mod_type);
2977                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2978                         ifmedia_set(media, m | IFM_UNKNOWN);
2979                         break;
2980                 }
2981                 break;
2982
2983         case FW_PORT_TYPE_QSFP:
2984                 switch (pi->mod_type) {
2985
2986                 case FW_PORT_MOD_TYPE_LR:
2987                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2988                         ifmedia_set(media, m | IFM_40G_LR4);
2989                         break;
2990
2991                 case FW_PORT_MOD_TYPE_SR:
2992                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2993                         ifmedia_set(media, m | IFM_40G_SR4);
2994                         break;
2995
2996                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2997                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2998                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2999                         ifmedia_set(media, m | IFM_40G_CR4);
3000                         break;
3001
3002                 case FW_PORT_MOD_TYPE_NONE:
3003                         m &= ~IFM_FDX;
3004                         ifmedia_add(media, m | IFM_NONE, data, NULL);
3005                         ifmedia_set(media, m | IFM_NONE);
3006                         break;
3007
3008                 default:
3009                         device_printf(pi->dev,
3010                             "unknown port_type (%d), mod_type (%d)\n",
3011                             pi->port_type, pi->mod_type);
3012                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
3013                         ifmedia_set(media, m | IFM_UNKNOWN);
3014                         break;
3015                 }
3016                 break;
3017
3018         default:
3019                 device_printf(pi->dev,
3020                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3021                     pi->mod_type);
3022                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
3023                 ifmedia_set(media, m | IFM_UNKNOWN);
3024                 break;
3025         }
3026
3027         PORT_UNLOCK(pi);
3028 }
3029
3030 #define FW_MAC_EXACT_CHUNK      7
3031
3032 /*
3033  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3034  * indicates which parameters should be programmed (the rest are left alone).
3035  */
3036 int
3037 update_mac_settings(struct ifnet *ifp, int flags)
3038 {
3039         int rc = 0;
3040         struct port_info *pi = ifp->if_softc;
3041         struct adapter *sc = pi->adapter;
3042         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3043         uint16_t viid = 0xffff;
3044         int16_t *xact_addr_filt = NULL;
3045
3046         ASSERT_SYNCHRONIZED_OP(sc);
3047         KASSERT(flags, ("%s: not told what to update.", __func__));
3048
3049         if (ifp == pi->ifp) {
3050                 viid = pi->viid;
3051                 xact_addr_filt = &pi->xact_addr_filt;
3052         }
3053 #ifdef DEV_NETMAP
3054         else if (ifp == pi->nm_ifp) {
3055                 viid = pi->nm_viid;
3056                 xact_addr_filt = &pi->nm_xact_addr_filt;
3057         }
3058 #endif
3059         if (flags & XGMAC_MTU)
3060                 mtu = ifp->if_mtu;
3061
3062         if (flags & XGMAC_PROMISC)
3063                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3064
3065         if (flags & XGMAC_ALLMULTI)
3066                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3067
3068         if (flags & XGMAC_VLANEX)
3069                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3070
3071         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3072                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3073                     1, vlanex, false);
3074                 if (rc) {
3075                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3076                             rc);
3077                         return (rc);
3078                 }
3079         }
3080
3081         if (flags & XGMAC_UCADDR) {
3082                 uint8_t ucaddr[ETHER_ADDR_LEN];
3083
3084                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3085                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3086                     true, true);
3087                 if (rc < 0) {
3088                         rc = -rc;
3089                         if_printf(ifp, "change_mac failed: %d\n", rc);
3090                         return (rc);
3091                 } else {
3092                         *xact_addr_filt = rc;
3093                         rc = 0;
3094                 }
3095         }
3096
3097         if (flags & XGMAC_MCADDRS) {
3098                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3099                 int del = 1;
3100                 uint64_t hash = 0;
3101                 struct ifmultiaddr *ifma;
3102                 int i = 0, j;
3103
3104                 if_maddr_rlock(ifp);
3105                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3106                         if (ifma->ifma_addr->sa_family != AF_LINK)
3107                                 continue;
3108                         mcaddr[i] =
3109                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3110                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3111                         i++;
3112
3113                         if (i == FW_MAC_EXACT_CHUNK) {
3114                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3115                                     i, mcaddr, NULL, &hash, 0);
3116                                 if (rc < 0) {
3117                                         rc = -rc;
3118                                         for (j = 0; j < i; j++) {
3119                                                 if_printf(ifp,
3120                                                     "failed to add mc address"
3121                                                     " %02x:%02x:%02x:"
3122                                                     "%02x:%02x:%02x rc=%d\n",
3123                                                     mcaddr[j][0], mcaddr[j][1],
3124                                                     mcaddr[j][2], mcaddr[j][3],
3125                                                     mcaddr[j][4], mcaddr[j][5],
3126                                                     rc);
3127                                         }
3128                                         goto mcfail;
3129                                 }
3130                                 del = 0;
3131                                 i = 0;
3132                         }
3133                 }
3134                 if (i > 0) {
3135                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3136                             mcaddr, NULL, &hash, 0);
3137                         if (rc < 0) {
3138                                 rc = -rc;
3139                                 for (j = 0; j < i; j++) {
3140                                         if_printf(ifp,
3141                                             "failed to add mc address"
3142                                             " %02x:%02x:%02x:"
3143                                             "%02x:%02x:%02x rc=%d\n",
3144                                             mcaddr[j][0], mcaddr[j][1],
3145                                             mcaddr[j][2], mcaddr[j][3],
3146                                             mcaddr[j][4], mcaddr[j][5],
3147                                             rc);
3148                                 }
3149                                 goto mcfail;
3150                         }
3151                 }
3152
3153                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3154                 if (rc != 0)
3155                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3156 mcfail:
3157                 if_maddr_runlock(ifp);
3158         }
3159
3160         return (rc);
3161 }
3162
3163 int
3164 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3165     char *wmesg)
3166 {
3167         int rc, pri;
3168
3169 #ifdef WITNESS
3170         /* the caller thinks it's ok to sleep, but is it really? */
3171         if (flags & SLEEP_OK)
3172                 pause("t4slptst", 1);
3173 #endif
3174
3175         if (INTR_OK)
3176                 pri = PCATCH;
3177         else
3178                 pri = 0;
3179
3180         ADAPTER_LOCK(sc);
3181         for (;;) {
3182
3183                 if (pi && IS_DOOMED(pi)) {
3184                         rc = ENXIO;
3185                         goto done;
3186                 }
3187
3188                 if (!IS_BUSY(sc)) {
3189                         rc = 0;
3190                         break;
3191                 }
3192
3193                 if (!(flags & SLEEP_OK)) {
3194                         rc = EBUSY;
3195                         goto done;
3196                 }
3197
3198                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3199                         rc = EINTR;
3200                         goto done;
3201                 }
3202         }
3203
3204         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3205         SET_BUSY(sc);
3206 #ifdef INVARIANTS
3207         sc->last_op = wmesg;
3208         sc->last_op_thr = curthread;
3209 #endif
3210
3211 done:
3212         if (!(flags & HOLD_LOCK) || rc)
3213                 ADAPTER_UNLOCK(sc);
3214
3215         return (rc);
3216 }
3217
3218 void
3219 end_synchronized_op(struct adapter *sc, int flags)
3220 {
3221
3222         if (flags & LOCK_HELD)
3223                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3224         else
3225                 ADAPTER_LOCK(sc);
3226
3227         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3228         CLR_BUSY(sc);
3229         wakeup(&sc->flags);
3230         ADAPTER_UNLOCK(sc);
3231 }
3232
3233 static int
3234 cxgbe_init_synchronized(struct port_info *pi)
3235 {
3236         struct adapter *sc = pi->adapter;
3237         struct ifnet *ifp = pi->ifp;
3238         int rc = 0;
3239
3240         ASSERT_SYNCHRONIZED_OP(sc);
3241
3242         if (isset(&sc->open_device_map, pi->port_id)) {
3243                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3244                     ("mismatch between open_device_map and if_drv_flags"));
3245                 return (0);     /* already running */
3246         }
3247
3248         if (!(sc->flags & FULL_INIT_DONE) &&
3249             ((rc = adapter_full_init(sc)) != 0))
3250                 return (rc);    /* error message displayed already */
3251
3252         if (!(pi->flags & PORT_INIT_DONE) &&
3253             ((rc = port_full_init(pi)) != 0))
3254                 return (rc); /* error message displayed already */
3255
3256         rc = update_mac_settings(ifp, XGMAC_ALL);
3257         if (rc)
3258                 goto done;      /* error message displayed already */
3259
3260         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3261         if (rc != 0) {
3262                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3263                 goto done;
3264         }
3265
3266         /*
3267          * The first iq of the first port to come up is used for tracing.
3268          */
3269         if (sc->traceq < 0) {
3270                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3271                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3272                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3273                     V_QUEUENUMBER(sc->traceq));
3274                 pi->flags |= HAS_TRACEQ;
3275         }
3276
3277         /* all ok */
3278         setbit(&sc->open_device_map, pi->port_id);
3279         PORT_LOCK(pi);
3280         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3281         PORT_UNLOCK(pi);
3282
3283         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3284 done:
3285         if (rc != 0)
3286                 cxgbe_uninit_synchronized(pi);
3287
3288         return (rc);
3289 }
3290
3291 /*
3292  * Idempotent.
3293  */
3294 static int
3295 cxgbe_uninit_synchronized(struct port_info *pi)
3296 {
3297         struct adapter *sc = pi->adapter;
3298         struct ifnet *ifp = pi->ifp;
3299         int rc;
3300
3301         ASSERT_SYNCHRONIZED_OP(sc);
3302
3303         /*
3304          * Disable the VI so that all its data in either direction is discarded
3305          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3306          * tick) intact as the TP can deliver negative advice or data that it's
3307          * holding in its RAM (for an offloaded connection) even after the VI is
3308          * disabled.
3309          */
3310         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3311         if (rc) {
3312                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3313                 return (rc);
3314         }
3315
3316         clrbit(&sc->open_device_map, pi->port_id);
3317         PORT_LOCK(pi);
3318         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3319         PORT_UNLOCK(pi);
3320
3321         pi->link_cfg.link_ok = 0;
3322         pi->link_cfg.speed = 0;
3323         pi->linkdnrc = -1;
3324         t4_os_link_changed(sc, pi->port_id, 0, -1);
3325
3326         return (0);
3327 }
3328
3329 /*
3330  * It is ok for this function to fail midway and return right away.  t4_detach
3331  * will walk the entire sc->irq list and clean up whatever is valid.
3332  */
3333 static int
3334 setup_intr_handlers(struct adapter *sc)
3335 {
3336         int rc, rid, p, q;
3337         char s[8];
3338         struct irq *irq;
3339         struct port_info *pi;
3340         struct sge_rxq *rxq;
3341 #ifdef TCP_OFFLOAD
3342         struct sge_ofld_rxq *ofld_rxq;
3343 #endif
3344 #ifdef DEV_NETMAP
3345         struct sge_nm_rxq *nm_rxq;
3346 #endif
3347
3348         /*
3349          * Setup interrupts.
3350          */
3351         irq = &sc->irq[0];
3352         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3353         if (sc->intr_count == 1)
3354                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3355
3356         /* Multiple interrupts. */
3357         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3358             ("%s: too few intr.", __func__));
3359
3360         /* The first one is always error intr */
3361         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3362         if (rc != 0)
3363                 return (rc);
3364         irq++;
3365         rid++;
3366
3367         /* The second one is always the firmware event queue */
3368         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3369         if (rc != 0)
3370                 return (rc);
3371         irq++;
3372         rid++;
3373
3374         for_each_port(sc, p) {
3375                 pi = sc->port[p];
3376
3377                 if (pi->flags & INTR_RXQ) {
3378                         for_each_rxq(pi, q, rxq) {
3379                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3380                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3381                                     s);
3382                                 if (rc != 0)
3383                                         return (rc);
3384                                 irq++;
3385                                 rid++;
3386                         }
3387                 }
3388 #ifdef TCP_OFFLOAD
3389                 if (pi->flags & INTR_OFLD_RXQ) {
3390                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3391                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3392                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3393                                     ofld_rxq, s);
3394                                 if (rc != 0)
3395                                         return (rc);
3396                                 irq++;
3397                                 rid++;
3398                         }
3399                 }
3400 #endif
3401 #ifdef DEV_NETMAP
3402                 if (pi->flags & INTR_NM_RXQ) {
3403                         for_each_nm_rxq(pi, q, nm_rxq) {
3404                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3405                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3406                                     nm_rxq, s);
3407                                 if (rc != 0)
3408                                         return (rc);
3409                                 irq++;
3410                                 rid++;
3411                         }
3412                 }
3413 #endif
3414         }
3415         MPASS(irq == &sc->irq[sc->intr_count]);
3416
3417         return (0);
3418 }
3419
3420 int
3421 adapter_full_init(struct adapter *sc)
3422 {
3423         int rc, i;
3424
3425         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3426         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3427             ("%s: FULL_INIT_DONE already", __func__));
3428
3429         /*
3430          * queues that belong to the adapter (not any particular port).
3431          */
3432         rc = t4_setup_adapter_queues(sc);
3433         if (rc != 0)
3434                 goto done;
3435
3436         for (i = 0; i < nitems(sc->tq); i++) {
3437                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3438                     taskqueue_thread_enqueue, &sc->tq[i]);
3439                 if (sc->tq[i] == NULL) {
3440                         device_printf(sc->dev,
3441                             "failed to allocate task queue %d\n", i);
3442                         rc = ENOMEM;
3443                         goto done;
3444                 }
3445                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3446                     device_get_nameunit(sc->dev), i);
3447         }
3448
3449         t4_intr_enable(sc);
3450         sc->flags |= FULL_INIT_DONE;
3451 done:
3452         if (rc != 0)
3453                 adapter_full_uninit(sc);
3454
3455         return (rc);
3456 }
3457
3458 int
3459 adapter_full_uninit(struct adapter *sc)
3460 {
3461         int i;
3462
3463         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3464
3465         t4_teardown_adapter_queues(sc);
3466
3467         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3468                 taskqueue_free(sc->tq[i]);
3469                 sc->tq[i] = NULL;
3470         }
3471
3472         sc->flags &= ~FULL_INIT_DONE;
3473
3474         return (0);
3475 }
3476
3477 int
3478 port_full_init(struct port_info *pi)
3479 {
3480         struct adapter *sc = pi->adapter;
3481         struct ifnet *ifp = pi->ifp;
3482         uint16_t *rss;
3483         struct sge_rxq *rxq;
3484         int rc, i, j;
3485
3486         ASSERT_SYNCHRONIZED_OP(sc);
3487         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3488             ("%s: PORT_INIT_DONE already", __func__));
3489
3490         sysctl_ctx_init(&pi->ctx);
3491         pi->flags |= PORT_SYSCTL_CTX;
3492
3493         /*
3494          * Allocate tx/rx/fl queues for this port.
3495          */
3496         rc = t4_setup_port_queues(pi);
3497         if (rc != 0)
3498                 goto done;      /* error message displayed already */
3499
3500         /*
3501          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3502          */
3503         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3504         for (i = 0; i < pi->rss_size;) {
3505                 for_each_rxq(pi, j, rxq) {
3506                         rss[i++] = rxq->iq.abs_id;
3507                         if (i == pi->rss_size)
3508                                 break;
3509                 }
3510         }
3511
3512         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3513             pi->rss_size);
3514         if (rc != 0) {
3515                 if_printf(ifp, "rss_config failed: %d\n", rc);
3516                 goto done;
3517         }
3518
3519         pi->rss = rss;
3520         pi->flags |= PORT_INIT_DONE;
3521 done:
3522         if (rc != 0)
3523                 port_full_uninit(pi);
3524
3525         return (rc);
3526 }
3527
3528 /*
3529  * Idempotent.
3530  */
3531 int
3532 port_full_uninit(struct port_info *pi)
3533 {
3534         struct adapter *sc = pi->adapter;
3535         int i;
3536         struct sge_rxq *rxq;
3537         struct sge_txq *txq;
3538 #ifdef TCP_OFFLOAD
3539         struct sge_ofld_rxq *ofld_rxq;
3540         struct sge_wrq *ofld_txq;
3541 #endif
3542
3543         if (pi->flags & PORT_INIT_DONE) {
3544
3545                 /* Need to quiesce queues.  XXX: ctrl queues? */
3546
3547                 for_each_txq(pi, i, txq) {
3548                         quiesce_eq(sc, &txq->eq);
3549                 }
3550
3551 #ifdef TCP_OFFLOAD
3552                 for_each_ofld_txq(pi, i, ofld_txq) {
3553                         quiesce_eq(sc, &ofld_txq->eq);
3554                 }
3555 #endif
3556
3557                 for_each_rxq(pi, i, rxq) {
3558                         quiesce_iq(sc, &rxq->iq);
3559                         quiesce_fl(sc, &rxq->fl);
3560                 }
3561
3562 #ifdef TCP_OFFLOAD
3563                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3564                         quiesce_iq(sc, &ofld_rxq->iq);
3565                         quiesce_fl(sc, &ofld_rxq->fl);
3566                 }
3567 #endif
3568                 free(pi->rss, M_CXGBE);
3569         }
3570
3571         t4_teardown_port_queues(pi);
3572         pi->flags &= ~PORT_INIT_DONE;
3573
3574         return (0);
3575 }
3576
3577 static void
3578 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3579 {
3580         EQ_LOCK(eq);
3581         eq->flags |= EQ_DOOMED;
3582
3583         /*
3584          * Wait for the response to a credit flush if one's
3585          * pending.
3586          */
3587         while (eq->flags & EQ_CRFLUSHED)
3588                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3589         EQ_UNLOCK(eq);
3590
3591         callout_drain(&eq->tx_callout); /* XXX: iffy */
3592         pause("callout", 10);           /* Still iffy */
3593
3594         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3595 }
3596
3597 static void
3598 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3599 {
3600         (void) sc;      /* unused */
3601
3602         /* Synchronize with the interrupt handler */
3603         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3604                 pause("iqfree", 1);
3605 }
3606
3607 static void
3608 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3609 {
3610         mtx_lock(&sc->sfl_lock);
3611         FL_LOCK(fl);
3612         fl->flags |= FL_DOOMED;
3613         FL_UNLOCK(fl);
3614         mtx_unlock(&sc->sfl_lock);
3615
3616         callout_drain(&sc->sfl_callout);
3617         KASSERT((fl->flags & FL_STARVING) == 0,
3618             ("%s: still starving", __func__));
3619 }
3620
3621 static int
3622 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3623     driver_intr_t *handler, void *arg, char *name)
3624 {
3625         int rc;
3626
3627         irq->rid = rid;
3628         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3629             RF_SHAREABLE | RF_ACTIVE);
3630         if (irq->res == NULL) {
3631                 device_printf(sc->dev,
3632                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3633                 return (ENOMEM);
3634         }
3635
3636         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3637             NULL, handler, arg, &irq->tag);
3638         if (rc != 0) {
3639                 device_printf(sc->dev,
3640                     "failed to setup interrupt for rid %d, name %s: %d\n",
3641                     rid, name, rc);
3642         } else if (name)
3643                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3644
3645         return (rc);
3646 }
3647
3648 static int
3649 t4_free_irq(struct adapter *sc, struct irq *irq)
3650 {
3651         if (irq->tag)
3652                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3653         if (irq->res)
3654                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3655
3656         bzero(irq, sizeof(*irq));
3657
3658         return (0);
3659 }
3660
3661 static void
3662 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3663     unsigned int end)
3664 {
3665         uint32_t *p = (uint32_t *)(buf + start);
3666
3667         for ( ; start <= end; start += sizeof(uint32_t))
3668                 *p++ = t4_read_reg(sc, start);
3669 }
3670
3671 static void
3672 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3673 {
3674         int i, n;
3675         const unsigned int *reg_ranges;
3676         static const unsigned int t4_reg_ranges[] = {
3677                 0x1008, 0x1108,
3678                 0x1180, 0x11b4,
3679                 0x11fc, 0x123c,
3680                 0x1300, 0x173c,
3681                 0x1800, 0x18fc,
3682                 0x3000, 0x30d8,
3683                 0x30e0, 0x5924,
3684                 0x5960, 0x59d4,
3685                 0x5a00, 0x5af8,
3686                 0x6000, 0x6098,
3687                 0x6100, 0x6150,
3688                 0x6200, 0x6208,
3689                 0x6240, 0x6248,
3690                 0x6280, 0x6338,
3691                 0x6370, 0x638c,
3692                 0x6400, 0x643c,
3693                 0x6500, 0x6524,
3694                 0x6a00, 0x6a38,
3695                 0x6a60, 0x6a78,
3696                 0x6b00, 0x6b84,
3697                 0x6bf0, 0x6c84,
3698                 0x6cf0, 0x6d84,
3699                 0x6df0, 0x6e84,
3700                 0x6ef0, 0x6f84,
3701                 0x6ff0, 0x7084,
3702                 0x70f0, 0x7184,
3703                 0x71f0, 0x7284,
3704                 0x72f0, 0x7384,
3705                 0x73f0, 0x7450,
3706                 0x7500, 0x7530,
3707                 0x7600, 0x761c,
3708                 0x7680, 0x76cc,
3709                 0x7700, 0x7798,
3710                 0x77c0, 0x77fc,
3711                 0x7900, 0x79fc,
3712                 0x7b00, 0x7c38,
3713                 0x7d00, 0x7efc,
3714                 0x8dc0, 0x8e1c,
3715                 0x8e30, 0x8e78,
3716                 0x8ea0, 0x8f6c,
3717                 0x8fc0, 0x9074,
3718                 0x90fc, 0x90fc,
3719                 0x9400, 0x9458,
3720                 0x9600, 0x96bc,
3721                 0x9800, 0x9808,
3722                 0x9820, 0x983c,
3723                 0x9850, 0x9864,
3724                 0x9c00, 0x9c6c,
3725                 0x9c80, 0x9cec,
3726                 0x9d00, 0x9d6c,
3727                 0x9d80, 0x9dec,
3728                 0x9e00, 0x9e6c,
3729                 0x9e80, 0x9eec,
3730                 0x9f00, 0x9f6c,
3731                 0x9f80, 0x9fec,
3732                 0xd004, 0xd03c,
3733                 0xdfc0, 0xdfe0,
3734                 0xe000, 0xea7c,
3735                 0xf000, 0x11110,
3736                 0x11118, 0x11190,
3737                 0x19040, 0x1906c,
3738                 0x19078, 0x19080,
3739                 0x1908c, 0x19124,
3740                 0x19150, 0x191b0,
3741                 0x191d0, 0x191e8,
3742                 0x19238, 0x1924c,
3743                 0x193f8, 0x19474,
3744                 0x19490, 0x194f8,
3745                 0x19800, 0x19f30,
3746                 0x1a000, 0x1a06c,
3747                 0x1a0b0, 0x1a120,
3748                 0x1a128, 0x1a138,
3749                 0x1a190, 0x1a1c4,
3750                 0x1a1fc, 0x1a1fc,
3751                 0x1e040, 0x1e04c,
3752                 0x1e284, 0x1e28c,
3753                 0x1e2c0, 0x1e2c0,
3754                 0x1e2e0, 0x1e2e0,
3755                 0x1e300, 0x1e384,
3756                 0x1e3c0, 0x1e3c8,
3757                 0x1e440, 0x1e44c,
3758                 0x1e684, 0x1e68c,
3759                 0x1e6c0, 0x1e6c0,
3760                 0x1e6e0, 0x1e6e0,
3761                 0x1e700, 0x1e784,
3762                 0x1e7c0, 0x1e7c8,
3763                 0x1e840, 0x1e84c,
3764                 0x1ea84, 0x1ea8c,
3765                 0x1eac0, 0x1eac0,
3766                 0x1eae0, 0x1eae0,
3767                 0x1eb00, 0x1eb84,
3768                 0x1ebc0, 0x1ebc8,
3769                 0x1ec40, 0x1ec4c,
3770                 0x1ee84, 0x1ee8c,
3771                 0x1eec0, 0x1eec0,
3772                 0x1eee0, 0x1eee0,
3773                 0x1ef00, 0x1ef84,
3774                 0x1efc0, 0x1efc8,
3775                 0x1f040, 0x1f04c,
3776                 0x1f284, 0x1f28c,
3777                 0x1f2c0, 0x1f2c0,
3778                 0x1f2e0, 0x1f2e0,
3779                 0x1f300, 0x1f384,
3780                 0x1f3c0, 0x1f3c8,
3781                 0x1f440, 0x1f44c,
3782                 0x1f684, 0x1f68c,
3783                 0x1f6c0, 0x1f6c0,
3784                 0x1f6e0, 0x1f6e0,
3785                 0x1f700, 0x1f784,
3786                 0x1f7c0, 0x1f7c8,
3787                 0x1f840, 0x1f84c,
3788                 0x1fa84, 0x1fa8c,
3789                 0x1fac0, 0x1fac0,
3790                 0x1fae0, 0x1fae0,
3791                 0x1fb00, 0x1fb84,
3792                 0x1fbc0, 0x1fbc8,
3793                 0x1fc40, 0x1fc4c,
3794                 0x1fe84, 0x1fe8c,
3795                 0x1fec0, 0x1fec0,
3796                 0x1fee0, 0x1fee0,
3797                 0x1ff00, 0x1ff84,
3798                 0x1ffc0, 0x1ffc8,
3799                 0x20000, 0x2002c,
3800                 0x20100, 0x2013c,
3801                 0x20190, 0x201c8,
3802                 0x20200, 0x20318,
3803                 0x20400, 0x20528,
3804                 0x20540, 0x20614,
3805                 0x21000, 0x21040,
3806                 0x2104c, 0x21060,
3807                 0x210c0, 0x210ec,
3808                 0x21200, 0x21268,
3809                 0x21270, 0x21284,
3810                 0x212fc, 0x21388,
3811                 0x21400, 0x21404,
3812                 0x21500, 0x21518,
3813                 0x2152c, 0x2153c,
3814                 0x21550, 0x21554,
3815                 0x21600, 0x21600,
3816                 0x21608, 0x21628,
3817                 0x21630, 0x2163c,
3818                 0x21700, 0x2171c,
3819                 0x21780, 0x2178c,
3820                 0x21800, 0x21c38,
3821                 0x21c80, 0x21d7c,
3822                 0x21e00, 0x21e04,
3823                 0x22000, 0x2202c,
3824                 0x22100, 0x2213c,
3825                 0x22190, 0x221c8,
3826                 0x22200, 0x22318,
3827                 0x22400, 0x22528,
3828                 0x22540, 0x22614,
3829                 0x23000, 0x23040,
3830                 0x2304c, 0x23060,
3831                 0x230c0, 0x230ec,
3832                 0x23200, 0x23268,
3833                 0x23270, 0x23284,
3834                 0x232fc, 0x23388,
3835                 0x23400, 0x23404,
3836                 0x23500, 0x23518,
3837                 0x2352c, 0x2353c,
3838                 0x23550, 0x23554,
3839                 0x23600, 0x23600,
3840                 0x23608, 0x23628,
3841                 0x23630, 0x2363c,
3842                 0x23700, 0x2371c,
3843                 0x23780, 0x2378c,
3844                 0x23800, 0x23c38,
3845                 0x23c80, 0x23d7c,
3846                 0x23e00, 0x23e04,
3847                 0x24000, 0x2402c,
3848                 0x24100, 0x2413c,
3849                 0x24190, 0x241c8,
3850                 0x24200, 0x24318,
3851                 0x24400, 0x24528,
3852                 0x24540, 0x24614,
3853                 0x25000, 0x25040,
3854                 0x2504c, 0x25060,
3855                 0x250c0, 0x250ec,
3856                 0x25200, 0x25268,
3857                 0x25270, 0x25284,
3858                 0x252fc, 0x25388,
3859                 0x25400, 0x25404,
3860                 0x25500, 0x25518,
3861                 0x2552c, 0x2553c,
3862                 0x25550, 0x25554,
3863                 0x25600, 0x25600,
3864                 0x25608, 0x25628,
3865                 0x25630, 0x2563c,
3866                 0x25700, 0x2571c,
3867                 0x25780, 0x2578c,
3868                 0x25800, 0x25c38,
3869                 0x25c80, 0x25d7c,
3870                 0x25e00, 0x25e04,
3871                 0x26000, 0x2602c,
3872                 0x26100, 0x2613c,
3873                 0x26190, 0x261c8,
3874                 0x26200, 0x26318,
3875                 0x26400, 0x26528,
3876                 0x26540, 0x26614,
3877                 0x27000, 0x27040,
3878                 0x2704c, 0x27060,
3879                 0x270c0, 0x270ec,
3880                 0x27200, 0x27268,
3881                 0x27270, 0x27284,
3882                 0x272fc, 0x27388,
3883                 0x27400, 0x27404,
3884                 0x27500, 0x27518,
3885                 0x2752c, 0x2753c,
3886                 0x27550, 0x27554,
3887                 0x27600, 0x27600,
3888                 0x27608, 0x27628,
3889                 0x27630, 0x2763c,
3890                 0x27700, 0x2771c,
3891                 0x27780, 0x2778c,
3892                 0x27800, 0x27c38,
3893                 0x27c80, 0x27d7c,
3894                 0x27e00, 0x27e04
3895         };
3896         static const unsigned int t5_reg_ranges[] = {
3897                 0x1008, 0x1148,
3898                 0x1180, 0x11b4,
3899                 0x11fc, 0x123c,
3900                 0x1280, 0x173c,
3901                 0x1800, 0x18fc,
3902                 0x3000, 0x3028,
3903                 0x3060, 0x30d8,
3904                 0x30e0, 0x30fc,
3905                 0x3140, 0x357c,
3906                 0x35a8, 0x35cc,
3907                 0x35ec, 0x35ec,
3908                 0x3600, 0x5624,
3909                 0x56cc, 0x575c,
3910                 0x580c, 0x5814,
3911                 0x5890, 0x58bc,
3912                 0x5940, 0x59dc,
3913                 0x59fc, 0x5a18,
3914                 0x5a60, 0x5a9c,
3915                 0x5b94, 0x5bfc,
3916                 0x6000, 0x6040,
3917                 0x6058, 0x614c,
3918                 0x7700, 0x7798,
3919                 0x77c0, 0x78fc,
3920                 0x7b00, 0x7c54,
3921                 0x7d00, 0x7efc,
3922                 0x8dc0, 0x8de0,
3923                 0x8df8, 0x8e84,
3924                 0x8ea0, 0x8f84,
3925                 0x8fc0, 0x90f8,
3926                 0x9400, 0x9470,
3927                 0x9600, 0x96f4,
3928                 0x9800, 0x9808,
3929                 0x9820, 0x983c,
3930                 0x9850, 0x9864,
3931                 0x9c00, 0x9c6c,
3932                 0x9c80, 0x9cec,
3933                 0x9d00, 0x9d6c,
3934                 0x9d80, 0x9dec,
3935                 0x9e00, 0x9e6c,
3936                 0x9e80, 0x9eec,
3937                 0x9f00, 0x9f6c,
3938                 0x9f80, 0xa020,
3939                 0xd004, 0xd03c,
3940                 0xdfc0, 0xdfe0,
3941                 0xe000, 0x11088,
3942                 0x1109c, 0x11110,
3943                 0x11118, 0x1117c,
3944                 0x11190, 0x11204,
3945                 0x19040, 0x1906c,
3946                 0x19078, 0x19080,
3947                 0x1908c, 0x19124,
3948                 0x19150, 0x191b0,
3949                 0x191d0, 0x191e8,
3950                 0x19238, 0x19290,
3951                 0x193f8, 0x19474,
3952                 0x19490, 0x194cc,
3953                 0x194f0, 0x194f8,
3954                 0x19c00, 0x19c60,
3955                 0x19c94, 0x19e10,
3956                 0x19e50, 0x19f34,
3957                 0x19f40, 0x19f50,
3958                 0x19f90, 0x19fe4,
3959                 0x1a000, 0x1a06c,
3960                 0x1a0b0, 0x1a120,
3961                 0x1a128, 0x1a138,
3962                 0x1a190, 0x1a1c4,
3963                 0x1a1fc, 0x1a1fc,
3964                 0x1e008, 0x1e00c,
3965                 0x1e040, 0x1e04c,
3966                 0x1e284, 0x1e290,
3967                 0x1e2c0, 0x1e2c0,
3968                 0x1e2e0, 0x1e2e0,
3969                 0x1e300, 0x1e384,
3970                 0x1e3c0, 0x1e3c8,
3971                 0x1e408, 0x1e40c,
3972                 0x1e440, 0x1e44c,
3973                 0x1e684, 0x1e690,
3974                 0x1e6c0, 0x1e6c0,
3975                 0x1e6e0, 0x1e6e0,
3976                 0x1e700, 0x1e784,
3977                 0x1e7c0, 0x1e7c8,
3978                 0x1e808, 0x1e80c,
3979                 0x1e840, 0x1e84c,
3980                 0x1ea84, 0x1ea90,
3981                 0x1eac0, 0x1eac0,
3982                 0x1eae0, 0x1eae0,
3983                 0x1eb00, 0x1eb84,
3984                 0x1ebc0, 0x1ebc8,
3985                 0x1ec08, 0x1ec0c,
3986                 0x1ec40, 0x1ec4c,
3987                 0x1ee84, 0x1ee90,
3988                 0x1eec0, 0x1eec0,
3989                 0x1eee0, 0x1eee0,
3990                 0x1ef00, 0x1ef84,
3991                 0x1efc0, 0x1efc8,
3992                 0x1f008, 0x1f00c,
3993                 0x1f040, 0x1f04c,
3994                 0x1f284, 0x1f290,
3995                 0x1f2c0, 0x1f2c0,
3996                 0x1f2e0, 0x1f2e0,
3997                 0x1f300, 0x1f384,
3998                 0x1f3c0, 0x1f3c8,
3999                 0x1f408, 0x1f40c,
4000                 0x1f440, 0x1f44c,
4001                 0x1f684, 0x1f690,
4002                 0x1f6c0, 0x1f6c0,
4003                 0x1f6e0, 0x1f6e0,
4004                 0x1f700, 0x1f784,
4005                 0x1f7c0, 0x1f7c8,
4006                 0x1f808, 0x1f80c,
4007                 0x1f840, 0x1f84c,
4008                 0x1fa84, 0x1fa90,
4009                 0x1fac0, 0x1fac0,
4010                 0x1fae0, 0x1fae0,
4011                 0x1fb00, 0x1fb84,
4012                 0x1fbc0, 0x1fbc8,
4013                 0x1fc08, 0x1fc0c,
4014                 0x1fc40, 0x1fc4c,
4015                 0x1fe84, 0x1fe90,
4016                 0x1fec0, 0x1fec0,
4017                 0x1fee0, 0x1fee0,
4018                 0x1ff00, 0x1ff84,
4019                 0x1ffc0, 0x1ffc8,
4020                 0x30000, 0x30030,
4021                 0x30100, 0x30144,
4022                 0x30190, 0x301d0,
4023                 0x30200, 0x30318,
4024                 0x30400, 0x3052c,
4025                 0x30540, 0x3061c,
4026                 0x30800, 0x30834,
4027                 0x308c0, 0x30908,
4028                 0x30910, 0x309ac,
4029                 0x30a00, 0x30a2c,
4030                 0x30a44, 0x30a50,
4031                 0x30a74, 0x30c24,
4032                 0x30d00, 0x30d00,
4033                 0x30d08, 0x30d14,
4034                 0x30d1c, 0x30d20,
4035                 0x30d3c, 0x30d50,
4036                 0x31200, 0x3120c,
4037                 0x31220, 0x31220,
4038                 0x31240, 0x31240,
4039                 0x31600, 0x3160c,
4040                 0x31a00, 0x31a1c,
4041                 0x31e00, 0x31e20,
4042                 0x31e38, 0x31e3c,
4043                 0x31e80, 0x31e80,
4044                 0x31e88, 0x31ea8,
4045                 0x31eb0, 0x31eb4,
4046                 0x31ec8, 0x31ed4,
4047                 0x31fb8, 0x32004,
4048                 0x32200, 0x32200,
4049                 0x32208, 0x32240,
4050                 0x32248, 0x32280,
4051                 0x32288, 0x322c0,
4052                 0x322c8, 0x322fc,
4053                 0x32600, 0x32630,
4054                 0x32a00, 0x32abc,
4055                 0x32b00, 0x32b70,
4056                 0x33000, 0x33048,
4057                 0x33060, 0x3309c,
4058                 0x330f0, 0x33148,
4059                 0x33160, 0x3319c,
4060                 0x331f0, 0x332e4,
4061                 0x332f8, 0x333e4,
4062                 0x333f8, 0x33448,
4063                 0x33460, 0x3349c,
4064                 0x334f0, 0x33548,
4065                 0x33560, 0x3359c,
4066                 0x335f0, 0x336e4,
4067                 0x336f8, 0x337e4,
4068                 0x337f8, 0x337fc,
4069                 0x33814, 0x33814,
4070                 0x3382c, 0x3382c,
4071                 0x33880, 0x3388c,
4072                 0x338e8, 0x338ec,
4073                 0x33900, 0x33948,
4074                 0x33960, 0x3399c,
4075                 0x339f0, 0x33ae4,
4076                 0x33af8, 0x33b10,
4077                 0x33b28, 0x33b28,
4078                 0x33b3c, 0x33b50,
4079                 0x33bf0, 0x33c10,
4080                 0x33c28, 0x33c28,
4081                 0x33c3c, 0x33c50,
4082                 0x33cf0, 0x33cfc,
4083                 0x34000, 0x34030,
4084                 0x34100, 0x34144,
4085                 0x34190, 0x341d0,
4086                 0x34200, 0x34318,
4087                 0x34400, 0x3452c,
4088                 0x34540, 0x3461c,
4089                 0x34800, 0x34834,
4090                 0x348c0, 0x34908,
4091                 0x34910, 0x349ac,
4092                 0x34a00, 0x34a2c,
4093                 0x34a44, 0x34a50,
4094                 0x34a74, 0x34c24,
4095                 0x34d00, 0x34d00,
4096                 0x34d08, 0x34d14,
4097                 0x34d1c, 0x34d20,
4098                 0x34d3c, 0x34d50,
4099                 0x35200, 0x3520c,
4100                 0x35220, 0x35220,
4101                 0x35240, 0x35240,
4102                 0x35600, 0x3560c,
4103                 0x35a00, 0x35a1c,
4104                 0x35e00, 0x35e20,
4105                 0x35e38, 0x35e3c,
4106                 0x35e80, 0x35e80,
4107                 0x35e88, 0x35ea8,
4108                 0x35eb0, 0x35eb4,
4109                 0x35ec8, 0x35ed4,
4110                 0x35fb8, 0x36004,
4111                 0x36200, 0x36200,
4112                 0x36208, 0x36240,
4113                 0x36248, 0x36280,
4114                 0x36288, 0x362c0,
4115                 0x362c8, 0x362fc,
4116                 0x36600, 0x36630,
4117                 0x36a00, 0x36abc,
4118                 0x36b00, 0x36b70,
4119                 0x37000, 0x37048,
4120                 0x37060, 0x3709c,
4121                 0x370f0, 0x37148,
4122                 0x37160, 0x3719c,
4123                 0x371f0, 0x372e4,
4124                 0x372f8, 0x373e4,
4125                 0x373f8, 0x37448,
4126                 0x37460, 0x3749c,
4127                 0x374f0, 0x37548,
4128                 0x37560, 0x3759c,
4129                 0x375f0, 0x376e4,
4130                 0x376f8, 0x377e4,
4131                 0x377f8, 0x377fc,
4132                 0x37814, 0x37814,
4133                 0x3782c, 0x3782c,
4134                 0x37880, 0x3788c,
4135                 0x378e8, 0x378ec,
4136                 0x37900, 0x37948,
4137                 0x37960, 0x3799c,
4138                 0x379f0, 0x37ae4,
4139                 0x37af8, 0x37b10,
4140                 0x37b28, 0x37b28,
4141                 0x37b3c, 0x37b50,
4142                 0x37bf0, 0x37c10,
4143                 0x37c28, 0x37c28,
4144                 0x37c3c, 0x37c50,
4145                 0x37cf0, 0x37cfc,
4146                 0x38000, 0x38030,
4147                 0x38100, 0x38144,
4148                 0x38190, 0x381d0,
4149                 0x38200, 0x38318,
4150                 0x38400, 0x3852c,
4151                 0x38540, 0x3861c,
4152                 0x38800, 0x38834,
4153                 0x388c0, 0x38908,
4154                 0x38910, 0x389ac,
4155                 0x38a00, 0x38a2c,
4156                 0x38a44, 0x38a50,
4157                 0x38a74, 0x38c24,
4158                 0x38d00, 0x38d00,
4159                 0x38d08, 0x38d14,
4160                 0x38d1c, 0x38d20,
4161                 0x38d3c, 0x38d50,
4162                 0x39200, 0x3920c,
4163                 0x39220, 0x39220,
4164                 0x39240, 0x39240,
4165                 0x39600, 0x3960c,
4166                 0x39a00, 0x39a1c,
4167                 0x39e00, 0x39e20,
4168                 0x39e38, 0x39e3c,
4169                 0x39e80, 0x39e80,
4170                 0x39e88, 0x39ea8,
4171                 0x39eb0, 0x39eb4,
4172                 0x39ec8, 0x39ed4,
4173                 0x39fb8, 0x3a004,
4174                 0x3a200, 0x3a200,
4175                 0x3a208, 0x3a240,
4176                 0x3a248, 0x3a280,
4177                 0x3a288, 0x3a2c0,
4178                 0x3a2c8, 0x3a2fc,
4179                 0x3a600, 0x3a630,
4180                 0x3aa00, 0x3aabc,
4181                 0x3ab00, 0x3ab70,
4182                 0x3b000, 0x3b048,
4183                 0x3b060, 0x3b09c,
4184                 0x3b0f0, 0x3b148,
4185                 0x3b160, 0x3b19c,
4186                 0x3b1f0, 0x3b2e4,
4187                 0x3b2f8, 0x3b3e4,
4188                 0x3b3f8, 0x3b448,
4189                 0x3b460, 0x3b49c,
4190                 0x3b4f0, 0x3b548,
4191                 0x3b560, 0x3b59c,
4192                 0x3b5f0, 0x3b6e4,
4193                 0x3b6f8, 0x3b7e4,
4194                 0x3b7f8, 0x3b7fc,
4195                 0x3b814, 0x3b814,
4196                 0x3b82c, 0x3b82c,
4197                 0x3b880, 0x3b88c,
4198                 0x3b8e8, 0x3b8ec,
4199                 0x3b900, 0x3b948,
4200                 0x3b960, 0x3b99c,
4201                 0x3b9f0, 0x3bae4,
4202                 0x3baf8, 0x3bb10,
4203                 0x3bb28, 0x3bb28,
4204                 0x3bb3c, 0x3bb50,
4205                 0x3bbf0, 0x3bc10,
4206                 0x3bc28, 0x3bc28,
4207                 0x3bc3c, 0x3bc50,
4208                 0x3bcf0, 0x3bcfc,
4209                 0x3c000, 0x3c030,
4210                 0x3c100, 0x3c144,
4211                 0x3c190, 0x3c1d0,
4212                 0x3c200, 0x3c318,
4213                 0x3c400, 0x3c52c,
4214                 0x3c540, 0x3c61c,
4215                 0x3c800, 0x3c834,
4216                 0x3c8c0, 0x3c908,
4217                 0x3c910, 0x3c9ac,
4218                 0x3ca00, 0x3ca2c,
4219                 0x3ca44, 0x3ca50,
4220                 0x3ca74, 0x3cc24,
4221                 0x3cd00, 0x3cd00,
4222                 0x3cd08, 0x3cd14,
4223                 0x3cd1c, 0x3cd20,
4224                 0x3cd3c, 0x3cd50,
4225                 0x3d200, 0x3d20c,
4226                 0x3d220, 0x3d220,
4227                 0x3d240, 0x3d240,
4228                 0x3d600, 0x3d60c,
4229                 0x3da00, 0x3da1c,
4230                 0x3de00, 0x3de20,
4231                 0x3de38, 0x3de3c,
4232                 0x3de80, 0x3de80,
4233                 0x3de88, 0x3dea8,
4234                 0x3deb0, 0x3deb4,
4235                 0x3dec8, 0x3ded4,
4236                 0x3dfb8, 0x3e004,
4237                 0x3e200, 0x3e200,
4238                 0x3e208, 0x3e240,
4239                 0x3e248, 0x3e280,
4240                 0x3e288, 0x3e2c0,
4241                 0x3e2c8, 0x3e2fc,
4242                 0x3e600, 0x3e630,
4243                 0x3ea00, 0x3eabc,
4244                 0x3eb00, 0x3eb70,
4245                 0x3f000, 0x3f048,
4246                 0x3f060, 0x3f09c,
4247                 0x3f0f0, 0x3f148,
4248                 0x3f160, 0x3f19c,
4249                 0x3f1f0, 0x3f2e4,
4250                 0x3f2f8, 0x3f3e4,
4251                 0x3f3f8, 0x3f448,
4252                 0x3f460, 0x3f49c,
4253                 0x3f4f0, 0x3f548,
4254                 0x3f560, 0x3f59c,
4255                 0x3f5f0, 0x3f6e4,
4256                 0x3f6f8, 0x3f7e4,
4257                 0x3f7f8, 0x3f7fc,
4258                 0x3f814, 0x3f814,
4259                 0x3f82c, 0x3f82c,
4260                 0x3f880, 0x3f88c,
4261                 0x3f8e8, 0x3f8ec,
4262                 0x3f900, 0x3f948,
4263                 0x3f960, 0x3f99c,
4264                 0x3f9f0, 0x3fae4,
4265                 0x3faf8, 0x3fb10,
4266                 0x3fb28, 0x3fb28,
4267                 0x3fb3c, 0x3fb50,
4268                 0x3fbf0, 0x3fc10,
4269                 0x3fc28, 0x3fc28,
4270                 0x3fc3c, 0x3fc50,
4271                 0x3fcf0, 0x3fcfc,
4272                 0x40000, 0x4000c,
4273                 0x40040, 0x40068,
4274                 0x4007c, 0x40144,
4275                 0x40180, 0x4018c,
4276                 0x40200, 0x40298,
4277                 0x402ac, 0x4033c,
4278                 0x403f8, 0x403fc,
4279                 0x41304, 0x413c4,
4280                 0x41400, 0x4141c,
4281                 0x41480, 0x414d0,
4282                 0x44000, 0x44078,
4283                 0x440c0, 0x44278,
4284                 0x442c0, 0x44478,
4285                 0x444c0, 0x44678,
4286                 0x446c0, 0x44878,
4287                 0x448c0, 0x449fc,
4288                 0x45000, 0x45068,
4289                 0x45080, 0x45084,
4290                 0x450a0, 0x450b0,
4291                 0x45200, 0x45268,
4292                 0x45280, 0x45284,
4293                 0x452a0, 0x452b0,
4294                 0x460c0, 0x460e4,
4295                 0x47000, 0x4708c,
4296                 0x47200, 0x47250,
4297                 0x47400, 0x47420,
4298                 0x47600, 0x47618,
4299                 0x47800, 0x47814,
4300                 0x48000, 0x4800c,
4301                 0x48040, 0x48068,
4302                 0x4807c, 0x48144,
4303                 0x48180, 0x4818c,
4304                 0x48200, 0x48298,
4305                 0x482ac, 0x4833c,
4306                 0x483f8, 0x483fc,
4307                 0x49304, 0x493c4,
4308                 0x49400, 0x4941c,
4309                 0x49480, 0x494d0,
4310                 0x4c000, 0x4c078,
4311                 0x4c0c0, 0x4c278,
4312                 0x4c2c0, 0x4c478,
4313                 0x4c4c0, 0x4c678,
4314                 0x4c6c0, 0x4c878,
4315                 0x4c8c0, 0x4c9fc,
4316                 0x4d000, 0x4d068,
4317                 0x4d080, 0x4d084,
4318                 0x4d0a0, 0x4d0b0,
4319                 0x4d200, 0x4d268,
4320                 0x4d280, 0x4d284,
4321                 0x4d2a0, 0x4d2b0,
4322                 0x4e0c0, 0x4e0e4,
4323                 0x4f000, 0x4f08c,
4324                 0x4f200, 0x4f250,
4325                 0x4f400, 0x4f420,
4326                 0x4f600, 0x4f618,
4327                 0x4f800, 0x4f814,
4328                 0x50000, 0x500cc,
4329                 0x50400, 0x50400,
4330                 0x50800, 0x508cc,
4331                 0x50c00, 0x50c00,
4332                 0x51000, 0x5101c,
4333                 0x51300, 0x51308,
4334         };
4335
4336         if (is_t4(sc)) {
4337                 reg_ranges = &t4_reg_ranges[0];
4338                 n = nitems(t4_reg_ranges);
4339         } else {
4340                 reg_ranges = &t5_reg_ranges[0];
4341                 n = nitems(t5_reg_ranges);
4342         }
4343
4344         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4345         for (i = 0; i < n; i += 2)
4346                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4347 }
4348
4349 static void
4350 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4351 {
4352         int i;
4353         u_int v, tnl_cong_drops;
4354         struct timeval tv;
4355         const struct timeval interval = {0, 250000};    /* 250ms */
4356
4357         getmicrotime(&tv);
4358         timevalsub(&tv, &interval);
4359         if (timevalcmp(&tv, &pi->last_refreshed, <))
4360                 return;
4361
4362         tnl_cong_drops = 0;
4363         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4364         for (i = 0; i < NCHAN; i++) {
4365                 if (pi->rx_chan_map & (1 << i)) {
4366                         mtx_lock(&sc->regwin_lock);
4367                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4368                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4369                         mtx_unlock(&sc->regwin_lock);
4370                         tnl_cong_drops += v;
4371                 }
4372         }
4373         pi->tnl_cong_drops = tnl_cong_drops;
4374         getmicrotime(&pi->last_refreshed);
4375 }
4376
4377 static void
4378 cxgbe_tick(void *arg)
4379 {
4380         struct port_info *pi = arg;
4381         struct adapter *sc = pi->adapter;
4382         struct ifnet *ifp = pi->ifp;
4383
4384         PORT_LOCK(pi);
4385         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4386                 PORT_UNLOCK(pi);
4387                 return; /* without scheduling another callout */
4388         }
4389
4390         cxgbe_refresh_stats(sc, pi);
4391
4392         callout_schedule(&pi->tick, hz);
4393         PORT_UNLOCK(pi);
4394 }
4395
4396 static void
4397 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4398 {
4399         struct ifnet *vlan;
4400
4401         if (arg != ifp || ifp->if_type != IFT_ETHER)
4402                 return;
4403
4404         vlan = VLAN_DEVAT(ifp, vid);
4405         VLAN_SETCOOKIE(vlan, ifp);
4406 }
4407
4408 static int
4409 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4410 {
4411
4412 #ifdef INVARIANTS
4413         panic("%s: opcode 0x%02x on iq %p with payload %p",
4414             __func__, rss->opcode, iq, m);
4415 #else
4416         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4417             __func__, rss->opcode, iq, m);
4418         m_freem(m);
4419 #endif
4420         return (EDOOFUS);
4421 }
4422
4423 int
4424 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4425 {
4426         uintptr_t *loc, new;
4427
4428         if (opcode >= nitems(sc->cpl_handler))
4429                 return (EINVAL);
4430
4431         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4432         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4433         atomic_store_rel_ptr(loc, new);
4434
4435         return (0);
4436 }
4437
4438 static int
4439 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4440 {
4441
4442 #ifdef INVARIANTS
4443         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4444 #else
4445         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4446             __func__, iq, ctrl);
4447 #endif
4448         return (EDOOFUS);
4449 }
4450
4451 int
4452 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4453 {
4454         uintptr_t *loc, new;
4455
4456         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4457         loc = (uintptr_t *) &sc->an_handler;
4458         atomic_store_rel_ptr(loc, new);
4459
4460         return (0);
4461 }
4462
4463 static int
4464 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4465 {
4466         const struct cpl_fw6_msg *cpl =
4467             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4468
4469 #ifdef INVARIANTS
4470         panic("%s: fw_msg type %d", __func__, cpl->type);
4471 #else
4472         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4473 #endif
4474         return (EDOOFUS);
4475 }
4476
4477 int
4478 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4479 {
4480         uintptr_t *loc, new;
4481
4482         if (type >= nitems(sc->fw_msg_handler))
4483                 return (EINVAL);
4484
4485         /*
4486          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4487          * handler dispatch table.  Reject any attempt to install a handler for
4488          * this subtype.
4489          */
4490         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4491                 return (EINVAL);
4492
4493         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4494         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4495         atomic_store_rel_ptr(loc, new);
4496
4497         return (0);
4498 }
4499
4500 static int
4501 t4_sysctls(struct adapter *sc)
4502 {
4503         struct sysctl_ctx_list *ctx;
4504         struct sysctl_oid *oid;
4505         struct sysctl_oid_list *children, *c0;
4506         static char *caps[] = {
4507                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4508                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4509                     "\6HASHFILTER\7ETHOFLD",
4510                 "\20\1TOE",                             /* caps[2] toecaps */
4511                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4512                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4513                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4514                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4515                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4516                     "\4PO_INITIAOR\5PO_TARGET"
4517         };
4518         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4519
4520         ctx = device_get_sysctl_ctx(sc->dev);
4521
4522         /*
4523          * dev.t4nex.X.
4524          */
4525         oid = device_get_sysctl_tree(sc->dev);
4526         c0 = children = SYSCTL_CHILDREN(oid);
4527
4528         sc->sc_do_rxcopy = 1;
4529         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4530             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4531
4532         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4533             sc->params.nports, "# of ports");
4534
4535         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4536             NULL, chip_rev(sc), "chip hardware revision");
4537
4538         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4539             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4540
4541         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4542             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4543
4544         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4545             sc->cfcsum, "config file checksum");
4546
4547         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4548             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4549             sysctl_bitfield, "A", "available doorbells");
4550
4551         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4552             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4553             sysctl_bitfield, "A", "available link capabilities");
4554
4555         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4556             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4557             sysctl_bitfield, "A", "available NIC capabilities");
4558
4559         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4560             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4561             sysctl_bitfield, "A", "available TCP offload capabilities");
4562
4563         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4564             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4565             sysctl_bitfield, "A", "available RDMA capabilities");
4566
4567         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4568             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4569             sysctl_bitfield, "A", "available iSCSI capabilities");
4570
4571         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4572             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4573             sysctl_bitfield, "A", "available FCoE capabilities");
4574
4575         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4576             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4577
4578         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4579             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4580             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4581             "interrupt holdoff timer values (us)");
4582
4583         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4584             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4585             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4586             "interrupt holdoff packet counter values");
4587
4588         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4589             NULL, sc->tids.nftids, "number of filters");
4590
4591         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4592             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4593             "chip temperature (in Celsius)");
4594
4595         t4_sge_sysctls(sc, ctx, children);
4596
4597         sc->lro_timeout = 100;
4598         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4599             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4600
4601 #ifdef SBUF_DRAIN
4602         /*
4603          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4604          */
4605         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4606             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4607             "logs and miscellaneous information");
4608         children = SYSCTL_CHILDREN(oid);
4609
4610         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4611             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4612             sysctl_cctrl, "A", "congestion control");
4613
4614         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4615             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4616             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4620             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4624             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4628             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4629
4630         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4631             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4632             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4633
4634         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4635             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4636             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4637
4638         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4639             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4640             sysctl_cim_la, "A", "CIM logic analyzer");
4641
4642         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4643             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4644             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4645
4646         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4647             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4648             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4649
4650         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4651             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4652             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4653
4654         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4655             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4656             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4657
4658         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4659             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4660             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4661
4662         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4663             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4664             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4665
4666         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4667             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4668             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4669
4670         if (is_t5(sc)) {
4671                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4672                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4673                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4674
4675                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4676                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4677                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4678         }
4679
4680         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4681             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4682             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4683
4684         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4685             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4686             sysctl_cim_qcfg, "A", "CIM queue configuration");
4687
4688         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4689             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4690             sysctl_cpl_stats, "A", "CPL statistics");
4691
4692         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4693             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694             sysctl_ddp_stats, "A", "DDP statistics");
4695
4696         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4697             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4698             sysctl_devlog, "A", "firmware's device log");
4699
4700         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4701             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4702             sysctl_fcoe_stats, "A", "FCoE statistics");
4703
4704         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4705             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4706             sysctl_hw_sched, "A", "hardware scheduler ");
4707
4708         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4709             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4710             sysctl_l2t, "A", "hardware L2 table");
4711
4712         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4713             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4714             sysctl_lb_stats, "A", "loopback statistics");
4715
4716         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4717             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4718             sysctl_meminfo, "A", "memory regions");
4719
4720         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4721             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4722             sysctl_mps_tcam, "A", "MPS TCAM entries");
4723
4724         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4725             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4726             sysctl_path_mtus, "A", "path MTUs");
4727
4728         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4729             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4730             sysctl_pm_stats, "A", "PM statistics");
4731
4732         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4733             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4734             sysctl_rdma_stats, "A", "RDMA statistics");
4735
4736         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4737             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4738             sysctl_tcp_stats, "A", "TCP statistics");
4739
4740         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4741             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4742             sysctl_tids, "A", "TID information");
4743
4744         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4745             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4746             sysctl_tp_err_stats, "A", "TP error statistics");
4747
4748         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4749             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4750             sysctl_tp_la, "A", "TP logic analyzer");
4751
4752         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4753             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4754             sysctl_tx_rate, "A", "Tx rate");
4755
4756         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4757             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4758             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4759
4760         if (is_t5(sc)) {
4761                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4762                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4763                     sysctl_wcwr_stats, "A", "write combined work requests");
4764         }
4765 #endif
4766
4767 #ifdef TCP_OFFLOAD
4768         if (is_offload(sc)) {
4769                 /*
4770                  * dev.t4nex.X.toe.
4771                  */
4772                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4773                     NULL, "TOE parameters");
4774                 children = SYSCTL_CHILDREN(oid);
4775
4776                 sc->tt.sndbuf = 256 * 1024;
4777                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4778                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4779
4780                 sc->tt.ddp = 0;
4781                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4782                     &sc->tt.ddp, 0, "DDP allowed");
4783
4784                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4785                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4786                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4787
4788                 sc->tt.ddp_thres =
4789                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4790                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4791                     &sc->tt.ddp_thres, 0, "DDP threshold");
4792
4793                 sc->tt.rx_coalesce = 1;
4794                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4795                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4796         }
4797 #endif
4798
4799
4800         return (0);
4801 }
4802
4803 static int
4804 cxgbe_sysctls(struct port_info *pi)
4805 {
4806         struct sysctl_ctx_list *ctx;
4807         struct sysctl_oid *oid;
4808         struct sysctl_oid_list *children;
4809         struct adapter *sc = pi->adapter;
4810
4811         ctx = device_get_sysctl_ctx(pi->dev);
4812
4813         /*
4814          * dev.cxgbe.X.
4815          */
4816         oid = device_get_sysctl_tree(pi->dev);
4817         children = SYSCTL_CHILDREN(oid);
4818
4819         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4820            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4821         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4822                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4823                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4824                     "PHY temperature (in Celsius)");
4825                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4826                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4827                     "PHY firmware version");
4828         }
4829         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4830             &pi->nrxq, 0, "# of rx queues");
4831         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4832             &pi->ntxq, 0, "# of tx queues");
4833         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4834             &pi->first_rxq, 0, "index of first rx queue");
4835         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4836             &pi->first_txq, 0, "index of first tx queue");
4837         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4838             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4839             "Reserve queue 0 for non-flowid packets");
4840
4841 #ifdef TCP_OFFLOAD
4842         if (is_offload(sc)) {
4843                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4844                     &pi->nofldrxq, 0,
4845                     "# of rx queues for offloaded TCP connections");
4846                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4847                     &pi->nofldtxq, 0,
4848                     "# of tx queues for offloaded TCP connections");
4849                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4850                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4851                     "index of first TOE rx queue");
4852                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4853                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4854                     "index of first TOE tx queue");
4855         }
4856 #endif
4857 #ifdef DEV_NETMAP
4858         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4859             &pi->nnmrxq, 0, "# of rx queues for netmap");
4860         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4861             &pi->nnmtxq, 0, "# of tx queues for netmap");
4862         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4863             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4864             "index of first netmap rx queue");
4865         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4866             CTLFLAG_RD, &pi->first_nm_txq, 0,
4867             "index of first netmap tx queue");
4868 #endif
4869
4870         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4871             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4872             "holdoff timer index");
4873         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4874             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4875             "holdoff packet counter index");
4876
4877         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4878             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4879             "rx queue size");
4880         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4881             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4882             "tx queue size");
4883
4884         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4885             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4886             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4887
4888         /*
4889          * dev.cxgbe.X.stats.
4890          */
4891         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4892             NULL, "port statistics");
4893         children = SYSCTL_CHILDREN(oid);
4894
4895 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4896         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4897             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4898             sysctl_handle_t4_reg64, "QU", desc)
4899
4900         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4901             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4902         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4903             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4904         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4905             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4906         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4907             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4908         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4909             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4910         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4911             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4912         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4913             "# of tx frames in this range",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4915         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4916             "# of tx frames in this range",
4917             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4918         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4919             "# of tx frames in this range",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4921         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4922             "# of tx frames in this range",
4923             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4924         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4925             "# of tx frames in this range",
4926             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4927         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4928             "# of tx frames in this range",
4929             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4930         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4931             "# of tx frames in this range",
4932             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4933         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4934             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4935         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4936             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4937         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4938             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4939         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4940             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4941         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4942             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4943         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4944             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4945         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4946             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4947         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4948             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4949         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4950             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4951         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4952             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4953
4954         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4956         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4958         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4960         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4962         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4964         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4965             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4966         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4967             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4968         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4969             "# of frames received with bad FCS",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4971         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4972             "# of frames received with length error",
4973             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4974         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4975             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4976         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4977             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4978         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4979             "# of rx frames in this range",
4980             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4981         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4982             "# of rx frames in this range",
4983             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4984         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4985             "# of rx frames in this range",
4986             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4987         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4988             "# of rx frames in this range",
4989             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4990         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4991             "# of rx frames in this range",
4992             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4993         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4994             "# of rx frames in this range",
4995             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4996         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4997             "# of rx frames in this range",
4998             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4999         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5000             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5001         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5002             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5003         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5004             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5005         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5006             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5007         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5008             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5009         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5010             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5011         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5012             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5013         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5014             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5015         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5016             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5017
5018 #undef SYSCTL_ADD_T4_REG64
5019
5020 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5021         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5022             &pi->stats.name, desc)
5023
5024         /* We get these from port_stats and they may be stale by upto 1s */
5025         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5026             "# drops due to buffer-group 0 overflows");
5027         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5028             "# drops due to buffer-group 1 overflows");
5029         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5030             "# drops due to buffer-group 2 overflows");
5031         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5032             "# drops due to buffer-group 3 overflows");
5033         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5034             "# of buffer-group 0 truncated packets");
5035         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5036             "# of buffer-group 1 truncated packets");
5037         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5038             "# of buffer-group 2 truncated packets");
5039         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5040             "# of buffer-group 3 truncated packets");
5041
5042 #undef SYSCTL_ADD_T4_PORTSTAT
5043
5044         return (0);
5045 }
5046
5047 static int
5048 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5049 {
5050         int rc, *i;
5051         struct sbuf sb;
5052
5053         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5054         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
5055                 sbuf_printf(&sb, "%d ", *i);
5056         sbuf_trim(&sb);
5057         sbuf_finish(&sb);
5058         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5059         sbuf_delete(&sb);
5060         return (rc);
5061 }
5062
5063 static int
5064 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5065 {
5066         int rc;
5067         struct sbuf *sb;
5068
5069         rc = sysctl_wire_old_buffer(req, 0);
5070         if (rc != 0)
5071                 return(rc);
5072
5073         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5074         if (sb == NULL)
5075                 return (ENOMEM);
5076
5077         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5078         rc = sbuf_finish(sb);
5079         sbuf_delete(sb);
5080
5081         return (rc);
5082 }
5083
5084 static int
5085 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5086 {
5087         struct port_info *pi = arg1;
5088         int op = arg2;
5089         struct adapter *sc = pi->adapter;
5090         u_int v;
5091         int rc;
5092
5093         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5094         if (rc)
5095                 return (rc);
5096         /* XXX: magic numbers */
5097         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5098             &v);
5099         end_synchronized_op(sc, 0);
5100         if (rc)
5101                 return (rc);
5102         if (op == 0)
5103                 v /= 256;
5104
5105         rc = sysctl_handle_int(oidp, &v, 0, req);
5106         return (rc);
5107 }
5108
5109 static int
5110 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5111 {
5112         struct port_info *pi = arg1;
5113         int rc, val;
5114
5115         val = pi->rsrv_noflowq;
5116         rc = sysctl_handle_int(oidp, &val, 0, req);
5117         if (rc != 0 || req->newptr == NULL)
5118                 return (rc);
5119
5120         if ((val >= 1) && (pi->ntxq > 1))
5121                 pi->rsrv_noflowq = 1;
5122         else
5123                 pi->rsrv_noflowq = 0;
5124
5125         return (rc);
5126 }
5127
5128 static int
5129 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5130 {
5131         struct port_info *pi = arg1;
5132         struct adapter *sc = pi->adapter;
5133         int idx, rc, i;
5134         struct sge_rxq *rxq;
5135 #ifdef TCP_OFFLOAD
5136         struct sge_ofld_rxq *ofld_rxq;
5137 #endif
5138         uint8_t v;
5139
5140         idx = pi->tmr_idx;
5141
5142         rc = sysctl_handle_int(oidp, &idx, 0, req);
5143         if (rc != 0 || req->newptr == NULL)
5144                 return (rc);
5145
5146         if (idx < 0 || idx >= SGE_NTIMERS)
5147                 return (EINVAL);
5148
5149         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5150             "t4tmr");
5151         if (rc)
5152                 return (rc);
5153
5154         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5155         for_each_rxq(pi, i, rxq) {
5156 #ifdef atomic_store_rel_8
5157                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5158 #else
5159                 rxq->iq.intr_params = v;
5160 #endif
5161         }
5162 #ifdef TCP_OFFLOAD
5163         for_each_ofld_rxq(pi, i, ofld_rxq) {
5164 #ifdef atomic_store_rel_8
5165                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5166 #else
5167                 ofld_rxq->iq.intr_params = v;
5168 #endif
5169         }
5170 #endif
5171         pi->tmr_idx = idx;
5172
5173         end_synchronized_op(sc, LOCK_HELD);
5174         return (0);
5175 }
5176
5177 static int
5178 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5179 {
5180         struct port_info *pi = arg1;
5181         struct adapter *sc = pi->adapter;
5182         int idx, rc;
5183
5184         idx = pi->pktc_idx;
5185
5186         rc = sysctl_handle_int(oidp, &idx, 0, req);
5187         if (rc != 0 || req->newptr == NULL)
5188                 return (rc);
5189
5190         if (idx < -1 || idx >= SGE_NCOUNTERS)
5191                 return (EINVAL);
5192
5193         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5194             "t4pktc");
5195         if (rc)
5196                 return (rc);
5197
5198         if (pi->flags & PORT_INIT_DONE)
5199                 rc = EBUSY; /* cannot be changed once the queues are created */
5200         else
5201                 pi->pktc_idx = idx;
5202
5203         end_synchronized_op(sc, LOCK_HELD);
5204         return (rc);
5205 }
5206
5207 static int
5208 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5209 {
5210         struct port_info *pi = arg1;
5211         struct adapter *sc = pi->adapter;
5212         int qsize, rc;
5213
5214         qsize = pi->qsize_rxq;
5215
5216         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5217         if (rc != 0 || req->newptr == NULL)
5218                 return (rc);
5219
5220         if (qsize < 128 || (qsize & 7))
5221                 return (EINVAL);
5222
5223         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5224             "t4rxqs");
5225         if (rc)
5226                 return (rc);
5227
5228         if (pi->flags & PORT_INIT_DONE)
5229                 rc = EBUSY; /* cannot be changed once the queues are created */
5230         else
5231                 pi->qsize_rxq = qsize;
5232
5233         end_synchronized_op(sc, LOCK_HELD);
5234         return (rc);
5235 }
5236
5237 static int
5238 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5239 {
5240         struct port_info *pi = arg1;
5241         struct adapter *sc = pi->adapter;
5242         int qsize, rc;
5243
5244         qsize = pi->qsize_txq;
5245
5246         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5247         if (rc != 0 || req->newptr == NULL)
5248                 return (rc);
5249
5250         /* bufring size must be powerof2 */
5251         if (qsize < 128 || !powerof2(qsize))
5252                 return (EINVAL);
5253
5254         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5255             "t4txqs");
5256         if (rc)
5257                 return (rc);
5258
5259         if (pi->flags & PORT_INIT_DONE)
5260                 rc = EBUSY; /* cannot be changed once the queues are created */
5261         else
5262                 pi->qsize_txq = qsize;
5263
5264         end_synchronized_op(sc, LOCK_HELD);
5265         return (rc);
5266 }
5267
5268 static int
5269 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5270 {
5271         struct port_info *pi = arg1;
5272         struct adapter *sc = pi->adapter;
5273         struct link_config *lc = &pi->link_cfg;
5274         int rc;
5275
5276         if (req->newptr == NULL) {
5277                 struct sbuf *sb;
5278                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5279
5280                 rc = sysctl_wire_old_buffer(req, 0);
5281                 if (rc != 0)
5282                         return(rc);
5283
5284                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5285                 if (sb == NULL)
5286                         return (ENOMEM);
5287
5288                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5289                 rc = sbuf_finish(sb);
5290                 sbuf_delete(sb);
5291         } else {
5292                 char s[2];
5293                 int n;
5294
5295                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5296                 s[1] = 0;
5297
5298                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5299                 if (rc != 0)
5300                         return(rc);
5301
5302                 if (s[1] != 0)
5303                         return (EINVAL);
5304                 if (s[0] < '0' || s[0] > '9')
5305                         return (EINVAL);        /* not a number */
5306                 n = s[0] - '0';
5307                 if (n & ~(PAUSE_TX | PAUSE_RX))
5308                         return (EINVAL);        /* some other bit is set too */
5309
5310                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5311                 if (rc)
5312                         return (rc);
5313                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5314                         int link_ok = lc->link_ok;
5315
5316                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5317                         lc->requested_fc |= n;
5318                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5319                         lc->link_ok = link_ok;  /* restore */
5320                 }
5321                 end_synchronized_op(sc, 0);
5322         }
5323
5324         return (rc);
5325 }
5326
5327 static int
5328 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5329 {
5330         struct adapter *sc = arg1;
5331         int reg = arg2;
5332         uint64_t val;
5333
5334         val = t4_read_reg64(sc, reg);
5335
5336         return (sysctl_handle_64(oidp, &val, 0, req));
5337 }
5338
5339 static int
5340 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5341 {
5342         struct adapter *sc = arg1;
5343         int rc, t;
5344         uint32_t param, val;
5345
5346         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5347         if (rc)
5348                 return (rc);
5349         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5350             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5351             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5352         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5353         end_synchronized_op(sc, 0);
5354         if (rc)
5355                 return (rc);
5356
5357         /* unknown is returned as 0 but we display -1 in that case */
5358         t = val == 0 ? -1 : val;
5359
5360         rc = sysctl_handle_int(oidp, &t, 0, req);
5361         return (rc);
5362 }
5363
5364 #ifdef SBUF_DRAIN
5365 static int
5366 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5367 {
5368         struct adapter *sc = arg1;
5369         struct sbuf *sb;
5370         int rc, i;
5371         uint16_t incr[NMTUS][NCCTRL_WIN];
5372         static const char *dec_fac[] = {
5373                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5374                 "0.9375"
5375         };
5376
5377         rc = sysctl_wire_old_buffer(req, 0);
5378         if (rc != 0)
5379                 return (rc);
5380
5381         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5382         if (sb == NULL)
5383                 return (ENOMEM);
5384
5385         t4_read_cong_tbl(sc, incr);
5386
5387         for (i = 0; i < NCCTRL_WIN; ++i) {
5388                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5389                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5390                     incr[5][i], incr[6][i], incr[7][i]);
5391                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5392                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5393                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5394                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5395         }
5396
5397         rc = sbuf_finish(sb);
5398         sbuf_delete(sb);
5399
5400         return (rc);
5401 }
5402
5403 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5404         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5405         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5406         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5407 };
5408
5409 static int
5410 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5411 {
5412         struct adapter *sc = arg1;
5413         struct sbuf *sb;
5414         int rc, i, n, qid = arg2;
5415         uint32_t *buf, *p;
5416         char *qtype;
5417         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5418
5419         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5420             ("%s: bad qid %d\n", __func__, qid));
5421
5422         if (qid < CIM_NUM_IBQ) {
5423                 /* inbound queue */
5424                 qtype = "IBQ";
5425                 n = 4 * CIM_IBQ_SIZE;
5426                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5427                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5428         } else {
5429                 /* outbound queue */
5430                 qtype = "OBQ";
5431                 qid -= CIM_NUM_IBQ;
5432                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5433                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5434                 rc = t4_read_cim_obq(sc, qid, buf, n);
5435         }
5436
5437         if (rc < 0) {
5438                 rc = -rc;
5439                 goto done;
5440         }
5441         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5442
5443         rc = sysctl_wire_old_buffer(req, 0);
5444         if (rc != 0)
5445                 goto done;
5446
5447         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5448         if (sb == NULL) {
5449                 rc = ENOMEM;
5450                 goto done;
5451         }
5452
5453         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5454         for (i = 0, p = buf; i < n; i += 16, p += 4)
5455                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5456                     p[2], p[3]);
5457
5458         rc = sbuf_finish(sb);
5459         sbuf_delete(sb);
5460 done:
5461         free(buf, M_CXGBE);
5462         return (rc);
5463 }
5464
5465 static int
5466 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5467 {
5468         struct adapter *sc = arg1;
5469         u_int cfg;
5470         struct sbuf *sb;
5471         uint32_t *buf, *p;
5472         int rc;
5473
5474         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5475         if (rc != 0)
5476                 return (rc);
5477
5478         rc = sysctl_wire_old_buffer(req, 0);
5479         if (rc != 0)
5480                 return (rc);
5481
5482         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5483         if (sb == NULL)
5484                 return (ENOMEM);
5485
5486         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5487             M_ZERO | M_WAITOK);
5488
5489         rc = -t4_cim_read_la(sc, buf, NULL);
5490         if (rc != 0)
5491                 goto done;
5492
5493         sbuf_printf(sb, "Status   Data      PC%s",
5494             cfg & F_UPDBGLACAPTPCONLY ? "" :
5495             "     LS0Stat  LS0Addr             LS0Data");
5496
5497         KASSERT((sc->params.cim_la_size & 7) == 0,
5498             ("%s: p will walk off the end of buf", __func__));
5499
5500         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5501                 if (cfg & F_UPDBGLACAPTPCONLY) {
5502                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5503                             p[6], p[7]);
5504                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5505                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5506                             p[4] & 0xff, p[5] >> 8);
5507                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5508                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5509                             p[1] & 0xf, p[2] >> 4);
5510                 } else {
5511                         sbuf_printf(sb,
5512                             "\n  %02x   %x%07x %x%07x %08x %08x "
5513                             "%08x%08x%08x%08x",
5514                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5515                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5516                             p[6], p[7]);
5517                 }
5518         }
5519
5520         rc = sbuf_finish(sb);
5521         sbuf_delete(sb);
5522 done:
5523         free(buf, M_CXGBE);
5524         return (rc);
5525 }
5526
5527 static int
5528 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5529 {
5530         struct adapter *sc = arg1;
5531         u_int i;
5532         struct sbuf *sb;
5533         uint32_t *buf, *p;
5534         int rc;
5535
5536         rc = sysctl_wire_old_buffer(req, 0);
5537         if (rc != 0)
5538                 return (rc);
5539
5540         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5541         if (sb == NULL)
5542                 return (ENOMEM);
5543
5544         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5545             M_ZERO | M_WAITOK);
5546
5547         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5548         p = buf;
5549
5550         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5551                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5552                     p[1], p[0]);
5553         }
5554
5555         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5556         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5557                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5558                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5559                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5560                     (p[1] >> 2) | ((p[2] & 3) << 30),
5561                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5562                     p[0] & 1);
5563         }
5564
5565         rc = sbuf_finish(sb);
5566         sbuf_delete(sb);
5567         free(buf, M_CXGBE);
5568         return (rc);
5569 }
5570
5571 static int
5572 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5573 {
5574         struct adapter *sc = arg1;
5575         u_int i;
5576         struct sbuf *sb;
5577         uint32_t *buf, *p;
5578         int rc;
5579
5580         rc = sysctl_wire_old_buffer(req, 0);
5581         if (rc != 0)
5582                 return (rc);
5583
5584         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5585         if (sb == NULL)
5586                 return (ENOMEM);
5587
5588         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5589             M_ZERO | M_WAITOK);
5590
5591         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5592         p = buf;
5593
5594         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5595         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5596                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5597                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5598                     p[4], p[3], p[2], p[1], p[0]);
5599         }
5600
5601         sbuf_printf(sb, "\n\nCntl ID               Data");
5602         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5603                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5604                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5605         }
5606
5607         rc = sbuf_finish(sb);
5608         sbuf_delete(sb);
5609         free(buf, M_CXGBE);
5610         return (rc);
5611 }
5612
5613 static int
5614 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5615 {
5616         struct adapter *sc = arg1;
5617         struct sbuf *sb;
5618         int rc, i;
5619         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5620         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5621         uint16_t thres[CIM_NUM_IBQ];
5622         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5623         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5624         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5625
5626         if (is_t4(sc)) {
5627                 cim_num_obq = CIM_NUM_OBQ;
5628                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5629                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5630         } else {
5631                 cim_num_obq = CIM_NUM_OBQ_T5;
5632                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5633                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5634         }
5635         nq = CIM_NUM_IBQ + cim_num_obq;
5636
5637         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5638         if (rc == 0)
5639                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5640         if (rc != 0)
5641                 return (rc);
5642
5643         t4_read_cimq_cfg(sc, base, size, thres);
5644
5645         rc = sysctl_wire_old_buffer(req, 0);
5646         if (rc != 0)
5647                 return (rc);
5648
5649         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5650         if (sb == NULL)
5651                 return (ENOMEM);
5652
5653         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5654
5655         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5656                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5657                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5658                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5659                     G_QUEREMFLITS(p[2]) * 16);
5660         for ( ; i < nq; i++, p += 4, wr += 2)
5661                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5662                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5663                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5664                     G_QUEREMFLITS(p[2]) * 16);
5665
5666         rc = sbuf_finish(sb);
5667         sbuf_delete(sb);
5668
5669         return (rc);
5670 }
5671
5672 static int
5673 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5674 {
5675         struct adapter *sc = arg1;
5676         struct sbuf *sb;
5677         int rc;
5678         struct tp_cpl_stats stats;
5679
5680         rc = sysctl_wire_old_buffer(req, 0);
5681         if (rc != 0)
5682                 return (rc);
5683
5684         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5685         if (sb == NULL)
5686                 return (ENOMEM);
5687
5688         t4_tp_get_cpl_stats(sc, &stats);
5689
5690         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5691             "channel 3\n");
5692         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5693                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5694         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5695                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5696
5697         rc = sbuf_finish(sb);
5698         sbuf_delete(sb);
5699
5700         return (rc);
5701 }
5702
5703 static int
5704 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5705 {
5706         struct adapter *sc = arg1;
5707         struct sbuf *sb;
5708         int rc;
5709         struct tp_usm_stats stats;
5710
5711         rc = sysctl_wire_old_buffer(req, 0);
5712         if (rc != 0)
5713                 return(rc);
5714
5715         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5716         if (sb == NULL)
5717                 return (ENOMEM);
5718
5719         t4_get_usm_stats(sc, &stats);
5720
5721         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5722         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5723         sbuf_printf(sb, "Drops:  %u", stats.drops);
5724
5725         rc = sbuf_finish(sb);
5726         sbuf_delete(sb);
5727
5728         return (rc);
5729 }
5730
5731 const char *devlog_level_strings[] = {
5732         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5733         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5734         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5735         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5736         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5737         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5738 };
5739
5740 const char *devlog_facility_strings[] = {
5741         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5742         [FW_DEVLOG_FACILITY_CF]         = "CF",
5743         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5744         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5745         [FW_DEVLOG_FACILITY_RES]        = "RES",
5746         [FW_DEVLOG_FACILITY_HW]         = "HW",
5747         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5748         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5749         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5750         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5751         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5752         [FW_DEVLOG_FACILITY_VI]         = "VI",
5753         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5754         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5755         [FW_DEVLOG_FACILITY_TM]         = "TM",
5756         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5757         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5758         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5759         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5760         [FW_DEVLOG_FACILITY_RI]         = "RI",
5761         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5762         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5763         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5764         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5765 };
5766
5767 static int
5768 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5769 {
5770         struct adapter *sc = arg1;
5771         struct devlog_params *dparams = &sc->params.devlog;
5772         struct fw_devlog_e *buf, *e;
5773         int i, j, rc, nentries, first = 0, m;
5774         struct sbuf *sb;
5775         uint64_t ftstamp = UINT64_MAX;
5776
5777         if (dparams->start == 0) {
5778                 dparams->memtype = FW_MEMTYPE_EDC0;
5779                 dparams->start = 0x84000;
5780                 dparams->size = 32768;
5781         }
5782
5783         nentries = dparams->size / sizeof(struct fw_devlog_e);
5784
5785         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5786         if (buf == NULL)
5787                 return (ENOMEM);
5788
5789         m = fwmtype_to_hwmtype(dparams->memtype);
5790         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5791         if (rc != 0)
5792                 goto done;
5793
5794         for (i = 0; i < nentries; i++) {
5795                 e = &buf[i];
5796
5797                 if (e->timestamp == 0)
5798                         break;  /* end */
5799
5800                 e->timestamp = be64toh(e->timestamp);
5801                 e->seqno = be32toh(e->seqno);
5802                 for (j = 0; j < 8; j++)
5803                         e->params[j] = be32toh(e->params[j]);
5804
5805                 if (e->timestamp < ftstamp) {
5806                         ftstamp = e->timestamp;
5807                         first = i;
5808                 }
5809         }
5810
5811         if (buf[first].timestamp == 0)
5812                 goto done;      /* nothing in the log */
5813
5814         rc = sysctl_wire_old_buffer(req, 0);
5815         if (rc != 0)
5816                 goto done;
5817
5818         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5819         if (sb == NULL) {
5820                 rc = ENOMEM;
5821                 goto done;
5822         }
5823         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5824             "Seq#", "Tstamp", "Level", "Facility", "Message");
5825
5826         i = first;
5827         do {
5828                 e = &buf[i];
5829                 if (e->timestamp == 0)
5830                         break;  /* end */
5831
5832                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5833                     e->seqno, e->timestamp,
5834                     (e->level < nitems(devlog_level_strings) ?
5835                         devlog_level_strings[e->level] : "UNKNOWN"),
5836                     (e->facility < nitems(devlog_facility_strings) ?
5837                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5838                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5839                     e->params[2], e->params[3], e->params[4],
5840                     e->params[5], e->params[6], e->params[7]);
5841
5842                 if (++i == nentries)
5843                         i = 0;
5844         } while (i != first);
5845
5846         rc = sbuf_finish(sb);
5847         sbuf_delete(sb);
5848 done:
5849         free(buf, M_CXGBE);
5850         return (rc);
5851 }
5852
5853 static int
5854 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5855 {
5856         struct adapter *sc = arg1;
5857         struct sbuf *sb;
5858         int rc;
5859         struct tp_fcoe_stats stats[4];
5860
5861         rc = sysctl_wire_old_buffer(req, 0);
5862         if (rc != 0)
5863                 return (rc);
5864
5865         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5866         if (sb == NULL)
5867                 return (ENOMEM);
5868
5869         t4_get_fcoe_stats(sc, 0, &stats[0]);
5870         t4_get_fcoe_stats(sc, 1, &stats[1]);
5871         t4_get_fcoe_stats(sc, 2, &stats[2]);
5872         t4_get_fcoe_stats(sc, 3, &stats[3]);
5873
5874         sbuf_printf(sb, "                   channel 0        channel 1        "
5875             "channel 2        channel 3\n");
5876         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5877             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5878             stats[3].octetsDDP);
5879         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5880             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5881         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5882             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5883             stats[3].framesDrop);
5884
5885         rc = sbuf_finish(sb);
5886         sbuf_delete(sb);
5887
5888         return (rc);
5889 }
5890
5891 static int
5892 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5893 {
5894         struct adapter *sc = arg1;
5895         struct sbuf *sb;
5896         int rc, i;
5897         unsigned int map, kbps, ipg, mode;
5898         unsigned int pace_tab[NTX_SCHED];
5899
5900         rc = sysctl_wire_old_buffer(req, 0);
5901         if (rc != 0)
5902                 return (rc);
5903
5904         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5905         if (sb == NULL)
5906                 return (ENOMEM);
5907
5908         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5909         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5910         t4_read_pace_tbl(sc, pace_tab);
5911
5912         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5913             "Class IPG (0.1 ns)   Flow IPG (us)");
5914
5915         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5916                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5917                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5918                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5919                 if (kbps)
5920                         sbuf_printf(sb, "%9u     ", kbps);
5921                 else
5922                         sbuf_printf(sb, " disabled     ");
5923
5924                 if (ipg)
5925                         sbuf_printf(sb, "%13u        ", ipg);
5926                 else
5927                         sbuf_printf(sb, "     disabled        ");
5928
5929                 if (pace_tab[i])
5930                         sbuf_printf(sb, "%10u", pace_tab[i]);
5931                 else
5932                         sbuf_printf(sb, "  disabled");
5933         }
5934
5935         rc = sbuf_finish(sb);
5936         sbuf_delete(sb);
5937
5938         return (rc);
5939 }
5940
5941 static int
5942 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5943 {
5944         struct adapter *sc = arg1;
5945         struct sbuf *sb;
5946         int rc, i, j;
5947         uint64_t *p0, *p1;
5948         struct lb_port_stats s[2];
5949         static const char *stat_name[] = {
5950                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5951                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5952                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5953                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5954                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5955                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5956                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5957         };
5958
5959         rc = sysctl_wire_old_buffer(req, 0);
5960         if (rc != 0)
5961                 return (rc);
5962
5963         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5964         if (sb == NULL)
5965                 return (ENOMEM);
5966
5967         memset(s, 0, sizeof(s));
5968
5969         for (i = 0; i < 4; i += 2) {
5970                 t4_get_lb_stats(sc, i, &s[0]);
5971                 t4_get_lb_stats(sc, i + 1, &s[1]);
5972
5973                 p0 = &s[0].octets;
5974                 p1 = &s[1].octets;
5975                 sbuf_printf(sb, "%s                       Loopback %u"
5976                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5977
5978                 for (j = 0; j < nitems(stat_name); j++)
5979                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5980                                    *p0++, *p1++);
5981         }
5982
5983         rc = sbuf_finish(sb);
5984         sbuf_delete(sb);
5985
5986         return (rc);
5987 }
5988
5989 static int
5990 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5991 {
5992         int rc = 0;
5993         struct port_info *pi = arg1;
5994         struct sbuf *sb;
5995         static const char *linkdnreasons[] = {
5996                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5997                 "PHY overheated", "unknown", "rx los", "reserved7"
5998         };
5999
6000         rc = sysctl_wire_old_buffer(req, 0);
6001         if (rc != 0)
6002                 return(rc);
6003         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6004         if (sb == NULL)
6005                 return (ENOMEM);
6006
6007         if (pi->linkdnrc < 0)
6008                 sbuf_printf(sb, "n/a");
6009         else if (pi->linkdnrc < nitems(linkdnreasons))
6010                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6011         else
6012                 sbuf_printf(sb, "%d", pi->linkdnrc);
6013
6014         rc = sbuf_finish(sb);
6015         sbuf_delete(sb);
6016
6017         return (rc);
6018 }
6019
6020 struct mem_desc {
6021         unsigned int base;
6022         unsigned int limit;
6023         unsigned int idx;
6024 };
6025
6026 static int
6027 mem_desc_cmp(const void *a, const void *b)
6028 {
6029         return ((const struct mem_desc *)a)->base -
6030                ((const struct mem_desc *)b)->base;
6031 }
6032
6033 static void
6034 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6035     unsigned int to)
6036 {
6037         unsigned int size;
6038
6039         size = to - from + 1;
6040         if (size == 0)
6041                 return;
6042
6043         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6044         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6045 }
6046
6047 static int
6048 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6049 {
6050         struct adapter *sc = arg1;
6051         struct sbuf *sb;
6052         int rc, i, n;
6053         uint32_t lo, hi, used, alloc;
6054         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6055         static const char *region[] = {
6056                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6057                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6058                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6059                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6060                 "RQUDP region:", "PBL region:", "TXPBL region:",
6061                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6062                 "On-chip queues:"
6063         };
6064         struct mem_desc avail[4];
6065         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6066         struct mem_desc *md = mem;
6067
6068         rc = sysctl_wire_old_buffer(req, 0);
6069         if (rc != 0)
6070                 return (rc);
6071
6072         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6073         if (sb == NULL)
6074                 return (ENOMEM);
6075
6076         for (i = 0; i < nitems(mem); i++) {
6077                 mem[i].limit = 0;
6078                 mem[i].idx = i;
6079         }
6080
6081         /* Find and sort the populated memory ranges */
6082         i = 0;
6083         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6084         if (lo & F_EDRAM0_ENABLE) {
6085                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6086                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6087                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6088                 avail[i].idx = 0;
6089                 i++;
6090         }
6091         if (lo & F_EDRAM1_ENABLE) {
6092                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6093                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6094                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6095                 avail[i].idx = 1;
6096                 i++;
6097         }
6098         if (lo & F_EXT_MEM_ENABLE) {
6099                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6100                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6101                 avail[i].limit = avail[i].base +
6102                     (G_EXT_MEM_SIZE(hi) << 20);
6103                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6104                 i++;
6105         }
6106         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6107                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6108                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6109                 avail[i].limit = avail[i].base +
6110                     (G_EXT_MEM1_SIZE(hi) << 20);
6111                 avail[i].idx = 4;
6112                 i++;
6113         }
6114         if (!i)                                    /* no memory available */
6115                 return 0;
6116         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6117
6118         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6119         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6120         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6121         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6122         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6123         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6124         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6125         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6126         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6127
6128         /* the next few have explicit upper bounds */
6129         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6130         md->limit = md->base - 1 +
6131                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6132                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6133         md++;
6134
6135         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6136         md->limit = md->base - 1 +
6137                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6138                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6139         md++;
6140
6141         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6142                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6143                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6144                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6145         } else {
6146                 md->base = 0;
6147                 md->idx = nitems(region);  /* hide it */
6148         }
6149         md++;
6150
6151 #define ulp_region(reg) \
6152         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6153         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6154
6155         ulp_region(RX_ISCSI);
6156         ulp_region(RX_TDDP);
6157         ulp_region(TX_TPT);
6158         ulp_region(RX_STAG);
6159         ulp_region(RX_RQ);
6160         ulp_region(RX_RQUDP);
6161         ulp_region(RX_PBL);
6162         ulp_region(TX_PBL);
6163 #undef ulp_region
6164
6165         md->base = 0;
6166         md->idx = nitems(region);
6167         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6168                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6169                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6170                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6171         }
6172         md++;
6173
6174         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6175         md->limit = md->base + sc->tids.ntids - 1;
6176         md++;
6177         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6178         md->limit = md->base + sc->tids.ntids - 1;
6179         md++;
6180
6181         md->base = sc->vres.ocq.start;
6182         if (sc->vres.ocq.size)
6183                 md->limit = md->base + sc->vres.ocq.size - 1;
6184         else
6185                 md->idx = nitems(region);  /* hide it */
6186         md++;
6187
6188         /* add any address-space holes, there can be up to 3 */
6189         for (n = 0; n < i - 1; n++)
6190                 if (avail[n].limit < avail[n + 1].base)
6191                         (md++)->base = avail[n].limit;
6192         if (avail[n].limit)
6193                 (md++)->base = avail[n].limit;
6194
6195         n = md - mem;
6196         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6197
6198         for (lo = 0; lo < i; lo++)
6199                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6200                                 avail[lo].limit - 1);
6201
6202         sbuf_printf(sb, "\n");
6203         for (i = 0; i < n; i++) {
6204                 if (mem[i].idx >= nitems(region))
6205                         continue;                        /* skip holes */
6206                 if (!mem[i].limit)
6207                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6208                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6209                                 mem[i].limit);
6210         }
6211
6212         sbuf_printf(sb, "\n");
6213         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6214         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6215         mem_region_show(sb, "uP RAM:", lo, hi);
6216
6217         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6218         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6219         mem_region_show(sb, "uP Extmem2:", lo, hi);
6220
6221         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6222         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6223                    G_PMRXMAXPAGE(lo),
6224                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6225                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6226
6227         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6228         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6229         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6230                    G_PMTXMAXPAGE(lo),
6231                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6232                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6233         sbuf_printf(sb, "%u p-structs\n",
6234                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6235
6236         for (i = 0; i < 4; i++) {
6237                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6238                 if (is_t4(sc)) {
6239                         used = G_USED(lo);
6240                         alloc = G_ALLOC(lo);
6241                 } else {
6242                         used = G_T5_USED(lo);
6243                         alloc = G_T5_ALLOC(lo);
6244                 }
6245                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6246                            i, used, alloc);
6247         }
6248         for (i = 0; i < 4; i++) {
6249                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6250                 if (is_t4(sc)) {
6251                         used = G_USED(lo);
6252                         alloc = G_ALLOC(lo);
6253                 } else {
6254                         used = G_T5_USED(lo);
6255                         alloc = G_T5_ALLOC(lo);
6256                 }
6257                 sbuf_printf(sb,
6258                            "\nLoopback %d using %u pages out of %u allocated",
6259                            i, used, alloc);
6260         }
6261
6262         rc = sbuf_finish(sb);
6263         sbuf_delete(sb);
6264
6265         return (rc);
6266 }
6267
6268 static inline void
6269 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6270 {
6271         *mask = x | y;
6272         y = htobe64(y);
6273         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6274 }
6275
6276 static int
6277 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6278 {
6279         struct adapter *sc = arg1;
6280         struct sbuf *sb;
6281         int rc, i, n;
6282
6283         rc = sysctl_wire_old_buffer(req, 0);
6284         if (rc != 0)
6285                 return (rc);
6286
6287         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6288         if (sb == NULL)
6289                 return (ENOMEM);
6290
6291         sbuf_printf(sb,
6292             "Idx  Ethernet address     Mask     Vld Ports PF"
6293             "  VF              Replication             P0 P1 P2 P3  ML");
6294         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6295             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6296         for (i = 0; i < n; i++) {
6297                 uint64_t tcamx, tcamy, mask;
6298                 uint32_t cls_lo, cls_hi;
6299                 uint8_t addr[ETHER_ADDR_LEN];
6300
6301                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6302                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6303                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6304                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6305
6306                 if (tcamx & tcamy)
6307                         continue;
6308
6309                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6310                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6311                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6312                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6313                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6314                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6315                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6316
6317                 if (cls_lo & F_REPLICATE) {
6318                         struct fw_ldst_cmd ldst_cmd;
6319
6320                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6321                         ldst_cmd.op_to_addrspace =
6322                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6323                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6324                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6325                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6326                         ldst_cmd.u.mps.fid_ctl =
6327                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6328                                 V_FW_LDST_CMD_CTL(i));
6329
6330                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6331                             "t4mps");
6332                         if (rc)
6333                                 break;
6334                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6335                             sizeof(ldst_cmd), &ldst_cmd);
6336                         end_synchronized_op(sc, 0);
6337
6338                         if (rc != 0) {
6339                                 sbuf_printf(sb,
6340                                     " ------------ error %3u ------------", rc);
6341                                 rc = 0;
6342                         } else {
6343                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6344                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6345                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6346                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6347                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6348                         }
6349                 } else
6350                         sbuf_printf(sb, "%36s", "");
6351
6352                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6353                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6354                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6355         }
6356
6357         if (rc)
6358                 (void) sbuf_finish(sb);
6359         else
6360                 rc = sbuf_finish(sb);
6361         sbuf_delete(sb);
6362
6363         return (rc);
6364 }
6365
6366 static int
6367 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6368 {
6369         struct adapter *sc = arg1;
6370         struct sbuf *sb;
6371         int rc;
6372         uint16_t mtus[NMTUS];
6373
6374         rc = sysctl_wire_old_buffer(req, 0);
6375         if (rc != 0)
6376                 return (rc);
6377
6378         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6379         if (sb == NULL)
6380                 return (ENOMEM);
6381
6382         t4_read_mtu_tbl(sc, mtus, NULL);
6383
6384         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6385             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6386             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6387             mtus[14], mtus[15]);
6388
6389         rc = sbuf_finish(sb);
6390         sbuf_delete(sb);
6391
6392         return (rc);
6393 }
6394
6395 static int
6396 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6397 {
6398         struct adapter *sc = arg1;
6399         struct sbuf *sb;
6400         int rc, i;
6401         uint32_t cnt[PM_NSTATS];
6402         uint64_t cyc[PM_NSTATS];
6403         static const char *rx_stats[] = {
6404                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6405         };
6406         static const char *tx_stats[] = {
6407                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6408         };
6409
6410         rc = sysctl_wire_old_buffer(req, 0);
6411         if (rc != 0)
6412                 return (rc);
6413
6414         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6415         if (sb == NULL)
6416                 return (ENOMEM);
6417
6418         t4_pmtx_get_stats(sc, cnt, cyc);
6419         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6420         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6421                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6422                     cyc[i]);
6423
6424         t4_pmrx_get_stats(sc, cnt, cyc);
6425         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6426         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6427                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6428                     cyc[i]);
6429
6430         rc = sbuf_finish(sb);
6431         sbuf_delete(sb);
6432
6433         return (rc);
6434 }
6435
6436 static int
6437 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6438 {
6439         struct adapter *sc = arg1;
6440         struct sbuf *sb;
6441         int rc;
6442         struct tp_rdma_stats stats;
6443
6444         rc = sysctl_wire_old_buffer(req, 0);
6445         if (rc != 0)
6446                 return (rc);
6447
6448         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6449         if (sb == NULL)
6450                 return (ENOMEM);
6451
6452         t4_tp_get_rdma_stats(sc, &stats);
6453         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6454         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6455
6456         rc = sbuf_finish(sb);
6457         sbuf_delete(sb);
6458
6459         return (rc);
6460 }
6461
6462 static int
6463 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6464 {
6465         struct adapter *sc = arg1;
6466         struct sbuf *sb;
6467         int rc;
6468         struct tp_tcp_stats v4, v6;
6469
6470         rc = sysctl_wire_old_buffer(req, 0);
6471         if (rc != 0)
6472                 return (rc);
6473
6474         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6475         if (sb == NULL)
6476                 return (ENOMEM);
6477
6478         t4_tp_get_tcp_stats(sc, &v4, &v6);
6479         sbuf_printf(sb,
6480             "                                IP                 IPv6\n");
6481         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6482             v4.tcpOutRsts, v6.tcpOutRsts);
6483         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6484             v4.tcpInSegs, v6.tcpInSegs);
6485         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6486             v4.tcpOutSegs, v6.tcpOutSegs);
6487         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6488             v4.tcpRetransSegs, v6.tcpRetransSegs);
6489
6490         rc = sbuf_finish(sb);
6491         sbuf_delete(sb);
6492
6493         return (rc);
6494 }
6495
6496 static int
6497 sysctl_tids(SYSCTL_HANDLER_ARGS)
6498 {
6499         struct adapter *sc = arg1;
6500         struct sbuf *sb;
6501         int rc;
6502         struct tid_info *t = &sc->tids;
6503
6504         rc = sysctl_wire_old_buffer(req, 0);
6505         if (rc != 0)
6506                 return (rc);
6507
6508         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6509         if (sb == NULL)
6510                 return (ENOMEM);
6511
6512         if (t->natids) {
6513                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6514                     t->atids_in_use);
6515         }
6516
6517         if (t->ntids) {
6518                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6519                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6520
6521                         if (b) {
6522                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6523                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6524                                     t->ntids - 1);
6525                         } else {
6526                                 sbuf_printf(sb, "TID range: %u-%u",
6527                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6528                                     t->ntids - 1);
6529                         }
6530                 } else
6531                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6532                 sbuf_printf(sb, ", in use: %u\n",
6533                     atomic_load_acq_int(&t->tids_in_use));
6534         }
6535
6536         if (t->nstids) {
6537                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6538                     t->stid_base + t->nstids - 1, t->stids_in_use);
6539         }
6540
6541         if (t->nftids) {
6542                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6543                     t->ftid_base + t->nftids - 1);
6544         }
6545
6546         if (t->netids) {
6547                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6548                     t->etid_base + t->netids - 1);
6549         }
6550
6551         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6552             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6553             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6554
6555         rc = sbuf_finish(sb);
6556         sbuf_delete(sb);
6557
6558         return (rc);
6559 }
6560
6561 static int
6562 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6563 {
6564         struct adapter *sc = arg1;
6565         struct sbuf *sb;
6566         int rc;
6567         struct tp_err_stats stats;
6568
6569         rc = sysctl_wire_old_buffer(req, 0);
6570         if (rc != 0)
6571                 return (rc);
6572
6573         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6574         if (sb == NULL)
6575                 return (ENOMEM);
6576
6577         t4_tp_get_err_stats(sc, &stats);
6578
6579         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6580                       "channel 3\n");
6581         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6582             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6583             stats.macInErrs[3]);
6584         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6585             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6586             stats.hdrInErrs[3]);
6587         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6588             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6589             stats.tcpInErrs[3]);
6590         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6591             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6592             stats.tcp6InErrs[3]);
6593         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6594             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6595             stats.tnlCongDrops[3]);
6596         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6597             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6598             stats.tnlTxDrops[3]);
6599         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6600             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6601             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6602         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6603             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6604             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6605         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6606             stats.ofldNoNeigh, stats.ofldCongDefer);
6607
6608         rc = sbuf_finish(sb);
6609         sbuf_delete(sb);
6610
6611         return (rc);
6612 }
6613
6614 struct field_desc {
6615         const char *name;
6616         u_int start;
6617         u_int width;
6618 };
6619
6620 static void
6621 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6622 {
6623         char buf[32];
6624         int line_size = 0;
6625
6626         while (f->name) {
6627                 uint64_t mask = (1ULL << f->width) - 1;
6628                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6629                     ((uintmax_t)v >> f->start) & mask);
6630
6631                 if (line_size + len >= 79) {
6632                         line_size = 8;
6633                         sbuf_printf(sb, "\n        ");
6634                 }
6635                 sbuf_printf(sb, "%s ", buf);
6636                 line_size += len + 1;
6637                 f++;
6638         }
6639         sbuf_printf(sb, "\n");
6640 }
6641
6642 static struct field_desc tp_la0[] = {
6643         { "RcfOpCodeOut", 60, 4 },
6644         { "State", 56, 4 },
6645         { "WcfState", 52, 4 },
6646         { "RcfOpcSrcOut", 50, 2 },
6647         { "CRxError", 49, 1 },
6648         { "ERxError", 48, 1 },
6649         { "SanityFailed", 47, 1 },
6650         { "SpuriousMsg", 46, 1 },
6651         { "FlushInputMsg", 45, 1 },
6652         { "FlushInputCpl", 44, 1 },
6653         { "RssUpBit", 43, 1 },
6654         { "RssFilterHit", 42, 1 },
6655         { "Tid", 32, 10 },
6656         { "InitTcb", 31, 1 },
6657         { "LineNumber", 24, 7 },
6658         { "Emsg", 23, 1 },
6659         { "EdataOut", 22, 1 },
6660         { "Cmsg", 21, 1 },
6661         { "CdataOut", 20, 1 },
6662         { "EreadPdu", 19, 1 },
6663         { "CreadPdu", 18, 1 },
6664         { "TunnelPkt", 17, 1 },
6665         { "RcfPeerFin", 16, 1 },
6666         { "RcfReasonOut", 12, 4 },
6667         { "TxCchannel", 10, 2 },
6668         { "RcfTxChannel", 8, 2 },
6669         { "RxEchannel", 6, 2 },
6670         { "RcfRxChannel", 5, 1 },
6671         { "RcfDataOutSrdy", 4, 1 },
6672         { "RxDvld", 3, 1 },
6673         { "RxOoDvld", 2, 1 },
6674         { "RxCongestion", 1, 1 },
6675         { "TxCongestion", 0, 1 },
6676         { NULL }
6677 };
6678
6679 static struct field_desc tp_la1[] = {
6680         { "CplCmdIn", 56, 8 },
6681         { "CplCmdOut", 48, 8 },
6682         { "ESynOut", 47, 1 },
6683         { "EAckOut", 46, 1 },
6684         { "EFinOut", 45, 1 },
6685         { "ERstOut", 44, 1 },
6686         { "SynIn", 43, 1 },
6687         { "AckIn", 42, 1 },
6688         { "FinIn", 41, 1 },
6689         { "RstIn", 40, 1 },
6690         { "DataIn", 39, 1 },
6691         { "DataInVld", 38, 1 },
6692         { "PadIn", 37, 1 },
6693         { "RxBufEmpty", 36, 1 },
6694         { "RxDdp", 35, 1 },
6695         { "RxFbCongestion", 34, 1 },
6696         { "TxFbCongestion", 33, 1 },
6697         { "TxPktSumSrdy", 32, 1 },
6698         { "RcfUlpType", 28, 4 },
6699         { "Eread", 27, 1 },
6700         { "Ebypass", 26, 1 },
6701         { "Esave", 25, 1 },
6702         { "Static0", 24, 1 },
6703         { "Cread", 23, 1 },
6704         { "Cbypass", 22, 1 },
6705         { "Csave", 21, 1 },
6706         { "CPktOut", 20, 1 },
6707         { "RxPagePoolFull", 18, 2 },
6708         { "RxLpbkPkt", 17, 1 },
6709         { "TxLpbkPkt", 16, 1 },
6710         { "RxVfValid", 15, 1 },
6711         { "SynLearned", 14, 1 },
6712         { "SetDelEntry", 13, 1 },
6713         { "SetInvEntry", 12, 1 },
6714         { "CpcmdDvld", 11, 1 },
6715         { "CpcmdSave", 10, 1 },
6716         { "RxPstructsFull", 8, 2 },
6717         { "EpcmdDvld", 7, 1 },
6718         { "EpcmdFlush", 6, 1 },
6719         { "EpcmdTrimPrefix", 5, 1 },
6720         { "EpcmdTrimPostfix", 4, 1 },
6721         { "ERssIp4Pkt", 3, 1 },
6722         { "ERssIp6Pkt", 2, 1 },
6723         { "ERssTcpUdpPkt", 1, 1 },
6724         { "ERssFceFipPkt", 0, 1 },
6725         { NULL }
6726 };
6727
6728 static struct field_desc tp_la2[] = {
6729         { "CplCmdIn", 56, 8 },
6730         { "MpsVfVld", 55, 1 },
6731         { "MpsPf", 52, 3 },
6732         { "MpsVf", 44, 8 },
6733         { "SynIn", 43, 1 },
6734         { "AckIn", 42, 1 },
6735         { "FinIn", 41, 1 },
6736         { "RstIn", 40, 1 },
6737         { "DataIn", 39, 1 },
6738         { "DataInVld", 38, 1 },
6739         { "PadIn", 37, 1 },
6740         { "RxBufEmpty", 36, 1 },
6741         { "RxDdp", 35, 1 },
6742         { "RxFbCongestion", 34, 1 },
6743         { "TxFbCongestion", 33, 1 },
6744         { "TxPktSumSrdy", 32, 1 },
6745         { "RcfUlpType", 28, 4 },
6746         { "Eread", 27, 1 },
6747         { "Ebypass", 26, 1 },
6748         { "Esave", 25, 1 },
6749         { "Static0", 24, 1 },
6750         { "Cread", 23, 1 },
6751         { "Cbypass", 22, 1 },
6752         { "Csave", 21, 1 },
6753         { "CPktOut", 20, 1 },
6754         { "RxPagePoolFull", 18, 2 },
6755         { "RxLpbkPkt", 17, 1 },
6756         { "TxLpbkPkt", 16, 1 },
6757         { "RxVfValid", 15, 1 },
6758         { "SynLearned", 14, 1 },
6759         { "SetDelEntry", 13, 1 },
6760         { "SetInvEntry", 12, 1 },
6761         { "CpcmdDvld", 11, 1 },
6762         { "CpcmdSave", 10, 1 },
6763         { "RxPstructsFull", 8, 2 },
6764         { "EpcmdDvld", 7, 1 },
6765         { "EpcmdFlush", 6, 1 },
6766         { "EpcmdTrimPrefix", 5, 1 },
6767         { "EpcmdTrimPostfix", 4, 1 },
6768         { "ERssIp4Pkt", 3, 1 },
6769         { "ERssIp6Pkt", 2, 1 },
6770         { "ERssTcpUdpPkt", 1, 1 },
6771         { "ERssFceFipPkt", 0, 1 },
6772         { NULL }
6773 };
6774
6775 static void
6776 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6777 {
6778
6779         field_desc_show(sb, *p, tp_la0);
6780 }
6781
6782 static void
6783 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6784 {
6785
6786         if (idx)
6787                 sbuf_printf(sb, "\n");
6788         field_desc_show(sb, p[0], tp_la0);
6789         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6790                 field_desc_show(sb, p[1], tp_la0);
6791 }
6792
6793 static void
6794 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6795 {
6796
6797         if (idx)
6798                 sbuf_printf(sb, "\n");
6799         field_desc_show(sb, p[0], tp_la0);
6800         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6801                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6802 }
6803
6804 static int
6805 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6806 {
6807         struct adapter *sc = arg1;
6808         struct sbuf *sb;
6809         uint64_t *buf, *p;
6810         int rc;
6811         u_int i, inc;
6812         void (*show_func)(struct sbuf *, uint64_t *, int);
6813
6814         rc = sysctl_wire_old_buffer(req, 0);
6815         if (rc != 0)
6816                 return (rc);
6817
6818         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6819         if (sb == NULL)
6820                 return (ENOMEM);
6821
6822         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6823
6824         t4_tp_read_la(sc, buf, NULL);
6825         p = buf;
6826
6827         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6828         case 2:
6829                 inc = 2;
6830                 show_func = tp_la_show2;
6831                 break;
6832         case 3:
6833                 inc = 2;
6834                 show_func = tp_la_show3;
6835                 break;
6836         default:
6837                 inc = 1;
6838                 show_func = tp_la_show;
6839         }
6840
6841         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6842                 (*show_func)(sb, p, i);
6843
6844         rc = sbuf_finish(sb);
6845         sbuf_delete(sb);
6846         free(buf, M_CXGBE);
6847         return (rc);
6848 }
6849
6850 static int
6851 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6852 {
6853         struct adapter *sc = arg1;
6854         struct sbuf *sb;
6855         int rc;
6856         u64 nrate[NCHAN], orate[NCHAN];
6857
6858         rc = sysctl_wire_old_buffer(req, 0);
6859         if (rc != 0)
6860                 return (rc);
6861
6862         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6863         if (sb == NULL)
6864                 return (ENOMEM);
6865
6866         t4_get_chan_txrate(sc, nrate, orate);
6867         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6868                  "channel 3\n");
6869         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6870             nrate[0], nrate[1], nrate[2], nrate[3]);
6871         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6872             orate[0], orate[1], orate[2], orate[3]);
6873
6874         rc = sbuf_finish(sb);
6875         sbuf_delete(sb);
6876
6877         return (rc);
6878 }
6879
6880 static int
6881 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6882 {
6883         struct adapter *sc = arg1;
6884         struct sbuf *sb;
6885         uint32_t *buf, *p;
6886         int rc, i;
6887
6888         rc = sysctl_wire_old_buffer(req, 0);
6889         if (rc != 0)
6890                 return (rc);
6891
6892         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6893         if (sb == NULL)
6894                 return (ENOMEM);
6895
6896         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6897             M_ZERO | M_WAITOK);
6898
6899         t4_ulprx_read_la(sc, buf);
6900         p = buf;
6901
6902         sbuf_printf(sb, "      Pcmd        Type   Message"
6903             "                Data");
6904         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6905                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6906                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6907         }
6908
6909         rc = sbuf_finish(sb);
6910         sbuf_delete(sb);
6911         free(buf, M_CXGBE);
6912         return (rc);
6913 }
6914
6915 static int
6916 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6917 {
6918         struct adapter *sc = arg1;
6919         struct sbuf *sb;
6920         int rc, v;
6921
6922         rc = sysctl_wire_old_buffer(req, 0);
6923         if (rc != 0)
6924                 return (rc);
6925
6926         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6927         if (sb == NULL)
6928                 return (ENOMEM);
6929
6930         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6931         if (G_STATSOURCE_T5(v) == 7) {
6932                 if (G_STATMODE(v) == 0) {
6933                         sbuf_printf(sb, "total %d, incomplete %d",
6934                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6935                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6936                 } else if (G_STATMODE(v) == 1) {
6937                         sbuf_printf(sb, "total %d, data overflow %d",
6938                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6939                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6940                 }
6941         }
6942         rc = sbuf_finish(sb);
6943         sbuf_delete(sb);
6944
6945         return (rc);
6946 }
6947 #endif
6948
6949 static inline void
6950 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6951 {
6952         struct buf_ring *br;
6953         struct mbuf *m;
6954
6955         TXQ_LOCK_ASSERT_OWNED(txq);
6956
6957         br = txq->br;
6958         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6959         if (m)
6960                 t4_eth_tx(ifp, txq, m);
6961 }
6962
6963 void
6964 t4_tx_callout(void *arg)
6965 {
6966         struct sge_eq *eq = arg;
6967         struct adapter *sc;
6968
6969         if (EQ_TRYLOCK(eq) == 0)
6970                 goto reschedule;
6971
6972         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6973                 EQ_UNLOCK(eq);
6974 reschedule:
6975                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6976                         callout_schedule(&eq->tx_callout, 1);
6977                 return;
6978         }
6979
6980         EQ_LOCK_ASSERT_OWNED(eq);
6981
6982         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6983
6984                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6985                         struct sge_txq *txq = arg;
6986                         struct port_info *pi = txq->ifp->if_softc;
6987
6988                         sc = pi->adapter;
6989                 } else {
6990                         struct sge_wrq *wrq = arg;
6991
6992                         sc = wrq->adapter;
6993                 }
6994
6995                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6996         }
6997
6998         EQ_UNLOCK(eq);
6999 }
7000
7001 void
7002 t4_tx_task(void *arg, int count)
7003 {
7004         struct sge_eq *eq = arg;
7005
7006         EQ_LOCK(eq);
7007         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
7008                 struct sge_txq *txq = arg;
7009                 txq_start(txq->ifp, txq);
7010         } else {
7011                 struct sge_wrq *wrq = arg;
7012                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
7013         }
7014         EQ_UNLOCK(eq);
7015 }
7016
7017 static uint32_t
7018 fconf_to_mode(uint32_t fconf)
7019 {
7020         uint32_t mode;
7021
7022         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
7023             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
7024
7025         if (fconf & F_FRAGMENTATION)
7026                 mode |= T4_FILTER_IP_FRAGMENT;
7027
7028         if (fconf & F_MPSHITTYPE)
7029                 mode |= T4_FILTER_MPS_HIT_TYPE;
7030
7031         if (fconf & F_MACMATCH)
7032                 mode |= T4_FILTER_MAC_IDX;
7033
7034         if (fconf & F_ETHERTYPE)
7035                 mode |= T4_FILTER_ETH_TYPE;
7036
7037         if (fconf & F_PROTOCOL)
7038                 mode |= T4_FILTER_IP_PROTO;
7039
7040         if (fconf & F_TOS)
7041                 mode |= T4_FILTER_IP_TOS;
7042
7043         if (fconf & F_VLAN)
7044                 mode |= T4_FILTER_VLAN;
7045
7046         if (fconf & F_VNIC_ID)
7047                 mode |= T4_FILTER_VNIC;
7048
7049         if (fconf & F_PORT)
7050                 mode |= T4_FILTER_PORT;
7051
7052         if (fconf & F_FCOE)
7053                 mode |= T4_FILTER_FCoE;
7054
7055         return (mode);
7056 }
7057
7058 static uint32_t
7059 mode_to_fconf(uint32_t mode)
7060 {
7061         uint32_t fconf = 0;
7062
7063         if (mode & T4_FILTER_IP_FRAGMENT)
7064                 fconf |= F_FRAGMENTATION;
7065
7066         if (mode & T4_FILTER_MPS_HIT_TYPE)
7067                 fconf |= F_MPSHITTYPE;
7068
7069         if (mode & T4_FILTER_MAC_IDX)
7070                 fconf |= F_MACMATCH;
7071
7072         if (mode & T4_FILTER_ETH_TYPE)
7073                 fconf |= F_ETHERTYPE;
7074
7075         if (mode & T4_FILTER_IP_PROTO)
7076                 fconf |= F_PROTOCOL;
7077
7078         if (mode & T4_FILTER_IP_TOS)
7079                 fconf |= F_TOS;
7080
7081         if (mode & T4_FILTER_VLAN)
7082                 fconf |= F_VLAN;
7083
7084         if (mode & T4_FILTER_VNIC)
7085                 fconf |= F_VNIC_ID;
7086
7087         if (mode & T4_FILTER_PORT)
7088                 fconf |= F_PORT;
7089
7090         if (mode & T4_FILTER_FCoE)
7091                 fconf |= F_FCOE;
7092
7093         return (fconf);
7094 }
7095
7096 static uint32_t
7097 fspec_to_fconf(struct t4_filter_specification *fs)
7098 {
7099         uint32_t fconf = 0;
7100
7101         if (fs->val.frag || fs->mask.frag)
7102                 fconf |= F_FRAGMENTATION;
7103
7104         if (fs->val.matchtype || fs->mask.matchtype)
7105                 fconf |= F_MPSHITTYPE;
7106
7107         if (fs->val.macidx || fs->mask.macidx)
7108                 fconf |= F_MACMATCH;
7109
7110         if (fs->val.ethtype || fs->mask.ethtype)
7111                 fconf |= F_ETHERTYPE;
7112
7113         if (fs->val.proto || fs->mask.proto)
7114                 fconf |= F_PROTOCOL;
7115
7116         if (fs->val.tos || fs->mask.tos)
7117                 fconf |= F_TOS;
7118
7119         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7120                 fconf |= F_VLAN;
7121
7122         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7123                 fconf |= F_VNIC_ID;
7124
7125         if (fs->val.iport || fs->mask.iport)
7126                 fconf |= F_PORT;
7127
7128         if (fs->val.fcoe || fs->mask.fcoe)
7129                 fconf |= F_FCOE;
7130
7131         return (fconf);
7132 }
7133
7134 static int
7135 get_filter_mode(struct adapter *sc, uint32_t *mode)
7136 {
7137         int rc;
7138         uint32_t fconf;
7139
7140         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7141             "t4getfm");
7142         if (rc)
7143                 return (rc);
7144
7145         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7146             A_TP_VLAN_PRI_MAP);
7147
7148         if (sc->params.tp.vlan_pri_map != fconf) {
7149                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7150                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7151                     fconf);
7152                 sc->params.tp.vlan_pri_map = fconf;
7153         }
7154
7155         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7156
7157         end_synchronized_op(sc, LOCK_HELD);
7158         return (0);
7159 }
7160
7161 static int
7162 set_filter_mode(struct adapter *sc, uint32_t mode)
7163 {
7164         uint32_t fconf;
7165         int rc;
7166
7167         fconf = mode_to_fconf(mode);
7168
7169         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7170             "t4setfm");
7171         if (rc)
7172                 return (rc);
7173
7174         if (sc->tids.ftids_in_use > 0) {
7175                 rc = EBUSY;
7176                 goto done;
7177         }
7178
7179 #ifdef TCP_OFFLOAD
7180         if (sc->offload_map) {
7181                 rc = EBUSY;
7182                 goto done;
7183         }
7184 #endif
7185
7186 #ifdef notyet
7187         rc = -t4_set_filter_mode(sc, fconf);
7188         if (rc == 0)
7189                 sc->filter_mode = fconf;
7190 #else
7191         rc = ENOTSUP;
7192 #endif
7193
7194 done:
7195         end_synchronized_op(sc, LOCK_HELD);
7196         return (rc);
7197 }
7198
7199 static inline uint64_t
7200 get_filter_hits(struct adapter *sc, uint32_t fid)
7201 {
7202         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7203         uint64_t hits;
7204
7205         memwin_info(sc, 0, &mw_base, NULL);
7206         off = position_memwin(sc, 0,
7207             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7208         if (is_t4(sc)) {
7209                 hits = t4_read_reg64(sc, mw_base + off + 16);
7210                 hits = be64toh(hits);
7211         } else {
7212                 hits = t4_read_reg(sc, mw_base + off + 24);
7213                 hits = be32toh(hits);
7214         }
7215
7216         return (hits);
7217 }
7218
7219 static int
7220 get_filter(struct adapter *sc, struct t4_filter *t)
7221 {
7222         int i, rc, nfilters = sc->tids.nftids;
7223         struct filter_entry *f;
7224
7225         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7226             "t4getf");
7227         if (rc)
7228                 return (rc);
7229
7230         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7231             t->idx >= nfilters) {
7232                 t->idx = 0xffffffff;
7233                 goto done;
7234         }
7235
7236         f = &sc->tids.ftid_tab[t->idx];
7237         for (i = t->idx; i < nfilters; i++, f++) {
7238                 if (f->valid) {
7239                         t->idx = i;
7240                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7241                         t->smtidx = f->smtidx;
7242                         if (f->fs.hitcnts)
7243                                 t->hits = get_filter_hits(sc, t->idx);
7244                         else
7245                                 t->hits = UINT64_MAX;
7246                         t->fs = f->fs;
7247
7248                         goto done;
7249                 }
7250         }
7251
7252         t->idx = 0xffffffff;
7253 done:
7254         end_synchronized_op(sc, LOCK_HELD);
7255         return (0);
7256 }
7257
7258 static int
7259 set_filter(struct adapter *sc, struct t4_filter *t)
7260 {
7261         unsigned int nfilters, nports;
7262         struct filter_entry *f;
7263         int i, rc;
7264
7265         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7266         if (rc)
7267                 return (rc);
7268
7269         nfilters = sc->tids.nftids;
7270         nports = sc->params.nports;
7271
7272         if (nfilters == 0) {
7273                 rc = ENOTSUP;
7274                 goto done;
7275         }
7276
7277         if (!(sc->flags & FULL_INIT_DONE)) {
7278                 rc = EAGAIN;
7279                 goto done;
7280         }
7281
7282         if (t->idx >= nfilters) {
7283                 rc = EINVAL;
7284                 goto done;
7285         }
7286
7287         /* Validate against the global filter mode */
7288         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7289             sc->params.tp.vlan_pri_map) {
7290                 rc = E2BIG;
7291                 goto done;
7292         }
7293
7294         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7295                 rc = EINVAL;
7296                 goto done;
7297         }
7298
7299         if (t->fs.val.iport >= nports) {
7300                 rc = EINVAL;
7301                 goto done;
7302         }
7303
7304         /* Can't specify an iq if not steering to it */
7305         if (!t->fs.dirsteer && t->fs.iq) {
7306                 rc = EINVAL;
7307                 goto done;
7308         }
7309
7310         /* IPv6 filter idx must be 4 aligned */
7311         if (t->fs.type == 1 &&
7312             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7313                 rc = EINVAL;
7314                 goto done;
7315         }
7316
7317         if (sc->tids.ftid_tab == NULL) {
7318                 KASSERT(sc->tids.ftids_in_use == 0,
7319                     ("%s: no memory allocated but filters_in_use > 0",
7320                     __func__));
7321
7322                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7323                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7324                 if (sc->tids.ftid_tab == NULL) {
7325                         rc = ENOMEM;
7326                         goto done;
7327                 }
7328                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7329         }
7330
7331         for (i = 0; i < 4; i++) {
7332                 f = &sc->tids.ftid_tab[t->idx + i];
7333
7334                 if (f->pending || f->valid) {
7335                         rc = EBUSY;
7336                         goto done;
7337                 }
7338                 if (f->locked) {
7339                         rc = EPERM;
7340                         goto done;
7341                 }
7342
7343                 if (t->fs.type == 0)
7344                         break;
7345         }
7346
7347         f = &sc->tids.ftid_tab[t->idx];
7348         f->fs = t->fs;
7349
7350         rc = set_filter_wr(sc, t->idx);
7351 done:
7352         end_synchronized_op(sc, 0);
7353
7354         if (rc == 0) {
7355                 mtx_lock(&sc->tids.ftid_lock);
7356                 for (;;) {
7357                         if (f->pending == 0) {
7358                                 rc = f->valid ? 0 : EIO;
7359                                 break;
7360                         }
7361
7362                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7363                             PCATCH, "t4setfw", 0)) {
7364                                 rc = EINPROGRESS;
7365                                 break;
7366                         }
7367                 }
7368                 mtx_unlock(&sc->tids.ftid_lock);
7369         }
7370         return (rc);
7371 }
7372
7373 static int
7374 del_filter(struct adapter *sc, struct t4_filter *t)
7375 {
7376         unsigned int nfilters;
7377         struct filter_entry *f;
7378         int rc;
7379
7380         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7381         if (rc)
7382                 return (rc);
7383
7384         nfilters = sc->tids.nftids;
7385
7386         if (nfilters == 0) {
7387                 rc = ENOTSUP;
7388                 goto done;
7389         }
7390
7391         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7392             t->idx >= nfilters) {
7393                 rc = EINVAL;
7394                 goto done;
7395         }
7396
7397         if (!(sc->flags & FULL_INIT_DONE)) {
7398                 rc = EAGAIN;
7399                 goto done;
7400         }
7401
7402         f = &sc->tids.ftid_tab[t->idx];
7403
7404         if (f->pending) {
7405                 rc = EBUSY;
7406                 goto done;
7407         }
7408         if (f->locked) {
7409                 rc = EPERM;
7410                 goto done;
7411         }
7412
7413         if (f->valid) {
7414                 t->fs = f->fs;  /* extra info for the caller */
7415                 rc = del_filter_wr(sc, t->idx);
7416         }
7417
7418 done:
7419         end_synchronized_op(sc, 0);
7420
7421         if (rc == 0) {
7422                 mtx_lock(&sc->tids.ftid_lock);
7423                 for (;;) {
7424                         if (f->pending == 0) {
7425                                 rc = f->valid ? EIO : 0;
7426                                 break;
7427                         }
7428
7429                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7430                             PCATCH, "t4delfw", 0)) {
7431                                 rc = EINPROGRESS;
7432                                 break;
7433                         }
7434                 }
7435                 mtx_unlock(&sc->tids.ftid_lock);
7436         }
7437
7438         return (rc);
7439 }
7440
7441 static void
7442 clear_filter(struct filter_entry *f)
7443 {
7444         if (f->l2t)
7445                 t4_l2t_release(f->l2t);
7446
7447         bzero(f, sizeof (*f));
7448 }
7449
7450 static int
7451 set_filter_wr(struct adapter *sc, int fidx)
7452 {
7453         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7454         struct wrqe *wr;
7455         struct fw_filter_wr *fwr;
7456         unsigned int ftid;
7457
7458         ASSERT_SYNCHRONIZED_OP(sc);
7459
7460         if (f->fs.newdmac || f->fs.newvlan) {
7461                 /* This filter needs an L2T entry; allocate one. */
7462                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7463                 if (f->l2t == NULL)
7464                         return (EAGAIN);
7465                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7466                     f->fs.dmac)) {
7467                         t4_l2t_release(f->l2t);
7468                         f->l2t = NULL;
7469                         return (ENOMEM);
7470                 }
7471         }
7472
7473         ftid = sc->tids.ftid_base + fidx;
7474
7475         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7476         if (wr == NULL)
7477                 return (ENOMEM);
7478
7479         fwr = wrtod(wr);
7480         bzero(fwr, sizeof (*fwr));
7481
7482         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7483         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7484         fwr->tid_to_iq =
7485             htobe32(V_FW_FILTER_WR_TID(ftid) |
7486                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7487                 V_FW_FILTER_WR_NOREPLY(0) |
7488                 V_FW_FILTER_WR_IQ(f->fs.iq));
7489         fwr->del_filter_to_l2tix =
7490             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7491                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7492                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7493                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7494                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7495                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7496                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7497                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7498                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7499                     f->fs.newvlan == VLAN_REWRITE) |
7500                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7501                     f->fs.newvlan == VLAN_REWRITE) |
7502                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7503                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7504                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7505                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7506         fwr->ethtype = htobe16(f->fs.val.ethtype);
7507         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7508         fwr->frag_to_ovlan_vldm =
7509             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7510                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7511                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7512                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7513                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7514                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7515         fwr->smac_sel = 0;
7516         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7517             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7518         fwr->maci_to_matchtypem =
7519             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7520                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7521                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7522                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7523                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7524                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7525                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7526                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7527         fwr->ptcl = f->fs.val.proto;
7528         fwr->ptclm = f->fs.mask.proto;
7529         fwr->ttyp = f->fs.val.tos;
7530         fwr->ttypm = f->fs.mask.tos;
7531         fwr->ivlan = htobe16(f->fs.val.vlan);
7532         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7533         fwr->ovlan = htobe16(f->fs.val.vnic);
7534         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7535         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7536         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7537         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7538         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7539         fwr->lp = htobe16(f->fs.val.dport);
7540         fwr->lpm = htobe16(f->fs.mask.dport);
7541         fwr->fp = htobe16(f->fs.val.sport);
7542         fwr->fpm = htobe16(f->fs.mask.sport);
7543         if (f->fs.newsmac)
7544                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7545
7546         f->pending = 1;
7547         sc->tids.ftids_in_use++;
7548
7549         t4_wrq_tx(sc, wr);
7550         return (0);
7551 }
7552
7553 static int
7554 del_filter_wr(struct adapter *sc, int fidx)
7555 {
7556         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7557         struct wrqe *wr;
7558         struct fw_filter_wr *fwr;
7559         unsigned int ftid;
7560
7561         ftid = sc->tids.ftid_base + fidx;
7562
7563         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7564         if (wr == NULL)
7565                 return (ENOMEM);
7566         fwr = wrtod(wr);
7567         bzero(fwr, sizeof (*fwr));
7568
7569         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7570
7571         f->pending = 1;
7572         t4_wrq_tx(sc, wr);
7573         return (0);
7574 }
7575
7576 int
7577 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7578 {
7579         struct adapter *sc = iq->adapter;
7580         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7581         unsigned int idx = GET_TID(rpl);
7582         unsigned int rc;
7583         struct filter_entry *f;
7584
7585         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7586             rss->opcode));
7587
7588         if (is_ftid(sc, idx)) {
7589
7590                 idx -= sc->tids.ftid_base;
7591                 f = &sc->tids.ftid_tab[idx];
7592                 rc = G_COOKIE(rpl->cookie);
7593
7594                 mtx_lock(&sc->tids.ftid_lock);
7595                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7596                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7597                             __func__, idx));
7598                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7599                         f->pending = 0;  /* asynchronous setup completed */
7600                         f->valid = 1;
7601                 } else {
7602                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7603                                 /* Add or delete failed, display an error */
7604                                 log(LOG_ERR,
7605                                     "filter %u setup failed with error %u\n",
7606                                     idx, rc);
7607                         }
7608
7609                         clear_filter(f);
7610                         sc->tids.ftids_in_use--;
7611                 }
7612                 wakeup(&sc->tids.ftid_tab);
7613                 mtx_unlock(&sc->tids.ftid_lock);
7614         }
7615
7616         return (0);
7617 }
7618
7619 static int
7620 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7621 {
7622         int rc;
7623
7624         if (cntxt->cid > M_CTXTQID)
7625                 return (EINVAL);
7626
7627         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7628             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7629                 return (EINVAL);
7630
7631         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7632         if (rc)
7633                 return (rc);
7634
7635         if (sc->flags & FW_OK) {
7636                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7637                     &cntxt->data[0]);
7638                 if (rc == 0)
7639                         goto done;
7640         }
7641
7642         /*
7643          * Read via firmware failed or wasn't even attempted.  Read directly via
7644          * the backdoor.
7645          */
7646         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7647 done:
7648         end_synchronized_op(sc, 0);
7649         return (rc);
7650 }
7651
7652 static int
7653 load_fw(struct adapter *sc, struct t4_data *fw)
7654 {
7655         int rc;
7656         uint8_t *fw_data;
7657
7658         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7659         if (rc)
7660                 return (rc);
7661
7662         if (sc->flags & FULL_INIT_DONE) {
7663                 rc = EBUSY;
7664                 goto done;
7665         }
7666
7667         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7668         if (fw_data == NULL) {
7669                 rc = ENOMEM;
7670                 goto done;
7671         }
7672
7673         rc = copyin(fw->data, fw_data, fw->len);
7674         if (rc == 0)
7675                 rc = -t4_load_fw(sc, fw_data, fw->len);
7676
7677         free(fw_data, M_CXGBE);
7678 done:
7679         end_synchronized_op(sc, 0);
7680         return (rc);
7681 }
7682
7683 static int
7684 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7685 {
7686         uint32_t addr, off, remaining, i, n;
7687         uint32_t *buf, *b;
7688         uint32_t mw_base, mw_aperture;
7689         int rc;
7690         uint8_t *dst;
7691
7692         rc = validate_mem_range(sc, mr->addr, mr->len);
7693         if (rc != 0)
7694                 return (rc);
7695
7696         memwin_info(sc, win, &mw_base, &mw_aperture);
7697         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7698         addr = mr->addr;
7699         remaining = mr->len;
7700         dst = (void *)mr->data;
7701
7702         while (remaining) {
7703                 off = position_memwin(sc, win, addr);
7704
7705                 /* number of bytes that we'll copy in the inner loop */
7706                 n = min(remaining, mw_aperture - off);
7707                 for (i = 0; i < n; i += 4)
7708                         *b++ = t4_read_reg(sc, mw_base + off + i);
7709
7710                 rc = copyout(buf, dst, n);
7711                 if (rc != 0)
7712                         break;
7713
7714                 b = buf;
7715                 dst += n;
7716                 remaining -= n;
7717                 addr += n;
7718         }
7719
7720         free(buf, M_CXGBE);
7721         return (rc);
7722 }
7723
7724 static int
7725 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7726 {
7727         int rc;
7728
7729         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7730                 return (EINVAL);
7731
7732         if (i2cd->len > sizeof(i2cd->data))
7733                 return (EFBIG);
7734
7735         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7736         if (rc)
7737                 return (rc);
7738         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7739             i2cd->offset, i2cd->len, &i2cd->data[0]);
7740         end_synchronized_op(sc, 0);
7741
7742         return (rc);
7743 }
7744
7745 static int
7746 in_range(int val, int lo, int hi)
7747 {
7748
7749         return (val < 0 || (val <= hi && val >= lo));
7750 }
7751
7752 static int
7753 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7754 {
7755         int fw_subcmd, fw_type, rc;
7756
7757         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7758         if (rc)
7759                 return (rc);
7760
7761         if (!(sc->flags & FULL_INIT_DONE)) {
7762                 rc = EAGAIN;
7763                 goto done;
7764         }
7765
7766         /*
7767          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7768          * sub-command and type are in common locations.)
7769          */
7770         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7771                 fw_subcmd = FW_SCHED_SC_CONFIG;
7772         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7773                 fw_subcmd = FW_SCHED_SC_PARAMS;
7774         else {
7775                 rc = EINVAL;
7776                 goto done;
7777         }
7778         if (p->type == SCHED_CLASS_TYPE_PACKET)
7779                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7780         else {
7781                 rc = EINVAL;
7782                 goto done;
7783         }
7784
7785         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7786                 /* Vet our parameters ..*/
7787                 if (p->u.config.minmax < 0) {
7788                         rc = EINVAL;
7789                         goto done;
7790                 }
7791
7792                 /* And pass the request to the firmware ...*/
7793                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7794                 goto done;
7795         }
7796
7797         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7798                 int fw_level;
7799                 int fw_mode;
7800                 int fw_rateunit;
7801                 int fw_ratemode;
7802
7803                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7804                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7805                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7806                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7807                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7808                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7809                 else {
7810                         rc = EINVAL;
7811                         goto done;
7812                 }
7813
7814                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7815                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7816                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7817                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7818                 else {
7819                         rc = EINVAL;
7820                         goto done;
7821                 }
7822
7823                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7824                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7825                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7826                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7827                 else {
7828                         rc = EINVAL;
7829                         goto done;
7830                 }
7831
7832                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7833                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7834                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7835                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7836                 else {
7837                         rc = EINVAL;
7838                         goto done;
7839                 }
7840
7841                 /* Vet our parameters ... */
7842                 if (!in_range(p->u.params.channel, 0, 3) ||
7843                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7844                     !in_range(p->u.params.minrate, 0, 10000000) ||
7845                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7846                     !in_range(p->u.params.weight, 0, 100)) {
7847                         rc = ERANGE;
7848                         goto done;
7849                 }
7850
7851                 /*
7852                  * Translate any unset parameters into the firmware's
7853                  * nomenclature and/or fail the call if the parameters
7854                  * are required ...
7855                  */
7856                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7857                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7858                         rc = EINVAL;
7859                         goto done;
7860                 }
7861                 if (p->u.params.minrate < 0)
7862                         p->u.params.minrate = 0;
7863                 if (p->u.params.maxrate < 0) {
7864                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7865                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7866                                 rc = EINVAL;
7867                                 goto done;
7868                         } else
7869                                 p->u.params.maxrate = 0;
7870                 }
7871                 if (p->u.params.weight < 0) {
7872                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7873                                 rc = EINVAL;
7874                                 goto done;
7875                         } else
7876                                 p->u.params.weight = 0;
7877                 }
7878                 if (p->u.params.pktsize < 0) {
7879                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7880                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7881                                 rc = EINVAL;
7882                                 goto done;
7883                         } else
7884                                 p->u.params.pktsize = 0;
7885                 }
7886
7887                 /* See what the firmware thinks of the request ... */
7888                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7889                     fw_rateunit, fw_ratemode, p->u.params.channel,
7890                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7891                     p->u.params.weight, p->u.params.pktsize, 1);
7892                 goto done;
7893         }
7894
7895         rc = EINVAL;
7896 done:
7897         end_synchronized_op(sc, 0);
7898         return (rc);
7899 }
7900
7901 static int
7902 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7903 {
7904         struct port_info *pi = NULL;
7905         struct sge_txq *txq;
7906         uint32_t fw_mnem, fw_queue, fw_class;
7907         int i, rc;
7908
7909         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7910         if (rc)
7911                 return (rc);
7912
7913         if (!(sc->flags & FULL_INIT_DONE)) {
7914                 rc = EAGAIN;
7915                 goto done;
7916         }
7917
7918         if (p->port >= sc->params.nports) {
7919                 rc = EINVAL;
7920                 goto done;
7921         }
7922
7923         pi = sc->port[p->port];
7924         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7925                 rc = EINVAL;
7926                 goto done;
7927         }
7928
7929         /*
7930          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7931          * Scheduling Class in this case).
7932          */
7933         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7934             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7935         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7936
7937         /*
7938          * If op.queue is non-negative, then we're only changing the scheduling
7939          * on a single specified TX queue.
7940          */
7941         if (p->queue >= 0) {
7942                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7943                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7944                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7945                     &fw_class);
7946                 goto done;
7947         }
7948
7949         /*
7950          * Change the scheduling on all the TX queues for the
7951          * interface.
7952          */
7953         for_each_txq(pi, i, txq) {
7954                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7955                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7956                     &fw_class);
7957                 if (rc)
7958                         goto done;
7959         }
7960
7961         rc = 0;
7962 done:
7963         end_synchronized_op(sc, 0);
7964         return (rc);
7965 }
7966
7967 int
7968 t4_os_find_pci_capability(struct adapter *sc, int cap)
7969 {
7970         int i;
7971
7972         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7973 }
7974
7975 int
7976 t4_os_pci_save_state(struct adapter *sc)
7977 {
7978         device_t dev;
7979         struct pci_devinfo *dinfo;
7980
7981         dev = sc->dev;
7982         dinfo = device_get_ivars(dev);
7983
7984         pci_cfg_save(dev, dinfo, 0);
7985         return (0);
7986 }
7987
7988 int
7989 t4_os_pci_restore_state(struct adapter *sc)
7990 {
7991         device_t dev;
7992         struct pci_devinfo *dinfo;
7993
7994         dev = sc->dev;
7995         dinfo = device_get_ivars(dev);
7996
7997         pci_cfg_restore(dev, dinfo);
7998         return (0);
7999 }
8000
8001 void
8002 t4_os_portmod_changed(const struct adapter *sc, int idx)
8003 {
8004         struct port_info *pi = sc->port[idx];
8005         static const char *mod_str[] = {
8006                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
8007         };
8008
8009         build_medialist(pi, &pi->media);
8010 #ifdef DEV_NETMAP
8011         build_medialist(pi, &pi->nm_media);
8012 #endif
8013
8014         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
8015                 if_printf(pi->ifp, "transceiver unplugged.\n");
8016         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
8017                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
8018         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
8019                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
8020         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
8021                 if_printf(pi->ifp, "%s transceiver inserted.\n",
8022                     mod_str[pi->mod_type]);
8023         } else {
8024                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
8025                     pi->mod_type);
8026         }
8027 }
8028
8029 void
8030 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
8031 {
8032         struct port_info *pi = sc->port[idx];
8033         struct ifnet *ifp = pi->ifp;
8034
8035         if (link_stat) {
8036                 pi->linkdnrc = -1;
8037                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
8038                 if_link_state_change(ifp, LINK_STATE_UP);
8039         } else {
8040                 if (reason >= 0)
8041                         pi->linkdnrc = reason;
8042                 if_link_state_change(ifp, LINK_STATE_DOWN);
8043         }
8044 }
8045
8046 void
8047 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
8048 {
8049         struct adapter *sc;
8050
8051         sx_slock(&t4_list_lock);
8052         SLIST_FOREACH(sc, &t4_list, link) {
8053                 /*
8054                  * func should not make any assumptions about what state sc is
8055                  * in - the only guarantee is that sc->sc_lock is a valid lock.
8056                  */
8057                 func(sc, arg);
8058         }
8059         sx_sunlock(&t4_list_lock);
8060 }
8061
8062 static int
8063 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8064 {
8065        return (0);
8066 }
8067
8068 static int
8069 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8070 {
8071        return (0);
8072 }
8073
8074 static int
8075 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8076     struct thread *td)
8077 {
8078         int rc;
8079         struct adapter *sc = dev->si_drv1;
8080
8081         rc = priv_check(td, PRIV_DRIVER);
8082         if (rc != 0)
8083                 return (rc);
8084
8085         switch (cmd) {
8086         case CHELSIO_T4_GETREG: {
8087                 struct t4_reg *edata = (struct t4_reg *)data;
8088
8089                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8090                         return (EFAULT);
8091
8092                 if (edata->size == 4)
8093                         edata->val = t4_read_reg(sc, edata->addr);
8094                 else if (edata->size == 8)
8095                         edata->val = t4_read_reg64(sc, edata->addr);
8096                 else
8097                         return (EINVAL);
8098
8099                 break;
8100         }
8101         case CHELSIO_T4_SETREG: {
8102                 struct t4_reg *edata = (struct t4_reg *)data;
8103
8104                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8105                         return (EFAULT);
8106
8107                 if (edata->size == 4) {
8108                         if (edata->val & 0xffffffff00000000)
8109                                 return (EINVAL);
8110                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8111                 } else if (edata->size == 8)
8112                         t4_write_reg64(sc, edata->addr, edata->val);
8113                 else
8114                         return (EINVAL);
8115                 break;
8116         }
8117         case CHELSIO_T4_REGDUMP: {
8118                 struct t4_regdump *regs = (struct t4_regdump *)data;
8119                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8120                 uint8_t *buf;
8121
8122                 if (regs->len < reglen) {
8123                         regs->len = reglen; /* hint to the caller */
8124                         return (ENOBUFS);
8125                 }
8126
8127                 regs->len = reglen;
8128                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8129                 t4_get_regs(sc, regs, buf);
8130                 rc = copyout(buf, regs->data, reglen);
8131                 free(buf, M_CXGBE);
8132                 break;
8133         }
8134         case CHELSIO_T4_GET_FILTER_MODE:
8135                 rc = get_filter_mode(sc, (uint32_t *)data);
8136                 break;
8137         case CHELSIO_T4_SET_FILTER_MODE:
8138                 rc = set_filter_mode(sc, *(uint32_t *)data);
8139                 break;
8140         case CHELSIO_T4_GET_FILTER:
8141                 rc = get_filter(sc, (struct t4_filter *)data);
8142                 break;
8143         case CHELSIO_T4_SET_FILTER:
8144                 rc = set_filter(sc, (struct t4_filter *)data);
8145                 break;
8146         case CHELSIO_T4_DEL_FILTER:
8147                 rc = del_filter(sc, (struct t4_filter *)data);
8148                 break;
8149         case CHELSIO_T4_GET_SGE_CONTEXT:
8150                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8151                 break;
8152         case CHELSIO_T4_LOAD_FW:
8153                 rc = load_fw(sc, (struct t4_data *)data);
8154                 break;
8155         case CHELSIO_T4_GET_MEM:
8156                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8157                 break;
8158         case CHELSIO_T4_GET_I2C:
8159                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8160                 break;
8161         case CHELSIO_T4_CLEAR_STATS: {
8162                 int i;
8163                 u_int port_id = *(uint32_t *)data;
8164                 struct port_info *pi;
8165
8166                 if (port_id >= sc->params.nports)
8167                         return (EINVAL);
8168                 pi = sc->port[port_id];
8169
8170                 /* MAC stats */
8171                 t4_clr_port_stats(sc, pi->tx_chan);
8172
8173                 if (pi->flags & PORT_INIT_DONE) {
8174                         struct sge_rxq *rxq;
8175                         struct sge_txq *txq;
8176                         struct sge_wrq *wrq;
8177
8178                         for_each_rxq(pi, i, rxq) {
8179 #if defined(INET) || defined(INET6)
8180                                 rxq->lro.lro_queued = 0;
8181                                 rxq->lro.lro_flushed = 0;
8182 #endif
8183                                 rxq->rxcsum = 0;
8184                                 rxq->vlan_extraction = 0;
8185                         }
8186
8187                         for_each_txq(pi, i, txq) {
8188                                 txq->txcsum = 0;
8189                                 txq->tso_wrs = 0;
8190                                 txq->vlan_insertion = 0;
8191                                 txq->imm_wrs = 0;
8192                                 txq->sgl_wrs = 0;
8193                                 txq->txpkt_wrs = 0;
8194                                 txq->txpkts_wrs = 0;
8195                                 txq->txpkts_pkts = 0;
8196                                 txq->br->br_drops = 0;
8197                                 txq->no_dmamap = 0;
8198                                 txq->no_desc = 0;
8199                         }
8200
8201 #ifdef TCP_OFFLOAD
8202                         /* nothing to clear for each ofld_rxq */
8203
8204                         for_each_ofld_txq(pi, i, wrq) {
8205                                 wrq->tx_wrs = 0;
8206                                 wrq->no_desc = 0;
8207                         }
8208 #endif
8209                         wrq = &sc->sge.ctrlq[pi->port_id];
8210                         wrq->tx_wrs = 0;
8211                         wrq->no_desc = 0;
8212                 }
8213                 break;
8214         }
8215         case CHELSIO_T4_SCHED_CLASS:
8216                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8217                 break;
8218         case CHELSIO_T4_SCHED_QUEUE:
8219                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8220                 break;
8221         case CHELSIO_T4_GET_TRACER:
8222                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8223                 break;
8224         case CHELSIO_T4_SET_TRACER:
8225                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8226                 break;
8227         default:
8228                 rc = EINVAL;
8229         }
8230
8231         return (rc);
8232 }
8233
8234 #ifdef TCP_OFFLOAD
8235 void
8236 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8237     const unsigned int *pgsz_order)
8238 {
8239         struct port_info *pi = ifp->if_softc;
8240         struct adapter *sc = pi->adapter;
8241
8242         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8243         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8244                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8245                 V_HPZ3(pgsz_order[3]));
8246 }
8247
8248 static int
8249 toe_capability(struct port_info *pi, int enable)
8250 {
8251         int rc;
8252         struct adapter *sc = pi->adapter;
8253
8254         ASSERT_SYNCHRONIZED_OP(sc);
8255
8256         if (!is_offload(sc))
8257                 return (ENODEV);
8258
8259         if (enable) {
8260                 if (!(sc->flags & FULL_INIT_DONE)) {
8261                         rc = cxgbe_init_synchronized(pi);
8262                         if (rc)
8263                                 return (rc);
8264                 }
8265
8266                 if (isset(&sc->offload_map, pi->port_id))
8267                         return (0);
8268
8269                 if (!(sc->flags & TOM_INIT_DONE)) {
8270                         rc = t4_activate_uld(sc, ULD_TOM);
8271                         if (rc == EAGAIN) {
8272                                 log(LOG_WARNING,
8273                                     "You must kldload t4_tom.ko before trying "
8274                                     "to enable TOE on a cxgbe interface.\n");
8275                         }
8276                         if (rc != 0)
8277                                 return (rc);
8278                         KASSERT(sc->tom_softc != NULL,
8279                             ("%s: TOM activated but softc NULL", __func__));
8280                         KASSERT(sc->flags & TOM_INIT_DONE,
8281                             ("%s: TOM activated but flag not set", __func__));
8282                 }
8283
8284                 setbit(&sc->offload_map, pi->port_id);
8285         } else {
8286                 if (!isset(&sc->offload_map, pi->port_id))
8287                         return (0);
8288
8289                 KASSERT(sc->flags & TOM_INIT_DONE,
8290                     ("%s: TOM never initialized?", __func__));
8291                 clrbit(&sc->offload_map, pi->port_id);
8292         }
8293
8294         return (0);
8295 }
8296
8297 /*
8298  * Add an upper layer driver to the global list.
8299  */
8300 int
8301 t4_register_uld(struct uld_info *ui)
8302 {
8303         int rc = 0;
8304         struct uld_info *u;
8305
8306         sx_xlock(&t4_uld_list_lock);
8307         SLIST_FOREACH(u, &t4_uld_list, link) {
8308             if (u->uld_id == ui->uld_id) {
8309                     rc = EEXIST;
8310                     goto done;
8311             }
8312         }
8313
8314         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8315         ui->refcount = 0;
8316 done:
8317         sx_xunlock(&t4_uld_list_lock);
8318         return (rc);
8319 }
8320
8321 int
8322 t4_unregister_uld(struct uld_info *ui)
8323 {
8324         int rc = EINVAL;
8325         struct uld_info *u;
8326
8327         sx_xlock(&t4_uld_list_lock);
8328
8329         SLIST_FOREACH(u, &t4_uld_list, link) {
8330             if (u == ui) {
8331                     if (ui->refcount > 0) {
8332                             rc = EBUSY;
8333                             goto done;
8334                     }
8335
8336                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8337                     rc = 0;
8338                     goto done;
8339             }
8340         }
8341 done:
8342         sx_xunlock(&t4_uld_list_lock);
8343         return (rc);
8344 }
8345
8346 int
8347 t4_activate_uld(struct adapter *sc, int id)
8348 {
8349         int rc = EAGAIN;
8350         struct uld_info *ui;
8351
8352         ASSERT_SYNCHRONIZED_OP(sc);
8353
8354         sx_slock(&t4_uld_list_lock);
8355
8356         SLIST_FOREACH(ui, &t4_uld_list, link) {
8357                 if (ui->uld_id == id) {
8358                         if (!(sc->flags & FULL_INIT_DONE)) {
8359                                 rc = adapter_full_init(sc);
8360                                 if (rc != 0)
8361                                         goto done;
8362                         }
8363
8364                         rc = ui->activate(sc);
8365                         if (rc == 0)
8366                                 ui->refcount++;
8367                         goto done;
8368                 }
8369         }
8370 done:
8371         sx_sunlock(&t4_uld_list_lock);
8372
8373         return (rc);
8374 }
8375
8376 int
8377 t4_deactivate_uld(struct adapter *sc, int id)
8378 {
8379         int rc = EINVAL;
8380         struct uld_info *ui;
8381
8382         ASSERT_SYNCHRONIZED_OP(sc);
8383
8384         sx_slock(&t4_uld_list_lock);
8385
8386         SLIST_FOREACH(ui, &t4_uld_list, link) {
8387                 if (ui->uld_id == id) {
8388                         rc = ui->deactivate(sc);
8389                         if (rc == 0)
8390                                 ui->refcount--;
8391                         goto done;
8392                 }
8393         }
8394 done:
8395         sx_sunlock(&t4_uld_list_lock);
8396
8397         return (rc);
8398 }
8399 #endif
8400
8401 /*
8402  * Come up with reasonable defaults for some of the tunables, provided they're
8403  * not set by the user (in which case we'll use the values as is).
8404  */
8405 static void
8406 tweak_tunables(void)
8407 {
8408         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8409
8410         if (t4_ntxq10g < 1)
8411                 t4_ntxq10g = min(nc, NTXQ_10G);
8412
8413         if (t4_ntxq1g < 1)
8414                 t4_ntxq1g = min(nc, NTXQ_1G);
8415
8416         if (t4_nrxq10g < 1)
8417                 t4_nrxq10g = min(nc, NRXQ_10G);
8418
8419         if (t4_nrxq1g < 1)
8420                 t4_nrxq1g = min(nc, NRXQ_1G);
8421
8422 #ifdef TCP_OFFLOAD
8423         if (t4_nofldtxq10g < 1)
8424                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8425
8426         if (t4_nofldtxq1g < 1)
8427                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8428
8429         if (t4_nofldrxq10g < 1)
8430                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8431
8432         if (t4_nofldrxq1g < 1)
8433                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8434
8435         if (t4_toecaps_allowed == -1)
8436                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8437 #else
8438         if (t4_toecaps_allowed == -1)
8439                 t4_toecaps_allowed = 0;
8440 #endif
8441
8442 #ifdef DEV_NETMAP
8443         if (t4_nnmtxq10g < 1)
8444                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8445
8446         if (t4_nnmtxq1g < 1)
8447                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8448
8449         if (t4_nnmrxq10g < 1)
8450                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8451
8452         if (t4_nnmrxq1g < 1)
8453                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8454 #endif
8455
8456         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8457                 t4_tmr_idx_10g = TMR_IDX_10G;
8458
8459         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8460                 t4_pktc_idx_10g = PKTC_IDX_10G;
8461
8462         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8463                 t4_tmr_idx_1g = TMR_IDX_1G;
8464
8465         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8466                 t4_pktc_idx_1g = PKTC_IDX_1G;
8467
8468         if (t4_qsize_txq < 128)
8469                 t4_qsize_txq = 128;
8470
8471         if (t4_qsize_rxq < 128)
8472                 t4_qsize_rxq = 128;
8473         while (t4_qsize_rxq & 7)
8474                 t4_qsize_rxq++;
8475
8476         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8477 }
8478
8479 static struct sx mlu;   /* mod load unload */
8480 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8481
8482 static int
8483 mod_event(module_t mod, int cmd, void *arg)
8484 {
8485         int rc = 0;
8486         static int loaded = 0;
8487
8488         switch (cmd) {
8489         case MOD_LOAD:
8490                 sx_xlock(&mlu);
8491                 if (loaded++ == 0) {
8492                         t4_sge_modload();
8493                         sx_init(&t4_list_lock, "T4/T5 adapters");
8494                         SLIST_INIT(&t4_list);
8495 #ifdef TCP_OFFLOAD
8496                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8497                         SLIST_INIT(&t4_uld_list);
8498 #endif
8499                         t4_tracer_modload();
8500                         tweak_tunables();
8501                 }
8502                 sx_xunlock(&mlu);
8503                 break;
8504
8505         case MOD_UNLOAD:
8506                 sx_xlock(&mlu);
8507                 if (--loaded == 0) {
8508                         int tries;
8509
8510                         sx_slock(&t4_list_lock);
8511                         if (!SLIST_EMPTY(&t4_list)) {
8512                                 rc = EBUSY;
8513                                 sx_sunlock(&t4_list_lock);
8514                                 goto done_unload;
8515                         }
8516 #ifdef TCP_OFFLOAD
8517                         sx_slock(&t4_uld_list_lock);
8518                         if (!SLIST_EMPTY(&t4_uld_list)) {
8519                                 rc = EBUSY;
8520                                 sx_sunlock(&t4_uld_list_lock);
8521                                 sx_sunlock(&t4_list_lock);
8522                                 goto done_unload;
8523                         }
8524 #endif
8525                         tries = 0;
8526                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8527                                 uprintf("%ju clusters with custom free routine "
8528                                     "still is use.\n", t4_sge_extfree_refs());
8529                                 pause("t4unload", 2 * hz);
8530                         }
8531 #ifdef TCP_OFFLOAD
8532                         sx_sunlock(&t4_uld_list_lock);
8533 #endif
8534                         sx_sunlock(&t4_list_lock);
8535
8536                         if (t4_sge_extfree_refs() == 0) {
8537                                 t4_tracer_modunload();
8538 #ifdef TCP_OFFLOAD
8539                                 sx_destroy(&t4_uld_list_lock);
8540 #endif
8541                                 sx_destroy(&t4_list_lock);
8542                                 t4_sge_modunload();
8543                                 loaded = 0;
8544                         } else {
8545                                 rc = EBUSY;
8546                                 loaded++;       /* undo earlier decrement */
8547                         }
8548                 }
8549 done_unload:
8550                 sx_xunlock(&mlu);
8551                 break;
8552         }
8553
8554         return (rc);
8555 }
8556
8557 static devclass_t t4_devclass, t5_devclass;
8558 static devclass_t cxgbe_devclass, cxl_devclass;
8559
8560 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8561 MODULE_VERSION(t4nex, 1);
8562 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8563
8564 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8565 MODULE_VERSION(t5nex, 1);
8566 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8567
8568 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8569 MODULE_VERSION(cxgbe, 1);
8570
8571 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8572 MODULE_VERSION(cxl, 1);