sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #if defined(__i386__) || defined(__amd64__)
  61 #include <vm/vm.h>
  62 #include <vm/pmap.h>
  63 #endif
  64
  65 #include "common/common.h"
  66 #include "common/t4_msg.h"
  67 #include "common/t4_regs.h"
  68 #include "common/t4_regs_values.h"
  69 #include "t4_ioctl.h"
  70 #include "t4_l2t.h"
  71 #include "t4_mp_ring.h"
  72
  73 /* T4 bus driver interface */
  74 static int t4_probe(device_t);
  75 static int t4_attach(device_t);
  76 static int t4_detach(device_t);
  77 static device_method_t t4_methods[] = {
  78         DEVMETHOD(device_probe,         t4_probe),
  79         DEVMETHOD(device_attach,        t4_attach),
  80         DEVMETHOD(device_detach,        t4_detach),
  81
  82         DEVMETHOD_END
  83 };
  84 static driver_t t4_driver = {
  85         "t4nex",
  86         t4_methods,
  87         sizeof(struct adapter)
  88 };
  89
  90
  91 /* T4 port (cxgbe) interface */
  92 static int cxgbe_probe(device_t);
  93 static int cxgbe_attach(device_t);
  94 static int cxgbe_detach(device_t);
  95 static device_method_t cxgbe_methods[] = {
  96         DEVMETHOD(device_probe,         cxgbe_probe),
  97         DEVMETHOD(device_attach,        cxgbe_attach),
  98         DEVMETHOD(device_detach,        cxgbe_detach),
  99         { 0, 0 }
 100 };
 101 static driver_t cxgbe_driver = {
 102         "cxgbe",
 103         cxgbe_methods,
 104         sizeof(struct port_info)
 105 };
 106
 107 static d_ioctl_t t4_ioctl;
 108 static d_open_t t4_open;
 109 static d_close_t t4_close;
 110
 111 static struct cdevsw t4_cdevsw = {
 112        .d_version = D_VERSION,
 113        .d_flags = 0,
 114        .d_open = t4_open,
 115        .d_close = t4_close,
 116        .d_ioctl = t4_ioctl,
 117        .d_name = "t4nex",
 118 };
 119
 120 /* T5 bus driver interface */
 121 static int t5_probe(device_t);
 122 static device_method_t t5_methods[] = {
 123         DEVMETHOD(device_probe,         t5_probe),
 124         DEVMETHOD(device_attach,        t4_attach),
 125         DEVMETHOD(device_detach,        t4_detach),
 126
 127         DEVMETHOD_END
 128 };
 129 static driver_t t5_driver = {
 130         "t5nex",
 131         t5_methods,
 132         sizeof(struct adapter)
 133 };
 134
 135
 136 /* T5 port (cxl) interface */
 137 static driver_t cxl_driver = {
 138         "cxl",
 139         cxgbe_methods,
 140         sizeof(struct port_info)
 141 };
 142
 143 static struct cdevsw t5_cdevsw = {
 144        .d_version = D_VERSION,
 145        .d_flags = 0,
 146        .d_open = t4_open,
 147        .d_close = t4_close,
 148        .d_ioctl = t4_ioctl,
 149        .d_name = "t5nex",
 150 };
 151
 152 /* ifnet + media interface */
 153 static void cxgbe_init(void *);
 154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 156 static void cxgbe_qflush(struct ifnet *);
 157 static int cxgbe_media_change(struct ifnet *);
 158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 159
 160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 161
 162 /*
 163  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 164  * then ADAPTER_LOCK, then t4_uld_list_lock.
 165  */
 166 static struct sx t4_list_lock;
 167 SLIST_HEAD(, adapter) t4_list;
 168 #ifdef TCP_OFFLOAD
 169 static struct sx t4_uld_list_lock;
 170 SLIST_HEAD(, uld_info) t4_uld_list;
 171 #endif
 172
 173 /*
 174  * Tunables.  See tweak_tunables() too.
 175  *
 176  * Each tunable is set to a default value here if it's known at compile-time.
 177  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 178  * provide a reasonable default when the driver is loaded.
 179  *
 180  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 181  * T5 are under hw.cxl.
 182  */
 183
 184 /*
 185  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 186  */
 187 #define NTXQ_10G 16
 188 static int t4_ntxq10g = -1;
 189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 190
 191 #define NRXQ_10G 8
 192 static int t4_nrxq10g = -1;
 193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 194
 195 #define NTXQ_1G 4
 196 static int t4_ntxq1g = -1;
 197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 198
 199 #define NRXQ_1G 2
 200 static int t4_nrxq1g = -1;
 201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 202
 203 static int t4_rsrv_noflowq = 0;
 204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 205
 206 #ifdef TCP_OFFLOAD
 207 #define NOFLDTXQ_10G 8
 208 static int t4_nofldtxq10g = -1;
 209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 210
 211 #define NOFLDRXQ_10G 2
 212 static int t4_nofldrxq10g = -1;
 213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 214
 215 #define NOFLDTXQ_1G 2
 216 static int t4_nofldtxq1g = -1;
 217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 218
 219 #define NOFLDRXQ_1G 1
 220 static int t4_nofldrxq1g = -1;
 221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 222 #endif
 223
 224 #ifdef DEV_NETMAP
 225 #define NNMTXQ_10G 2
 226 static int t4_nnmtxq10g = -1;
 227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 228
 229 #define NNMRXQ_10G 2
 230 static int t4_nnmrxq10g = -1;
 231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 232
 233 #define NNMTXQ_1G 1
 234 static int t4_nnmtxq1g = -1;
 235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 236
 237 #define NNMRXQ_1G 1
 238 static int t4_nnmrxq1g = -1;
 239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 240 #endif
 241
 242 /*
 243  * Holdoff parameters for 10G and 1G ports.
 244  */
 245 #define TMR_IDX_10G 1
 246 static int t4_tmr_idx_10g = TMR_IDX_10G;
 247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 248
 249 #define PKTC_IDX_10G (-1)
 250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 252
 253 #define TMR_IDX_1G 1
 254 static int t4_tmr_idx_1g = TMR_IDX_1G;
 255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 256
 257 #define PKTC_IDX_1G (-1)
 258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 260
 261 /*
 262  * Size (# of entries) of each tx and rx queue.
 263  */
 264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 266
 267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 269
 270 /*
 271  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 272  */
 273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 275
 276 /*
 277  * Configuration file.
 278  */
 279 #define DEFAULT_CF      "default"
 280 #define FLASH_CF        "flash"
 281 #define UWIRE_CF        "uwire"
 282 #define FPGA_CF         "fpga"
 283 static char t4_cfg_file[32] = DEFAULT_CF;
 284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 285
 286 /*
 287  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 288  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 289  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 290  *            mark or when signalled to do so, 0 to never emit PAUSE.
 291  */
 292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 294
 295 /*
 296  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 297  * encouraged respectively).
 298  */
 299 static unsigned int t4_fw_install = 1;
 300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 301
 302 /*
 303  * ASIC features that will be used.  Disable the ones you don't want so that the
 304  * chip resources aren't wasted on features that will not be used.
 305  */
 306 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 308
 309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 311
 312 static int t4_toecaps_allowed = -1;
 313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 314
 315 static int t4_rdmacaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 317
 318 static int t4_iscsicaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 320
 321 static int t4_fcoecaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 323
 324 static int t5_write_combine = 0;
 325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 326
 327 struct intrs_and_queues {
 328         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 329         uint16_t nirq;          /* Total # of vectors */
 330         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 331         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 332         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 333         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 334         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 335         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 336         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 337 #ifdef TCP_OFFLOAD
 338         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 339         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 340         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 341         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 342 #endif
 343 #ifdef DEV_NETMAP
 344         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 345         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 346         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 347         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 348 #endif
 349 };
 350
 351 struct filter_entry {
 352         uint32_t valid:1;       /* filter allocated and valid */
 353         uint32_t locked:1;      /* filter is administratively locked */
 354         uint32_t pending:1;     /* filter action is pending firmware reply */
 355         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 356         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 357
 358         struct t4_filter_specification fs;
 359 };
 360
 361 static int map_bars_0_and_4(struct adapter *);
 362 static int map_bar_2(struct adapter *);
 363 static void setup_memwin(struct adapter *);
 364 static int validate_mem_range(struct adapter *, uint32_t, int);
 365 static int fwmtype_to_hwmtype(int);
 366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 367     uint32_t *);
 368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 371     struct intrs_and_queues *);
 372 static int prep_firmware(struct adapter *);
 373 static int partition_resources(struct adapter *, const struct firmware *,
 374     const char *);
 375 static int get_params__pre_init(struct adapter *);
 376 static int get_params__post_init(struct adapter *);
 377 static int set_params__post_init(struct adapter *);
 378 static void t4_set_desc(struct adapter *);
 379 static void build_medialist(struct port_info *, struct ifmedia *);
 380 static int cxgbe_init_synchronized(struct port_info *);
 381 static int cxgbe_uninit_synchronized(struct port_info *);
 382 static int setup_intr_handlers(struct adapter *);
 383 static void quiesce_txq(struct adapter *, struct sge_txq *);
 384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 385 static void quiesce_iq(struct adapter *, struct sge_iq *);
 386 static void quiesce_fl(struct adapter *, struct sge_fl *);
 387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 388     driver_intr_t *, void *, char *);
 389 static int t4_free_irq(struct adapter *, struct irq *);
 390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 391     unsigned int);
 392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 394 static void cxgbe_tick(void *);
 395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 397     struct mbuf *);
 398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 400 static int t4_sysctls(struct adapter *);
 401 static int cxgbe_sysctls(struct port_info *);
 402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 413 #ifdef SBUF_DRAIN
 414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 439 #endif
 440 static uint32_t fconf_to_mode(uint32_t);
 441 static uint32_t mode_to_fconf(uint32_t);
 442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 443 static int get_filter_mode(struct adapter *, uint32_t *);
 444 static int set_filter_mode(struct adapter *, uint32_t);
 445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 446 static int get_filter(struct adapter *, struct t4_filter *);
 447 static int set_filter(struct adapter *, struct t4_filter *);
 448 static int del_filter(struct adapter *, struct t4_filter *);
 449 static void clear_filter(struct filter_entry *);
 450 static int set_filter_wr(struct adapter *, int);
 451 static int del_filter_wr(struct adapter *, int);
 452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 453 static int load_fw(struct adapter *, struct t4_data *);
 454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 458 #ifdef TCP_OFFLOAD
 459 static int toe_capability(struct port_info *, int);
 460 #endif
 461 static int mod_event(module_t, int, void *);
 462
 463 struct {
 464         uint16_t device;
 465         char *desc;
 466 } t4_pciids[] = {
 467         {0xa000, "Chelsio Terminator 4 FPGA"},
 468         {0x4400, "Chelsio T440-dbg"},
 469         {0x4401, "Chelsio T420-CR"},
 470         {0x4402, "Chelsio T422-CR"},
 471         {0x4403, "Chelsio T440-CR"},
 472         {0x4404, "Chelsio T420-BCH"},
 473         {0x4405, "Chelsio T440-BCH"},
 474         {0x4406, "Chelsio T440-CH"},
 475         {0x4407, "Chelsio T420-SO"},
 476         {0x4408, "Chelsio T420-CX"},
 477         {0x4409, "Chelsio T420-BT"},
 478         {0x440a, "Chelsio T404-BT"},
 479         {0x440e, "Chelsio T440-LP-CR"},
 480 }, t5_pciids[] = {
 481         {0xb000, "Chelsio Terminator 5 FPGA"},
 482         {0x5400, "Chelsio T580-dbg"},
 483         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 484         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 485         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 486         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 487         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 488         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 489         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 490         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 491         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 492         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 493         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 494         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 495         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 496 #ifdef notyet
 497         {0x5404,  "Chelsio T520-BCH"},
 498         {0x5405,  "Chelsio T540-BCH"},
 499         {0x5406,  "Chelsio T540-CH"},
 500         {0x5408,  "Chelsio T520-CX"},
 501         {0x540b,  "Chelsio B520-SR"},
 502         {0x540c,  "Chelsio B504-BT"},
 503         {0x540f,  "Chelsio Amsterdam"},
 504         {0x5413,  "Chelsio T580-CHR"},
 505 #endif
 506 };
 507
 508 #ifdef TCP_OFFLOAD
 509 /*
 510  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 511  * exactly the same for both rxq and ofld_rxq.
 512  */
 513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 515 #endif
 516
 517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 520
 521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 522
 523 static int
 524 t4_probe(device_t dev)
 525 {
 526         int i;
 527         uint16_t v = pci_get_vendor(dev);
 528         uint16_t d = pci_get_device(dev);
 529         uint8_t f = pci_get_function(dev);
 530
 531         if (v != PCI_VENDOR_ID_CHELSIO)
 532                 return (ENXIO);
 533
 534         /* Attach only to PF0 of the FPGA */
 535         if (d == 0xa000 && f != 0)
 536                 return (ENXIO);
 537
 538         for (i = 0; i < nitems(t4_pciids); i++) {
 539                 if (d == t4_pciids[i].device) {
 540                         device_set_desc(dev, t4_pciids[i].desc);
 541                         return (BUS_PROBE_DEFAULT);
 542                 }
 543         }
 544
 545         return (ENXIO);
 546 }
 547
 548 static int
 549 t5_probe(device_t dev)
 550 {
 551         int i;
 552         uint16_t v = pci_get_vendor(dev);
 553         uint16_t d = pci_get_device(dev);
 554         uint8_t f = pci_get_function(dev);
 555
 556         if (v != PCI_VENDOR_ID_CHELSIO)
 557                 return (ENXIO);
 558
 559         /* Attach only to PF0 of the FPGA */
 560         if (d == 0xb000 && f != 0)
 561                 return (ENXIO);
 562
 563         for (i = 0; i < nitems(t5_pciids); i++) {
 564                 if (d == t5_pciids[i].device) {
 565                         device_set_desc(dev, t5_pciids[i].desc);
 566                         return (BUS_PROBE_DEFAULT);
 567                 }
 568         }
 569
 570         return (ENXIO);
 571 }
 572
 573 static void
 574 t5_attribute_workaround(device_t dev)
 575 {
 576         device_t root_port;
 577         uint32_t v;
 578
 579         /*
 580          * The T5 chips do not properly echo the No Snoop and Relaxed
 581          * Ordering attributes when replying to a TLP from a Root
 582          * Port.  As a workaround, find the parent Root Port and
 583          * disable No Snoop and Relaxed Ordering.  Note that this
 584          * affects all devices under this root port.
 585          */
 586         root_port = pci_find_pcie_root_port(dev);
 587         if (root_port == NULL) {
 588                 device_printf(dev, "Unable to find parent root port\n");
 589                 return;
 590         }
 591
 592         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 593             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 594         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 595             0)
 596                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 597                     device_get_nameunit(root_port));
 598 }
 599
 600 static int
 601 t4_attach(device_t dev)
 602 {
 603         struct adapter *sc;
 604         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 605         struct intrs_and_queues iaq;
 606         struct sge *s;
 607 #ifdef TCP_OFFLOAD
 608         int ofld_rqidx, ofld_tqidx;
 609 #endif
 610 #ifdef DEV_NETMAP
 611         int nm_rqidx, nm_tqidx;
 612 #endif
 613
 614         sc = device_get_softc(dev);
 615         sc->dev = dev;
 616         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 617
 618         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 619                 t5_attribute_workaround(dev);
 620         pci_enable_busmaster(dev);
 621         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 622                 uint32_t v;
 623
 624                 pci_set_max_read_req(dev, 4096);
 625                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 626                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 627                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 628
 629                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 630         }
 631
 632         sc->traceq = -1;
 633         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 634         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 635             device_get_nameunit(dev));
 636
 637         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 638             device_get_nameunit(dev));
 639         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 640         sx_xlock(&t4_list_lock);
 641         SLIST_INSERT_HEAD(&t4_list, sc, link);
 642         sx_xunlock(&t4_list_lock);
 643
 644         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 645         TAILQ_INIT(&sc->sfl);
 646         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 647
 648         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 649
 650         rc = map_bars_0_and_4(sc);
 651         if (rc != 0)
 652                 goto done; /* error message displayed already */
 653
 654         /*
 655          * This is the real PF# to which we're attaching.  Works from within PCI
 656          * passthrough environments too, where pci_get_function() could return a
 657          * different PF# depending on the passthrough configuration.  We need to
 658          * use the real PF# in all our communication with the firmware.
 659          */
 660         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 661         sc->mbox = sc->pf;
 662
 663         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 664         sc->an_handler = an_not_handled;
 665         for (i = 0; i < nitems(sc->cpl_handler); i++)
 666                 sc->cpl_handler[i] = cpl_not_handled;
 667         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 668                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 669         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 670         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 671         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 672         t4_init_sge_cpl_handlers(sc);
 673
 674         /* Prepare the adapter for operation */
 675         rc = -t4_prep_adapter(sc);
 676         if (rc != 0) {
 677                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 678                 goto done;
 679         }
 680
 681         /*
 682          * Do this really early, with the memory windows set up even before the
 683          * character device.  The userland tool's register i/o and mem read
 684          * will work even in "recovery mode".
 685          */
 686         setup_memwin(sc);
 687         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 688             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 689             device_get_nameunit(dev));
 690         if (sc->cdev == NULL)
 691                 device_printf(dev, "failed to create nexus char device.\n");
 692         else
 693                 sc->cdev->si_drv1 = sc;
 694
 695         /* Go no further if recovery mode has been requested. */
 696         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 697                 device_printf(dev, "recovery mode.\n");
 698                 goto done;
 699         }
 700
 701 #if defined(__i386__)
 702         if ((cpu_feature & CPUID_CX8) == 0) {
 703                 device_printf(dev, "64 bit atomics not available.\n");
 704                 rc = ENOTSUP;
 705                 goto done;
 706         }
 707 #endif
 708
 709         /* Prepare the firmware for operation */
 710         rc = prep_firmware(sc);
 711         if (rc != 0)
 712                 goto done; /* error message displayed already */
 713
 714         rc = get_params__post_init(sc);
 715         if (rc != 0)
 716                 goto done; /* error message displayed already */
 717
 718         rc = set_params__post_init(sc);
 719         if (rc != 0)
 720                 goto done; /* error message displayed already */
 721
 722         rc = map_bar_2(sc);
 723         if (rc != 0)
 724                 goto done; /* error message displayed already */
 725
 726         rc = t4_create_dma_tag(sc);
 727         if (rc != 0)
 728                 goto done; /* error message displayed already */
 729
 730         /*
 731          * First pass over all the ports - allocate VIs and initialize some
 732          * basic parameters like mac address, port type, etc.  We also figure
 733          * out whether a port is 10G or 1G and use that information when
 734          * calculating how many interrupts to attempt to allocate.
 735          */
 736         n10g = n1g = 0;
 737         for_each_port(sc, i) {
 738                 struct port_info *pi;
 739
 740                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 741                 sc->port[i] = pi;
 742
 743                 /* These must be set before t4_port_init */
 744                 pi->adapter = sc;
 745                 pi->port_id = i;
 746
 747                 /* Allocate the vi and initialize parameters like mac addr */
 748                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 749                 if (rc != 0) {
 750                         device_printf(dev, "unable to initialize port %d: %d\n",
 751                             i, rc);
 752                         free(pi, M_CXGBE);
 753                         sc->port[i] = NULL;
 754                         goto done;
 755                 }
 756
 757                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 758                 pi->link_cfg.requested_fc |= t4_pause_settings;
 759                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 760                 pi->link_cfg.fc |= t4_pause_settings;
 761
 762                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 763                 if (rc != 0) {
 764                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 765                         free(pi, M_CXGBE);
 766                         sc->port[i] = NULL;
 767                         goto done;
 768                 }
 769
 770                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 771                     device_get_nameunit(dev), i);
 772                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 773                 sc->chan_map[pi->tx_chan] = i;
 774
 775                 if (is_10G_port(pi) || is_40G_port(pi)) {
 776                         n10g++;
 777                         pi->tmr_idx = t4_tmr_idx_10g;
 778                         pi->pktc_idx = t4_pktc_idx_10g;
 779                 } else {
 780                         n1g++;
 781                         pi->tmr_idx = t4_tmr_idx_1g;
 782                         pi->pktc_idx = t4_pktc_idx_1g;
 783                 }
 784
 785                 pi->xact_addr_filt = -1;
 786                 pi->linkdnrc = -1;
 787
 788                 pi->qsize_rxq = t4_qsize_rxq;
 789                 pi->qsize_txq = t4_qsize_txq;
 790
 791                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 792                 if (pi->dev == NULL) {
 793                         device_printf(dev,
 794                             "failed to add device for port %d.\n", i);
 795                         rc = ENXIO;
 796                         goto done;
 797                 }
 798                 device_set_softc(pi->dev, pi);
 799         }
 800
 801         /*
 802          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 803          */
 804         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 805         if (rc != 0)
 806                 goto done; /* error message displayed already */
 807
 808         sc->intr_type = iaq.intr_type;
 809         sc->intr_count = iaq.nirq;
 810
 811         s = &sc->sge;
 812         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 813         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 814         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 815         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 816         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 817 #ifdef TCP_OFFLOAD
 818         if (is_offload(sc)) {
 819                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 820                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 821                 s->neq += s->nofldtxq + s->nofldrxq;
 822                 s->niq += s->nofldrxq;
 823
 824                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 825                     M_CXGBE, M_ZERO | M_WAITOK);
 826                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 827                     M_CXGBE, M_ZERO | M_WAITOK);
 828         }
 829 #endif
 830 #ifdef DEV_NETMAP
 831         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 832         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 833         s->neq += s->nnmtxq + s->nnmrxq;
 834         s->niq += s->nnmrxq;
 835
 836         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 837             M_CXGBE, M_ZERO | M_WAITOK);
 838         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 839             M_CXGBE, M_ZERO | M_WAITOK);
 840 #endif
 841
 842         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 843             M_ZERO | M_WAITOK);
 844         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 845             M_ZERO | M_WAITOK);
 846         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 847             M_ZERO | M_WAITOK);
 848         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 849             M_ZERO | M_WAITOK);
 850         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 851             M_ZERO | M_WAITOK);
 852
 853         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 854             M_ZERO | M_WAITOK);
 855
 856         t4_init_l2t(sc, M_WAITOK);
 857
 858         /*
 859          * Second pass over the ports.  This time we know the number of rx and
 860          * tx queues that each port should get.
 861          */
 862         rqidx = tqidx = 0;
 863 #ifdef TCP_OFFLOAD
 864         ofld_rqidx = ofld_tqidx = 0;
 865 #endif
 866 #ifdef DEV_NETMAP
 867         nm_rqidx = nm_tqidx = 0;
 868 #endif
 869         for_each_port(sc, i) {
 870                 struct port_info *pi = sc->port[i];
 871
 872                 if (pi == NULL)
 873                         continue;
 874
 875                 pi->first_rxq = rqidx;
 876                 pi->first_txq = tqidx;
 877                 if (is_10G_port(pi) || is_40G_port(pi)) {
 878                         pi->flags |= iaq.intr_flags_10g;
 879                         pi->nrxq = iaq.nrxq10g;
 880                         pi->ntxq = iaq.ntxq10g;
 881                 } else {
 882                         pi->flags |= iaq.intr_flags_1g;
 883                         pi->nrxq = iaq.nrxq1g;
 884                         pi->ntxq = iaq.ntxq1g;
 885                 }
 886
 887                 if (pi->ntxq > 1)
 888                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 889                 else
 890                         pi->rsrv_noflowq = 0;
 891
 892                 rqidx += pi->nrxq;
 893                 tqidx += pi->ntxq;
 894 #ifdef TCP_OFFLOAD
 895                 if (is_offload(sc)) {
 896                         pi->first_ofld_rxq = ofld_rqidx;
 897                         pi->first_ofld_txq = ofld_tqidx;
 898                         if (is_10G_port(pi) || is_40G_port(pi)) {
 899                                 pi->nofldrxq = iaq.nofldrxq10g;
 900                                 pi->nofldtxq = iaq.nofldtxq10g;
 901                         } else {
 902                                 pi->nofldrxq = iaq.nofldrxq1g;
 903                                 pi->nofldtxq = iaq.nofldtxq1g;
 904                         }
 905                         ofld_rqidx += pi->nofldrxq;
 906                         ofld_tqidx += pi->nofldtxq;
 907                 }
 908 #endif
 909 #ifdef DEV_NETMAP
 910                 pi->first_nm_rxq = nm_rqidx;
 911                 pi->first_nm_txq = nm_tqidx;
 912                 if (is_10G_port(pi) || is_40G_port(pi)) {
 913                         pi->nnmrxq = iaq.nnmrxq10g;
 914                         pi->nnmtxq = iaq.nnmtxq10g;
 915                 } else {
 916                         pi->nnmrxq = iaq.nnmrxq1g;
 917                         pi->nnmtxq = iaq.nnmtxq1g;
 918                 }
 919                 nm_rqidx += pi->nnmrxq;
 920                 nm_tqidx += pi->nnmtxq;
 921 #endif
 922         }
 923
 924         rc = setup_intr_handlers(sc);
 925         if (rc != 0) {
 926                 device_printf(dev,
 927                     "failed to setup interrupt handlers: %d\n", rc);
 928                 goto done;
 929         }
 930
 931         rc = bus_generic_attach(dev);
 932         if (rc != 0) {
 933                 device_printf(dev,
 934                     "failed to attach all child ports: %d\n", rc);
 935                 goto done;
 936         }
 937
 938         device_printf(dev,
 939             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 940             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 941             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 942             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 943             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 944
 945         t4_set_desc(sc);
 946
 947 done:
 948         if (rc != 0 && sc->cdev) {
 949                 /* cdev was created and so cxgbetool works; recover that way. */
 950                 device_printf(dev,
 951                     "error during attach, adapter is now in recovery mode.\n");
 952                 rc = 0;
 953         }
 954
 955         if (rc != 0)
 956                 t4_detach(dev);
 957         else
 958                 t4_sysctls(sc);
 959
 960         return (rc);
 961 }
 962
 963 /*
 964  * Idempotent
 965  */
 966 static int
 967 t4_detach(device_t dev)
 968 {
 969         struct adapter *sc;
 970         struct port_info *pi;
 971         int i, rc;
 972
 973         sc = device_get_softc(dev);
 974
 975         if (sc->flags & FULL_INIT_DONE)
 976                 t4_intr_disable(sc);
 977
 978         if (sc->cdev) {
 979                 destroy_dev(sc->cdev);
 980                 sc->cdev = NULL;
 981         }
 982
 983         rc = bus_generic_detach(dev);
 984         if (rc) {
 985                 device_printf(dev,
 986                     "failed to detach child devices: %d\n", rc);
 987                 return (rc);
 988         }
 989
 990         for (i = 0; i < sc->intr_count; i++)
 991                 t4_free_irq(sc, &sc->irq[i]);
 992
 993         for (i = 0; i < MAX_NPORTS; i++) {
 994                 pi = sc->port[i];
 995                 if (pi) {
 996                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 997                         if (pi->dev)
 998                                 device_delete_child(dev, pi->dev);
 999
1000                         mtx_destroy(&pi->pi_lock);
1001                         free(pi, M_CXGBE);
1002                 }
1003         }
1004
1005         if (sc->flags & FULL_INIT_DONE)
1006                 adapter_full_uninit(sc);
1007
1008         if (sc->flags & FW_OK)
1009                 t4_fw_bye(sc, sc->mbox);
1010
1011         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1012                 pci_release_msi(dev);
1013
1014         if (sc->regs_res)
1015                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1016                     sc->regs_res);
1017
1018         if (sc->udbs_res)
1019                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1020                     sc->udbs_res);
1021
1022         if (sc->msix_res)
1023                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1024                     sc->msix_res);
1025
1026         if (sc->l2t)
1027                 t4_free_l2t(sc->l2t);
1028
1029 #ifdef TCP_OFFLOAD
1030         free(sc->sge.ofld_rxq, M_CXGBE);
1031         free(sc->sge.ofld_txq, M_CXGBE);
1032 #endif
1033 #ifdef DEV_NETMAP
1034         free(sc->sge.nm_rxq, M_CXGBE);
1035         free(sc->sge.nm_txq, M_CXGBE);
1036 #endif
1037         free(sc->irq, M_CXGBE);
1038         free(sc->sge.rxq, M_CXGBE);
1039         free(sc->sge.txq, M_CXGBE);
1040         free(sc->sge.ctrlq, M_CXGBE);
1041         free(sc->sge.iqmap, M_CXGBE);
1042         free(sc->sge.eqmap, M_CXGBE);
1043         free(sc->tids.ftid_tab, M_CXGBE);
1044         t4_destroy_dma_tag(sc);
1045         if (mtx_initialized(&sc->sc_lock)) {
1046                 sx_xlock(&t4_list_lock);
1047                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1048                 sx_xunlock(&t4_list_lock);
1049                 mtx_destroy(&sc->sc_lock);
1050         }
1051
1052         if (mtx_initialized(&sc->tids.ftid_lock))
1053                 mtx_destroy(&sc->tids.ftid_lock);
1054         if (mtx_initialized(&sc->sfl_lock))
1055                 mtx_destroy(&sc->sfl_lock);
1056         if (mtx_initialized(&sc->ifp_lock))
1057                 mtx_destroy(&sc->ifp_lock);
1058         if (mtx_initialized(&sc->regwin_lock))
1059                 mtx_destroy(&sc->regwin_lock);
1060
1061         bzero(sc, sizeof(*sc));
1062
1063         return (0);
1064 }
1065
1066 static int
1067 cxgbe_probe(device_t dev)
1068 {
1069         char buf[128];
1070         struct port_info *pi = device_get_softc(dev);
1071
1072         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1073         device_set_desc_copy(dev, buf);
1074
1075         return (BUS_PROBE_DEFAULT);
1076 }
1077
1078 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1079     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1080     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1081 #define T4_CAP_ENABLE (T4_CAP)
1082
1083 static int
1084 cxgbe_attach(device_t dev)
1085 {
1086         struct port_info *pi = device_get_softc(dev);
1087         struct ifnet *ifp;
1088         char *s;
1089         int n, o;
1090
1091         /* Allocate an ifnet and set it up */
1092         ifp = if_alloc(IFT_ETHER);
1093         if (ifp == NULL) {
1094                 device_printf(dev, "Cannot allocate ifnet\n");
1095                 return (ENOMEM);
1096         }
1097         pi->ifp = ifp;
1098         ifp->if_softc = pi;
1099
1100         callout_init(&pi->tick, CALLOUT_MPSAFE);
1101
1102         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1103         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1104
1105         ifp->if_init = cxgbe_init;
1106         ifp->if_ioctl = cxgbe_ioctl;
1107         ifp->if_transmit = cxgbe_transmit;
1108         ifp->if_qflush = cxgbe_qflush;
1109
1110         ifp->if_capabilities = T4_CAP;
1111 #ifdef TCP_OFFLOAD
1112         if (is_offload(pi->adapter))
1113                 ifp->if_capabilities |= IFCAP_TOE;
1114 #endif
1115         ifp->if_capenable = T4_CAP_ENABLE;
1116         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1117             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1118
1119         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1120         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1121         ifp->if_hw_tsomaxsegsize = 65536;
1122
1123         /* Initialize ifmedia for this port */
1124         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1125             cxgbe_media_status);
1126         build_medialist(pi, &pi->media);
1127
1128         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1129             EVENTHANDLER_PRI_ANY);
1130
1131         ether_ifattach(ifp, pi->hw_addr);
1132
1133         n = 128;
1134         s = malloc(n, M_CXGBE, M_WAITOK);
1135         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1136         MPASS(n > o);
1137 #ifdef TCP_OFFLOAD
1138         if (is_offload(pi->adapter)) {
1139                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1140                     pi->nofldtxq, pi->nofldrxq);
1141                 MPASS(n > o);
1142         }
1143 #endif
1144 #ifdef DEV_NETMAP
1145         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1146             pi->nnmrxq);
1147         MPASS(n > o);
1148 #endif
1149         device_printf(dev, "%s\n", s);
1150         free(s, M_CXGBE);
1151
1152 #ifdef DEV_NETMAP
1153         /* nm_media handled here to keep implementation private to this file */
1154         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1155             cxgbe_media_status);
1156         build_medialist(pi, &pi->nm_media);
1157         create_netmap_ifnet(pi);        /* logs errors it something fails */
1158 #endif
1159         cxgbe_sysctls(pi);
1160
1161         return (0);
1162 }
1163
1164 static int
1165 cxgbe_detach(device_t dev)
1166 {
1167         struct port_info *pi = device_get_softc(dev);
1168         struct adapter *sc = pi->adapter;
1169         struct ifnet *ifp = pi->ifp;
1170
1171         /* Tell if_ioctl and if_init that the port is going away */
1172         ADAPTER_LOCK(sc);
1173         SET_DOOMED(pi);
1174         wakeup(&sc->flags);
1175         while (IS_BUSY(sc))
1176                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1177         SET_BUSY(sc);
1178 #ifdef INVARIANTS
1179         sc->last_op = "t4detach";
1180         sc->last_op_thr = curthread;
1181         sc->last_op_flags = 0;
1182 #endif
1183         ADAPTER_UNLOCK(sc);
1184
1185         if (pi->flags & HAS_TRACEQ) {
1186                 sc->traceq = -1;        /* cloner should not create ifnet */
1187                 t4_tracer_port_detach(sc);
1188         }
1189
1190         if (pi->vlan_c)
1191                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1192
1193         PORT_LOCK(pi);
1194         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1195         callout_stop(&pi->tick);
1196         PORT_UNLOCK(pi);
1197         callout_drain(&pi->tick);
1198
1199         /* Let detach proceed even if these fail. */
1200         cxgbe_uninit_synchronized(pi);
1201         port_full_uninit(pi);
1202
1203         ifmedia_removeall(&pi->media);
1204         ether_ifdetach(pi->ifp);
1205         if_free(pi->ifp);
1206
1207 #ifdef DEV_NETMAP
1208         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1209         destroy_netmap_ifnet(pi);
1210 #endif
1211
1212         ADAPTER_LOCK(sc);
1213         CLR_BUSY(sc);
1214         wakeup(&sc->flags);
1215         ADAPTER_UNLOCK(sc);
1216
1217         return (0);
1218 }
1219
1220 static void
1221 cxgbe_init(void *arg)
1222 {
1223         struct port_info *pi = arg;
1224         struct adapter *sc = pi->adapter;
1225
1226         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1227                 return;
1228         cxgbe_init_synchronized(pi);
1229         end_synchronized_op(sc, 0);
1230 }
1231
1232 static int
1233 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1234 {
1235         int rc = 0, mtu, flags, can_sleep;
1236         struct port_info *pi = ifp->if_softc;
1237         struct adapter *sc = pi->adapter;
1238         struct ifreq *ifr = (struct ifreq *)data;
1239         uint32_t mask;
1240
1241         switch (cmd) {
1242         case SIOCSIFMTU:
1243                 mtu = ifr->ifr_mtu;
1244                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1245                         return (EINVAL);
1246
1247                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1248                 if (rc)
1249                         return (rc);
1250                 ifp->if_mtu = mtu;
1251                 if (pi->flags & PORT_INIT_DONE) {
1252                         t4_update_fl_bufsize(ifp);
1253                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1254                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1255                 }
1256                 end_synchronized_op(sc, 0);
1257                 break;
1258
1259         case SIOCSIFFLAGS:
1260                 can_sleep = 0;
1261 redo_sifflags:
1262                 rc = begin_synchronized_op(sc, pi,
1263                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1264                 if (rc)
1265                         return (rc);
1266
1267                 if (ifp->if_flags & IFF_UP) {
1268                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1269                                 flags = pi->if_flags;
1270                                 if ((ifp->if_flags ^ flags) &
1271                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1272                                         if (can_sleep == 1) {
1273                                                 end_synchronized_op(sc, 0);
1274                                                 can_sleep = 0;
1275                                                 goto redo_sifflags;
1276                                         }
1277                                         rc = update_mac_settings(ifp,
1278                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1279                                 }
1280                         } else {
1281                                 if (can_sleep == 0) {
1282                                         end_synchronized_op(sc, LOCK_HELD);
1283                                         can_sleep = 1;
1284                                         goto redo_sifflags;
1285                                 }
1286                                 rc = cxgbe_init_synchronized(pi);
1287                         }
1288                         pi->if_flags = ifp->if_flags;
1289                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1290                         if (can_sleep == 0) {
1291                                 end_synchronized_op(sc, LOCK_HELD);
1292                                 can_sleep = 1;
1293                                 goto redo_sifflags;
1294                         }
1295                         rc = cxgbe_uninit_synchronized(pi);
1296                 }
1297                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1298                 break;
1299
1300         case SIOCADDMULTI:
1301         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1302                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1303                 if (rc)
1304                         return (rc);
1305                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1306                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1307                 end_synchronized_op(sc, LOCK_HELD);
1308                 break;
1309
1310         case SIOCSIFCAP:
1311                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1312                 if (rc)
1313                         return (rc);
1314
1315                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1316                 if (mask & IFCAP_TXCSUM) {
1317                         ifp->if_capenable ^= IFCAP_TXCSUM;
1318                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1319
1320                         if (IFCAP_TSO4 & ifp->if_capenable &&
1321                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1322                                 ifp->if_capenable &= ~IFCAP_TSO4;
1323                                 if_printf(ifp,
1324                                     "tso4 disabled due to -txcsum.\n");
1325                         }
1326                 }
1327                 if (mask & IFCAP_TXCSUM_IPV6) {
1328                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1329                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1330
1331                         if (IFCAP_TSO6 & ifp->if_capenable &&
1332                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1333                                 ifp->if_capenable &= ~IFCAP_TSO6;
1334                                 if_printf(ifp,
1335                                     "tso6 disabled due to -txcsum6.\n");
1336                         }
1337                 }
1338                 if (mask & IFCAP_RXCSUM)
1339                         ifp->if_capenable ^= IFCAP_RXCSUM;
1340                 if (mask & IFCAP_RXCSUM_IPV6)
1341                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1342
1343                 /*
1344                  * Note that we leave CSUM_TSO alone (it is always set).  The
1345                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1346                  * sending a TSO request our way, so it's sufficient to toggle
1347                  * IFCAP_TSOx only.
1348                  */
1349                 if (mask & IFCAP_TSO4) {
1350                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1351                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1352                                 if_printf(ifp, "enable txcsum first.\n");
1353                                 rc = EAGAIN;
1354                                 goto fail;
1355                         }
1356                         ifp->if_capenable ^= IFCAP_TSO4;
1357                 }
1358                 if (mask & IFCAP_TSO6) {
1359                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1360                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1361                                 if_printf(ifp, "enable txcsum6 first.\n");
1362                                 rc = EAGAIN;
1363                                 goto fail;
1364                         }
1365                         ifp->if_capenable ^= IFCAP_TSO6;
1366                 }
1367                 if (mask & IFCAP_LRO) {
1368 #if defined(INET) || defined(INET6)
1369                         int i;
1370                         struct sge_rxq *rxq;
1371
1372                         ifp->if_capenable ^= IFCAP_LRO;
1373                         for_each_rxq(pi, i, rxq) {
1374                                 if (ifp->if_capenable & IFCAP_LRO)
1375                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1376                                 else
1377                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1378                         }
1379 #endif
1380                 }
1381 #ifdef TCP_OFFLOAD
1382                 if (mask & IFCAP_TOE) {
1383                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1384
1385                         rc = toe_capability(pi, enable);
1386                         if (rc != 0)
1387                                 goto fail;
1388
1389                         ifp->if_capenable ^= mask;
1390                 }
1391 #endif
1392                 if (mask & IFCAP_VLAN_HWTAGGING) {
1393                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1394                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1395                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1396                 }
1397                 if (mask & IFCAP_VLAN_MTU) {
1398                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1399
1400                         /* Need to find out how to disable auto-mtu-inflation */
1401                 }
1402                 if (mask & IFCAP_VLAN_HWTSO)
1403                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1404                 if (mask & IFCAP_VLAN_HWCSUM)
1405                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1406
1407 #ifdef VLAN_CAPABILITIES
1408                 VLAN_CAPABILITIES(ifp);
1409 #endif
1410 fail:
1411                 end_synchronized_op(sc, 0);
1412                 break;
1413
1414         case SIOCSIFMEDIA:
1415         case SIOCGIFMEDIA:
1416                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1417                 break;
1418
1419         case SIOCGI2C: {
1420                 struct ifi2creq i2c;
1421
1422                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1423                 if (rc != 0)
1424                         break;
1425                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1426                         rc = EPERM;
1427                         break;
1428                 }
1429                 if (i2c.len > sizeof(i2c.data)) {
1430                         rc = EINVAL;
1431                         break;
1432                 }
1433                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1434                 if (rc)
1435                         return (rc);
1436                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1437                     i2c.offset, i2c.len, &i2c.data[0]);
1438                 end_synchronized_op(sc, 0);
1439                 if (rc == 0)
1440                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1441                 break;
1442         }
1443
1444         default:
1445                 rc = ether_ioctl(ifp, cmd, data);
1446         }
1447
1448         return (rc);
1449 }
1450
1451 static int
1452 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1453 {
1454         struct port_info *pi = ifp->if_softc;
1455         struct adapter *sc = pi->adapter;
1456         struct sge_txq *txq;
1457         void *items[1];
1458         int rc;
1459
1460         M_ASSERTPKTHDR(m);
1461         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1462
1463         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1464                 m_freem(m);
1465                 return (ENETDOWN);
1466         }
1467
1468         rc = parse_pkt(&m);
1469         if (__predict_false(rc != 0)) {
1470                 MPASS(m == NULL);                       /* was freed already */
1471                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1472                 return (rc);
1473         }
1474
1475         /* Select a txq. */
1476         txq = &sc->sge.txq[pi->first_txq];
1477         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1478                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1479                     pi->rsrv_noflowq);
1480
1481         items[0] = m;
1482         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1483         if (__predict_false(rc != 0))
1484                 m_freem(m);
1485
1486         return (rc);
1487 }
1488
1489 static void
1490 cxgbe_qflush(struct ifnet *ifp)
1491 {
1492         struct port_info *pi = ifp->if_softc;
1493         struct sge_txq *txq;
1494         int i;
1495
1496         /* queues do not exist if !PORT_INIT_DONE. */
1497         if (pi->flags & PORT_INIT_DONE) {
1498                 for_each_txq(pi, i, txq) {
1499                         TXQ_LOCK(txq);
1500                         txq->eq.flags &= ~EQ_ENABLED;
1501                         TXQ_UNLOCK(txq);
1502                         while (!mp_ring_is_idle(txq->r)) {
1503                                 mp_ring_check_drainage(txq->r, 0);
1504                                 pause("qflush", 1);
1505                         }
1506                 }
1507         }
1508         if_qflush(ifp);
1509 }
1510
1511 static int
1512 cxgbe_media_change(struct ifnet *ifp)
1513 {
1514         struct port_info *pi = ifp->if_softc;
1515
1516         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1517
1518         return (EOPNOTSUPP);
1519 }
1520
1521 static void
1522 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1523 {
1524         struct port_info *pi = ifp->if_softc;
1525         struct ifmedia *media = NULL;
1526         struct ifmedia_entry *cur;
1527         int speed = pi->link_cfg.speed;
1528 #ifdef INVARIANTS
1529         int data = (pi->port_type << 8) | pi->mod_type;
1530 #endif
1531
1532         if (ifp == pi->ifp)
1533                 media = &pi->media;
1534 #ifdef DEV_NETMAP
1535         else if (ifp == pi->nm_ifp)
1536                 media = &pi->nm_media;
1537 #endif
1538         MPASS(media != NULL);
1539
1540         cur = media->ifm_cur;
1541         MPASS(cur->ifm_data == data);
1542
1543         ifmr->ifm_status = IFM_AVALID;
1544         if (!pi->link_cfg.link_ok)
1545                 return;
1546
1547         ifmr->ifm_status |= IFM_ACTIVE;
1548
1549         /* active and current will differ iff current media is autoselect. */
1550         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1551                 return;
1552
1553         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1554         if (speed == SPEED_10000)
1555                 ifmr->ifm_active |= IFM_10G_T;
1556         else if (speed == SPEED_1000)
1557                 ifmr->ifm_active |= IFM_1000_T;
1558         else if (speed == SPEED_100)
1559                 ifmr->ifm_active |= IFM_100_TX;
1560         else if (speed == SPEED_10)
1561                 ifmr->ifm_active |= IFM_10_T;
1562         else
1563                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1564                             speed));
1565 }
1566
1567 void
1568 t4_fatal_err(struct adapter *sc)
1569 {
1570         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1571         t4_intr_disable(sc);
1572         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1573             device_get_nameunit(sc->dev));
1574 }
1575
1576 static int
1577 map_bars_0_and_4(struct adapter *sc)
1578 {
1579         sc->regs_rid = PCIR_BAR(0);
1580         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1581             &sc->regs_rid, RF_ACTIVE);
1582         if (sc->regs_res == NULL) {
1583                 device_printf(sc->dev, "cannot map registers.\n");
1584                 return (ENXIO);
1585         }
1586         sc->bt = rman_get_bustag(sc->regs_res);
1587         sc->bh = rman_get_bushandle(sc->regs_res);
1588         sc->mmio_len = rman_get_size(sc->regs_res);
1589         setbit(&sc->doorbells, DOORBELL_KDB);
1590
1591         sc->msix_rid = PCIR_BAR(4);
1592         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1593             &sc->msix_rid, RF_ACTIVE);
1594         if (sc->msix_res == NULL) {
1595                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1596                 return (ENXIO);
1597         }
1598
1599         return (0);
1600 }
1601
1602 static int
1603 map_bar_2(struct adapter *sc)
1604 {
1605
1606         /*
1607          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1608          * to map it if RDMA is disabled.
1609          */
1610         if (is_t4(sc) && sc->rdmacaps == 0)
1611                 return (0);
1612
1613         sc->udbs_rid = PCIR_BAR(2);
1614         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1615             &sc->udbs_rid, RF_ACTIVE);
1616         if (sc->udbs_res == NULL) {
1617                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1618                 return (ENXIO);
1619         }
1620         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1621
1622         if (is_t5(sc)) {
1623                 setbit(&sc->doorbells, DOORBELL_UDB);
1624 #if defined(__i386__) || defined(__amd64__)
1625                 if (t5_write_combine) {
1626                         int rc;
1627
1628                         /*
1629                          * Enable write combining on BAR2.  This is the
1630                          * userspace doorbell BAR and is split into 128B
1631                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1632                          * with an egress queue.  The first 64B has the doorbell
1633                          * and the second 64B can be used to submit a tx work
1634                          * request with an implicit doorbell.
1635                          */
1636
1637                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1638                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1639                         if (rc == 0) {
1640                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1641                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1642                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1643                         } else {
1644                                 device_printf(sc->dev,
1645                                     "couldn't enable write combining: %d\n",
1646                                     rc);
1647                         }
1648
1649                         t4_write_reg(sc, A_SGE_STAT_CFG,
1650                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1651                 }
1652 #endif
1653         }
1654
1655         return (0);
1656 }
1657
1658 static const struct memwin t4_memwin[] = {
1659         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1660         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1661         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1662 };
1663
1664 static const struct memwin t5_memwin[] = {
1665         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1666         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1667         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1668 };
1669
1670 static void
1671 setup_memwin(struct adapter *sc)
1672 {
1673         const struct memwin *mw;
1674         int i, n;
1675         uint32_t bar0;
1676
1677         if (is_t4(sc)) {
1678                 /*
1679                  * Read low 32b of bar0 indirectly via the hardware backdoor
1680                  * mechanism.  Works from within PCI passthrough environments
1681                  * too, where rman_get_start() can return a different value.  We
1682                  * need to program the T4 memory window decoders with the actual
1683                  * addresses that will be coming across the PCIe link.
1684                  */
1685                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1686                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1687
1688                 mw = &t4_memwin[0];
1689                 n = nitems(t4_memwin);
1690         } else {
1691                 /* T5 uses the relative offset inside the PCIe BAR */
1692                 bar0 = 0;
1693
1694                 mw = &t5_memwin[0];
1695                 n = nitems(t5_memwin);
1696         }
1697
1698         for (i = 0; i < n; i++, mw++) {
1699                 t4_write_reg(sc,
1700                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1701                     (mw->base + bar0) | V_BIR(0) |
1702                     V_WINDOW(ilog2(mw->aperture) - 10));
1703         }
1704
1705         /* flush */
1706         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1707 }
1708
1709 /*
1710  * Verify that the memory range specified by the addr/len pair is valid and lies
1711  * entirely within a single region (EDCx or MCx).
1712  */
1713 static int
1714 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1715 {
1716         uint32_t em, addr_len, maddr, mlen;
1717
1718         /* Memory can only be accessed in naturally aligned 4 byte units */
1719         if (addr & 3 || len & 3 || len == 0)
1720                 return (EINVAL);
1721
1722         /* Enabled memories */
1723         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1724         if (em & F_EDRAM0_ENABLE) {
1725                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1726                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1727                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1728                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1729                     addr + len <= maddr + mlen)
1730                         return (0);
1731         }
1732         if (em & F_EDRAM1_ENABLE) {
1733                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1734                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1735                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1736                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1737                     addr + len <= maddr + mlen)
1738                         return (0);
1739         }
1740         if (em & F_EXT_MEM_ENABLE) {
1741                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1742                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1743                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1744                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1745                     addr + len <= maddr + mlen)
1746                         return (0);
1747         }
1748         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1749                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1750                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1751                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1752                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1753                     addr + len <= maddr + mlen)
1754                         return (0);
1755         }
1756
1757         return (EFAULT);
1758 }
1759
1760 static int
1761 fwmtype_to_hwmtype(int mtype)
1762 {
1763
1764         switch (mtype) {
1765         case FW_MEMTYPE_EDC0:
1766                 return (MEM_EDC0);
1767         case FW_MEMTYPE_EDC1:
1768                 return (MEM_EDC1);
1769         case FW_MEMTYPE_EXTMEM:
1770                 return (MEM_MC0);
1771         case FW_MEMTYPE_EXTMEM1:
1772                 return (MEM_MC1);
1773         default:
1774                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1775         }
1776 }
1777
1778 /*
1779  * Verify that the memory range specified by the memtype/offset/len pair is
1780  * valid and lies entirely within the memtype specified.  The global address of
1781  * the start of the range is returned in addr.
1782  */
1783 static int
1784 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1785     uint32_t *addr)
1786 {
1787         uint32_t em, addr_len, maddr, mlen;
1788
1789         /* Memory can only be accessed in naturally aligned 4 byte units */
1790         if (off & 3 || len & 3 || len == 0)
1791                 return (EINVAL);
1792
1793         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1794         switch (fwmtype_to_hwmtype(mtype)) {
1795         case MEM_EDC0:
1796                 if (!(em & F_EDRAM0_ENABLE))
1797                         return (EINVAL);
1798                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1799                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1800                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1801                 break;
1802         case MEM_EDC1:
1803                 if (!(em & F_EDRAM1_ENABLE))
1804                         return (EINVAL);
1805                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1806                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1807                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1808                 break;
1809         case MEM_MC:
1810                 if (!(em & F_EXT_MEM_ENABLE))
1811                         return (EINVAL);
1812                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1813                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1814                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1815                 break;
1816         case MEM_MC1:
1817                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1818                         return (EINVAL);
1819                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1820                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1821                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1822                 break;
1823         default:
1824                 return (EINVAL);
1825         }
1826
1827         if (mlen > 0 && off < mlen && off + len <= mlen) {
1828                 *addr = maddr + off;    /* global address */
1829                 return (0);
1830         }
1831
1832         return (EFAULT);
1833 }
1834
1835 static void
1836 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1837 {
1838         const struct memwin *mw;
1839
1840         if (is_t4(sc)) {
1841                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1842                     ("%s: incorrect memwin# (%d)", __func__, win));
1843                 mw = &t4_memwin[win];
1844         } else {
1845                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1846                     ("%s: incorrect memwin# (%d)", __func__, win));
1847                 mw = &t5_memwin[win];
1848         }
1849
1850         if (base != NULL)
1851                 *base = mw->base;
1852         if (aperture != NULL)
1853                 *aperture = mw->aperture;
1854 }
1855
1856 /*
1857  * Positions the memory window such that it can be used to access the specified
1858  * address in the chip's address space.  The return value is the offset of addr
1859  * from the start of the window.
1860  */
1861 static uint32_t
1862 position_memwin(struct adapter *sc, int n, uint32_t addr)
1863 {
1864         uint32_t start, pf;
1865         uint32_t reg;
1866
1867         KASSERT(n >= 0 && n <= 3,
1868             ("%s: invalid window %d.", __func__, n));
1869         KASSERT((addr & 3) == 0,
1870             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1871
1872         if (is_t4(sc)) {
1873                 pf = 0;
1874                 start = addr & ~0xf;    /* start must be 16B aligned */
1875         } else {
1876                 pf = V_PFNUM(sc->pf);
1877                 start = addr & ~0x7f;   /* start must be 128B aligned */
1878         }
1879         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1880
1881         t4_write_reg(sc, reg, start | pf);
1882         t4_read_reg(sc, reg);
1883
1884         return (addr - start);
1885 }
1886
1887 static int
1888 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1889     struct intrs_and_queues *iaq)
1890 {
1891         int rc, itype, navail, nrxq10g, nrxq1g, n;
1892         int nofldrxq10g = 0, nofldrxq1g = 0;
1893         int nnmrxq10g = 0, nnmrxq1g = 0;
1894
1895         bzero(iaq, sizeof(*iaq));
1896
1897         iaq->ntxq10g = t4_ntxq10g;
1898         iaq->ntxq1g = t4_ntxq1g;
1899         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1900         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1901         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1902 #ifdef TCP_OFFLOAD
1903         if (is_offload(sc)) {
1904                 iaq->nofldtxq10g = t4_nofldtxq10g;
1905                 iaq->nofldtxq1g = t4_nofldtxq1g;
1906                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1907                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1908         }
1909 #endif
1910 #ifdef DEV_NETMAP
1911         iaq->nnmtxq10g = t4_nnmtxq10g;
1912         iaq->nnmtxq1g = t4_nnmtxq1g;
1913         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1914         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1915 #endif
1916
1917         for (itype = INTR_MSIX; itype; itype >>= 1) {
1918
1919                 if ((itype & t4_intr_types) == 0)
1920                         continue;       /* not allowed */
1921
1922                 if (itype == INTR_MSIX)
1923                         navail = pci_msix_count(sc->dev);
1924                 else if (itype == INTR_MSI)
1925                         navail = pci_msi_count(sc->dev);
1926                 else
1927                         navail = 1;
1928 restart:
1929                 if (navail == 0)
1930                         continue;
1931
1932                 iaq->intr_type = itype;
1933                 iaq->intr_flags_10g = 0;
1934                 iaq->intr_flags_1g = 0;
1935
1936                 /*
1937                  * Best option: an interrupt vector for errors, one for the
1938                  * firmware event queue, and one for every rxq (NIC, TOE, and
1939                  * netmap).
1940                  */
1941                 iaq->nirq = T4_EXTRA_INTR;
1942                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1943                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1944                 if (iaq->nirq <= navail &&
1945                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1946                         iaq->intr_flags_10g = INTR_ALL;
1947                         iaq->intr_flags_1g = INTR_ALL;
1948                         goto allocate;
1949                 }
1950
1951                 /*
1952                  * Second best option: a vector for errors, one for the firmware
1953                  * event queue, and vectors for either all the NIC rx queues or
1954                  * all the TOE rx queues.  The queues that don't get vectors
1955                  * will forward their interrupts to those that do.
1956                  *
1957                  * Note: netmap rx queues cannot be created early and so they
1958                  * can't be setup to receive forwarded interrupts for others.
1959                  */
1960                 iaq->nirq = T4_EXTRA_INTR;
1961                 if (nrxq10g >= nofldrxq10g) {
1962                         iaq->intr_flags_10g = INTR_RXQ;
1963                         iaq->nirq += n10g * nrxq10g;
1964 #ifdef DEV_NETMAP
1965                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1966 #endif
1967                 } else {
1968                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1969                         iaq->nirq += n10g * nofldrxq10g;
1970 #ifdef DEV_NETMAP
1971                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1972 #endif
1973                 }
1974                 if (nrxq1g >= nofldrxq1g) {
1975                         iaq->intr_flags_1g = INTR_RXQ;
1976                         iaq->nirq += n1g * nrxq1g;
1977 #ifdef DEV_NETMAP
1978                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1979 #endif
1980                 } else {
1981                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1982                         iaq->nirq += n1g * nofldrxq1g;
1983 #ifdef DEV_NETMAP
1984                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1985 #endif
1986                 }
1987                 if (iaq->nirq <= navail &&
1988                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1989                         goto allocate;
1990
1991                 /*
1992                  * Next best option: an interrupt vector for errors, one for the
1993                  * firmware event queue, and at least one per port.  At this
1994                  * point we know we'll have to downsize nrxq and/or nofldrxq
1995                  * and/or nnmrxq to fit what's available to us.
1996                  */
1997                 iaq->nirq = T4_EXTRA_INTR;
1998                 iaq->nirq += n10g + n1g;
1999                 if (iaq->nirq <= navail) {
2000                         int leftover = navail - iaq->nirq;
2001
2002                         if (n10g > 0) {
2003                                 int target = max(nrxq10g, nofldrxq10g);
2004
2005                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2006                                     INTR_RXQ : INTR_OFLD_RXQ;
2007
2008                                 n = 1;
2009                                 while (n < target && leftover >= n10g) {
2010                                         leftover -= n10g;
2011                                         iaq->nirq += n10g;
2012                                         n++;
2013                                 }
2014                                 iaq->nrxq10g = min(n, nrxq10g);
2015 #ifdef TCP_OFFLOAD
2016                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2017 #endif
2018 #ifdef DEV_NETMAP
2019                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2020 #endif
2021                         }
2022
2023                         if (n1g > 0) {
2024                                 int target = max(nrxq1g, nofldrxq1g);
2025
2026                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2027                                     INTR_RXQ : INTR_OFLD_RXQ;
2028
2029                                 n = 1;
2030                                 while (n < target && leftover >= n1g) {
2031                                         leftover -= n1g;
2032                                         iaq->nirq += n1g;
2033                                         n++;
2034                                 }
2035                                 iaq->nrxq1g = min(n, nrxq1g);
2036 #ifdef TCP_OFFLOAD
2037                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2038 #endif
2039 #ifdef DEV_NETMAP
2040                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2041 #endif
2042                         }
2043
2044                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2045                                 goto allocate;
2046                 }
2047
2048                 /*
2049                  * Least desirable option: one interrupt vector for everything.
2050                  */
2051                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2052                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2053 #ifdef TCP_OFFLOAD
2054                 if (is_offload(sc))
2055                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2056 #endif
2057 #ifdef DEV_NETMAP
2058                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2059 #endif
2060
2061 allocate:
2062                 navail = iaq->nirq;
2063                 rc = 0;
2064                 if (itype == INTR_MSIX)
2065                         rc = pci_alloc_msix(sc->dev, &navail);
2066                 else if (itype == INTR_MSI)
2067                         rc = pci_alloc_msi(sc->dev, &navail);
2068
2069                 if (rc == 0) {
2070                         if (navail == iaq->nirq)
2071                                 return (0);
2072
2073                         /*
2074                          * Didn't get the number requested.  Use whatever number
2075                          * the kernel is willing to allocate (it's in navail).
2076                          */
2077                         device_printf(sc->dev, "fewer vectors than requested, "
2078                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2079                             itype, iaq->nirq, navail);
2080                         pci_release_msi(sc->dev);
2081                         goto restart;
2082                 }
2083
2084                 device_printf(sc->dev,
2085                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2086                     itype, rc, iaq->nirq, navail);
2087         }
2088
2089         device_printf(sc->dev,
2090             "failed to find a usable interrupt type.  "
2091             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2092             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2093
2094         return (ENXIO);
2095 }
2096
2097 #define FW_VERSION(chip) ( \
2098     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2099     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2100     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2101     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2102 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2103
2104 struct fw_info {
2105         uint8_t chip;
2106         char *kld_name;
2107         char *fw_mod_name;
2108         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2109 } fw_info[] = {
2110         {
2111                 .chip = CHELSIO_T4,
2112                 .kld_name = "t4fw_cfg",
2113                 .fw_mod_name = "t4fw",
2114                 .fw_hdr = {
2115                         .chip = FW_HDR_CHIP_T4,
2116                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2117                         .intfver_nic = FW_INTFVER(T4, NIC),
2118                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2119                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2120                         .intfver_ri = FW_INTFVER(T4, RI),
2121                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2122                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2123                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2124                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2125                 },
2126         }, {
2127                 .chip = CHELSIO_T5,
2128                 .kld_name = "t5fw_cfg",
2129                 .fw_mod_name = "t5fw",
2130                 .fw_hdr = {
2131                         .chip = FW_HDR_CHIP_T5,
2132                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2133                         .intfver_nic = FW_INTFVER(T5, NIC),
2134                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2135                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2136                         .intfver_ri = FW_INTFVER(T5, RI),
2137                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2138                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2139                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2140                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2141                 },
2142         }
2143 };
2144
2145 static struct fw_info *
2146 find_fw_info(int chip)
2147 {
2148         int i;
2149
2150         for (i = 0; i < nitems(fw_info); i++) {
2151                 if (fw_info[i].chip == chip)
2152                         return (&fw_info[i]);
2153         }
2154         return (NULL);
2155 }
2156
2157 /*
2158  * Is the given firmware API compatible with the one the driver was compiled
2159  * with?
2160  */
2161 static int
2162 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2163 {
2164
2165         /* short circuit if it's the exact same firmware version */
2166         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2167                 return (1);
2168
2169         /*
2170          * XXX: Is this too conservative?  Perhaps I should limit this to the
2171          * features that are supported in the driver.
2172          */
2173 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2174         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2175             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2176             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2177                 return (1);
2178 #undef SAME_INTF
2179
2180         return (0);
2181 }
2182
2183 /*
2184  * The firmware in the KLD is usable, but should it be installed?  This routine
2185  * explains itself in detail if it indicates the KLD firmware should be
2186  * installed.
2187  */
2188 static int
2189 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2190 {
2191         const char *reason;
2192
2193         if (!card_fw_usable) {
2194                 reason = "incompatible or unusable";
2195                 goto install;
2196         }
2197
2198         if (k > c) {
2199                 reason = "older than the version bundled with this driver";
2200                 goto install;
2201         }
2202
2203         if (t4_fw_install == 2 && k != c) {
2204                 reason = "different than the version bundled with this driver";
2205                 goto install;
2206         }
2207
2208         return (0);
2209
2210 install:
2211         if (t4_fw_install == 0) {
2212                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2213                     "but the driver is prohibited from installing a different "
2214                     "firmware on the card.\n",
2215                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2216                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2217
2218                 return (0);
2219         }
2220
2221         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2222             "installing firmware %u.%u.%u.%u on card.\n",
2223             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2224             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2225             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2226             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2227
2228         return (1);
2229 }
2230 /*
2231  * Establish contact with the firmware and determine if we are the master driver
2232  * or not, and whether we are responsible for chip initialization.
2233  */
2234 static int
2235 prep_firmware(struct adapter *sc)
2236 {
2237         const struct firmware *fw = NULL, *default_cfg;
2238         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2239         enum dev_state state;
2240         struct fw_info *fw_info;
2241         struct fw_hdr *card_fw;         /* fw on the card */
2242         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2243         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2244                                            against */
2245
2246         /* Contact firmware. */
2247         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2248         if (rc < 0 || state == DEV_STATE_ERR) {
2249                 rc = -rc;
2250                 device_printf(sc->dev,
2251                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2252                 return (rc);
2253         }
2254         pf = rc;
2255         if (pf == sc->mbox)
2256                 sc->flags |= MASTER_PF;
2257         else if (state == DEV_STATE_UNINIT) {
2258                 /*
2259                  * We didn't get to be the master so we definitely won't be
2260                  * configuring the chip.  It's a bug if someone else hasn't
2261                  * configured it already.
2262                  */
2263                 device_printf(sc->dev, "couldn't be master(%d), "
2264                     "device not already initialized either(%d).\n", rc, state);
2265                 return (EDOOFUS);
2266         }
2267
2268         /* This is the firmware whose headers the driver was compiled against */
2269         fw_info = find_fw_info(chip_id(sc));
2270         if (fw_info == NULL) {
2271                 device_printf(sc->dev,
2272                     "unable to look up firmware information for chip %d.\n",
2273                     chip_id(sc));
2274                 return (EINVAL);
2275         }
2276         drv_fw = &fw_info->fw_hdr;
2277
2278         /*
2279          * The firmware KLD contains many modules.  The KLD name is also the
2280          * name of the module that contains the default config file.
2281          */
2282         default_cfg = firmware_get(fw_info->kld_name);
2283
2284         /* Read the header of the firmware on the card */
2285         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2286         rc = -t4_read_flash(sc, FLASH_FW_START,
2287             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2288         if (rc == 0)
2289                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2290         else {
2291                 device_printf(sc->dev,
2292                     "Unable to read card's firmware header: %d\n", rc);
2293                 card_fw_usable = 0;
2294         }
2295
2296         /* This is the firmware in the KLD */
2297         fw = firmware_get(fw_info->fw_mod_name);
2298         if (fw != NULL) {
2299                 kld_fw = (const void *)fw->data;
2300                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2301         } else {
2302                 kld_fw = NULL;
2303                 kld_fw_usable = 0;
2304         }
2305
2306         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2307             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2308                 /*
2309                  * Common case: the firmware on the card is an exact match and
2310                  * the KLD is an exact match too, or the KLD is
2311                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2312                  * here -- use cxgbetool loadfw if you want to reinstall the
2313                  * same firmware as the one on the card.
2314                  */
2315         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2316             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2317             be32toh(card_fw->fw_ver))) {
2318
2319                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2320                 if (rc != 0) {
2321                         device_printf(sc->dev,
2322                             "failed to install firmware: %d\n", rc);
2323                         goto done;
2324                 }
2325
2326                 /* Installed successfully, update the cached header too. */
2327                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2328                 card_fw_usable = 1;
2329                 need_fw_reset = 0;      /* already reset as part of load_fw */
2330         }
2331
2332         if (!card_fw_usable) {
2333                 uint32_t d, c, k;
2334
2335                 d = ntohl(drv_fw->fw_ver);
2336                 c = ntohl(card_fw->fw_ver);
2337                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2338
2339                 device_printf(sc->dev, "Cannot find a usable firmware: "
2340                     "fw_install %d, chip state %d, "
2341                     "driver compiled with %d.%d.%d.%d, "
2342                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2343                     t4_fw_install, state,
2344                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2345                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2346                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2347                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2348                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2349                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2350                 rc = EINVAL;
2351                 goto done;
2352         }
2353
2354         /* We're using whatever's on the card and it's known to be good. */
2355         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2356         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2357             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2358             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2359             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2360             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2361         t4_get_tp_version(sc, &sc->params.tp_vers);
2362
2363         /* Reset device */
2364         if (need_fw_reset &&
2365             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2366                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2367                 if (rc != ETIMEDOUT && rc != EIO)
2368                         t4_fw_bye(sc, sc->mbox);
2369                 goto done;
2370         }
2371         sc->flags |= FW_OK;
2372
2373         rc = get_params__pre_init(sc);
2374         if (rc != 0)
2375                 goto done; /* error message displayed already */
2376
2377         /* Partition adapter resources as specified in the config file. */
2378         if (state == DEV_STATE_UNINIT) {
2379
2380                 KASSERT(sc->flags & MASTER_PF,
2381                     ("%s: trying to change chip settings when not master.",
2382                     __func__));
2383
2384                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2385                 if (rc != 0)
2386                         goto done;      /* error message displayed already */
2387
2388                 t4_tweak_chip_settings(sc);
2389
2390                 /* get basic stuff going */
2391                 rc = -t4_fw_initialize(sc, sc->mbox);
2392                 if (rc != 0) {
2393                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2394                         goto done;
2395                 }
2396         } else {
2397                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2398                 sc->cfcsum = 0;
2399         }
2400
2401 done:
2402         free(card_fw, M_CXGBE);
2403         if (fw != NULL)
2404                 firmware_put(fw, FIRMWARE_UNLOAD);
2405         if (default_cfg != NULL)
2406                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2407
2408         return (rc);
2409 }
2410
2411 #define FW_PARAM_DEV(param) \
2412         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2413          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2414 #define FW_PARAM_PFVF(param) \
2415         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2416          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2417
2418 /*
2419  * Partition chip resources for use between various PFs, VFs, etc.
2420  */
2421 static int
2422 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2423     const char *name_prefix)
2424 {
2425         const struct firmware *cfg = NULL;
2426         int rc = 0;
2427         struct fw_caps_config_cmd caps;
2428         uint32_t mtype, moff, finicsum, cfcsum;
2429
2430         /*
2431          * Figure out what configuration file to use.  Pick the default config
2432          * file for the card if the user hasn't specified one explicitly.
2433          */
2434         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2435         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2436                 /* Card specific overrides go here. */
2437                 if (pci_get_device(sc->dev) == 0x440a)
2438                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2439                 if (is_fpga(sc))
2440                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2441         }
2442
2443         /*
2444          * We need to load another module if the profile is anything except
2445          * "default" or "flash".
2446          */
2447         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2448             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2449                 char s[32];
2450
2451                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2452                 cfg = firmware_get(s);
2453                 if (cfg == NULL) {
2454                         if (default_cfg != NULL) {
2455                                 device_printf(sc->dev,
2456                                     "unable to load module \"%s\" for "
2457                                     "configuration profile \"%s\", will use "
2458                                     "the default config file instead.\n",
2459                                     s, sc->cfg_file);
2460                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2461                                     "%s", DEFAULT_CF);
2462                         } else {
2463                                 device_printf(sc->dev,
2464                                     "unable to load module \"%s\" for "
2465                                     "configuration profile \"%s\", will use "
2466                                     "the config file on the card's flash "
2467                                     "instead.\n", s, sc->cfg_file);
2468                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2469                                     "%s", FLASH_CF);
2470                         }
2471                 }
2472         }
2473
2474         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2475             default_cfg == NULL) {
2476                 device_printf(sc->dev,
2477                     "default config file not available, will use the config "
2478                     "file on the card's flash instead.\n");
2479                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2480         }
2481
2482         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2483                 u_int cflen, i, n;
2484                 const uint32_t *cfdata;
2485                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2486
2487                 KASSERT(cfg != NULL || default_cfg != NULL,
2488                     ("%s: no config to upload", __func__));
2489
2490                 /*
2491                  * Ask the firmware where it wants us to upload the config file.
2492                  */
2493                 param = FW_PARAM_DEV(CF);
2494                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2495                 if (rc != 0) {
2496                         /* No support for config file?  Shouldn't happen. */
2497                         device_printf(sc->dev,
2498                             "failed to query config file location: %d.\n", rc);
2499                         goto done;
2500                 }
2501                 mtype = G_FW_PARAMS_PARAM_Y(val);
2502                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2503
2504                 /*
2505                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2506                  * useless stuffing/comments at the end of the config file so
2507                  * it's ok to simply throw away the last remaining bytes when
2508                  * the config file is not an exact multiple of 4.  This also
2509                  * helps with the validate_mt_off_len check.
2510                  */
2511                 if (cfg != NULL) {
2512                         cflen = cfg->datasize & ~3;
2513                         cfdata = cfg->data;
2514                 } else {
2515                         cflen = default_cfg->datasize & ~3;
2516                         cfdata = default_cfg->data;
2517                 }
2518
2519                 if (cflen > FLASH_CFG_MAX_SIZE) {
2520                         device_printf(sc->dev,
2521                             "config file too long (%d, max allowed is %d).  "
2522                             "Will try to use the config on the card, if any.\n",
2523                             cflen, FLASH_CFG_MAX_SIZE);
2524                         goto use_config_on_flash;
2525                 }
2526
2527                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2528                 if (rc != 0) {
2529                         device_printf(sc->dev,
2530                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2531                             "Will try to use the config on the card, if any.\n",
2532                             __func__, mtype, moff, cflen, rc);
2533                         goto use_config_on_flash;
2534                 }
2535
2536                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2537                 while (cflen) {
2538                         off = position_memwin(sc, 2, addr);
2539                         n = min(cflen, mw_aperture - off);
2540                         for (i = 0; i < n; i += 4)
2541                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2542                         cflen -= n;
2543                         addr += n;
2544                 }
2545         } else {
2546 use_config_on_flash:
2547                 mtype = FW_MEMTYPE_FLASH;
2548                 moff = t4_flash_cfg_addr(sc);
2549         }
2550
2551         bzero(&caps, sizeof(caps));
2552         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2553             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2554         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2555             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2556             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2557         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2558         if (rc != 0) {
2559                 device_printf(sc->dev,
2560                     "failed to pre-process config file: %d "
2561                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2562                 goto done;
2563         }
2564
2565         finicsum = be32toh(caps.finicsum);
2566         cfcsum = be32toh(caps.cfcsum);
2567         if (finicsum != cfcsum) {
2568                 device_printf(sc->dev,
2569                     "WARNING: config file checksum mismatch: %08x %08x\n",
2570                     finicsum, cfcsum);
2571         }
2572         sc->cfcsum = cfcsum;
2573
2574 #define LIMIT_CAPS(x) do { \
2575         caps.x &= htobe16(t4_##x##_allowed); \
2576 } while (0)
2577
2578         /*
2579          * Let the firmware know what features will (not) be used so it can tune
2580          * things accordingly.
2581          */
2582         LIMIT_CAPS(linkcaps);
2583         LIMIT_CAPS(niccaps);
2584         LIMIT_CAPS(toecaps);
2585         LIMIT_CAPS(rdmacaps);
2586         LIMIT_CAPS(iscsicaps);
2587         LIMIT_CAPS(fcoecaps);
2588 #undef LIMIT_CAPS
2589
2590         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2591             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2592         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2593         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2594         if (rc != 0) {
2595                 device_printf(sc->dev,
2596                     "failed to process config file: %d.\n", rc);
2597         }
2598 done:
2599         if (cfg != NULL)
2600                 firmware_put(cfg, FIRMWARE_UNLOAD);
2601         return (rc);
2602 }
2603
2604 /*
2605  * Retrieve parameters that are needed (or nice to have) very early.
2606  */
2607 static int
2608 get_params__pre_init(struct adapter *sc)
2609 {
2610         int rc;
2611         uint32_t param[2], val[2];
2612         struct fw_devlog_cmd cmd;
2613         struct devlog_params *dlog = &sc->params.devlog;
2614
2615         param[0] = FW_PARAM_DEV(PORTVEC);
2616         param[1] = FW_PARAM_DEV(CCLK);
2617         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2618         if (rc != 0) {
2619                 device_printf(sc->dev,
2620                     "failed to query parameters (pre_init): %d.\n", rc);
2621                 return (rc);
2622         }
2623
2624         sc->params.portvec = val[0];
2625         sc->params.nports = bitcount32(val[0]);
2626         sc->params.vpd.cclk = val[1];
2627
2628         /* Read device log parameters. */
2629         bzero(&cmd, sizeof(cmd));
2630         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2631             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2632         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2633         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2634         if (rc != 0) {
2635                 device_printf(sc->dev,
2636                     "failed to get devlog parameters: %d.\n", rc);
2637                 bzero(dlog, sizeof (*dlog));
2638                 rc = 0; /* devlog isn't critical for device operation */
2639         } else {
2640                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2641                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2642                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2643                 dlog->size = be32toh(cmd.memsize_devlog);
2644         }
2645
2646         return (rc);
2647 }
2648
2649 /*
2650  * Retrieve various parameters that are of interest to the driver.  The device
2651  * has been initialized by the firmware at this point.
2652  */
2653 static int
2654 get_params__post_init(struct adapter *sc)
2655 {
2656         int rc;
2657         uint32_t param[7], val[7];
2658         struct fw_caps_config_cmd caps;
2659
2660         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2661         param[1] = FW_PARAM_PFVF(EQ_START);
2662         param[2] = FW_PARAM_PFVF(FILTER_START);
2663         param[3] = FW_PARAM_PFVF(FILTER_END);
2664         param[4] = FW_PARAM_PFVF(L2T_START);
2665         param[5] = FW_PARAM_PFVF(L2T_END);
2666         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2667         if (rc != 0) {
2668                 device_printf(sc->dev,
2669                     "failed to query parameters (post_init): %d.\n", rc);
2670                 return (rc);
2671         }
2672
2673         sc->sge.iq_start = val[0];
2674         sc->sge.eq_start = val[1];
2675         sc->tids.ftid_base = val[2];
2676         sc->tids.nftids = val[3] - val[2] + 1;
2677         sc->params.ftid_min = val[2];
2678         sc->params.ftid_max = val[3];
2679         sc->vres.l2t.start = val[4];
2680         sc->vres.l2t.size = val[5] - val[4] + 1;
2681         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2682             ("%s: L2 table size (%u) larger than expected (%u)",
2683             __func__, sc->vres.l2t.size, L2T_SIZE));
2684
2685         /* get capabilites */
2686         bzero(&caps, sizeof(caps));
2687         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2688             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2689         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2690         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2691         if (rc != 0) {
2692                 device_printf(sc->dev,
2693                     "failed to get card capabilities: %d.\n", rc);
2694                 return (rc);
2695         }
2696
2697 #define READ_CAPS(x) do { \
2698         sc->x = htobe16(caps.x); \
2699 } while (0)
2700         READ_CAPS(linkcaps);
2701         READ_CAPS(niccaps);
2702         READ_CAPS(toecaps);
2703         READ_CAPS(rdmacaps);
2704         READ_CAPS(iscsicaps);
2705         READ_CAPS(fcoecaps);
2706
2707         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2708                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2709                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2710                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2711                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2712                 if (rc != 0) {
2713                         device_printf(sc->dev,
2714                             "failed to query NIC parameters: %d.\n", rc);
2715                         return (rc);
2716                 }
2717                 sc->tids.etid_base = val[0];
2718                 sc->params.etid_min = val[0];
2719                 sc->tids.netids = val[1] - val[0] + 1;
2720                 sc->params.netids = sc->tids.netids;
2721                 sc->params.eo_wr_cred = val[2];
2722                 sc->params.ethoffload = 1;
2723         }
2724
2725         if (sc->toecaps) {
2726                 /* query offload-related parameters */
2727                 param[0] = FW_PARAM_DEV(NTID);
2728                 param[1] = FW_PARAM_PFVF(SERVER_START);
2729                 param[2] = FW_PARAM_PFVF(SERVER_END);
2730                 param[3] = FW_PARAM_PFVF(TDDP_START);
2731                 param[4] = FW_PARAM_PFVF(TDDP_END);
2732                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2733                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2734                 if (rc != 0) {
2735                         device_printf(sc->dev,
2736                             "failed to query TOE parameters: %d.\n", rc);
2737                         return (rc);
2738                 }
2739                 sc->tids.ntids = val[0];
2740                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2741                 sc->tids.stid_base = val[1];
2742                 sc->tids.nstids = val[2] - val[1] + 1;
2743                 sc->vres.ddp.start = val[3];
2744                 sc->vres.ddp.size = val[4] - val[3] + 1;
2745                 sc->params.ofldq_wr_cred = val[5];
2746                 sc->params.offload = 1;
2747         }
2748         if (sc->rdmacaps) {
2749                 param[0] = FW_PARAM_PFVF(STAG_START);
2750                 param[1] = FW_PARAM_PFVF(STAG_END);
2751                 param[2] = FW_PARAM_PFVF(RQ_START);
2752                 param[3] = FW_PARAM_PFVF(RQ_END);
2753                 param[4] = FW_PARAM_PFVF(PBL_START);
2754                 param[5] = FW_PARAM_PFVF(PBL_END);
2755                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2756                 if (rc != 0) {
2757                         device_printf(sc->dev,
2758                             "failed to query RDMA parameters(1): %d.\n", rc);
2759                         return (rc);
2760                 }
2761                 sc->vres.stag.start = val[0];
2762                 sc->vres.stag.size = val[1] - val[0] + 1;
2763                 sc->vres.rq.start = val[2];
2764                 sc->vres.rq.size = val[3] - val[2] + 1;
2765                 sc->vres.pbl.start = val[4];
2766                 sc->vres.pbl.size = val[5] - val[4] + 1;
2767
2768                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2769                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2770                 param[2] = FW_PARAM_PFVF(CQ_START);
2771                 param[3] = FW_PARAM_PFVF(CQ_END);
2772                 param[4] = FW_PARAM_PFVF(OCQ_START);
2773                 param[5] = FW_PARAM_PFVF(OCQ_END);
2774                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2775                 if (rc != 0) {
2776                         device_printf(sc->dev,
2777                             "failed to query RDMA parameters(2): %d.\n", rc);
2778                         return (rc);
2779                 }
2780                 sc->vres.qp.start = val[0];
2781                 sc->vres.qp.size = val[1] - val[0] + 1;
2782                 sc->vres.cq.start = val[2];
2783                 sc->vres.cq.size = val[3] - val[2] + 1;
2784                 sc->vres.ocq.start = val[4];
2785                 sc->vres.ocq.size = val[5] - val[4] + 1;
2786         }
2787         if (sc->iscsicaps) {
2788                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2789                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2790                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2791                 if (rc != 0) {
2792                         device_printf(sc->dev,
2793                             "failed to query iSCSI parameters: %d.\n", rc);
2794                         return (rc);
2795                 }
2796                 sc->vres.iscsi.start = val[0];
2797                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2798         }
2799
2800         /*
2801          * We've got the params we wanted to query via the firmware.  Now grab
2802          * some others directly from the chip.
2803          */
2804         rc = t4_read_chip_settings(sc);
2805
2806         return (rc);
2807 }
2808
2809 static int
2810 set_params__post_init(struct adapter *sc)
2811 {
2812         uint32_t param, val;
2813
2814         /* ask for encapsulated CPLs */
2815         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2816         val = 1;
2817         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2818
2819         return (0);
2820 }
2821
2822 #undef FW_PARAM_PFVF
2823 #undef FW_PARAM_DEV
2824
2825 static void
2826 t4_set_desc(struct adapter *sc)
2827 {
2828         char buf[128];
2829         struct adapter_params *p = &sc->params;
2830
2831         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2832             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2833             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2834
2835         device_set_desc_copy(sc->dev, buf);
2836 }
2837
2838 static void
2839 build_medialist(struct port_info *pi, struct ifmedia *media)
2840 {
2841         int data, m;
2842
2843         PORT_LOCK(pi);
2844
2845         ifmedia_removeall(media);
2846
2847         m = IFM_ETHER | IFM_FDX;
2848         data = (pi->port_type << 8) | pi->mod_type;
2849
2850         switch(pi->port_type) {
2851         case FW_PORT_TYPE_BT_XFI:
2852         case FW_PORT_TYPE_BT_XAUI:
2853                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2854                 /* fall through */
2855
2856         case FW_PORT_TYPE_BT_SGMII:
2857                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2858                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2859                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2860                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2861                 break;
2862
2863         case FW_PORT_TYPE_CX4:
2864                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2865                 ifmedia_set(media, m | IFM_10G_CX4);
2866                 break;
2867
2868         case FW_PORT_TYPE_QSFP_10G:
2869         case FW_PORT_TYPE_SFP:
2870         case FW_PORT_TYPE_FIBER_XFI:
2871         case FW_PORT_TYPE_FIBER_XAUI:
2872                 switch (pi->mod_type) {
2873
2874                 case FW_PORT_MOD_TYPE_LR:
2875                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2876                         ifmedia_set(media, m | IFM_10G_LR);
2877                         break;
2878
2879                 case FW_PORT_MOD_TYPE_SR:
2880                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2881                         ifmedia_set(media, m | IFM_10G_SR);
2882                         break;
2883
2884                 case FW_PORT_MOD_TYPE_LRM:
2885                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2886                         ifmedia_set(media, m | IFM_10G_LRM);
2887                         break;
2888
2889                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2890                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2891                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2892                         ifmedia_set(media, m | IFM_10G_TWINAX);
2893                         break;
2894
2895                 case FW_PORT_MOD_TYPE_NONE:
2896                         m &= ~IFM_FDX;
2897                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2898                         ifmedia_set(media, m | IFM_NONE);
2899                         break;
2900
2901                 case FW_PORT_MOD_TYPE_NA:
2902                 case FW_PORT_MOD_TYPE_ER:
2903                 default:
2904                         device_printf(pi->dev,
2905                             "unknown port_type (%d), mod_type (%d)\n",
2906                             pi->port_type, pi->mod_type);
2907                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2908                         ifmedia_set(media, m | IFM_UNKNOWN);
2909                         break;
2910                 }
2911                 break;
2912
2913         case FW_PORT_TYPE_QSFP:
2914                 switch (pi->mod_type) {
2915
2916                 case FW_PORT_MOD_TYPE_LR:
2917                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2918                         ifmedia_set(media, m | IFM_40G_LR4);
2919                         break;
2920
2921                 case FW_PORT_MOD_TYPE_SR:
2922                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2923                         ifmedia_set(media, m | IFM_40G_SR4);
2924                         break;
2925
2926                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2927                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2928                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2929                         ifmedia_set(media, m | IFM_40G_CR4);
2930                         break;
2931
2932                 case FW_PORT_MOD_TYPE_NONE:
2933                         m &= ~IFM_FDX;
2934                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2935                         ifmedia_set(media, m | IFM_NONE);
2936                         break;
2937
2938                 default:
2939                         device_printf(pi->dev,
2940                             "unknown port_type (%d), mod_type (%d)\n",
2941                             pi->port_type, pi->mod_type);
2942                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2943                         ifmedia_set(media, m | IFM_UNKNOWN);
2944                         break;
2945                 }
2946                 break;
2947
2948         default:
2949                 device_printf(pi->dev,
2950                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2951                     pi->mod_type);
2952                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2953                 ifmedia_set(media, m | IFM_UNKNOWN);
2954                 break;
2955         }
2956
2957         PORT_UNLOCK(pi);
2958 }
2959
2960 #define FW_MAC_EXACT_CHUNK      7
2961
2962 /*
2963  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2964  * indicates which parameters should be programmed (the rest are left alone).
2965  */
2966 int
2967 update_mac_settings(struct ifnet *ifp, int flags)
2968 {
2969         int rc = 0;
2970         struct port_info *pi = ifp->if_softc;
2971         struct adapter *sc = pi->adapter;
2972         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2973         uint16_t viid = 0xffff;
2974         int16_t *xact_addr_filt = NULL;
2975
2976         ASSERT_SYNCHRONIZED_OP(sc);
2977         KASSERT(flags, ("%s: not told what to update.", __func__));
2978
2979         if (ifp == pi->ifp) {
2980                 viid = pi->viid;
2981                 xact_addr_filt = &pi->xact_addr_filt;
2982         }
2983 #ifdef DEV_NETMAP
2984         else if (ifp == pi->nm_ifp) {
2985                 viid = pi->nm_viid;
2986                 xact_addr_filt = &pi->nm_xact_addr_filt;
2987         }
2988 #endif
2989         if (flags & XGMAC_MTU)
2990                 mtu = ifp->if_mtu;
2991
2992         if (flags & XGMAC_PROMISC)
2993                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2994
2995         if (flags & XGMAC_ALLMULTI)
2996                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2997
2998         if (flags & XGMAC_VLANEX)
2999                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3000
3001         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3002                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3003                     1, vlanex, false);
3004                 if (rc) {
3005                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3006                             rc);
3007                         return (rc);
3008                 }
3009         }
3010
3011         if (flags & XGMAC_UCADDR) {
3012                 uint8_t ucaddr[ETHER_ADDR_LEN];
3013
3014                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3015                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3016                     true, true);
3017                 if (rc < 0) {
3018                         rc = -rc;
3019                         if_printf(ifp, "change_mac failed: %d\n", rc);
3020                         return (rc);
3021                 } else {
3022                         *xact_addr_filt = rc;
3023                         rc = 0;
3024                 }
3025         }
3026
3027         if (flags & XGMAC_MCADDRS) {
3028                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3029                 int del = 1;
3030                 uint64_t hash = 0;
3031                 struct ifmultiaddr *ifma;
3032                 int i = 0, j;
3033
3034                 if_maddr_rlock(ifp);
3035                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3036                         if (ifma->ifma_addr->sa_family != AF_LINK)
3037                                 continue;
3038                         mcaddr[i] =
3039                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3040                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3041                         i++;
3042
3043                         if (i == FW_MAC_EXACT_CHUNK) {
3044                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3045                                     i, mcaddr, NULL, &hash, 0);
3046                                 if (rc < 0) {
3047                                         rc = -rc;
3048                                         for (j = 0; j < i; j++) {
3049                                                 if_printf(ifp,
3050                                                     "failed to add mc address"
3051                                                     " %02x:%02x:%02x:"
3052                                                     "%02x:%02x:%02x rc=%d\n",
3053                                                     mcaddr[j][0], mcaddr[j][1],
3054                                                     mcaddr[j][2], mcaddr[j][3],
3055                                                     mcaddr[j][4], mcaddr[j][5],
3056                                                     rc);
3057                                         }
3058                                         goto mcfail;
3059                                 }
3060                                 del = 0;
3061                                 i = 0;
3062                         }
3063                 }
3064                 if (i > 0) {
3065                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3066                             mcaddr, NULL, &hash, 0);
3067                         if (rc < 0) {
3068                                 rc = -rc;
3069                                 for (j = 0; j < i; j++) {
3070                                         if_printf(ifp,
3071                                             "failed to add mc address"
3072                                             " %02x:%02x:%02x:"
3073                                             "%02x:%02x:%02x rc=%d\n",
3074                                             mcaddr[j][0], mcaddr[j][1],
3075                                             mcaddr[j][2], mcaddr[j][3],
3076                                             mcaddr[j][4], mcaddr[j][5],
3077                                             rc);
3078                                 }
3079                                 goto mcfail;
3080                         }
3081                 }
3082
3083                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3084                 if (rc != 0)
3085                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3086 mcfail:
3087                 if_maddr_runlock(ifp);
3088         }
3089
3090         return (rc);
3091 }
3092
3093 /*
3094  * {begin|end}_synchronized_op must be called from the same thread.
3095  */
3096 int
3097 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3098     char *wmesg)
3099 {
3100         int rc, pri;
3101
3102 #ifdef WITNESS
3103         /* the caller thinks it's ok to sleep, but is it really? */
3104         if (flags & SLEEP_OK)
3105                 pause("t4slptst", 1);
3106 #endif
3107
3108         if (INTR_OK)
3109                 pri = PCATCH;
3110         else
3111                 pri = 0;
3112
3113         ADAPTER_LOCK(sc);
3114         for (;;) {
3115
3116                 if (pi && IS_DOOMED(pi)) {
3117                         rc = ENXIO;
3118                         goto done;
3119                 }
3120
3121                 if (!IS_BUSY(sc)) {
3122                         rc = 0;
3123                         break;
3124                 }
3125
3126                 if (!(flags & SLEEP_OK)) {
3127                         rc = EBUSY;
3128                         goto done;
3129                 }
3130
3131                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3132                         rc = EINTR;
3133                         goto done;
3134                 }
3135         }
3136
3137         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3138         SET_BUSY(sc);
3139 #ifdef INVARIANTS
3140         sc->last_op = wmesg;
3141         sc->last_op_thr = curthread;
3142         sc->last_op_flags = flags;
3143 #endif
3144
3145 done:
3146         if (!(flags & HOLD_LOCK) || rc)
3147                 ADAPTER_UNLOCK(sc);
3148
3149         return (rc);
3150 }
3151
3152 /*
3153  * {begin|end}_synchronized_op must be called from the same thread.
3154  */
3155 void
3156 end_synchronized_op(struct adapter *sc, int flags)
3157 {
3158
3159         if (flags & LOCK_HELD)
3160                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3161         else
3162                 ADAPTER_LOCK(sc);
3163
3164         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3165         CLR_BUSY(sc);
3166         wakeup(&sc->flags);
3167         ADAPTER_UNLOCK(sc);
3168 }
3169
3170 static int
3171 cxgbe_init_synchronized(struct port_info *pi)
3172 {
3173         struct adapter *sc = pi->adapter;
3174         struct ifnet *ifp = pi->ifp;
3175         int rc = 0, i;
3176         struct sge_txq *txq;
3177
3178         ASSERT_SYNCHRONIZED_OP(sc);
3179
3180         if (isset(&sc->open_device_map, pi->port_id)) {
3181                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3182                     ("mismatch between open_device_map and if_drv_flags"));
3183                 return (0);     /* already running */
3184         }
3185
3186         if (!(sc->flags & FULL_INIT_DONE) &&
3187             ((rc = adapter_full_init(sc)) != 0))
3188                 return (rc);    /* error message displayed already */
3189
3190         if (!(pi->flags & PORT_INIT_DONE) &&
3191             ((rc = port_full_init(pi)) != 0))
3192                 return (rc); /* error message displayed already */
3193
3194         rc = update_mac_settings(ifp, XGMAC_ALL);
3195         if (rc)
3196                 goto done;      /* error message displayed already */
3197
3198         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3199         if (rc != 0) {
3200                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3201                 goto done;
3202         }
3203
3204         /*
3205          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3206          * if this changes.
3207          */
3208
3209         for_each_txq(pi, i, txq) {
3210                 TXQ_LOCK(txq);
3211                 txq->eq.flags |= EQ_ENABLED;
3212                 TXQ_UNLOCK(txq);
3213         }
3214
3215         /*
3216          * The first iq of the first port to come up is used for tracing.
3217          */
3218         if (sc->traceq < 0) {
3219                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3220                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3221                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3222                     V_QUEUENUMBER(sc->traceq));
3223                 pi->flags |= HAS_TRACEQ;
3224         }
3225
3226         /* all ok */
3227         setbit(&sc->open_device_map, pi->port_id);
3228         PORT_LOCK(pi);
3229         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3230         PORT_UNLOCK(pi);
3231
3232         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3233 done:
3234         if (rc != 0)
3235                 cxgbe_uninit_synchronized(pi);
3236
3237         return (rc);
3238 }
3239
3240 /*
3241  * Idempotent.
3242  */
3243 static int
3244 cxgbe_uninit_synchronized(struct port_info *pi)
3245 {
3246         struct adapter *sc = pi->adapter;
3247         struct ifnet *ifp = pi->ifp;
3248         int rc, i;
3249         struct sge_txq *txq;
3250
3251         ASSERT_SYNCHRONIZED_OP(sc);
3252
3253         if (!(pi->flags & PORT_INIT_DONE)) {
3254                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3255                     ("uninited port is running"));
3256                 return (0);
3257         }
3258
3259         /*
3260          * Disable the VI so that all its data in either direction is discarded
3261          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3262          * tick) intact as the TP can deliver negative advice or data that it's
3263          * holding in its RAM (for an offloaded connection) even after the VI is
3264          * disabled.
3265          */
3266         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3267         if (rc) {
3268                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3269                 return (rc);
3270         }
3271
3272         for_each_txq(pi, i, txq) {
3273                 TXQ_LOCK(txq);
3274                 txq->eq.flags &= ~EQ_ENABLED;
3275                 TXQ_UNLOCK(txq);
3276         }
3277
3278         clrbit(&sc->open_device_map, pi->port_id);
3279         PORT_LOCK(pi);
3280         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3281         PORT_UNLOCK(pi);
3282
3283         pi->link_cfg.link_ok = 0;
3284         pi->link_cfg.speed = 0;
3285         pi->linkdnrc = -1;
3286         t4_os_link_changed(sc, pi->port_id, 0, -1);
3287
3288         return (0);
3289 }
3290
3291 /*
3292  * It is ok for this function to fail midway and return right away.  t4_detach
3293  * will walk the entire sc->irq list and clean up whatever is valid.
3294  */
3295 static int
3296 setup_intr_handlers(struct adapter *sc)
3297 {
3298         int rc, rid, p, q;
3299         char s[8];
3300         struct irq *irq;
3301         struct port_info *pi;
3302         struct sge_rxq *rxq;
3303 #ifdef TCP_OFFLOAD
3304         struct sge_ofld_rxq *ofld_rxq;
3305 #endif
3306 #ifdef DEV_NETMAP
3307         struct sge_nm_rxq *nm_rxq;
3308 #endif
3309
3310         /*
3311          * Setup interrupts.
3312          */
3313         irq = &sc->irq[0];
3314         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3315         if (sc->intr_count == 1)
3316                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3317
3318         /* Multiple interrupts. */
3319         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3320             ("%s: too few intr.", __func__));
3321
3322         /* The first one is always error intr */
3323         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3324         if (rc != 0)
3325                 return (rc);
3326         irq++;
3327         rid++;
3328
3329         /* The second one is always the firmware event queue */
3330         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3331         if (rc != 0)
3332                 return (rc);
3333         irq++;
3334         rid++;
3335
3336         for_each_port(sc, p) {
3337                 pi = sc->port[p];
3338
3339                 if (pi->flags & INTR_RXQ) {
3340                         for_each_rxq(pi, q, rxq) {
3341                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3342                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3343                                     s);
3344                                 if (rc != 0)
3345                                         return (rc);
3346                                 irq++;
3347                                 rid++;
3348                         }
3349                 }
3350 #ifdef TCP_OFFLOAD
3351                 if (pi->flags & INTR_OFLD_RXQ) {
3352                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3353                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3354                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3355                                     ofld_rxq, s);
3356                                 if (rc != 0)
3357                                         return (rc);
3358                                 irq++;
3359                                 rid++;
3360                         }
3361                 }
3362 #endif
3363 #ifdef DEV_NETMAP
3364                 if (pi->flags & INTR_NM_RXQ) {
3365                         for_each_nm_rxq(pi, q, nm_rxq) {
3366                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3367                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3368                                     nm_rxq, s);
3369                                 if (rc != 0)
3370                                         return (rc);
3371                                 irq++;
3372                                 rid++;
3373                         }
3374                 }
3375 #endif
3376         }
3377         MPASS(irq == &sc->irq[sc->intr_count]);
3378
3379         return (0);
3380 }
3381
3382 int
3383 adapter_full_init(struct adapter *sc)
3384 {
3385         int rc, i;
3386
3387         ASSERT_SYNCHRONIZED_OP(sc);
3388         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3389         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3390             ("%s: FULL_INIT_DONE already", __func__));
3391
3392         /*
3393          * queues that belong to the adapter (not any particular port).
3394          */
3395         rc = t4_setup_adapter_queues(sc);
3396         if (rc != 0)
3397                 goto done;
3398
3399         for (i = 0; i < nitems(sc->tq); i++) {
3400                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3401                     taskqueue_thread_enqueue, &sc->tq[i]);
3402                 if (sc->tq[i] == NULL) {
3403                         device_printf(sc->dev,
3404                             "failed to allocate task queue %d\n", i);
3405                         rc = ENOMEM;
3406                         goto done;
3407                 }
3408                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3409                     device_get_nameunit(sc->dev), i);
3410         }
3411
3412         t4_intr_enable(sc);
3413         sc->flags |= FULL_INIT_DONE;
3414 done:
3415         if (rc != 0)
3416                 adapter_full_uninit(sc);
3417
3418         return (rc);
3419 }
3420
3421 int
3422 adapter_full_uninit(struct adapter *sc)
3423 {
3424         int i;
3425
3426         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3427
3428         t4_teardown_adapter_queues(sc);
3429
3430         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3431                 taskqueue_free(sc->tq[i]);
3432                 sc->tq[i] = NULL;
3433         }
3434
3435         sc->flags &= ~FULL_INIT_DONE;
3436
3437         return (0);
3438 }
3439
3440 int
3441 port_full_init(struct port_info *pi)
3442 {
3443         struct adapter *sc = pi->adapter;
3444         struct ifnet *ifp = pi->ifp;
3445         uint16_t *rss;
3446         struct sge_rxq *rxq;
3447         int rc, i, j;
3448
3449         ASSERT_SYNCHRONIZED_OP(sc);
3450         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3451             ("%s: PORT_INIT_DONE already", __func__));
3452
3453         sysctl_ctx_init(&pi->ctx);
3454         pi->flags |= PORT_SYSCTL_CTX;
3455
3456         /*
3457          * Allocate tx/rx/fl queues for this port.
3458          */
3459         rc = t4_setup_port_queues(pi);
3460         if (rc != 0)
3461                 goto done;      /* error message displayed already */
3462
3463         /*
3464          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3465          */
3466         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3467         for (i = 0; i < pi->rss_size;) {
3468                 for_each_rxq(pi, j, rxq) {
3469                         rss[i++] = rxq->iq.abs_id;
3470                         if (i == pi->rss_size)
3471                                 break;
3472                 }
3473         }
3474
3475         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3476             pi->rss_size);
3477         if (rc != 0) {
3478                 if_printf(ifp, "rss_config failed: %d\n", rc);
3479                 goto done;
3480         }
3481
3482         pi->rss = rss;
3483         pi->flags |= PORT_INIT_DONE;
3484 done:
3485         if (rc != 0)
3486                 port_full_uninit(pi);
3487
3488         return (rc);
3489 }
3490
3491 /*
3492  * Idempotent.
3493  */
3494 int
3495 port_full_uninit(struct port_info *pi)
3496 {
3497         struct adapter *sc = pi->adapter;
3498         int i;
3499         struct sge_rxq *rxq;
3500         struct sge_txq *txq;
3501 #ifdef TCP_OFFLOAD
3502         struct sge_ofld_rxq *ofld_rxq;
3503         struct sge_wrq *ofld_txq;
3504 #endif
3505
3506         if (pi->flags & PORT_INIT_DONE) {
3507
3508                 /* Need to quiesce queues.  */
3509
3510                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3511
3512                 for_each_txq(pi, i, txq) {
3513                         quiesce_txq(sc, txq);
3514                 }
3515
3516 #ifdef TCP_OFFLOAD
3517                 for_each_ofld_txq(pi, i, ofld_txq) {
3518                         quiesce_wrq(sc, ofld_txq);
3519                 }
3520 #endif
3521
3522                 for_each_rxq(pi, i, rxq) {
3523                         quiesce_iq(sc, &rxq->iq);
3524                         quiesce_fl(sc, &rxq->fl);
3525                 }
3526
3527 #ifdef TCP_OFFLOAD
3528                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3529                         quiesce_iq(sc, &ofld_rxq->iq);
3530                         quiesce_fl(sc, &ofld_rxq->fl);
3531                 }
3532 #endif
3533                 free(pi->rss, M_CXGBE);
3534         }
3535
3536         t4_teardown_port_queues(pi);
3537         pi->flags &= ~PORT_INIT_DONE;
3538
3539         return (0);
3540 }
3541
3542 static void
3543 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3544 {
3545         struct sge_eq *eq = &txq->eq;
3546         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3547
3548         (void) sc;      /* unused */
3549
3550 #ifdef INVARIANTS
3551         TXQ_LOCK(txq);
3552         MPASS((eq->flags & EQ_ENABLED) == 0);
3553         TXQ_UNLOCK(txq);
3554 #endif
3555
3556         /* Wait for the mp_ring to empty. */
3557         while (!mp_ring_is_idle(txq->r)) {
3558                 mp_ring_check_drainage(txq->r, 0);
3559                 pause("rquiesce", 1);
3560         }
3561
3562         /* Then wait for the hardware to finish. */
3563         while (spg->cidx != htobe16(eq->pidx))
3564                 pause("equiesce", 1);
3565
3566         /* Finally, wait for the driver to reclaim all descriptors. */
3567         while (eq->cidx != eq->pidx)
3568                 pause("dquiesce", 1);
3569 }
3570
3571 static void
3572 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3573 {
3574
3575         /* XXXTX */
3576 }
3577
3578 static void
3579 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3580 {
3581         (void) sc;      /* unused */
3582
3583         /* Synchronize with the interrupt handler */
3584         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3585                 pause("iqfree", 1);
3586 }
3587
3588 static void
3589 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3590 {
3591         mtx_lock(&sc->sfl_lock);
3592         FL_LOCK(fl);
3593         fl->flags |= FL_DOOMED;
3594         FL_UNLOCK(fl);
3595         mtx_unlock(&sc->sfl_lock);
3596
3597         callout_drain(&sc->sfl_callout);
3598         KASSERT((fl->flags & FL_STARVING) == 0,
3599             ("%s: still starving", __func__));
3600 }
3601
3602 static int
3603 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3604     driver_intr_t *handler, void *arg, char *name)
3605 {
3606         int rc;
3607
3608         irq->rid = rid;
3609         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3610             RF_SHAREABLE | RF_ACTIVE);
3611         if (irq->res == NULL) {
3612                 device_printf(sc->dev,
3613                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3614                 return (ENOMEM);
3615         }
3616
3617         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3618             NULL, handler, arg, &irq->tag);
3619         if (rc != 0) {
3620                 device_printf(sc->dev,
3621                     "failed to setup interrupt for rid %d, name %s: %d\n",
3622                     rid, name, rc);
3623         } else if (name)
3624                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3625
3626         return (rc);
3627 }
3628
3629 static int
3630 t4_free_irq(struct adapter *sc, struct irq *irq)
3631 {
3632         if (irq->tag)
3633                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3634         if (irq->res)
3635                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3636
3637         bzero(irq, sizeof(*irq));
3638
3639         return (0);
3640 }
3641
3642 static void
3643 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3644     unsigned int end)
3645 {
3646         uint32_t *p = (uint32_t *)(buf + start);
3647
3648         for ( ; start <= end; start += sizeof(uint32_t))
3649                 *p++ = t4_read_reg(sc, start);
3650 }
3651
3652 static void
3653 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3654 {
3655         int i, n;
3656         const unsigned int *reg_ranges;
3657         static const unsigned int t4_reg_ranges[] = {
3658                 0x1008, 0x1108,
3659                 0x1180, 0x11b4,
3660                 0x11fc, 0x123c,
3661                 0x1300, 0x173c,
3662                 0x1800, 0x18fc,
3663                 0x3000, 0x30d8,
3664                 0x30e0, 0x5924,
3665                 0x5960, 0x59d4,
3666                 0x5a00, 0x5af8,
3667                 0x6000, 0x6098,
3668                 0x6100, 0x6150,
3669                 0x6200, 0x6208,
3670                 0x6240, 0x6248,
3671                 0x6280, 0x6338,
3672                 0x6370, 0x638c,
3673                 0x6400, 0x643c,
3674                 0x6500, 0x6524,
3675                 0x6a00, 0x6a38,
3676                 0x6a60, 0x6a78,
3677                 0x6b00, 0x6b84,
3678                 0x6bf0, 0x6c84,
3679                 0x6cf0, 0x6d84,
3680                 0x6df0, 0x6e84,
3681                 0x6ef0, 0x6f84,
3682                 0x6ff0, 0x7084,
3683                 0x70f0, 0x7184,
3684                 0x71f0, 0x7284,
3685                 0x72f0, 0x7384,
3686                 0x73f0, 0x7450,
3687                 0x7500, 0x7530,
3688                 0x7600, 0x761c,
3689                 0x7680, 0x76cc,
3690                 0x7700, 0x7798,
3691                 0x77c0, 0x77fc,
3692                 0x7900, 0x79fc,
3693                 0x7b00, 0x7c38,
3694                 0x7d00, 0x7efc,
3695                 0x8dc0, 0x8e1c,
3696                 0x8e30, 0x8e78,
3697                 0x8ea0, 0x8f6c,
3698                 0x8fc0, 0x9074,
3699                 0x90fc, 0x90fc,
3700                 0x9400, 0x9458,
3701                 0x9600, 0x96bc,
3702                 0x9800, 0x9808,
3703                 0x9820, 0x983c,
3704                 0x9850, 0x9864,
3705                 0x9c00, 0x9c6c,
3706                 0x9c80, 0x9cec,
3707                 0x9d00, 0x9d6c,
3708                 0x9d80, 0x9dec,
3709                 0x9e00, 0x9e6c,
3710                 0x9e80, 0x9eec,
3711                 0x9f00, 0x9f6c,
3712                 0x9f80, 0x9fec,
3713                 0xd004, 0xd03c,
3714                 0xdfc0, 0xdfe0,
3715                 0xe000, 0xea7c,
3716                 0xf000, 0x11110,
3717                 0x11118, 0x11190,
3718                 0x19040, 0x1906c,
3719                 0x19078, 0x19080,
3720                 0x1908c, 0x19124,
3721                 0x19150, 0x191b0,
3722                 0x191d0, 0x191e8,
3723                 0x19238, 0x1924c,
3724                 0x193f8, 0x19474,
3725                 0x19490, 0x194f8,
3726                 0x19800, 0x19f30,
3727                 0x1a000, 0x1a06c,
3728                 0x1a0b0, 0x1a120,
3729                 0x1a128, 0x1a138,
3730                 0x1a190, 0x1a1c4,
3731                 0x1a1fc, 0x1a1fc,
3732                 0x1e040, 0x1e04c,
3733                 0x1e284, 0x1e28c,
3734                 0x1e2c0, 0x1e2c0,
3735                 0x1e2e0, 0x1e2e0,
3736                 0x1e300, 0x1e384,
3737                 0x1e3c0, 0x1e3c8,
3738                 0x1e440, 0x1e44c,
3739                 0x1e684, 0x1e68c,
3740                 0x1e6c0, 0x1e6c0,
3741                 0x1e6e0, 0x1e6e0,
3742                 0x1e700, 0x1e784,
3743                 0x1e7c0, 0x1e7c8,
3744                 0x1e840, 0x1e84c,
3745                 0x1ea84, 0x1ea8c,
3746                 0x1eac0, 0x1eac0,
3747                 0x1eae0, 0x1eae0,
3748                 0x1eb00, 0x1eb84,
3749                 0x1ebc0, 0x1ebc8,
3750                 0x1ec40, 0x1ec4c,
3751                 0x1ee84, 0x1ee8c,
3752                 0x1eec0, 0x1eec0,
3753                 0x1eee0, 0x1eee0,
3754                 0x1ef00, 0x1ef84,
3755                 0x1efc0, 0x1efc8,
3756                 0x1f040, 0x1f04c,
3757                 0x1f284, 0x1f28c,
3758                 0x1f2c0, 0x1f2c0,
3759                 0x1f2e0, 0x1f2e0,
3760                 0x1f300, 0x1f384,
3761                 0x1f3c0, 0x1f3c8,
3762                 0x1f440, 0x1f44c,
3763                 0x1f684, 0x1f68c,
3764                 0x1f6c0, 0x1f6c0,
3765                 0x1f6e0, 0x1f6e0,
3766                 0x1f700, 0x1f784,
3767                 0x1f7c0, 0x1f7c8,
3768                 0x1f840, 0x1f84c,
3769                 0x1fa84, 0x1fa8c,
3770                 0x1fac0, 0x1fac0,
3771                 0x1fae0, 0x1fae0,
3772                 0x1fb00, 0x1fb84,
3773                 0x1fbc0, 0x1fbc8,
3774                 0x1fc40, 0x1fc4c,
3775                 0x1fe84, 0x1fe8c,
3776                 0x1fec0, 0x1fec0,
3777                 0x1fee0, 0x1fee0,
3778                 0x1ff00, 0x1ff84,
3779                 0x1ffc0, 0x1ffc8,
3780                 0x20000, 0x2002c,
3781                 0x20100, 0x2013c,
3782                 0x20190, 0x201c8,
3783                 0x20200, 0x20318,
3784                 0x20400, 0x20528,
3785                 0x20540, 0x20614,
3786                 0x21000, 0x21040,
3787                 0x2104c, 0x21060,
3788                 0x210c0, 0x210ec,
3789                 0x21200, 0x21268,
3790                 0x21270, 0x21284,
3791                 0x212fc, 0x21388,
3792                 0x21400, 0x21404,
3793                 0x21500, 0x21518,
3794                 0x2152c, 0x2153c,
3795                 0x21550, 0x21554,
3796                 0x21600, 0x21600,
3797                 0x21608, 0x21628,
3798                 0x21630, 0x2163c,
3799                 0x21700, 0x2171c,
3800                 0x21780, 0x2178c,
3801                 0x21800, 0x21c38,
3802                 0x21c80, 0x21d7c,
3803                 0x21e00, 0x21e04,
3804                 0x22000, 0x2202c,
3805                 0x22100, 0x2213c,
3806                 0x22190, 0x221c8,
3807                 0x22200, 0x22318,
3808                 0x22400, 0x22528,
3809                 0x22540, 0x22614,
3810                 0x23000, 0x23040,
3811                 0x2304c, 0x23060,
3812                 0x230c0, 0x230ec,
3813                 0x23200, 0x23268,
3814                 0x23270, 0x23284,
3815                 0x232fc, 0x23388,
3816                 0x23400, 0x23404,
3817                 0x23500, 0x23518,
3818                 0x2352c, 0x2353c,
3819                 0x23550, 0x23554,
3820                 0x23600, 0x23600,
3821                 0x23608, 0x23628,
3822                 0x23630, 0x2363c,
3823                 0x23700, 0x2371c,
3824                 0x23780, 0x2378c,
3825                 0x23800, 0x23c38,
3826                 0x23c80, 0x23d7c,
3827                 0x23e00, 0x23e04,
3828                 0x24000, 0x2402c,
3829                 0x24100, 0x2413c,
3830                 0x24190, 0x241c8,
3831                 0x24200, 0x24318,
3832                 0x24400, 0x24528,
3833                 0x24540, 0x24614,
3834                 0x25000, 0x25040,
3835                 0x2504c, 0x25060,
3836                 0x250c0, 0x250ec,
3837                 0x25200, 0x25268,
3838                 0x25270, 0x25284,
3839                 0x252fc, 0x25388,
3840                 0x25400, 0x25404,
3841                 0x25500, 0x25518,
3842                 0x2552c, 0x2553c,
3843                 0x25550, 0x25554,
3844                 0x25600, 0x25600,
3845                 0x25608, 0x25628,
3846                 0x25630, 0x2563c,
3847                 0x25700, 0x2571c,
3848                 0x25780, 0x2578c,
3849                 0x25800, 0x25c38,
3850                 0x25c80, 0x25d7c,
3851                 0x25e00, 0x25e04,
3852                 0x26000, 0x2602c,
3853                 0x26100, 0x2613c,
3854                 0x26190, 0x261c8,
3855                 0x26200, 0x26318,
3856                 0x26400, 0x26528,
3857                 0x26540, 0x26614,
3858                 0x27000, 0x27040,
3859                 0x2704c, 0x27060,
3860                 0x270c0, 0x270ec,
3861                 0x27200, 0x27268,
3862                 0x27270, 0x27284,
3863                 0x272fc, 0x27388,
3864                 0x27400, 0x27404,
3865                 0x27500, 0x27518,
3866                 0x2752c, 0x2753c,
3867                 0x27550, 0x27554,
3868                 0x27600, 0x27600,
3869                 0x27608, 0x27628,
3870                 0x27630, 0x2763c,
3871                 0x27700, 0x2771c,
3872                 0x27780, 0x2778c,
3873                 0x27800, 0x27c38,
3874                 0x27c80, 0x27d7c,
3875                 0x27e00, 0x27e04
3876         };
3877         static const unsigned int t5_reg_ranges[] = {
3878                 0x1008, 0x1148,
3879                 0x1180, 0x11b4,
3880                 0x11fc, 0x123c,
3881                 0x1280, 0x173c,
3882                 0x1800, 0x18fc,
3883                 0x3000, 0x3028,
3884                 0x3060, 0x30d8,
3885                 0x30e0, 0x30fc,
3886                 0x3140, 0x357c,
3887                 0x35a8, 0x35cc,
3888                 0x35ec, 0x35ec,
3889                 0x3600, 0x5624,
3890                 0x56cc, 0x575c,
3891                 0x580c, 0x5814,
3892                 0x5890, 0x58bc,
3893                 0x5940, 0x59dc,
3894                 0x59fc, 0x5a18,
3895                 0x5a60, 0x5a9c,
3896                 0x5b94, 0x5bfc,
3897                 0x6000, 0x6040,
3898                 0x6058, 0x614c,
3899                 0x7700, 0x7798,
3900                 0x77c0, 0x78fc,
3901                 0x7b00, 0x7c54,
3902                 0x7d00, 0x7efc,
3903                 0x8dc0, 0x8de0,
3904                 0x8df8, 0x8e84,
3905                 0x8ea0, 0x8f84,
3906                 0x8fc0, 0x90f8,
3907                 0x9400, 0x9470,
3908                 0x9600, 0x96f4,
3909                 0x9800, 0x9808,
3910                 0x9820, 0x983c,
3911                 0x9850, 0x9864,
3912                 0x9c00, 0x9c6c,
3913                 0x9c80, 0x9cec,
3914                 0x9d00, 0x9d6c,
3915                 0x9d80, 0x9dec,
3916                 0x9e00, 0x9e6c,
3917                 0x9e80, 0x9eec,
3918                 0x9f00, 0x9f6c,
3919                 0x9f80, 0xa020,
3920                 0xd004, 0xd03c,
3921                 0xdfc0, 0xdfe0,
3922                 0xe000, 0x11088,
3923                 0x1109c, 0x11110,
3924                 0x11118, 0x1117c,
3925                 0x11190, 0x11204,
3926                 0x19040, 0x1906c,
3927                 0x19078, 0x19080,
3928                 0x1908c, 0x19124,
3929                 0x19150, 0x191b0,
3930                 0x191d0, 0x191e8,
3931                 0x19238, 0x19290,
3932                 0x193f8, 0x19474,
3933                 0x19490, 0x194cc,
3934                 0x194f0, 0x194f8,
3935                 0x19c00, 0x19c60,
3936                 0x19c94, 0x19e10,
3937                 0x19e50, 0x19f34,
3938                 0x19f40, 0x19f50,
3939                 0x19f90, 0x19fe4,
3940                 0x1a000, 0x1a06c,
3941                 0x1a0b0, 0x1a120,
3942                 0x1a128, 0x1a138,
3943                 0x1a190, 0x1a1c4,
3944                 0x1a1fc, 0x1a1fc,
3945                 0x1e008, 0x1e00c,
3946                 0x1e040, 0x1e04c,
3947                 0x1e284, 0x1e290,
3948                 0x1e2c0, 0x1e2c0,
3949                 0x1e2e0, 0x1e2e0,
3950                 0x1e300, 0x1e384,
3951                 0x1e3c0, 0x1e3c8,
3952                 0x1e408, 0x1e40c,
3953                 0x1e440, 0x1e44c,
3954                 0x1e684, 0x1e690,
3955                 0x1e6c0, 0x1e6c0,
3956                 0x1e6e0, 0x1e6e0,
3957                 0x1e700, 0x1e784,
3958                 0x1e7c0, 0x1e7c8,
3959                 0x1e808, 0x1e80c,
3960                 0x1e840, 0x1e84c,
3961                 0x1ea84, 0x1ea90,
3962                 0x1eac0, 0x1eac0,
3963                 0x1eae0, 0x1eae0,
3964                 0x1eb00, 0x1eb84,
3965                 0x1ebc0, 0x1ebc8,
3966                 0x1ec08, 0x1ec0c,
3967                 0x1ec40, 0x1ec4c,
3968                 0x1ee84, 0x1ee90,
3969                 0x1eec0, 0x1eec0,
3970                 0x1eee0, 0x1eee0,
3971                 0x1ef00, 0x1ef84,
3972                 0x1efc0, 0x1efc8,
3973                 0x1f008, 0x1f00c,
3974                 0x1f040, 0x1f04c,
3975                 0x1f284, 0x1f290,
3976                 0x1f2c0, 0x1f2c0,
3977                 0x1f2e0, 0x1f2e0,
3978                 0x1f300, 0x1f384,
3979                 0x1f3c0, 0x1f3c8,
3980                 0x1f408, 0x1f40c,
3981                 0x1f440, 0x1f44c,
3982                 0x1f684, 0x1f690,
3983                 0x1f6c0, 0x1f6c0,
3984                 0x1f6e0, 0x1f6e0,
3985                 0x1f700, 0x1f784,
3986                 0x1f7c0, 0x1f7c8,
3987                 0x1f808, 0x1f80c,
3988                 0x1f840, 0x1f84c,
3989                 0x1fa84, 0x1fa90,
3990                 0x1fac0, 0x1fac0,
3991                 0x1fae0, 0x1fae0,
3992                 0x1fb00, 0x1fb84,
3993                 0x1fbc0, 0x1fbc8,
3994                 0x1fc08, 0x1fc0c,
3995                 0x1fc40, 0x1fc4c,
3996                 0x1fe84, 0x1fe90,
3997                 0x1fec0, 0x1fec0,
3998                 0x1fee0, 0x1fee0,
3999                 0x1ff00, 0x1ff84,
4000                 0x1ffc0, 0x1ffc8,
4001                 0x30000, 0x30030,
4002                 0x30100, 0x30144,
4003                 0x30190, 0x301d0,
4004                 0x30200, 0x30318,
4005                 0x30400, 0x3052c,
4006                 0x30540, 0x3061c,
4007                 0x30800, 0x30834,
4008                 0x308c0, 0x30908,
4009                 0x30910, 0x309ac,
4010                 0x30a00, 0x30a2c,
4011                 0x30a44, 0x30a50,
4012                 0x30a74, 0x30c24,
4013                 0x30d00, 0x30d00,
4014                 0x30d08, 0x30d14,
4015                 0x30d1c, 0x30d20,
4016                 0x30d3c, 0x30d50,
4017                 0x31200, 0x3120c,
4018                 0x31220, 0x31220,
4019                 0x31240, 0x31240,
4020                 0x31600, 0x3160c,
4021                 0x31a00, 0x31a1c,
4022                 0x31e00, 0x31e20,
4023                 0x31e38, 0x31e3c,
4024                 0x31e80, 0x31e80,
4025                 0x31e88, 0x31ea8,
4026                 0x31eb0, 0x31eb4,
4027                 0x31ec8, 0x31ed4,
4028                 0x31fb8, 0x32004,
4029                 0x32200, 0x32200,
4030                 0x32208, 0x32240,
4031                 0x32248, 0x32280,
4032                 0x32288, 0x322c0,
4033                 0x322c8, 0x322fc,
4034                 0x32600, 0x32630,
4035                 0x32a00, 0x32abc,
4036                 0x32b00, 0x32b70,
4037                 0x33000, 0x33048,
4038                 0x33060, 0x3309c,
4039                 0x330f0, 0x33148,
4040                 0x33160, 0x3319c,
4041                 0x331f0, 0x332e4,
4042                 0x332f8, 0x333e4,
4043                 0x333f8, 0x33448,
4044                 0x33460, 0x3349c,
4045                 0x334f0, 0x33548,
4046                 0x33560, 0x3359c,
4047                 0x335f0, 0x336e4,
4048                 0x336f8, 0x337e4,
4049                 0x337f8, 0x337fc,
4050                 0x33814, 0x33814,
4051                 0x3382c, 0x3382c,
4052                 0x33880, 0x3388c,
4053                 0x338e8, 0x338ec,
4054                 0x33900, 0x33948,
4055                 0x33960, 0x3399c,
4056                 0x339f0, 0x33ae4,
4057                 0x33af8, 0x33b10,
4058                 0x33b28, 0x33b28,
4059                 0x33b3c, 0x33b50,
4060                 0x33bf0, 0x33c10,
4061                 0x33c28, 0x33c28,
4062                 0x33c3c, 0x33c50,
4063                 0x33cf0, 0x33cfc,
4064                 0x34000, 0x34030,
4065                 0x34100, 0x34144,
4066                 0x34190, 0x341d0,
4067                 0x34200, 0x34318,
4068                 0x34400, 0x3452c,
4069                 0x34540, 0x3461c,
4070                 0x34800, 0x34834,
4071                 0x348c0, 0x34908,
4072                 0x34910, 0x349ac,
4073                 0x34a00, 0x34a2c,
4074                 0x34a44, 0x34a50,
4075                 0x34a74, 0x34c24,
4076                 0x34d00, 0x34d00,
4077                 0x34d08, 0x34d14,
4078                 0x34d1c, 0x34d20,
4079                 0x34d3c, 0x34d50,
4080                 0x35200, 0x3520c,
4081                 0x35220, 0x35220,
4082                 0x35240, 0x35240,
4083                 0x35600, 0x3560c,
4084                 0x35a00, 0x35a1c,
4085                 0x35e00, 0x35e20,
4086                 0x35e38, 0x35e3c,
4087                 0x35e80, 0x35e80,
4088                 0x35e88, 0x35ea8,
4089                 0x35eb0, 0x35eb4,
4090                 0x35ec8, 0x35ed4,
4091                 0x35fb8, 0x36004,
4092                 0x36200, 0x36200,
4093                 0x36208, 0x36240,
4094                 0x36248, 0x36280,
4095                 0x36288, 0x362c0,
4096                 0x362c8, 0x362fc,
4097                 0x36600, 0x36630,
4098                 0x36a00, 0x36abc,
4099                 0x36b00, 0x36b70,
4100                 0x37000, 0x37048,
4101                 0x37060, 0x3709c,
4102                 0x370f0, 0x37148,
4103                 0x37160, 0x3719c,
4104                 0x371f0, 0x372e4,
4105                 0x372f8, 0x373e4,
4106                 0x373f8, 0x37448,
4107                 0x37460, 0x3749c,
4108                 0x374f0, 0x37548,
4109                 0x37560, 0x3759c,
4110                 0x375f0, 0x376e4,
4111                 0x376f8, 0x377e4,
4112                 0x377f8, 0x377fc,
4113                 0x37814, 0x37814,
4114                 0x3782c, 0x3782c,
4115                 0x37880, 0x3788c,
4116                 0x378e8, 0x378ec,
4117                 0x37900, 0x37948,
4118                 0x37960, 0x3799c,
4119                 0x379f0, 0x37ae4,
4120                 0x37af8, 0x37b10,
4121                 0x37b28, 0x37b28,
4122                 0x37b3c, 0x37b50,
4123                 0x37bf0, 0x37c10,
4124                 0x37c28, 0x37c28,
4125                 0x37c3c, 0x37c50,
4126                 0x37cf0, 0x37cfc,
4127                 0x38000, 0x38030,
4128                 0x38100, 0x38144,
4129                 0x38190, 0x381d0,
4130                 0x38200, 0x38318,
4131                 0x38400, 0x3852c,
4132                 0x38540, 0x3861c,
4133                 0x38800, 0x38834,
4134                 0x388c0, 0x38908,
4135                 0x38910, 0x389ac,
4136                 0x38a00, 0x38a2c,
4137                 0x38a44, 0x38a50,
4138                 0x38a74, 0x38c24,
4139                 0x38d00, 0x38d00,
4140                 0x38d08, 0x38d14,
4141                 0x38d1c, 0x38d20,
4142                 0x38d3c, 0x38d50,
4143                 0x39200, 0x3920c,
4144                 0x39220, 0x39220,
4145                 0x39240, 0x39240,
4146                 0x39600, 0x3960c,
4147                 0x39a00, 0x39a1c,
4148                 0x39e00, 0x39e20,
4149                 0x39e38, 0x39e3c,
4150                 0x39e80, 0x39e80,
4151                 0x39e88, 0x39ea8,
4152                 0x39eb0, 0x39eb4,
4153                 0x39ec8, 0x39ed4,
4154                 0x39fb8, 0x3a004,
4155                 0x3a200, 0x3a200,
4156                 0x3a208, 0x3a240,
4157                 0x3a248, 0x3a280,
4158                 0x3a288, 0x3a2c0,
4159                 0x3a2c8, 0x3a2fc,
4160                 0x3a600, 0x3a630,
4161                 0x3aa00, 0x3aabc,
4162                 0x3ab00, 0x3ab70,
4163                 0x3b000, 0x3b048,
4164                 0x3b060, 0x3b09c,
4165                 0x3b0f0, 0x3b148,
4166                 0x3b160, 0x3b19c,
4167                 0x3b1f0, 0x3b2e4,
4168                 0x3b2f8, 0x3b3e4,
4169                 0x3b3f8, 0x3b448,
4170                 0x3b460, 0x3b49c,
4171                 0x3b4f0, 0x3b548,
4172                 0x3b560, 0x3b59c,
4173                 0x3b5f0, 0x3b6e4,
4174                 0x3b6f8, 0x3b7e4,
4175                 0x3b7f8, 0x3b7fc,
4176                 0x3b814, 0x3b814,
4177                 0x3b82c, 0x3b82c,
4178                 0x3b880, 0x3b88c,
4179                 0x3b8e8, 0x3b8ec,
4180                 0x3b900, 0x3b948,
4181                 0x3b960, 0x3b99c,
4182                 0x3b9f0, 0x3bae4,
4183                 0x3baf8, 0x3bb10,
4184                 0x3bb28, 0x3bb28,
4185                 0x3bb3c, 0x3bb50,
4186                 0x3bbf0, 0x3bc10,
4187                 0x3bc28, 0x3bc28,
4188                 0x3bc3c, 0x3bc50,
4189                 0x3bcf0, 0x3bcfc,
4190                 0x3c000, 0x3c030,
4191                 0x3c100, 0x3c144,
4192                 0x3c190, 0x3c1d0,
4193                 0x3c200, 0x3c318,
4194                 0x3c400, 0x3c52c,
4195                 0x3c540, 0x3c61c,
4196                 0x3c800, 0x3c834,
4197                 0x3c8c0, 0x3c908,
4198                 0x3c910, 0x3c9ac,
4199                 0x3ca00, 0x3ca2c,
4200                 0x3ca44, 0x3ca50,
4201                 0x3ca74, 0x3cc24,
4202                 0x3cd00, 0x3cd00,
4203                 0x3cd08, 0x3cd14,
4204                 0x3cd1c, 0x3cd20,
4205                 0x3cd3c, 0x3cd50,
4206                 0x3d200, 0x3d20c,
4207                 0x3d220, 0x3d220,
4208                 0x3d240, 0x3d240,
4209                 0x3d600, 0x3d60c,
4210                 0x3da00, 0x3da1c,
4211                 0x3de00, 0x3de20,
4212                 0x3de38, 0x3de3c,
4213                 0x3de80, 0x3de80,
4214                 0x3de88, 0x3dea8,
4215                 0x3deb0, 0x3deb4,
4216                 0x3dec8, 0x3ded4,
4217                 0x3dfb8, 0x3e004,
4218                 0x3e200, 0x3e200,
4219                 0x3e208, 0x3e240,
4220                 0x3e248, 0x3e280,
4221                 0x3e288, 0x3e2c0,
4222                 0x3e2c8, 0x3e2fc,
4223                 0x3e600, 0x3e630,
4224                 0x3ea00, 0x3eabc,
4225                 0x3eb00, 0x3eb70,
4226                 0x3f000, 0x3f048,
4227                 0x3f060, 0x3f09c,
4228                 0x3f0f0, 0x3f148,
4229                 0x3f160, 0x3f19c,
4230                 0x3f1f0, 0x3f2e4,
4231                 0x3f2f8, 0x3f3e4,
4232                 0x3f3f8, 0x3f448,
4233                 0x3f460, 0x3f49c,
4234                 0x3f4f0, 0x3f548,
4235                 0x3f560, 0x3f59c,
4236                 0x3f5f0, 0x3f6e4,
4237                 0x3f6f8, 0x3f7e4,
4238                 0x3f7f8, 0x3f7fc,
4239                 0x3f814, 0x3f814,
4240                 0x3f82c, 0x3f82c,
4241                 0x3f880, 0x3f88c,
4242                 0x3f8e8, 0x3f8ec,
4243                 0x3f900, 0x3f948,
4244                 0x3f960, 0x3f99c,
4245                 0x3f9f0, 0x3fae4,
4246                 0x3faf8, 0x3fb10,
4247                 0x3fb28, 0x3fb28,
4248                 0x3fb3c, 0x3fb50,
4249                 0x3fbf0, 0x3fc10,
4250                 0x3fc28, 0x3fc28,
4251                 0x3fc3c, 0x3fc50,
4252                 0x3fcf0, 0x3fcfc,
4253                 0x40000, 0x4000c,
4254                 0x40040, 0x40068,
4255                 0x4007c, 0x40144,
4256                 0x40180, 0x4018c,
4257                 0x40200, 0x40298,
4258                 0x402ac, 0x4033c,
4259                 0x403f8, 0x403fc,
4260                 0x41304, 0x413c4,
4261                 0x41400, 0x4141c,
4262                 0x41480, 0x414d0,
4263                 0x44000, 0x44078,
4264                 0x440c0, 0x44278,
4265                 0x442c0, 0x44478,
4266                 0x444c0, 0x44678,
4267                 0x446c0, 0x44878,
4268                 0x448c0, 0x449fc,
4269                 0x45000, 0x45068,
4270                 0x45080, 0x45084,
4271                 0x450a0, 0x450b0,
4272                 0x45200, 0x45268,
4273                 0x45280, 0x45284,
4274                 0x452a0, 0x452b0,
4275                 0x460c0, 0x460e4,
4276                 0x47000, 0x4708c,
4277                 0x47200, 0x47250,
4278                 0x47400, 0x47420,
4279                 0x47600, 0x47618,
4280                 0x47800, 0x47814,
4281                 0x48000, 0x4800c,
4282                 0x48040, 0x48068,
4283                 0x4807c, 0x48144,
4284                 0x48180, 0x4818c,
4285                 0x48200, 0x48298,
4286                 0x482ac, 0x4833c,
4287                 0x483f8, 0x483fc,
4288                 0x49304, 0x493c4,
4289                 0x49400, 0x4941c,
4290                 0x49480, 0x494d0,
4291                 0x4c000, 0x4c078,
4292                 0x4c0c0, 0x4c278,
4293                 0x4c2c0, 0x4c478,
4294                 0x4c4c0, 0x4c678,
4295                 0x4c6c0, 0x4c878,
4296                 0x4c8c0, 0x4c9fc,
4297                 0x4d000, 0x4d068,
4298                 0x4d080, 0x4d084,
4299                 0x4d0a0, 0x4d0b0,
4300                 0x4d200, 0x4d268,
4301                 0x4d280, 0x4d284,
4302                 0x4d2a0, 0x4d2b0,
4303                 0x4e0c0, 0x4e0e4,
4304                 0x4f000, 0x4f08c,
4305                 0x4f200, 0x4f250,
4306                 0x4f400, 0x4f420,
4307                 0x4f600, 0x4f618,
4308                 0x4f800, 0x4f814,
4309                 0x50000, 0x500cc,
4310                 0x50400, 0x50400,
4311                 0x50800, 0x508cc,
4312                 0x50c00, 0x50c00,
4313                 0x51000, 0x5101c,
4314                 0x51300, 0x51308,
4315         };
4316
4317         if (is_t4(sc)) {
4318                 reg_ranges = &t4_reg_ranges[0];
4319                 n = nitems(t4_reg_ranges);
4320         } else {
4321                 reg_ranges = &t5_reg_ranges[0];
4322                 n = nitems(t5_reg_ranges);
4323         }
4324
4325         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4326         for (i = 0; i < n; i += 2)
4327                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4328 }
4329
4330 static void
4331 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4332 {
4333         struct ifnet *ifp = pi->ifp;
4334         struct sge_txq *txq;
4335         int i, drops;
4336         struct port_stats *s = &pi->stats;
4337         struct timeval tv;
4338         const struct timeval interval = {0, 250000};    /* 250ms */
4339
4340         getmicrotime(&tv);
4341         timevalsub(&tv, &interval);
4342         if (timevalcmp(&tv, &pi->last_refreshed, <))
4343                 return;
4344
4345         t4_get_port_stats(sc, pi->tx_chan, s);
4346
4347         ifp->if_opackets = s->tx_frames - s->tx_pause;
4348         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4349         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4350         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4351         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4352         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4353         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4354             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4355             s->rx_trunc3;
4356         for (i = 0; i < NCHAN; i++) {
4357                 if (pi->rx_chan_map & (1 << i)) {
4358                         uint32_t v;
4359
4360                         mtx_lock(&sc->regwin_lock);
4361                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4362                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4363                         mtx_unlock(&sc->regwin_lock);
4364                         ifp->if_iqdrops += v;
4365                 }
4366         }
4367
4368         drops = s->tx_drop;
4369         for_each_txq(pi, i, txq)
4370                 drops += counter_u64_fetch(txq->r->drops);
4371         ifp->if_snd.ifq_drops = drops;
4372
4373         ifp->if_oerrors = s->tx_error_frames;
4374         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4375             s->rx_fcs_err + s->rx_len_err;
4376
4377         getmicrotime(&pi->last_refreshed);
4378 }
4379
4380 static void
4381 cxgbe_tick(void *arg)
4382 {
4383         struct port_info *pi = arg;
4384         struct adapter *sc = pi->adapter;
4385         struct ifnet *ifp = pi->ifp;
4386
4387         PORT_LOCK(pi);
4388         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4389                 PORT_UNLOCK(pi);
4390                 return; /* without scheduling another callout */
4391         }
4392
4393         cxgbe_refresh_stats(sc, pi);
4394
4395         callout_schedule(&pi->tick, hz);
4396         PORT_UNLOCK(pi);
4397 }
4398
4399 static void
4400 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4401 {
4402         struct ifnet *vlan;
4403
4404         if (arg != ifp || ifp->if_type != IFT_ETHER)
4405                 return;
4406
4407         vlan = VLAN_DEVAT(ifp, vid);
4408         VLAN_SETCOOKIE(vlan, ifp);
4409 }
4410
4411 static int
4412 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4413 {
4414
4415 #ifdef INVARIANTS
4416         panic("%s: opcode 0x%02x on iq %p with payload %p",
4417             __func__, rss->opcode, iq, m);
4418 #else
4419         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4420             __func__, rss->opcode, iq, m);
4421         m_freem(m);
4422 #endif
4423         return (EDOOFUS);
4424 }
4425
4426 int
4427 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4428 {
4429         uintptr_t *loc, new;
4430
4431         if (opcode >= nitems(sc->cpl_handler))
4432                 return (EINVAL);
4433
4434         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4435         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4436         atomic_store_rel_ptr(loc, new);
4437
4438         return (0);
4439 }
4440
4441 static int
4442 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4443 {
4444
4445 #ifdef INVARIANTS
4446         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4447 #else
4448         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4449             __func__, iq, ctrl);
4450 #endif
4451         return (EDOOFUS);
4452 }
4453
4454 int
4455 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4456 {
4457         uintptr_t *loc, new;
4458
4459         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4460         loc = (uintptr_t *) &sc->an_handler;
4461         atomic_store_rel_ptr(loc, new);
4462
4463         return (0);
4464 }
4465
4466 static int
4467 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4468 {
4469         const struct cpl_fw6_msg *cpl =
4470             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4471
4472 #ifdef INVARIANTS
4473         panic("%s: fw_msg type %d", __func__, cpl->type);
4474 #else
4475         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4476 #endif
4477         return (EDOOFUS);
4478 }
4479
4480 int
4481 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4482 {
4483         uintptr_t *loc, new;
4484
4485         if (type >= nitems(sc->fw_msg_handler))
4486                 return (EINVAL);
4487
4488         /*
4489          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4490          * handler dispatch table.  Reject any attempt to install a handler for
4491          * this subtype.
4492          */
4493         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4494                 return (EINVAL);
4495
4496         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4497         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4498         atomic_store_rel_ptr(loc, new);
4499
4500         return (0);
4501 }
4502
4503 static int
4504 t4_sysctls(struct adapter *sc)
4505 {
4506         struct sysctl_ctx_list *ctx;
4507         struct sysctl_oid *oid;
4508         struct sysctl_oid_list *children, *c0;
4509         static char *caps[] = {
4510                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4511                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4512                     "\6HASHFILTER\7ETHOFLD",
4513                 "\20\1TOE",                             /* caps[2] toecaps */
4514                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4515                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4516                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4517                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4518                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4519                     "\4PO_INITIAOR\5PO_TARGET"
4520         };
4521         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4522
4523         ctx = device_get_sysctl_ctx(sc->dev);
4524
4525         /*
4526          * dev.t4nex.X.
4527          */
4528         oid = device_get_sysctl_tree(sc->dev);
4529         c0 = children = SYSCTL_CHILDREN(oid);
4530
4531         sc->sc_do_rxcopy = 1;
4532         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4533             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4534
4535         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4536             sc->params.nports, "# of ports");
4537
4538         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4539             NULL, chip_rev(sc), "chip hardware revision");
4540
4541         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4542             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4543
4544         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4545             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4546
4547         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4548             sc->cfcsum, "config file checksum");
4549
4550         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4551             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4552             sysctl_bitfield, "A", "available doorbells");
4553
4554         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4555             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4556             sysctl_bitfield, "A", "available link capabilities");
4557
4558         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4559             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4560             sysctl_bitfield, "A", "available NIC capabilities");
4561
4562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4563             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4564             sysctl_bitfield, "A", "available TCP offload capabilities");
4565
4566         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4567             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4568             sysctl_bitfield, "A", "available RDMA capabilities");
4569
4570         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4571             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4572             sysctl_bitfield, "A", "available iSCSI capabilities");
4573
4574         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4575             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4576             sysctl_bitfield, "A", "available FCoE capabilities");
4577
4578         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4579             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4582             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4583             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4584             "interrupt holdoff timer values (us)");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4588             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4589             "interrupt holdoff packet counter values");
4590
4591         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4592             NULL, sc->tids.nftids, "number of filters");
4593
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4595             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4596             "chip temperature (in Celsius)");
4597
4598         t4_sge_sysctls(sc, ctx, children);
4599
4600         sc->lro_timeout = 100;
4601         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4602             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4603
4604         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4605             &sc->debug_flags, 0, "flags to enable runtime debugging");
4606
4607 #ifdef SBUF_DRAIN
4608         /*
4609          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4610          */
4611         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4612             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4613             "logs and miscellaneous information");
4614         children = SYSCTL_CHILDREN(oid);
4615
4616         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4617             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4618             sysctl_cctrl, "A", "congestion control");
4619
4620         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4621             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4622             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4623
4624         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4625             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4626             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4627
4628         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4629             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4630             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4631
4632         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4633             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4634             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4635
4636         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4637             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4638             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4639
4640         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4641             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4642             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4643
4644         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4645             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4646             sysctl_cim_la, "A", "CIM logic analyzer");
4647
4648         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4649             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4650             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4651
4652         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4653             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4654             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4655
4656         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4657             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4658             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4659
4660         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4661             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4662             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4663
4664         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4665             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4666             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4667
4668         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4669             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4670             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4671
4672         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4673             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4674             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4675
4676         if (is_t5(sc)) {
4677                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4678                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4679                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4680
4681                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4682                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4683                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4684         }
4685
4686         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4687             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4688             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4689
4690         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4691             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4692             sysctl_cim_qcfg, "A", "CIM queue configuration");
4693
4694         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4695             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4696             sysctl_cpl_stats, "A", "CPL statistics");
4697
4698         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4699             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4700             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4701
4702         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4703             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4704             sysctl_devlog, "A", "firmware's device log");
4705
4706         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4707             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4708             sysctl_fcoe_stats, "A", "FCoE statistics");
4709
4710         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4711             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4712             sysctl_hw_sched, "A", "hardware scheduler ");
4713
4714         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4715             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4716             sysctl_l2t, "A", "hardware L2 table");
4717
4718         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4719             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4720             sysctl_lb_stats, "A", "loopback statistics");
4721
4722         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4723             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4724             sysctl_meminfo, "A", "memory regions");
4725
4726         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4727             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4728             sysctl_mps_tcam, "A", "MPS TCAM entries");
4729
4730         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4731             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4732             sysctl_path_mtus, "A", "path MTUs");
4733
4734         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4735             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4736             sysctl_pm_stats, "A", "PM statistics");
4737
4738         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4739             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4740             sysctl_rdma_stats, "A", "RDMA statistics");
4741
4742         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4743             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4744             sysctl_tcp_stats, "A", "TCP statistics");
4745
4746         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4747             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4748             sysctl_tids, "A", "TID information");
4749
4750         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4751             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4752             sysctl_tp_err_stats, "A", "TP error statistics");
4753
4754         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4755             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4756             sysctl_tp_la, "A", "TP logic analyzer");
4757
4758         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4759             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4760             sysctl_tx_rate, "A", "Tx rate");
4761
4762         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4763             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4764             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4765
4766         if (is_t5(sc)) {
4767                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4768                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4769                     sysctl_wcwr_stats, "A", "write combined work requests");
4770         }
4771 #endif
4772
4773 #ifdef TCP_OFFLOAD
4774         if (is_offload(sc)) {
4775                 /*
4776                  * dev.t4nex.X.toe.
4777                  */
4778                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4779                     NULL, "TOE parameters");
4780                 children = SYSCTL_CHILDREN(oid);
4781
4782                 sc->tt.sndbuf = 256 * 1024;
4783                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4784                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4785
4786                 sc->tt.ddp = 0;
4787                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4788                     &sc->tt.ddp, 0, "DDP allowed");
4789
4790                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4791                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4792                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4793
4794                 sc->tt.ddp_thres =
4795                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4796                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4797                     &sc->tt.ddp_thres, 0, "DDP threshold");
4798
4799                 sc->tt.rx_coalesce = 1;
4800                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4801                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4802
4803                 sc->tt.tx_align = 1;
4804                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4805                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4806         }
4807 #endif
4808
4809
4810         return (0);
4811 }
4812
4813 static int
4814 cxgbe_sysctls(struct port_info *pi)
4815 {
4816         struct sysctl_ctx_list *ctx;
4817         struct sysctl_oid *oid;
4818         struct sysctl_oid_list *children;
4819         struct adapter *sc = pi->adapter;
4820
4821         ctx = device_get_sysctl_ctx(pi->dev);
4822
4823         /*
4824          * dev.cxgbe.X.
4825          */
4826         oid = device_get_sysctl_tree(pi->dev);
4827         children = SYSCTL_CHILDREN(oid);
4828
4829         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4830            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4831         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4832                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4833                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4834                     "PHY temperature (in Celsius)");
4835                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4836                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4837                     "PHY firmware version");
4838         }
4839         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4840             &pi->nrxq, 0, "# of rx queues");
4841         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4842             &pi->ntxq, 0, "# of tx queues");
4843         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4844             &pi->first_rxq, 0, "index of first rx queue");
4845         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4846             &pi->first_txq, 0, "index of first tx queue");
4847         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4848             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4849             "Reserve queue 0 for non-flowid packets");
4850
4851 #ifdef TCP_OFFLOAD
4852         if (is_offload(sc)) {
4853                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4854                     &pi->nofldrxq, 0,
4855                     "# of rx queues for offloaded TCP connections");
4856                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4857                     &pi->nofldtxq, 0,
4858                     "# of tx queues for offloaded TCP connections");
4859                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4860                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4861                     "index of first TOE rx queue");
4862                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4863                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4864                     "index of first TOE tx queue");
4865         }
4866 #endif
4867 #ifdef DEV_NETMAP
4868         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4869             &pi->nnmrxq, 0, "# of rx queues for netmap");
4870         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4871             &pi->nnmtxq, 0, "# of tx queues for netmap");
4872         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4873             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4874             "index of first netmap rx queue");
4875         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4876             CTLFLAG_RD, &pi->first_nm_txq, 0,
4877             "index of first netmap tx queue");
4878 #endif
4879
4880         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4881             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4882             "holdoff timer index");
4883         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4884             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4885             "holdoff packet counter index");
4886
4887         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4888             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4889             "rx queue size");
4890         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4891             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4892             "tx queue size");
4893
4894         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4895             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4896             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4897
4898         /*
4899          * dev.cxgbe.X.stats.
4900          */
4901         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4902             NULL, "port statistics");
4903         children = SYSCTL_CHILDREN(oid);
4904         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4905             &pi->tx_parse_error, 0,
4906             "# of tx packets with invalid length or # of segments");
4907
4908 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4909         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4910             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4911             sysctl_handle_t4_reg64, "QU", desc)
4912
4913         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4915         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4917         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4919         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4921         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4923         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4925         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4926             "# of tx frames in this range",
4927             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4928         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4929             "# of tx frames in this range",
4930             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4931         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4932             "# of tx frames in this range",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4934         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4935             "# of tx frames in this range",
4936             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4937         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4938             "# of tx frames in this range",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4940         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4941             "# of tx frames in this range",
4942             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4943         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4944             "# of tx frames in this range",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4946         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4948         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4950         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4952         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4954         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4956         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4958         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4960         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4962         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4964         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4965             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4966
4967         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4968             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4969         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4971         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4972             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4973         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4974             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4975         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4976             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4977         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4979         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4980             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4981         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4982             "# of frames received with bad FCS",
4983             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4984         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4985             "# of frames received with length error",
4986             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4987         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4990             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4991         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4992             "# of rx frames in this range",
4993             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4994         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4995             "# of rx frames in this range",
4996             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4997         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4998             "# of rx frames in this range",
4999             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5000         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5001             "# of rx frames in this range",
5002             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5003         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5004             "# of rx frames in this range",
5005             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5006         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5007             "# of rx frames in this range",
5008             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5009         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5010             "# of rx frames in this range",
5011             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5012         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5013             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5014         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5015             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5016         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5017             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5018         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5019             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5020         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5021             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5022         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5023             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5024         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5025             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5026         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5027             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5028         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5029             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5030
5031 #undef SYSCTL_ADD_T4_REG64
5032
5033 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5034         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5035             &pi->stats.name, desc)
5036
5037         /* We get these from port_stats and they may be stale by upto 1s */
5038         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5039             "# drops due to buffer-group 0 overflows");
5040         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5041             "# drops due to buffer-group 1 overflows");
5042         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5043             "# drops due to buffer-group 2 overflows");
5044         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5045             "# drops due to buffer-group 3 overflows");
5046         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5047             "# of buffer-group 0 truncated packets");
5048         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5049             "# of buffer-group 1 truncated packets");
5050         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5051             "# of buffer-group 2 truncated packets");
5052         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5053             "# of buffer-group 3 truncated packets");
5054
5055 #undef SYSCTL_ADD_T4_PORTSTAT
5056
5057         return (0);
5058 }
5059
5060 static int
5061 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5062 {
5063         int rc, *i, space = 0;
5064         struct sbuf sb;
5065
5066         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5067         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5068                 if (space)
5069                         sbuf_printf(&sb, " ");
5070                 sbuf_printf(&sb, "%d", *i);
5071                 space = 1;
5072         }
5073         sbuf_finish(&sb);
5074         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5075         sbuf_delete(&sb);
5076         return (rc);
5077 }
5078
5079 static int
5080 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5081 {
5082         int rc;
5083         struct sbuf *sb;
5084
5085         rc = sysctl_wire_old_buffer(req, 0);
5086         if (rc != 0)
5087                 return(rc);
5088
5089         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5090         if (sb == NULL)
5091                 return (ENOMEM);
5092
5093         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5094         rc = sbuf_finish(sb);
5095         sbuf_delete(sb);
5096
5097         return (rc);
5098 }
5099
5100 static int
5101 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5102 {
5103         struct port_info *pi = arg1;
5104         int op = arg2;
5105         struct adapter *sc = pi->adapter;
5106         u_int v;
5107         int rc;
5108
5109         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5110         if (rc)
5111                 return (rc);
5112         /* XXX: magic numbers */
5113         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5114             &v);
5115         end_synchronized_op(sc, 0);
5116         if (rc)
5117                 return (rc);
5118         if (op == 0)
5119                 v /= 256;
5120
5121         rc = sysctl_handle_int(oidp, &v, 0, req);
5122         return (rc);
5123 }
5124
5125 static int
5126 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5127 {
5128         struct port_info *pi = arg1;
5129         int rc, val;
5130
5131         val = pi->rsrv_noflowq;
5132         rc = sysctl_handle_int(oidp, &val, 0, req);
5133         if (rc != 0 || req->newptr == NULL)
5134                 return (rc);
5135
5136         if ((val >= 1) && (pi->ntxq > 1))
5137                 pi->rsrv_noflowq = 1;
5138         else
5139                 pi->rsrv_noflowq = 0;
5140
5141         return (rc);
5142 }
5143
5144 static int
5145 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5146 {
5147         struct port_info *pi = arg1;
5148         struct adapter *sc = pi->adapter;
5149         int idx, rc, i;
5150         struct sge_rxq *rxq;
5151 #ifdef TCP_OFFLOAD
5152         struct sge_ofld_rxq *ofld_rxq;
5153 #endif
5154         uint8_t v;
5155
5156         idx = pi->tmr_idx;
5157
5158         rc = sysctl_handle_int(oidp, &idx, 0, req);
5159         if (rc != 0 || req->newptr == NULL)
5160                 return (rc);
5161
5162         if (idx < 0 || idx >= SGE_NTIMERS)
5163                 return (EINVAL);
5164
5165         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5166             "t4tmr");
5167         if (rc)
5168                 return (rc);
5169
5170         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5171         for_each_rxq(pi, i, rxq) {
5172 #ifdef atomic_store_rel_8
5173                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5174 #else
5175                 rxq->iq.intr_params = v;
5176 #endif
5177         }
5178 #ifdef TCP_OFFLOAD
5179         for_each_ofld_rxq(pi, i, ofld_rxq) {
5180 #ifdef atomic_store_rel_8
5181                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5182 #else
5183                 ofld_rxq->iq.intr_params = v;
5184 #endif
5185         }
5186 #endif
5187         pi->tmr_idx = idx;
5188
5189         end_synchronized_op(sc, LOCK_HELD);
5190         return (0);
5191 }
5192
5193 static int
5194 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5195 {
5196         struct port_info *pi = arg1;
5197         struct adapter *sc = pi->adapter;
5198         int idx, rc;
5199
5200         idx = pi->pktc_idx;
5201
5202         rc = sysctl_handle_int(oidp, &idx, 0, req);
5203         if (rc != 0 || req->newptr == NULL)
5204                 return (rc);
5205
5206         if (idx < -1 || idx >= SGE_NCOUNTERS)
5207                 return (EINVAL);
5208
5209         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5210             "t4pktc");
5211         if (rc)
5212                 return (rc);
5213
5214         if (pi->flags & PORT_INIT_DONE)
5215                 rc = EBUSY; /* cannot be changed once the queues are created */
5216         else
5217                 pi->pktc_idx = idx;
5218
5219         end_synchronized_op(sc, LOCK_HELD);
5220         return (rc);
5221 }
5222
5223 static int
5224 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5225 {
5226         struct port_info *pi = arg1;
5227         struct adapter *sc = pi->adapter;
5228         int qsize, rc;
5229
5230         qsize = pi->qsize_rxq;
5231
5232         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5233         if (rc != 0 || req->newptr == NULL)
5234                 return (rc);
5235
5236         if (qsize < 128 || (qsize & 7))
5237                 return (EINVAL);
5238
5239         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5240             "t4rxqs");
5241         if (rc)
5242                 return (rc);
5243
5244         if (pi->flags & PORT_INIT_DONE)
5245                 rc = EBUSY; /* cannot be changed once the queues are created */
5246         else
5247                 pi->qsize_rxq = qsize;
5248
5249         end_synchronized_op(sc, LOCK_HELD);
5250         return (rc);
5251 }
5252
5253 static int
5254 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5255 {
5256         struct port_info *pi = arg1;
5257         struct adapter *sc = pi->adapter;
5258         int qsize, rc;
5259
5260         qsize = pi->qsize_txq;
5261
5262         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5263         if (rc != 0 || req->newptr == NULL)
5264                 return (rc);
5265
5266         if (qsize < 128 || qsize > 65536)
5267                 return (EINVAL);
5268
5269         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5270             "t4txqs");
5271         if (rc)
5272                 return (rc);
5273
5274         if (pi->flags & PORT_INIT_DONE)
5275                 rc = EBUSY; /* cannot be changed once the queues are created */
5276         else
5277                 pi->qsize_txq = qsize;
5278
5279         end_synchronized_op(sc, LOCK_HELD);
5280         return (rc);
5281 }
5282
5283 static int
5284 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5285 {
5286         struct port_info *pi = arg1;
5287         struct adapter *sc = pi->adapter;
5288         struct link_config *lc = &pi->link_cfg;
5289         int rc;
5290
5291         if (req->newptr == NULL) {
5292                 struct sbuf *sb;
5293                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5294
5295                 rc = sysctl_wire_old_buffer(req, 0);
5296                 if (rc != 0)
5297                         return(rc);
5298
5299                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5300                 if (sb == NULL)
5301                         return (ENOMEM);
5302
5303                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5304                 rc = sbuf_finish(sb);
5305                 sbuf_delete(sb);
5306         } else {
5307                 char s[2];
5308                 int n;
5309
5310                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5311                 s[1] = 0;
5312
5313                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5314                 if (rc != 0)
5315                         return(rc);
5316
5317                 if (s[1] != 0)
5318                         return (EINVAL);
5319                 if (s[0] < '0' || s[0] > '9')
5320                         return (EINVAL);        /* not a number */
5321                 n = s[0] - '0';
5322                 if (n & ~(PAUSE_TX | PAUSE_RX))
5323                         return (EINVAL);        /* some other bit is set too */
5324
5325                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5326                 if (rc)
5327                         return (rc);
5328                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5329                         int link_ok = lc->link_ok;
5330
5331                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5332                         lc->requested_fc |= n;
5333                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5334                         lc->link_ok = link_ok;  /* restore */
5335                 }
5336                 end_synchronized_op(sc, 0);
5337         }
5338
5339         return (rc);
5340 }
5341
5342 static int
5343 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5344 {
5345         struct adapter *sc = arg1;
5346         int reg = arg2;
5347         uint64_t val;
5348
5349         val = t4_read_reg64(sc, reg);
5350
5351         return (sysctl_handle_64(oidp, &val, 0, req));
5352 }
5353
5354 static int
5355 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5356 {
5357         struct adapter *sc = arg1;
5358         int rc, t;
5359         uint32_t param, val;
5360
5361         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5362         if (rc)
5363                 return (rc);
5364         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5365             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5366             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5367         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5368         end_synchronized_op(sc, 0);
5369         if (rc)
5370                 return (rc);
5371
5372         /* unknown is returned as 0 but we display -1 in that case */
5373         t = val == 0 ? -1 : val;
5374
5375         rc = sysctl_handle_int(oidp, &t, 0, req);
5376         return (rc);
5377 }
5378
5379 #ifdef SBUF_DRAIN
5380 static int
5381 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5382 {
5383         struct adapter *sc = arg1;
5384         struct sbuf *sb;
5385         int rc, i;
5386         uint16_t incr[NMTUS][NCCTRL_WIN];
5387         static const char *dec_fac[] = {
5388                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5389                 "0.9375"
5390         };
5391
5392         rc = sysctl_wire_old_buffer(req, 0);
5393         if (rc != 0)
5394                 return (rc);
5395
5396         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5397         if (sb == NULL)
5398                 return (ENOMEM);
5399
5400         t4_read_cong_tbl(sc, incr);
5401
5402         for (i = 0; i < NCCTRL_WIN; ++i) {
5403                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5404                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5405                     incr[5][i], incr[6][i], incr[7][i]);
5406                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5407                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5408                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5409                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5410         }
5411
5412         rc = sbuf_finish(sb);
5413         sbuf_delete(sb);
5414
5415         return (rc);
5416 }
5417
5418 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5419         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5420         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5421         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5422 };
5423
5424 static int
5425 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5426 {
5427         struct adapter *sc = arg1;
5428         struct sbuf *sb;
5429         int rc, i, n, qid = arg2;
5430         uint32_t *buf, *p;
5431         char *qtype;
5432         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5433
5434         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5435             ("%s: bad qid %d\n", __func__, qid));
5436
5437         if (qid < CIM_NUM_IBQ) {
5438                 /* inbound queue */
5439                 qtype = "IBQ";
5440                 n = 4 * CIM_IBQ_SIZE;
5441                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5442                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5443         } else {
5444                 /* outbound queue */
5445                 qtype = "OBQ";
5446                 qid -= CIM_NUM_IBQ;
5447                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5448                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5449                 rc = t4_read_cim_obq(sc, qid, buf, n);
5450         }
5451
5452         if (rc < 0) {
5453                 rc = -rc;
5454                 goto done;
5455         }
5456         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5457
5458         rc = sysctl_wire_old_buffer(req, 0);
5459         if (rc != 0)
5460                 goto done;
5461
5462         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5463         if (sb == NULL) {
5464                 rc = ENOMEM;
5465                 goto done;
5466         }
5467
5468         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5469         for (i = 0, p = buf; i < n; i += 16, p += 4)
5470                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5471                     p[2], p[3]);
5472
5473         rc = sbuf_finish(sb);
5474         sbuf_delete(sb);
5475 done:
5476         free(buf, M_CXGBE);
5477         return (rc);
5478 }
5479
5480 static int
5481 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5482 {
5483         struct adapter *sc = arg1;
5484         u_int cfg;
5485         struct sbuf *sb;
5486         uint32_t *buf, *p;
5487         int rc;
5488
5489         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5490         if (rc != 0)
5491                 return (rc);
5492
5493         rc = sysctl_wire_old_buffer(req, 0);
5494         if (rc != 0)
5495                 return (rc);
5496
5497         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5498         if (sb == NULL)
5499                 return (ENOMEM);
5500
5501         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5502             M_ZERO | M_WAITOK);
5503
5504         rc = -t4_cim_read_la(sc, buf, NULL);
5505         if (rc != 0)
5506                 goto done;
5507
5508         sbuf_printf(sb, "Status   Data      PC%s",
5509             cfg & F_UPDBGLACAPTPCONLY ? "" :
5510             "     LS0Stat  LS0Addr             LS0Data");
5511
5512         KASSERT((sc->params.cim_la_size & 7) == 0,
5513             ("%s: p will walk off the end of buf", __func__));
5514
5515         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5516                 if (cfg & F_UPDBGLACAPTPCONLY) {
5517                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5518                             p[6], p[7]);
5519                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5520                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5521                             p[4] & 0xff, p[5] >> 8);
5522                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5523                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5524                             p[1] & 0xf, p[2] >> 4);
5525                 } else {
5526                         sbuf_printf(sb,
5527                             "\n  %02x   %x%07x %x%07x %08x %08x "
5528                             "%08x%08x%08x%08x",
5529                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5530                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5531                             p[6], p[7]);
5532                 }
5533         }
5534
5535         rc = sbuf_finish(sb);
5536         sbuf_delete(sb);
5537 done:
5538         free(buf, M_CXGBE);
5539         return (rc);
5540 }
5541
5542 static int
5543 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5544 {
5545         struct adapter *sc = arg1;
5546         u_int i;
5547         struct sbuf *sb;
5548         uint32_t *buf, *p;
5549         int rc;
5550
5551         rc = sysctl_wire_old_buffer(req, 0);
5552         if (rc != 0)
5553                 return (rc);
5554
5555         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5556         if (sb == NULL)
5557                 return (ENOMEM);
5558
5559         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5560             M_ZERO | M_WAITOK);
5561
5562         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5563         p = buf;
5564
5565         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5566                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5567                     p[1], p[0]);
5568         }
5569
5570         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5571         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5572                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5573                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5574                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5575                     (p[1] >> 2) | ((p[2] & 3) << 30),
5576                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5577                     p[0] & 1);
5578         }
5579
5580         rc = sbuf_finish(sb);
5581         sbuf_delete(sb);
5582         free(buf, M_CXGBE);
5583         return (rc);
5584 }
5585
5586 static int
5587 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5588 {
5589         struct adapter *sc = arg1;
5590         u_int i;
5591         struct sbuf *sb;
5592         uint32_t *buf, *p;
5593         int rc;
5594
5595         rc = sysctl_wire_old_buffer(req, 0);
5596         if (rc != 0)
5597                 return (rc);
5598
5599         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5600         if (sb == NULL)
5601                 return (ENOMEM);
5602
5603         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5604             M_ZERO | M_WAITOK);
5605
5606         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5607         p = buf;
5608
5609         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5610         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5611                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5612                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5613                     p[4], p[3], p[2], p[1], p[0]);
5614         }
5615
5616         sbuf_printf(sb, "\n\nCntl ID               Data");
5617         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5618                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5619                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5620         }
5621
5622         rc = sbuf_finish(sb);
5623         sbuf_delete(sb);
5624         free(buf, M_CXGBE);
5625         return (rc);
5626 }
5627
5628 static int
5629 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5630 {
5631         struct adapter *sc = arg1;
5632         struct sbuf *sb;
5633         int rc, i;
5634         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5635         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5636         uint16_t thres[CIM_NUM_IBQ];
5637         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5638         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5639         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5640
5641         if (is_t4(sc)) {
5642                 cim_num_obq = CIM_NUM_OBQ;
5643                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5644                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5645         } else {
5646                 cim_num_obq = CIM_NUM_OBQ_T5;
5647                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5648                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5649         }
5650         nq = CIM_NUM_IBQ + cim_num_obq;
5651
5652         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5653         if (rc == 0)
5654                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5655         if (rc != 0)
5656                 return (rc);
5657
5658         t4_read_cimq_cfg(sc, base, size, thres);
5659
5660         rc = sysctl_wire_old_buffer(req, 0);
5661         if (rc != 0)
5662                 return (rc);
5663
5664         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5665         if (sb == NULL)
5666                 return (ENOMEM);
5667
5668         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5669
5670         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5671                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5672                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5673                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5674                     G_QUEREMFLITS(p[2]) * 16);
5675         for ( ; i < nq; i++, p += 4, wr += 2)
5676                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5677                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5678                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5679                     G_QUEREMFLITS(p[2]) * 16);
5680
5681         rc = sbuf_finish(sb);
5682         sbuf_delete(sb);
5683
5684         return (rc);
5685 }
5686
5687 static int
5688 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5689 {
5690         struct adapter *sc = arg1;
5691         struct sbuf *sb;
5692         int rc;
5693         struct tp_cpl_stats stats;
5694
5695         rc = sysctl_wire_old_buffer(req, 0);
5696         if (rc != 0)
5697                 return (rc);
5698
5699         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5700         if (sb == NULL)
5701                 return (ENOMEM);
5702
5703         t4_tp_get_cpl_stats(sc, &stats);
5704
5705         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5706             "channel 3\n");
5707         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5708                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5709         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5710                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5711
5712         rc = sbuf_finish(sb);
5713         sbuf_delete(sb);
5714
5715         return (rc);
5716 }
5717
5718 static int
5719 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5720 {
5721         struct adapter *sc = arg1;
5722         struct sbuf *sb;
5723         int rc;
5724         struct tp_usm_stats stats;
5725
5726         rc = sysctl_wire_old_buffer(req, 0);
5727         if (rc != 0)
5728                 return(rc);
5729
5730         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5731         if (sb == NULL)
5732                 return (ENOMEM);
5733
5734         t4_get_usm_stats(sc, &stats);
5735
5736         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5737         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5738         sbuf_printf(sb, "Drops:  %u", stats.drops);
5739
5740         rc = sbuf_finish(sb);
5741         sbuf_delete(sb);
5742
5743         return (rc);
5744 }
5745
5746 const char *devlog_level_strings[] = {
5747         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5748         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5749         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5750         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5751         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5752         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5753 };
5754
5755 const char *devlog_facility_strings[] = {
5756         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5757         [FW_DEVLOG_FACILITY_CF]         = "CF",
5758         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5759         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5760         [FW_DEVLOG_FACILITY_RES]        = "RES",
5761         [FW_DEVLOG_FACILITY_HW]         = "HW",
5762         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5763         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5764         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5765         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5766         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5767         [FW_DEVLOG_FACILITY_VI]         = "VI",
5768         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5769         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5770         [FW_DEVLOG_FACILITY_TM]         = "TM",
5771         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5772         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5773         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5774         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5775         [FW_DEVLOG_FACILITY_RI]         = "RI",
5776         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5777         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5778         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5779         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5780 };
5781
5782 static int
5783 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5784 {
5785         struct adapter *sc = arg1;
5786         struct devlog_params *dparams = &sc->params.devlog;
5787         struct fw_devlog_e *buf, *e;
5788         int i, j, rc, nentries, first = 0, m;
5789         struct sbuf *sb;
5790         uint64_t ftstamp = UINT64_MAX;
5791
5792         if (dparams->start == 0) {
5793                 dparams->memtype = FW_MEMTYPE_EDC0;
5794                 dparams->start = 0x84000;
5795                 dparams->size = 32768;
5796         }
5797
5798         nentries = dparams->size / sizeof(struct fw_devlog_e);
5799
5800         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5801         if (buf == NULL)
5802                 return (ENOMEM);
5803
5804         m = fwmtype_to_hwmtype(dparams->memtype);
5805         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5806         if (rc != 0)
5807                 goto done;
5808
5809         for (i = 0; i < nentries; i++) {
5810                 e = &buf[i];
5811
5812                 if (e->timestamp == 0)
5813                         break;  /* end */
5814
5815                 e->timestamp = be64toh(e->timestamp);
5816                 e->seqno = be32toh(e->seqno);
5817                 for (j = 0; j < 8; j++)
5818                         e->params[j] = be32toh(e->params[j]);
5819
5820                 if (e->timestamp < ftstamp) {
5821                         ftstamp = e->timestamp;
5822                         first = i;
5823                 }
5824         }
5825
5826         if (buf[first].timestamp == 0)
5827                 goto done;      /* nothing in the log */
5828
5829         rc = sysctl_wire_old_buffer(req, 0);
5830         if (rc != 0)
5831                 goto done;
5832
5833         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5834         if (sb == NULL) {
5835                 rc = ENOMEM;
5836                 goto done;
5837         }
5838         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5839             "Seq#", "Tstamp", "Level", "Facility", "Message");
5840
5841         i = first;
5842         do {
5843                 e = &buf[i];
5844                 if (e->timestamp == 0)
5845                         break;  /* end */
5846
5847                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5848                     e->seqno, e->timestamp,
5849                     (e->level < nitems(devlog_level_strings) ?
5850                         devlog_level_strings[e->level] : "UNKNOWN"),
5851                     (e->facility < nitems(devlog_facility_strings) ?
5852                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5853                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5854                     e->params[2], e->params[3], e->params[4],
5855                     e->params[5], e->params[6], e->params[7]);
5856
5857                 if (++i == nentries)
5858                         i = 0;
5859         } while (i != first);
5860
5861         rc = sbuf_finish(sb);
5862         sbuf_delete(sb);
5863 done:
5864         free(buf, M_CXGBE);
5865         return (rc);
5866 }
5867
5868 static int
5869 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5870 {
5871         struct adapter *sc = arg1;
5872         struct sbuf *sb;
5873         int rc;
5874         struct tp_fcoe_stats stats[4];
5875
5876         rc = sysctl_wire_old_buffer(req, 0);
5877         if (rc != 0)
5878                 return (rc);
5879
5880         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5881         if (sb == NULL)
5882                 return (ENOMEM);
5883
5884         t4_get_fcoe_stats(sc, 0, &stats[0]);
5885         t4_get_fcoe_stats(sc, 1, &stats[1]);
5886         t4_get_fcoe_stats(sc, 2, &stats[2]);
5887         t4_get_fcoe_stats(sc, 3, &stats[3]);
5888
5889         sbuf_printf(sb, "                   channel 0        channel 1        "
5890             "channel 2        channel 3\n");
5891         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5892             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5893             stats[3].octetsDDP);
5894         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5895             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5896         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5897             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5898             stats[3].framesDrop);
5899
5900         rc = sbuf_finish(sb);
5901         sbuf_delete(sb);
5902
5903         return (rc);
5904 }
5905
5906 static int
5907 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5908 {
5909         struct adapter *sc = arg1;
5910         struct sbuf *sb;
5911         int rc, i;
5912         unsigned int map, kbps, ipg, mode;
5913         unsigned int pace_tab[NTX_SCHED];
5914
5915         rc = sysctl_wire_old_buffer(req, 0);
5916         if (rc != 0)
5917                 return (rc);
5918
5919         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5920         if (sb == NULL)
5921                 return (ENOMEM);
5922
5923         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5924         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5925         t4_read_pace_tbl(sc, pace_tab);
5926
5927         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5928             "Class IPG (0.1 ns)   Flow IPG (us)");
5929
5930         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5931                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5932                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5933                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5934                 if (kbps)
5935                         sbuf_printf(sb, "%9u     ", kbps);
5936                 else
5937                         sbuf_printf(sb, " disabled     ");
5938
5939                 if (ipg)
5940                         sbuf_printf(sb, "%13u        ", ipg);
5941                 else
5942                         sbuf_printf(sb, "     disabled        ");
5943
5944                 if (pace_tab[i])
5945                         sbuf_printf(sb, "%10u", pace_tab[i]);
5946                 else
5947                         sbuf_printf(sb, "  disabled");
5948         }
5949
5950         rc = sbuf_finish(sb);
5951         sbuf_delete(sb);
5952
5953         return (rc);
5954 }
5955
5956 static int
5957 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5958 {
5959         struct adapter *sc = arg1;
5960         struct sbuf *sb;
5961         int rc, i, j;
5962         uint64_t *p0, *p1;
5963         struct lb_port_stats s[2];
5964         static const char *stat_name[] = {
5965                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5966                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5967                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5968                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5969                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5970                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5971                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5972         };
5973
5974         rc = sysctl_wire_old_buffer(req, 0);
5975         if (rc != 0)
5976                 return (rc);
5977
5978         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5979         if (sb == NULL)
5980                 return (ENOMEM);
5981
5982         memset(s, 0, sizeof(s));
5983
5984         for (i = 0; i < 4; i += 2) {
5985                 t4_get_lb_stats(sc, i, &s[0]);
5986                 t4_get_lb_stats(sc, i + 1, &s[1]);
5987
5988                 p0 = &s[0].octets;
5989                 p1 = &s[1].octets;
5990                 sbuf_printf(sb, "%s                       Loopback %u"
5991                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5992
5993                 for (j = 0; j < nitems(stat_name); j++)
5994                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5995                                    *p0++, *p1++);
5996         }
5997
5998         rc = sbuf_finish(sb);
5999         sbuf_delete(sb);
6000
6001         return (rc);
6002 }
6003
6004 static int
6005 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6006 {
6007         int rc = 0;
6008         struct port_info *pi = arg1;
6009         struct sbuf *sb;
6010         static const char *linkdnreasons[] = {
6011                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6012                 "PHY overheated", "unknown", "rx los", "reserved7"
6013         };
6014
6015         rc = sysctl_wire_old_buffer(req, 0);
6016         if (rc != 0)
6017                 return(rc);
6018         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6019         if (sb == NULL)
6020                 return (ENOMEM);
6021
6022         if (pi->linkdnrc < 0)
6023                 sbuf_printf(sb, "n/a");
6024         else if (pi->linkdnrc < nitems(linkdnreasons))
6025                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6026         else
6027                 sbuf_printf(sb, "%d", pi->linkdnrc);
6028
6029         rc = sbuf_finish(sb);
6030         sbuf_delete(sb);
6031
6032         return (rc);
6033 }
6034
6035 struct mem_desc {
6036         unsigned int base;
6037         unsigned int limit;
6038         unsigned int idx;
6039 };
6040
6041 static int
6042 mem_desc_cmp(const void *a, const void *b)
6043 {
6044         return ((const struct mem_desc *)a)->base -
6045                ((const struct mem_desc *)b)->base;
6046 }
6047
6048 static void
6049 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6050     unsigned int to)
6051 {
6052         unsigned int size;
6053
6054         size = to - from + 1;
6055         if (size == 0)
6056                 return;
6057
6058         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6059         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6060 }
6061
6062 static int
6063 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6064 {
6065         struct adapter *sc = arg1;
6066         struct sbuf *sb;
6067         int rc, i, n;
6068         uint32_t lo, hi, used, alloc;
6069         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6070         static const char *region[] = {
6071                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6072                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6073                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6074                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6075                 "RQUDP region:", "PBL region:", "TXPBL region:",
6076                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6077                 "On-chip queues:"
6078         };
6079         struct mem_desc avail[4];
6080         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6081         struct mem_desc *md = mem;
6082
6083         rc = sysctl_wire_old_buffer(req, 0);
6084         if (rc != 0)
6085                 return (rc);
6086
6087         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6088         if (sb == NULL)
6089                 return (ENOMEM);
6090
6091         for (i = 0; i < nitems(mem); i++) {
6092                 mem[i].limit = 0;
6093                 mem[i].idx = i;
6094         }
6095
6096         /* Find and sort the populated memory ranges */
6097         i = 0;
6098         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6099         if (lo & F_EDRAM0_ENABLE) {
6100                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6101                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6102                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6103                 avail[i].idx = 0;
6104                 i++;
6105         }
6106         if (lo & F_EDRAM1_ENABLE) {
6107                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6108                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6109                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6110                 avail[i].idx = 1;
6111                 i++;
6112         }
6113         if (lo & F_EXT_MEM_ENABLE) {
6114                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6115                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6116                 avail[i].limit = avail[i].base +
6117                     (G_EXT_MEM_SIZE(hi) << 20);
6118                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6119                 i++;
6120         }
6121         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6122                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6123                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6124                 avail[i].limit = avail[i].base +
6125                     (G_EXT_MEM1_SIZE(hi) << 20);
6126                 avail[i].idx = 4;
6127                 i++;
6128         }
6129         if (!i)                                    /* no memory available */
6130                 return 0;
6131         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6132
6133         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6134         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6135         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6136         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6137         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6138         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6139         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6140         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6141         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6142
6143         /* the next few have explicit upper bounds */
6144         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6145         md->limit = md->base - 1 +
6146                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6147                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6148         md++;
6149
6150         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6151         md->limit = md->base - 1 +
6152                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6153                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6154         md++;
6155
6156         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6157                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6158                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6159                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6160         } else {
6161                 md->base = 0;
6162                 md->idx = nitems(region);  /* hide it */
6163         }
6164         md++;
6165
6166 #define ulp_region(reg) \
6167         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6168         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6169
6170         ulp_region(RX_ISCSI);
6171         ulp_region(RX_TDDP);
6172         ulp_region(TX_TPT);
6173         ulp_region(RX_STAG);
6174         ulp_region(RX_RQ);
6175         ulp_region(RX_RQUDP);
6176         ulp_region(RX_PBL);
6177         ulp_region(TX_PBL);
6178 #undef ulp_region
6179
6180         md->base = 0;
6181         md->idx = nitems(region);
6182         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6183                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6184                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6185                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6186         }
6187         md++;
6188
6189         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6190         md->limit = md->base + sc->tids.ntids - 1;
6191         md++;
6192         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6193         md->limit = md->base + sc->tids.ntids - 1;
6194         md++;
6195
6196         md->base = sc->vres.ocq.start;
6197         if (sc->vres.ocq.size)
6198                 md->limit = md->base + sc->vres.ocq.size - 1;
6199         else
6200                 md->idx = nitems(region);  /* hide it */
6201         md++;
6202
6203         /* add any address-space holes, there can be up to 3 */
6204         for (n = 0; n < i - 1; n++)
6205                 if (avail[n].limit < avail[n + 1].base)
6206                         (md++)->base = avail[n].limit;
6207         if (avail[n].limit)
6208                 (md++)->base = avail[n].limit;
6209
6210         n = md - mem;
6211         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6212
6213         for (lo = 0; lo < i; lo++)
6214                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6215                                 avail[lo].limit - 1);
6216
6217         sbuf_printf(sb, "\n");
6218         for (i = 0; i < n; i++) {
6219                 if (mem[i].idx >= nitems(region))
6220                         continue;                        /* skip holes */
6221                 if (!mem[i].limit)
6222                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6223                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6224                                 mem[i].limit);
6225         }
6226
6227         sbuf_printf(sb, "\n");
6228         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6229         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6230         mem_region_show(sb, "uP RAM:", lo, hi);
6231
6232         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6233         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6234         mem_region_show(sb, "uP Extmem2:", lo, hi);
6235
6236         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6237         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6238                    G_PMRXMAXPAGE(lo),
6239                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6240                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6241
6242         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6243         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6244         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6245                    G_PMTXMAXPAGE(lo),
6246                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6247                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6248         sbuf_printf(sb, "%u p-structs\n",
6249                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6250
6251         for (i = 0; i < 4; i++) {
6252                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6253                 if (is_t4(sc)) {
6254                         used = G_USED(lo);
6255                         alloc = G_ALLOC(lo);
6256                 } else {
6257                         used = G_T5_USED(lo);
6258                         alloc = G_T5_ALLOC(lo);
6259                 }
6260                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6261                            i, used, alloc);
6262         }
6263         for (i = 0; i < 4; i++) {
6264                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6265                 if (is_t4(sc)) {
6266                         used = G_USED(lo);
6267                         alloc = G_ALLOC(lo);
6268                 } else {
6269                         used = G_T5_USED(lo);
6270                         alloc = G_T5_ALLOC(lo);
6271                 }
6272                 sbuf_printf(sb,
6273                            "\nLoopback %d using %u pages out of %u allocated",
6274                            i, used, alloc);
6275         }
6276
6277         rc = sbuf_finish(sb);
6278         sbuf_delete(sb);
6279
6280         return (rc);
6281 }
6282
6283 static inline void
6284 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6285 {
6286         *mask = x | y;
6287         y = htobe64(y);
6288         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6289 }
6290
6291 static int
6292 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6293 {
6294         struct adapter *sc = arg1;
6295         struct sbuf *sb;
6296         int rc, i, n;
6297
6298         rc = sysctl_wire_old_buffer(req, 0);
6299         if (rc != 0)
6300                 return (rc);
6301
6302         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6303         if (sb == NULL)
6304                 return (ENOMEM);
6305
6306         sbuf_printf(sb,
6307             "Idx  Ethernet address     Mask     Vld Ports PF"
6308             "  VF              Replication             P0 P1 P2 P3  ML");
6309         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6310             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6311         for (i = 0; i < n; i++) {
6312                 uint64_t tcamx, tcamy, mask;
6313                 uint32_t cls_lo, cls_hi;
6314                 uint8_t addr[ETHER_ADDR_LEN];
6315
6316                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6317                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6318                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6319                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6320
6321                 if (tcamx & tcamy)
6322                         continue;
6323
6324                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6325                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6326                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6327                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6328                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6329                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6330                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6331
6332                 if (cls_lo & F_REPLICATE) {
6333                         struct fw_ldst_cmd ldst_cmd;
6334
6335                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6336                         ldst_cmd.op_to_addrspace =
6337                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6338                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6339                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6340                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6341                         ldst_cmd.u.mps.rplc.fid_idx =
6342                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6343                                 V_FW_LDST_CMD_IDX(i));
6344
6345                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6346                             "t4mps");
6347                         if (rc)
6348                                 break;
6349                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6350                             sizeof(ldst_cmd), &ldst_cmd);
6351                         end_synchronized_op(sc, 0);
6352
6353                         if (rc != 0) {
6354                                 sbuf_printf(sb,
6355                                     " ------------ error %3u ------------", rc);
6356                                 rc = 0;
6357                         } else {
6358                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6359                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6360                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6361                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6362                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6363                         }
6364                 } else
6365                         sbuf_printf(sb, "%36s", "");
6366
6367                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6368                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6369                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6370         }
6371
6372         if (rc)
6373                 (void) sbuf_finish(sb);
6374         else
6375                 rc = sbuf_finish(sb);
6376         sbuf_delete(sb);
6377
6378         return (rc);
6379 }
6380
6381 static int
6382 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6383 {
6384         struct adapter *sc = arg1;
6385         struct sbuf *sb;
6386         int rc;
6387         uint16_t mtus[NMTUS];
6388
6389         rc = sysctl_wire_old_buffer(req, 0);
6390         if (rc != 0)
6391                 return (rc);
6392
6393         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6394         if (sb == NULL)
6395                 return (ENOMEM);
6396
6397         t4_read_mtu_tbl(sc, mtus, NULL);
6398
6399         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6400             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6401             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6402             mtus[14], mtus[15]);
6403
6404         rc = sbuf_finish(sb);
6405         sbuf_delete(sb);
6406
6407         return (rc);
6408 }
6409
6410 static int
6411 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6412 {
6413         struct adapter *sc = arg1;
6414         struct sbuf *sb;
6415         int rc, i;
6416         uint32_t cnt[PM_NSTATS];
6417         uint64_t cyc[PM_NSTATS];
6418         static const char *rx_stats[] = {
6419                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6420         };
6421         static const char *tx_stats[] = {
6422                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6423         };
6424
6425         rc = sysctl_wire_old_buffer(req, 0);
6426         if (rc != 0)
6427                 return (rc);
6428
6429         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6430         if (sb == NULL)
6431                 return (ENOMEM);
6432
6433         t4_pmtx_get_stats(sc, cnt, cyc);
6434         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6435         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6436                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6437                     cyc[i]);
6438
6439         t4_pmrx_get_stats(sc, cnt, cyc);
6440         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6441         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6442                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6443                     cyc[i]);
6444
6445         rc = sbuf_finish(sb);
6446         sbuf_delete(sb);
6447
6448         return (rc);
6449 }
6450
6451 static int
6452 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6453 {
6454         struct adapter *sc = arg1;
6455         struct sbuf *sb;
6456         int rc;
6457         struct tp_rdma_stats stats;
6458
6459         rc = sysctl_wire_old_buffer(req, 0);
6460         if (rc != 0)
6461                 return (rc);
6462
6463         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6464         if (sb == NULL)
6465                 return (ENOMEM);
6466
6467         t4_tp_get_rdma_stats(sc, &stats);
6468         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6469         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6470
6471         rc = sbuf_finish(sb);
6472         sbuf_delete(sb);
6473
6474         return (rc);
6475 }
6476
6477 static int
6478 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6479 {
6480         struct adapter *sc = arg1;
6481         struct sbuf *sb;
6482         int rc;
6483         struct tp_tcp_stats v4, v6;
6484
6485         rc = sysctl_wire_old_buffer(req, 0);
6486         if (rc != 0)
6487                 return (rc);
6488
6489         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6490         if (sb == NULL)
6491                 return (ENOMEM);
6492
6493         t4_tp_get_tcp_stats(sc, &v4, &v6);
6494         sbuf_printf(sb,
6495             "                                IP                 IPv6\n");
6496         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6497             v4.tcpOutRsts, v6.tcpOutRsts);
6498         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6499             v4.tcpInSegs, v6.tcpInSegs);
6500         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6501             v4.tcpOutSegs, v6.tcpOutSegs);
6502         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6503             v4.tcpRetransSegs, v6.tcpRetransSegs);
6504
6505         rc = sbuf_finish(sb);
6506         sbuf_delete(sb);
6507
6508         return (rc);
6509 }
6510
6511 static int
6512 sysctl_tids(SYSCTL_HANDLER_ARGS)
6513 {
6514         struct adapter *sc = arg1;
6515         struct sbuf *sb;
6516         int rc;
6517         struct tid_info *t = &sc->tids;
6518
6519         rc = sysctl_wire_old_buffer(req, 0);
6520         if (rc != 0)
6521                 return (rc);
6522
6523         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6524         if (sb == NULL)
6525                 return (ENOMEM);
6526
6527         if (t->natids) {
6528                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6529                     t->atids_in_use);
6530         }
6531
6532         if (t->ntids) {
6533                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6534                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6535
6536                         if (b) {
6537                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6538                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6539                                     t->ntids - 1);
6540                         } else {
6541                                 sbuf_printf(sb, "TID range: %u-%u",
6542                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6543                                     t->ntids - 1);
6544                         }
6545                 } else
6546                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6547                 sbuf_printf(sb, ", in use: %u\n",
6548                     atomic_load_acq_int(&t->tids_in_use));
6549         }
6550
6551         if (t->nstids) {
6552                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6553                     t->stid_base + t->nstids - 1, t->stids_in_use);
6554         }
6555
6556         if (t->nftids) {
6557                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6558                     t->ftid_base + t->nftids - 1);
6559         }
6560
6561         if (t->netids) {
6562                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6563                     t->etid_base + t->netids - 1);
6564         }
6565
6566         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6567             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6568             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6569
6570         rc = sbuf_finish(sb);
6571         sbuf_delete(sb);
6572
6573         return (rc);
6574 }
6575
6576 static int
6577 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6578 {
6579         struct adapter *sc = arg1;
6580         struct sbuf *sb;
6581         int rc;
6582         struct tp_err_stats stats;
6583
6584         rc = sysctl_wire_old_buffer(req, 0);
6585         if (rc != 0)
6586                 return (rc);
6587
6588         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6589         if (sb == NULL)
6590                 return (ENOMEM);
6591
6592         t4_tp_get_err_stats(sc, &stats);
6593
6594         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6595                       "channel 3\n");
6596         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6597             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6598             stats.macInErrs[3]);
6599         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6600             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6601             stats.hdrInErrs[3]);
6602         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6603             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6604             stats.tcpInErrs[3]);
6605         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6606             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6607             stats.tcp6InErrs[3]);
6608         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6609             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6610             stats.tnlCongDrops[3]);
6611         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6612             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6613             stats.tnlTxDrops[3]);
6614         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6615             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6616             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6617         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6618             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6619             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6620         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6621             stats.ofldNoNeigh, stats.ofldCongDefer);
6622
6623         rc = sbuf_finish(sb);
6624         sbuf_delete(sb);
6625
6626         return (rc);
6627 }
6628
6629 struct field_desc {
6630         const char *name;
6631         u_int start;
6632         u_int width;
6633 };
6634
6635 static void
6636 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6637 {
6638         char buf[32];
6639         int line_size = 0;
6640
6641         while (f->name) {
6642                 uint64_t mask = (1ULL << f->width) - 1;
6643                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6644                     ((uintmax_t)v >> f->start) & mask);
6645
6646                 if (line_size + len >= 79) {
6647                         line_size = 8;
6648                         sbuf_printf(sb, "\n        ");
6649                 }
6650                 sbuf_printf(sb, "%s ", buf);
6651                 line_size += len + 1;
6652                 f++;
6653         }
6654         sbuf_printf(sb, "\n");
6655 }
6656
6657 static struct field_desc tp_la0[] = {
6658         { "RcfOpCodeOut", 60, 4 },
6659         { "State", 56, 4 },
6660         { "WcfState", 52, 4 },
6661         { "RcfOpcSrcOut", 50, 2 },
6662         { "CRxError", 49, 1 },
6663         { "ERxError", 48, 1 },
6664         { "SanityFailed", 47, 1 },
6665         { "SpuriousMsg", 46, 1 },
6666         { "FlushInputMsg", 45, 1 },
6667         { "FlushInputCpl", 44, 1 },
6668         { "RssUpBit", 43, 1 },
6669         { "RssFilterHit", 42, 1 },
6670         { "Tid", 32, 10 },
6671         { "InitTcb", 31, 1 },
6672         { "LineNumber", 24, 7 },
6673         { "Emsg", 23, 1 },
6674         { "EdataOut", 22, 1 },
6675         { "Cmsg", 21, 1 },
6676         { "CdataOut", 20, 1 },
6677         { "EreadPdu", 19, 1 },
6678         { "CreadPdu", 18, 1 },
6679         { "TunnelPkt", 17, 1 },
6680         { "RcfPeerFin", 16, 1 },
6681         { "RcfReasonOut", 12, 4 },
6682         { "TxCchannel", 10, 2 },
6683         { "RcfTxChannel", 8, 2 },
6684         { "RxEchannel", 6, 2 },
6685         { "RcfRxChannel", 5, 1 },
6686         { "RcfDataOutSrdy", 4, 1 },
6687         { "RxDvld", 3, 1 },
6688         { "RxOoDvld", 2, 1 },
6689         { "RxCongestion", 1, 1 },
6690         { "TxCongestion", 0, 1 },
6691         { NULL }
6692 };
6693
6694 static struct field_desc tp_la1[] = {
6695         { "CplCmdIn", 56, 8 },
6696         { "CplCmdOut", 48, 8 },
6697         { "ESynOut", 47, 1 },
6698         { "EAckOut", 46, 1 },
6699         { "EFinOut", 45, 1 },
6700         { "ERstOut", 44, 1 },
6701         { "SynIn", 43, 1 },
6702         { "AckIn", 42, 1 },
6703         { "FinIn", 41, 1 },
6704         { "RstIn", 40, 1 },
6705         { "DataIn", 39, 1 },
6706         { "DataInVld", 38, 1 },
6707         { "PadIn", 37, 1 },
6708         { "RxBufEmpty", 36, 1 },
6709         { "RxDdp", 35, 1 },
6710         { "RxFbCongestion", 34, 1 },
6711         { "TxFbCongestion", 33, 1 },
6712         { "TxPktSumSrdy", 32, 1 },
6713         { "RcfUlpType", 28, 4 },
6714         { "Eread", 27, 1 },
6715         { "Ebypass", 26, 1 },
6716         { "Esave", 25, 1 },
6717         { "Static0", 24, 1 },
6718         { "Cread", 23, 1 },
6719         { "Cbypass", 22, 1 },
6720         { "Csave", 21, 1 },
6721         { "CPktOut", 20, 1 },
6722         { "RxPagePoolFull", 18, 2 },
6723         { "RxLpbkPkt", 17, 1 },
6724         { "TxLpbkPkt", 16, 1 },
6725         { "RxVfValid", 15, 1 },
6726         { "SynLearned", 14, 1 },
6727         { "SetDelEntry", 13, 1 },
6728         { "SetInvEntry", 12, 1 },
6729         { "CpcmdDvld", 11, 1 },
6730         { "CpcmdSave", 10, 1 },
6731         { "RxPstructsFull", 8, 2 },
6732         { "EpcmdDvld", 7, 1 },
6733         { "EpcmdFlush", 6, 1 },
6734         { "EpcmdTrimPrefix", 5, 1 },
6735         { "EpcmdTrimPostfix", 4, 1 },
6736         { "ERssIp4Pkt", 3, 1 },
6737         { "ERssIp6Pkt", 2, 1 },
6738         { "ERssTcpUdpPkt", 1, 1 },
6739         { "ERssFceFipPkt", 0, 1 },
6740         { NULL }
6741 };
6742
6743 static struct field_desc tp_la2[] = {
6744         { "CplCmdIn", 56, 8 },
6745         { "MpsVfVld", 55, 1 },
6746         { "MpsPf", 52, 3 },
6747         { "MpsVf", 44, 8 },
6748         { "SynIn", 43, 1 },
6749         { "AckIn", 42, 1 },
6750         { "FinIn", 41, 1 },
6751         { "RstIn", 40, 1 },
6752         { "DataIn", 39, 1 },
6753         { "DataInVld", 38, 1 },
6754         { "PadIn", 37, 1 },
6755         { "RxBufEmpty", 36, 1 },
6756         { "RxDdp", 35, 1 },
6757         { "RxFbCongestion", 34, 1 },
6758         { "TxFbCongestion", 33, 1 },
6759         { "TxPktSumSrdy", 32, 1 },
6760         { "RcfUlpType", 28, 4 },
6761         { "Eread", 27, 1 },
6762         { "Ebypass", 26, 1 },
6763         { "Esave", 25, 1 },
6764         { "Static0", 24, 1 },
6765         { "Cread", 23, 1 },
6766         { "Cbypass", 22, 1 },
6767         { "Csave", 21, 1 },
6768         { "CPktOut", 20, 1 },
6769         { "RxPagePoolFull", 18, 2 },
6770         { "RxLpbkPkt", 17, 1 },
6771         { "TxLpbkPkt", 16, 1 },
6772         { "RxVfValid", 15, 1 },
6773         { "SynLearned", 14, 1 },
6774         { "SetDelEntry", 13, 1 },
6775         { "SetInvEntry", 12, 1 },
6776         { "CpcmdDvld", 11, 1 },
6777         { "CpcmdSave", 10, 1 },
6778         { "RxPstructsFull", 8, 2 },
6779         { "EpcmdDvld", 7, 1 },
6780         { "EpcmdFlush", 6, 1 },
6781         { "EpcmdTrimPrefix", 5, 1 },
6782         { "EpcmdTrimPostfix", 4, 1 },
6783         { "ERssIp4Pkt", 3, 1 },
6784         { "ERssIp6Pkt", 2, 1 },
6785         { "ERssTcpUdpPkt", 1, 1 },
6786         { "ERssFceFipPkt", 0, 1 },
6787         { NULL }
6788 };
6789
6790 static void
6791 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6792 {
6793
6794         field_desc_show(sb, *p, tp_la0);
6795 }
6796
6797 static void
6798 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6799 {
6800
6801         if (idx)
6802                 sbuf_printf(sb, "\n");
6803         field_desc_show(sb, p[0], tp_la0);
6804         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6805                 field_desc_show(sb, p[1], tp_la0);
6806 }
6807
6808 static void
6809 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6810 {
6811
6812         if (idx)
6813                 sbuf_printf(sb, "\n");
6814         field_desc_show(sb, p[0], tp_la0);
6815         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6816                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6817 }
6818
6819 static int
6820 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6821 {
6822         struct adapter *sc = arg1;
6823         struct sbuf *sb;
6824         uint64_t *buf, *p;
6825         int rc;
6826         u_int i, inc;
6827         void (*show_func)(struct sbuf *, uint64_t *, int);
6828
6829         rc = sysctl_wire_old_buffer(req, 0);
6830         if (rc != 0)
6831                 return (rc);
6832
6833         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6834         if (sb == NULL)
6835                 return (ENOMEM);
6836
6837         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6838
6839         t4_tp_read_la(sc, buf, NULL);
6840         p = buf;
6841
6842         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6843         case 2:
6844                 inc = 2;
6845                 show_func = tp_la_show2;
6846                 break;
6847         case 3:
6848                 inc = 2;
6849                 show_func = tp_la_show3;
6850                 break;
6851         default:
6852                 inc = 1;
6853                 show_func = tp_la_show;
6854         }
6855
6856         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6857                 (*show_func)(sb, p, i);
6858
6859         rc = sbuf_finish(sb);
6860         sbuf_delete(sb);
6861         free(buf, M_CXGBE);
6862         return (rc);
6863 }
6864
6865 static int
6866 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6867 {
6868         struct adapter *sc = arg1;
6869         struct sbuf *sb;
6870         int rc;
6871         u64 nrate[NCHAN], orate[NCHAN];
6872
6873         rc = sysctl_wire_old_buffer(req, 0);
6874         if (rc != 0)
6875                 return (rc);
6876
6877         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6878         if (sb == NULL)
6879                 return (ENOMEM);
6880
6881         t4_get_chan_txrate(sc, nrate, orate);
6882         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6883                  "channel 3\n");
6884         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6885             nrate[0], nrate[1], nrate[2], nrate[3]);
6886         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6887             orate[0], orate[1], orate[2], orate[3]);
6888
6889         rc = sbuf_finish(sb);
6890         sbuf_delete(sb);
6891
6892         return (rc);
6893 }
6894
6895 static int
6896 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6897 {
6898         struct adapter *sc = arg1;
6899         struct sbuf *sb;
6900         uint32_t *buf, *p;
6901         int rc, i;
6902
6903         rc = sysctl_wire_old_buffer(req, 0);
6904         if (rc != 0)
6905                 return (rc);
6906
6907         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6908         if (sb == NULL)
6909                 return (ENOMEM);
6910
6911         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6912             M_ZERO | M_WAITOK);
6913
6914         t4_ulprx_read_la(sc, buf);
6915         p = buf;
6916
6917         sbuf_printf(sb, "      Pcmd        Type   Message"
6918             "                Data");
6919         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6920                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6921                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6922         }
6923
6924         rc = sbuf_finish(sb);
6925         sbuf_delete(sb);
6926         free(buf, M_CXGBE);
6927         return (rc);
6928 }
6929
6930 static int
6931 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6932 {
6933         struct adapter *sc = arg1;
6934         struct sbuf *sb;
6935         int rc, v;
6936
6937         rc = sysctl_wire_old_buffer(req, 0);
6938         if (rc != 0)
6939                 return (rc);
6940
6941         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6942         if (sb == NULL)
6943                 return (ENOMEM);
6944
6945         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6946         if (G_STATSOURCE_T5(v) == 7) {
6947                 if (G_STATMODE(v) == 0) {
6948                         sbuf_printf(sb, "total %d, incomplete %d",
6949                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6950                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6951                 } else if (G_STATMODE(v) == 1) {
6952                         sbuf_printf(sb, "total %d, data overflow %d",
6953                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6954                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6955                 }
6956         }
6957         rc = sbuf_finish(sb);
6958         sbuf_delete(sb);
6959
6960         return (rc);
6961 }
6962 #endif
6963
6964 static uint32_t
6965 fconf_to_mode(uint32_t fconf)
6966 {
6967         uint32_t mode;
6968
6969         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6970             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6971
6972         if (fconf & F_FRAGMENTATION)
6973                 mode |= T4_FILTER_IP_FRAGMENT;
6974
6975         if (fconf & F_MPSHITTYPE)
6976                 mode |= T4_FILTER_MPS_HIT_TYPE;
6977
6978         if (fconf & F_MACMATCH)
6979                 mode |= T4_FILTER_MAC_IDX;
6980
6981         if (fconf & F_ETHERTYPE)
6982                 mode |= T4_FILTER_ETH_TYPE;
6983
6984         if (fconf & F_PROTOCOL)
6985                 mode |= T4_FILTER_IP_PROTO;
6986
6987         if (fconf & F_TOS)
6988                 mode |= T4_FILTER_IP_TOS;
6989
6990         if (fconf & F_VLAN)
6991                 mode |= T4_FILTER_VLAN;
6992
6993         if (fconf & F_VNIC_ID)
6994                 mode |= T4_FILTER_VNIC;
6995
6996         if (fconf & F_PORT)
6997                 mode |= T4_FILTER_PORT;
6998
6999         if (fconf & F_FCOE)
7000                 mode |= T4_FILTER_FCoE;
7001
7002         return (mode);
7003 }
7004
7005 static uint32_t
7006 mode_to_fconf(uint32_t mode)
7007 {
7008         uint32_t fconf = 0;
7009
7010         if (mode & T4_FILTER_IP_FRAGMENT)
7011                 fconf |= F_FRAGMENTATION;
7012
7013         if (mode & T4_FILTER_MPS_HIT_TYPE)
7014                 fconf |= F_MPSHITTYPE;
7015
7016         if (mode & T4_FILTER_MAC_IDX)
7017                 fconf |= F_MACMATCH;
7018
7019         if (mode & T4_FILTER_ETH_TYPE)
7020                 fconf |= F_ETHERTYPE;
7021
7022         if (mode & T4_FILTER_IP_PROTO)
7023                 fconf |= F_PROTOCOL;
7024
7025         if (mode & T4_FILTER_IP_TOS)
7026                 fconf |= F_TOS;
7027
7028         if (mode & T4_FILTER_VLAN)
7029                 fconf |= F_VLAN;
7030
7031         if (mode & T4_FILTER_VNIC)
7032                 fconf |= F_VNIC_ID;
7033
7034         if (mode & T4_FILTER_PORT)
7035                 fconf |= F_PORT;
7036
7037         if (mode & T4_FILTER_FCoE)
7038                 fconf |= F_FCOE;
7039
7040         return (fconf);
7041 }
7042
7043 static uint32_t
7044 fspec_to_fconf(struct t4_filter_specification *fs)
7045 {
7046         uint32_t fconf = 0;
7047
7048         if (fs->val.frag || fs->mask.frag)
7049                 fconf |= F_FRAGMENTATION;
7050
7051         if (fs->val.matchtype || fs->mask.matchtype)
7052                 fconf |= F_MPSHITTYPE;
7053
7054         if (fs->val.macidx || fs->mask.macidx)
7055                 fconf |= F_MACMATCH;
7056
7057         if (fs->val.ethtype || fs->mask.ethtype)
7058                 fconf |= F_ETHERTYPE;
7059
7060         if (fs->val.proto || fs->mask.proto)
7061                 fconf |= F_PROTOCOL;
7062
7063         if (fs->val.tos || fs->mask.tos)
7064                 fconf |= F_TOS;
7065
7066         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7067                 fconf |= F_VLAN;
7068
7069         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7070                 fconf |= F_VNIC_ID;
7071
7072         if (fs->val.iport || fs->mask.iport)
7073                 fconf |= F_PORT;
7074
7075         if (fs->val.fcoe || fs->mask.fcoe)
7076                 fconf |= F_FCOE;
7077
7078         return (fconf);
7079 }
7080
7081 static int
7082 get_filter_mode(struct adapter *sc, uint32_t *mode)
7083 {
7084         int rc;
7085         uint32_t fconf;
7086
7087         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7088             "t4getfm");
7089         if (rc)
7090                 return (rc);
7091
7092         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7093             A_TP_VLAN_PRI_MAP);
7094
7095         if (sc->params.tp.vlan_pri_map != fconf) {
7096                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7097                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7098                     fconf);
7099         }
7100
7101         *mode = fconf_to_mode(fconf);
7102
7103         end_synchronized_op(sc, LOCK_HELD);
7104         return (0);
7105 }
7106
7107 static int
7108 set_filter_mode(struct adapter *sc, uint32_t mode)
7109 {
7110         uint32_t fconf;
7111         int rc;
7112
7113         fconf = mode_to_fconf(mode);
7114
7115         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7116             "t4setfm");
7117         if (rc)
7118                 return (rc);
7119
7120         if (sc->tids.ftids_in_use > 0) {
7121                 rc = EBUSY;
7122                 goto done;
7123         }
7124
7125 #ifdef TCP_OFFLOAD
7126         if (uld_active(sc, ULD_TOM)) {
7127                 rc = EBUSY;
7128                 goto done;
7129         }
7130 #endif
7131
7132         rc = -t4_set_filter_mode(sc, fconf);
7133 done:
7134         end_synchronized_op(sc, LOCK_HELD);
7135         return (rc);
7136 }
7137
7138 static inline uint64_t
7139 get_filter_hits(struct adapter *sc, uint32_t fid)
7140 {
7141         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7142         uint64_t hits;
7143
7144         memwin_info(sc, 0, &mw_base, NULL);
7145         off = position_memwin(sc, 0,
7146             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7147         if (is_t4(sc)) {
7148                 hits = t4_read_reg64(sc, mw_base + off + 16);
7149                 hits = be64toh(hits);
7150         } else {
7151                 hits = t4_read_reg(sc, mw_base + off + 24);
7152                 hits = be32toh(hits);
7153         }
7154
7155         return (hits);
7156 }
7157
7158 static int
7159 get_filter(struct adapter *sc, struct t4_filter *t)
7160 {
7161         int i, rc, nfilters = sc->tids.nftids;
7162         struct filter_entry *f;
7163
7164         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7165             "t4getf");
7166         if (rc)
7167                 return (rc);
7168
7169         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7170             t->idx >= nfilters) {
7171                 t->idx = 0xffffffff;
7172                 goto done;
7173         }
7174
7175         f = &sc->tids.ftid_tab[t->idx];
7176         for (i = t->idx; i < nfilters; i++, f++) {
7177                 if (f->valid) {
7178                         t->idx = i;
7179                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7180                         t->smtidx = f->smtidx;
7181                         if (f->fs.hitcnts)
7182                                 t->hits = get_filter_hits(sc, t->idx);
7183                         else
7184                                 t->hits = UINT64_MAX;
7185                         t->fs = f->fs;
7186
7187                         goto done;
7188                 }
7189         }
7190
7191         t->idx = 0xffffffff;
7192 done:
7193         end_synchronized_op(sc, LOCK_HELD);
7194         return (0);
7195 }
7196
7197 static int
7198 set_filter(struct adapter *sc, struct t4_filter *t)
7199 {
7200         unsigned int nfilters, nports;
7201         struct filter_entry *f;
7202         int i, rc;
7203
7204         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7205         if (rc)
7206                 return (rc);
7207
7208         nfilters = sc->tids.nftids;
7209         nports = sc->params.nports;
7210
7211         if (nfilters == 0) {
7212                 rc = ENOTSUP;
7213                 goto done;
7214         }
7215
7216         if (!(sc->flags & FULL_INIT_DONE)) {
7217                 rc = EAGAIN;
7218                 goto done;
7219         }
7220
7221         if (t->idx >= nfilters) {
7222                 rc = EINVAL;
7223                 goto done;
7224         }
7225
7226         /* Validate against the global filter mode */
7227         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7228             sc->params.tp.vlan_pri_map) {
7229                 rc = E2BIG;
7230                 goto done;
7231         }
7232
7233         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7234                 rc = EINVAL;
7235                 goto done;
7236         }
7237
7238         if (t->fs.val.iport >= nports) {
7239                 rc = EINVAL;
7240                 goto done;
7241         }
7242
7243         /* Can't specify an iq if not steering to it */
7244         if (!t->fs.dirsteer && t->fs.iq) {
7245                 rc = EINVAL;
7246                 goto done;
7247         }
7248
7249         /* IPv6 filter idx must be 4 aligned */
7250         if (t->fs.type == 1 &&
7251             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7252                 rc = EINVAL;
7253                 goto done;
7254         }
7255
7256         if (sc->tids.ftid_tab == NULL) {
7257                 KASSERT(sc->tids.ftids_in_use == 0,
7258                     ("%s: no memory allocated but filters_in_use > 0",
7259                     __func__));
7260
7261                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7262                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7263                 if (sc->tids.ftid_tab == NULL) {
7264                         rc = ENOMEM;
7265                         goto done;
7266                 }
7267                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7268         }
7269
7270         for (i = 0; i < 4; i++) {
7271                 f = &sc->tids.ftid_tab[t->idx + i];
7272
7273                 if (f->pending || f->valid) {
7274                         rc = EBUSY;
7275                         goto done;
7276                 }
7277                 if (f->locked) {
7278                         rc = EPERM;
7279                         goto done;
7280                 }
7281
7282                 if (t->fs.type == 0)
7283                         break;
7284         }
7285
7286         f = &sc->tids.ftid_tab[t->idx];
7287         f->fs = t->fs;
7288
7289         rc = set_filter_wr(sc, t->idx);
7290 done:
7291         end_synchronized_op(sc, 0);
7292
7293         if (rc == 0) {
7294                 mtx_lock(&sc->tids.ftid_lock);
7295                 for (;;) {
7296                         if (f->pending == 0) {
7297                                 rc = f->valid ? 0 : EIO;
7298                                 break;
7299                         }
7300
7301                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7302                             PCATCH, "t4setfw", 0)) {
7303                                 rc = EINPROGRESS;
7304                                 break;
7305                         }
7306                 }
7307                 mtx_unlock(&sc->tids.ftid_lock);
7308         }
7309         return (rc);
7310 }
7311
7312 static int
7313 del_filter(struct adapter *sc, struct t4_filter *t)
7314 {
7315         unsigned int nfilters;
7316         struct filter_entry *f;
7317         int rc;
7318
7319         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7320         if (rc)
7321                 return (rc);
7322
7323         nfilters = sc->tids.nftids;
7324
7325         if (nfilters == 0) {
7326                 rc = ENOTSUP;
7327                 goto done;
7328         }
7329
7330         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7331             t->idx >= nfilters) {
7332                 rc = EINVAL;
7333                 goto done;
7334         }
7335
7336         if (!(sc->flags & FULL_INIT_DONE)) {
7337                 rc = EAGAIN;
7338                 goto done;
7339         }
7340
7341         f = &sc->tids.ftid_tab[t->idx];
7342
7343         if (f->pending) {
7344                 rc = EBUSY;
7345                 goto done;
7346         }
7347         if (f->locked) {
7348                 rc = EPERM;
7349                 goto done;
7350         }
7351
7352         if (f->valid) {
7353                 t->fs = f->fs;  /* extra info for the caller */
7354                 rc = del_filter_wr(sc, t->idx);
7355         }
7356
7357 done:
7358         end_synchronized_op(sc, 0);
7359
7360         if (rc == 0) {
7361                 mtx_lock(&sc->tids.ftid_lock);
7362                 for (;;) {
7363                         if (f->pending == 0) {
7364                                 rc = f->valid ? EIO : 0;
7365                                 break;
7366                         }
7367
7368                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7369                             PCATCH, "t4delfw", 0)) {
7370                                 rc = EINPROGRESS;
7371                                 break;
7372                         }
7373                 }
7374                 mtx_unlock(&sc->tids.ftid_lock);
7375         }
7376
7377         return (rc);
7378 }
7379
7380 static void
7381 clear_filter(struct filter_entry *f)
7382 {
7383         if (f->l2t)
7384                 t4_l2t_release(f->l2t);
7385
7386         bzero(f, sizeof (*f));
7387 }
7388
7389 static int
7390 set_filter_wr(struct adapter *sc, int fidx)
7391 {
7392         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7393         struct fw_filter_wr *fwr;
7394         unsigned int ftid;
7395         struct wrq_cookie cookie;
7396
7397         ASSERT_SYNCHRONIZED_OP(sc);
7398
7399         if (f->fs.newdmac || f->fs.newvlan) {
7400                 /* This filter needs an L2T entry; allocate one. */
7401                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7402                 if (f->l2t == NULL)
7403                         return (EAGAIN);
7404                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7405                     f->fs.dmac)) {
7406                         t4_l2t_release(f->l2t);
7407                         f->l2t = NULL;
7408                         return (ENOMEM);
7409                 }
7410         }
7411
7412         ftid = sc->tids.ftid_base + fidx;
7413
7414         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7415         if (fwr == NULL)
7416                 return (ENOMEM);
7417         bzero(fwr, sizeof(*fwr));
7418
7419         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7420         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7421         fwr->tid_to_iq =
7422             htobe32(V_FW_FILTER_WR_TID(ftid) |
7423                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7424                 V_FW_FILTER_WR_NOREPLY(0) |
7425                 V_FW_FILTER_WR_IQ(f->fs.iq));
7426         fwr->del_filter_to_l2tix =
7427             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7428                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7429                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7430                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7431                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7432                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7433                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7434                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7435                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7436                     f->fs.newvlan == VLAN_REWRITE) |
7437                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7438                     f->fs.newvlan == VLAN_REWRITE) |
7439                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7440                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7441                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7442                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7443         fwr->ethtype = htobe16(f->fs.val.ethtype);
7444         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7445         fwr->frag_to_ovlan_vldm =
7446             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7447                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7448                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7449                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7450                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7451                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7452         fwr->smac_sel = 0;
7453         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7454             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7455         fwr->maci_to_matchtypem =
7456             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7457                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7458                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7459                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7460                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7461                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7462                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7463                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7464         fwr->ptcl = f->fs.val.proto;
7465         fwr->ptclm = f->fs.mask.proto;
7466         fwr->ttyp = f->fs.val.tos;
7467         fwr->ttypm = f->fs.mask.tos;
7468         fwr->ivlan = htobe16(f->fs.val.vlan);
7469         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7470         fwr->ovlan = htobe16(f->fs.val.vnic);
7471         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7472         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7473         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7474         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7475         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7476         fwr->lp = htobe16(f->fs.val.dport);
7477         fwr->lpm = htobe16(f->fs.mask.dport);
7478         fwr->fp = htobe16(f->fs.val.sport);
7479         fwr->fpm = htobe16(f->fs.mask.sport);
7480         if (f->fs.newsmac)
7481                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7482
7483         f->pending = 1;
7484         sc->tids.ftids_in_use++;
7485
7486         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7487         return (0);
7488 }
7489
7490 static int
7491 del_filter_wr(struct adapter *sc, int fidx)
7492 {
7493         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7494         struct fw_filter_wr *fwr;
7495         unsigned int ftid;
7496         struct wrq_cookie cookie;
7497
7498         ftid = sc->tids.ftid_base + fidx;
7499
7500         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7501         if (fwr == NULL)
7502                 return (ENOMEM);
7503         bzero(fwr, sizeof (*fwr));
7504
7505         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7506
7507         f->pending = 1;
7508         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7509         return (0);
7510 }
7511
7512 int
7513 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7514 {
7515         struct adapter *sc = iq->adapter;
7516         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7517         unsigned int idx = GET_TID(rpl);
7518         unsigned int rc;
7519         struct filter_entry *f;
7520
7521         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7522             rss->opcode));
7523
7524         if (is_ftid(sc, idx)) {
7525
7526                 idx -= sc->tids.ftid_base;
7527                 f = &sc->tids.ftid_tab[idx];
7528                 rc = G_COOKIE(rpl->cookie);
7529
7530                 mtx_lock(&sc->tids.ftid_lock);
7531                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7532                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7533                             __func__, idx));
7534                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7535                         f->pending = 0;  /* asynchronous setup completed */
7536                         f->valid = 1;
7537                 } else {
7538                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7539                                 /* Add or delete failed, display an error */
7540                                 log(LOG_ERR,
7541                                     "filter %u setup failed with error %u\n",
7542                                     idx, rc);
7543                         }
7544
7545                         clear_filter(f);
7546                         sc->tids.ftids_in_use--;
7547                 }
7548                 wakeup(&sc->tids.ftid_tab);
7549                 mtx_unlock(&sc->tids.ftid_lock);
7550         }
7551
7552         return (0);
7553 }
7554
7555 static int
7556 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7557 {
7558         int rc;
7559
7560         if (cntxt->cid > M_CTXTQID)
7561                 return (EINVAL);
7562
7563         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7564             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7565                 return (EINVAL);
7566
7567         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7568         if (rc)
7569                 return (rc);
7570
7571         if (sc->flags & FW_OK) {
7572                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7573                     &cntxt->data[0]);
7574                 if (rc == 0)
7575                         goto done;
7576         }
7577
7578         /*
7579          * Read via firmware failed or wasn't even attempted.  Read directly via
7580          * the backdoor.
7581          */
7582         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7583 done:
7584         end_synchronized_op(sc, 0);
7585         return (rc);
7586 }
7587
7588 static int
7589 load_fw(struct adapter *sc, struct t4_data *fw)
7590 {
7591         int rc;
7592         uint8_t *fw_data;
7593
7594         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7595         if (rc)
7596                 return (rc);
7597
7598         if (sc->flags & FULL_INIT_DONE) {
7599                 rc = EBUSY;
7600                 goto done;
7601         }
7602
7603         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7604         if (fw_data == NULL) {
7605                 rc = ENOMEM;
7606                 goto done;
7607         }
7608
7609         rc = copyin(fw->data, fw_data, fw->len);
7610         if (rc == 0)
7611                 rc = -t4_load_fw(sc, fw_data, fw->len);
7612
7613         free(fw_data, M_CXGBE);
7614 done:
7615         end_synchronized_op(sc, 0);
7616         return (rc);
7617 }
7618
7619 static int
7620 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7621 {
7622         uint32_t addr, off, remaining, i, n;
7623         uint32_t *buf, *b;
7624         uint32_t mw_base, mw_aperture;
7625         int rc;
7626         uint8_t *dst;
7627
7628         rc = validate_mem_range(sc, mr->addr, mr->len);
7629         if (rc != 0)
7630                 return (rc);
7631
7632         memwin_info(sc, win, &mw_base, &mw_aperture);
7633         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7634         addr = mr->addr;
7635         remaining = mr->len;
7636         dst = (void *)mr->data;
7637
7638         while (remaining) {
7639                 off = position_memwin(sc, win, addr);
7640
7641                 /* number of bytes that we'll copy in the inner loop */
7642                 n = min(remaining, mw_aperture - off);
7643                 for (i = 0; i < n; i += 4)
7644                         *b++ = t4_read_reg(sc, mw_base + off + i);
7645
7646                 rc = copyout(buf, dst, n);
7647                 if (rc != 0)
7648                         break;
7649
7650                 b = buf;
7651                 dst += n;
7652                 remaining -= n;
7653                 addr += n;
7654         }
7655
7656         free(buf, M_CXGBE);
7657         return (rc);
7658 }
7659
7660 static int
7661 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7662 {
7663         int rc;
7664
7665         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7666                 return (EINVAL);
7667
7668         if (i2cd->len > sizeof(i2cd->data))
7669                 return (EFBIG);
7670
7671         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7672         if (rc)
7673                 return (rc);
7674         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7675             i2cd->offset, i2cd->len, &i2cd->data[0]);
7676         end_synchronized_op(sc, 0);
7677
7678         return (rc);
7679 }
7680
7681 static int
7682 in_range(int val, int lo, int hi)
7683 {
7684
7685         return (val < 0 || (val <= hi && val >= lo));
7686 }
7687
7688 static int
7689 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7690 {
7691         int fw_subcmd, fw_type, rc;
7692
7693         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7694         if (rc)
7695                 return (rc);
7696
7697         if (!(sc->flags & FULL_INIT_DONE)) {
7698                 rc = EAGAIN;
7699                 goto done;
7700         }
7701
7702         /*
7703          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7704          * sub-command and type are in common locations.)
7705          */
7706         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7707                 fw_subcmd = FW_SCHED_SC_CONFIG;
7708         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7709                 fw_subcmd = FW_SCHED_SC_PARAMS;
7710         else {
7711                 rc = EINVAL;
7712                 goto done;
7713         }
7714         if (p->type == SCHED_CLASS_TYPE_PACKET)
7715                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7716         else {
7717                 rc = EINVAL;
7718                 goto done;
7719         }
7720
7721         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7722                 /* Vet our parameters ..*/
7723                 if (p->u.config.minmax < 0) {
7724                         rc = EINVAL;
7725                         goto done;
7726                 }
7727
7728                 /* And pass the request to the firmware ...*/
7729                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7730                 goto done;
7731         }
7732
7733         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7734                 int fw_level;
7735                 int fw_mode;
7736                 int fw_rateunit;
7737                 int fw_ratemode;
7738
7739                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7740                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7741                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7742                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7743                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7744                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7745                 else {
7746                         rc = EINVAL;
7747                         goto done;
7748                 }
7749
7750                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7751                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7752                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7753                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7754                 else {
7755                         rc = EINVAL;
7756                         goto done;
7757                 }
7758
7759                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7760                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7761                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7762                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7763                 else {
7764                         rc = EINVAL;
7765                         goto done;
7766                 }
7767
7768                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7769                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7770                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7771                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7772                 else {
7773                         rc = EINVAL;
7774                         goto done;
7775                 }
7776
7777                 /* Vet our parameters ... */
7778                 if (!in_range(p->u.params.channel, 0, 3) ||
7779                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7780                     !in_range(p->u.params.minrate, 0, 10000000) ||
7781                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7782                     !in_range(p->u.params.weight, 0, 100)) {
7783                         rc = ERANGE;
7784                         goto done;
7785                 }
7786
7787                 /*
7788                  * Translate any unset parameters into the firmware's
7789                  * nomenclature and/or fail the call if the parameters
7790                  * are required ...
7791                  */
7792                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7793                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7794                         rc = EINVAL;
7795                         goto done;
7796                 }
7797                 if (p->u.params.minrate < 0)
7798                         p->u.params.minrate = 0;
7799                 if (p->u.params.maxrate < 0) {
7800                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7801                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7802                                 rc = EINVAL;
7803                                 goto done;
7804                         } else
7805                                 p->u.params.maxrate = 0;
7806                 }
7807                 if (p->u.params.weight < 0) {
7808                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7809                                 rc = EINVAL;
7810                                 goto done;
7811                         } else
7812                                 p->u.params.weight = 0;
7813                 }
7814                 if (p->u.params.pktsize < 0) {
7815                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7816                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7817                                 rc = EINVAL;
7818                                 goto done;
7819                         } else
7820                                 p->u.params.pktsize = 0;
7821                 }
7822
7823                 /* See what the firmware thinks of the request ... */
7824                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7825                     fw_rateunit, fw_ratemode, p->u.params.channel,
7826                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7827                     p->u.params.weight, p->u.params.pktsize, 1);
7828                 goto done;
7829         }
7830
7831         rc = EINVAL;
7832 done:
7833         end_synchronized_op(sc, 0);
7834         return (rc);
7835 }
7836
7837 static int
7838 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7839 {
7840         struct port_info *pi = NULL;
7841         struct sge_txq *txq;
7842         uint32_t fw_mnem, fw_queue, fw_class;
7843         int i, rc;
7844
7845         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7846         if (rc)
7847                 return (rc);
7848
7849         if (!(sc->flags & FULL_INIT_DONE)) {
7850                 rc = EAGAIN;
7851                 goto done;
7852         }
7853
7854         if (p->port >= sc->params.nports) {
7855                 rc = EINVAL;
7856                 goto done;
7857         }
7858
7859         pi = sc->port[p->port];
7860         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7861                 rc = EINVAL;
7862                 goto done;
7863         }
7864
7865         /*
7866          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7867          * Scheduling Class in this case).
7868          */
7869         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7870             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7871         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7872
7873         /*
7874          * If op.queue is non-negative, then we're only changing the scheduling
7875          * on a single specified TX queue.
7876          */
7877         if (p->queue >= 0) {
7878                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7879                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7880                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7881                     &fw_class);
7882                 goto done;
7883         }
7884
7885         /*
7886          * Change the scheduling on all the TX queues for the
7887          * interface.
7888          */
7889         for_each_txq(pi, i, txq) {
7890                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7891                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7892                     &fw_class);
7893                 if (rc)
7894                         goto done;
7895         }
7896
7897         rc = 0;
7898 done:
7899         end_synchronized_op(sc, 0);
7900         return (rc);
7901 }
7902
7903 int
7904 t4_os_find_pci_capability(struct adapter *sc, int cap)
7905 {
7906         int i;
7907
7908         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7909 }
7910
7911 int
7912 t4_os_pci_save_state(struct adapter *sc)
7913 {
7914         device_t dev;
7915         struct pci_devinfo *dinfo;
7916
7917         dev = sc->dev;
7918         dinfo = device_get_ivars(dev);
7919
7920         pci_cfg_save(dev, dinfo, 0);
7921         return (0);
7922 }
7923
7924 int
7925 t4_os_pci_restore_state(struct adapter *sc)
7926 {
7927         device_t dev;
7928         struct pci_devinfo *dinfo;
7929
7930         dev = sc->dev;
7931         dinfo = device_get_ivars(dev);
7932
7933         pci_cfg_restore(dev, dinfo);
7934         return (0);
7935 }
7936
7937 void
7938 t4_os_portmod_changed(const struct adapter *sc, int idx)
7939 {
7940         struct port_info *pi = sc->port[idx];
7941         static const char *mod_str[] = {
7942                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7943         };
7944
7945         build_medialist(pi, &pi->media);
7946 #ifdef DEV_NETMAP
7947         build_medialist(pi, &pi->nm_media);
7948 #endif
7949
7950         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7951                 if_printf(pi->ifp, "transceiver unplugged.\n");
7952         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7953                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7954         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7955                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7956         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7957                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7958                     mod_str[pi->mod_type]);
7959         } else {
7960                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7961                     pi->mod_type);
7962         }
7963 }
7964
7965 void
7966 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7967 {
7968         struct port_info *pi = sc->port[idx];
7969         struct ifnet *ifp = pi->ifp;
7970
7971         if (link_stat) {
7972                 pi->linkdnrc = -1;
7973                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7974                 if_link_state_change(ifp, LINK_STATE_UP);
7975         } else {
7976                 if (reason >= 0)
7977                         pi->linkdnrc = reason;
7978                 if_link_state_change(ifp, LINK_STATE_DOWN);
7979         }
7980 }
7981
7982 void
7983 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7984 {
7985         struct adapter *sc;
7986
7987         sx_slock(&t4_list_lock);
7988         SLIST_FOREACH(sc, &t4_list, link) {
7989                 /*
7990                  * func should not make any assumptions about what state sc is
7991                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7992                  */
7993                 func(sc, arg);
7994         }
7995         sx_sunlock(&t4_list_lock);
7996 }
7997
7998 static int
7999 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8000 {
8001        return (0);
8002 }
8003
8004 static int
8005 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8006 {
8007        return (0);
8008 }
8009
8010 static int
8011 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8012     struct thread *td)
8013 {
8014         int rc;
8015         struct adapter *sc = dev->si_drv1;
8016
8017         rc = priv_check(td, PRIV_DRIVER);
8018         if (rc != 0)
8019                 return (rc);
8020
8021         switch (cmd) {
8022         case CHELSIO_T4_GETREG: {
8023                 struct t4_reg *edata = (struct t4_reg *)data;
8024
8025                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8026                         return (EFAULT);
8027
8028                 if (edata->size == 4)
8029                         edata->val = t4_read_reg(sc, edata->addr);
8030                 else if (edata->size == 8)
8031                         edata->val = t4_read_reg64(sc, edata->addr);
8032                 else
8033                         return (EINVAL);
8034
8035                 break;
8036         }
8037         case CHELSIO_T4_SETREG: {
8038                 struct t4_reg *edata = (struct t4_reg *)data;
8039
8040                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8041                         return (EFAULT);
8042
8043                 if (edata->size == 4) {
8044                         if (edata->val & 0xffffffff00000000)
8045                                 return (EINVAL);
8046                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8047                 } else if (edata->size == 8)
8048                         t4_write_reg64(sc, edata->addr, edata->val);
8049                 else
8050                         return (EINVAL);
8051                 break;
8052         }
8053         case CHELSIO_T4_REGDUMP: {
8054                 struct t4_regdump *regs = (struct t4_regdump *)data;
8055                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8056                 uint8_t *buf;
8057
8058                 if (regs->len < reglen) {
8059                         regs->len = reglen; /* hint to the caller */
8060                         return (ENOBUFS);
8061                 }
8062
8063                 regs->len = reglen;
8064                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8065                 t4_get_regs(sc, regs, buf);
8066                 rc = copyout(buf, regs->data, reglen);
8067                 free(buf, M_CXGBE);
8068                 break;
8069         }
8070         case CHELSIO_T4_GET_FILTER_MODE:
8071                 rc = get_filter_mode(sc, (uint32_t *)data);
8072                 break;
8073         case CHELSIO_T4_SET_FILTER_MODE:
8074                 rc = set_filter_mode(sc, *(uint32_t *)data);
8075                 break;
8076         case CHELSIO_T4_GET_FILTER:
8077                 rc = get_filter(sc, (struct t4_filter *)data);
8078                 break;
8079         case CHELSIO_T4_SET_FILTER:
8080                 rc = set_filter(sc, (struct t4_filter *)data);
8081                 break;
8082         case CHELSIO_T4_DEL_FILTER:
8083                 rc = del_filter(sc, (struct t4_filter *)data);
8084                 break;
8085         case CHELSIO_T4_GET_SGE_CONTEXT:
8086                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8087                 break;
8088         case CHELSIO_T4_LOAD_FW:
8089                 rc = load_fw(sc, (struct t4_data *)data);
8090                 break;
8091         case CHELSIO_T4_GET_MEM:
8092                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8093                 break;
8094         case CHELSIO_T4_GET_I2C:
8095                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8096                 break;
8097         case CHELSIO_T4_CLEAR_STATS: {
8098                 int i;
8099                 u_int port_id = *(uint32_t *)data;
8100                 struct port_info *pi;
8101
8102                 if (port_id >= sc->params.nports)
8103                         return (EINVAL);
8104                 pi = sc->port[port_id];
8105
8106                 /* MAC stats */
8107                 t4_clr_port_stats(sc, pi->tx_chan);
8108                 pi->tx_parse_error = 0;
8109
8110                 if (pi->flags & PORT_INIT_DONE) {
8111                         struct sge_rxq *rxq;
8112                         struct sge_txq *txq;
8113                         struct sge_wrq *wrq;
8114
8115                         for_each_rxq(pi, i, rxq) {
8116 #if defined(INET) || defined(INET6)
8117                                 rxq->lro.lro_queued = 0;
8118                                 rxq->lro.lro_flushed = 0;
8119 #endif
8120                                 rxq->rxcsum = 0;
8121                                 rxq->vlan_extraction = 0;
8122                         }
8123
8124                         for_each_txq(pi, i, txq) {
8125                                 txq->txcsum = 0;
8126                                 txq->tso_wrs = 0;
8127                                 txq->vlan_insertion = 0;
8128                                 txq->imm_wrs = 0;
8129                                 txq->sgl_wrs = 0;
8130                                 txq->txpkt_wrs = 0;
8131                                 txq->txpkts0_wrs = 0;
8132                                 txq->txpkts1_wrs = 0;
8133                                 txq->txpkts0_pkts = 0;
8134                                 txq->txpkts1_pkts = 0;
8135                                 mp_ring_reset_stats(txq->r);
8136                         }
8137
8138 #ifdef TCP_OFFLOAD
8139                         /* nothing to clear for each ofld_rxq */
8140
8141                         for_each_ofld_txq(pi, i, wrq) {
8142                                 wrq->tx_wrs_direct = 0;
8143                                 wrq->tx_wrs_copied = 0;
8144                         }
8145 #endif
8146                         wrq = &sc->sge.ctrlq[pi->port_id];
8147                         wrq->tx_wrs_direct = 0;
8148                         wrq->tx_wrs_copied = 0;
8149                 }
8150                 break;
8151         }
8152         case CHELSIO_T4_SCHED_CLASS:
8153                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8154                 break;
8155         case CHELSIO_T4_SCHED_QUEUE:
8156                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8157                 break;
8158         case CHELSIO_T4_GET_TRACER:
8159                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8160                 break;
8161         case CHELSIO_T4_SET_TRACER:
8162                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8163                 break;
8164         default:
8165                 rc = EINVAL;
8166         }
8167
8168         return (rc);
8169 }
8170
8171 #ifdef TCP_OFFLOAD
8172 void
8173 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8174     const unsigned int *pgsz_order)
8175 {
8176         struct port_info *pi = ifp->if_softc;
8177         struct adapter *sc = pi->adapter;
8178
8179         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8180         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8181                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8182                 V_HPZ3(pgsz_order[3]));
8183 }
8184
8185 static int
8186 toe_capability(struct port_info *pi, int enable)
8187 {
8188         int rc;
8189         struct adapter *sc = pi->adapter;
8190
8191         ASSERT_SYNCHRONIZED_OP(sc);
8192
8193         if (!is_offload(sc))
8194                 return (ENODEV);
8195
8196         if (enable) {
8197                 /*
8198                  * We need the port's queues around so that we're able to send
8199                  * and receive CPLs to/from the TOE even if the ifnet for this
8200                  * port has never been UP'd administratively.
8201                  */
8202                 if (!(pi->flags & PORT_INIT_DONE)) {
8203                         rc = cxgbe_init_synchronized(pi);
8204                         if (rc)
8205                                 return (rc);
8206                 }
8207
8208                 if (isset(&sc->offload_map, pi->port_id))
8209                         return (0);
8210
8211                 if (!uld_active(sc, ULD_TOM)) {
8212                         rc = t4_activate_uld(sc, ULD_TOM);
8213                         if (rc == EAGAIN) {
8214                                 log(LOG_WARNING,
8215                                     "You must kldload t4_tom.ko before trying "
8216                                     "to enable TOE on a cxgbe interface.\n");
8217                         }
8218                         if (rc != 0)
8219                                 return (rc);
8220                         KASSERT(sc->tom_softc != NULL,
8221                             ("%s: TOM activated but softc NULL", __func__));
8222                         KASSERT(uld_active(sc, ULD_TOM),
8223                             ("%s: TOM activated but flag not set", __func__));
8224                 }
8225
8226                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8227                 if (!uld_active(sc, ULD_IWARP))
8228                         (void) t4_activate_uld(sc, ULD_IWARP);
8229                 if (!uld_active(sc, ULD_ISCSI))
8230                         (void) t4_activate_uld(sc, ULD_ISCSI);
8231
8232                 setbit(&sc->offload_map, pi->port_id);
8233         } else {
8234                 if (!isset(&sc->offload_map, pi->port_id))
8235                         return (0);
8236
8237                 KASSERT(uld_active(sc, ULD_TOM),
8238                     ("%s: TOM never initialized?", __func__));
8239                 clrbit(&sc->offload_map, pi->port_id);
8240         }
8241
8242         return (0);
8243 }
8244
8245 /*
8246  * Add an upper layer driver to the global list.
8247  */
8248 int
8249 t4_register_uld(struct uld_info *ui)
8250 {
8251         int rc = 0;
8252         struct uld_info *u;
8253
8254         sx_xlock(&t4_uld_list_lock);
8255         SLIST_FOREACH(u, &t4_uld_list, link) {
8256             if (u->uld_id == ui->uld_id) {
8257                     rc = EEXIST;
8258                     goto done;
8259             }
8260         }
8261
8262         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8263         ui->refcount = 0;
8264 done:
8265         sx_xunlock(&t4_uld_list_lock);
8266         return (rc);
8267 }
8268
8269 int
8270 t4_unregister_uld(struct uld_info *ui)
8271 {
8272         int rc = EINVAL;
8273         struct uld_info *u;
8274
8275         sx_xlock(&t4_uld_list_lock);
8276
8277         SLIST_FOREACH(u, &t4_uld_list, link) {
8278             if (u == ui) {
8279                     if (ui->refcount > 0) {
8280                             rc = EBUSY;
8281                             goto done;
8282                     }
8283
8284                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8285                     rc = 0;
8286                     goto done;
8287             }
8288         }
8289 done:
8290         sx_xunlock(&t4_uld_list_lock);
8291         return (rc);
8292 }
8293
8294 int
8295 t4_activate_uld(struct adapter *sc, int id)
8296 {
8297         int rc;
8298         struct uld_info *ui;
8299
8300         ASSERT_SYNCHRONIZED_OP(sc);
8301
8302         if (id < 0 || id > ULD_MAX)
8303                 return (EINVAL);
8304         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8305
8306         sx_slock(&t4_uld_list_lock);
8307
8308         SLIST_FOREACH(ui, &t4_uld_list, link) {
8309                 if (ui->uld_id == id) {
8310                         if (!(sc->flags & FULL_INIT_DONE)) {
8311                                 rc = adapter_full_init(sc);
8312                                 if (rc != 0)
8313                                         break;
8314                         }
8315
8316                         rc = ui->activate(sc);
8317                         if (rc == 0) {
8318                                 setbit(&sc->active_ulds, id);
8319                                 ui->refcount++;
8320                         }
8321                         break;
8322                 }
8323         }
8324
8325         sx_sunlock(&t4_uld_list_lock);
8326
8327         return (rc);
8328 }
8329
8330 int
8331 t4_deactivate_uld(struct adapter *sc, int id)
8332 {
8333         int rc;
8334         struct uld_info *ui;
8335
8336         ASSERT_SYNCHRONIZED_OP(sc);
8337
8338         if (id < 0 || id > ULD_MAX)
8339                 return (EINVAL);
8340         rc = ENXIO;
8341
8342         sx_slock(&t4_uld_list_lock);
8343
8344         SLIST_FOREACH(ui, &t4_uld_list, link) {
8345                 if (ui->uld_id == id) {
8346                         rc = ui->deactivate(sc);
8347                         if (rc == 0) {
8348                                 clrbit(&sc->active_ulds, id);
8349                                 ui->refcount--;
8350                         }
8351                         break;
8352                 }
8353         }
8354
8355         sx_sunlock(&t4_uld_list_lock);
8356
8357         return (rc);
8358 }
8359
8360 int
8361 uld_active(struct adapter *sc, int uld_id)
8362 {
8363
8364         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8365
8366         return (isset(&sc->active_ulds, uld_id));
8367 }
8368 #endif
8369
8370 /*
8371  * Come up with reasonable defaults for some of the tunables, provided they're
8372  * not set by the user (in which case we'll use the values as is).
8373  */
8374 static void
8375 tweak_tunables(void)
8376 {
8377         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8378
8379         if (t4_ntxq10g < 1)
8380                 t4_ntxq10g = min(nc, NTXQ_10G);
8381
8382         if (t4_ntxq1g < 1)
8383                 t4_ntxq1g = min(nc, NTXQ_1G);
8384
8385         if (t4_nrxq10g < 1)
8386                 t4_nrxq10g = min(nc, NRXQ_10G);
8387
8388         if (t4_nrxq1g < 1)
8389                 t4_nrxq1g = min(nc, NRXQ_1G);
8390
8391 #ifdef TCP_OFFLOAD
8392         if (t4_nofldtxq10g < 1)
8393                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8394
8395         if (t4_nofldtxq1g < 1)
8396                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8397
8398         if (t4_nofldrxq10g < 1)
8399                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8400
8401         if (t4_nofldrxq1g < 1)
8402                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8403
8404         if (t4_toecaps_allowed == -1)
8405                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8406 #else
8407         if (t4_toecaps_allowed == -1)
8408                 t4_toecaps_allowed = 0;
8409 #endif
8410
8411 #ifdef DEV_NETMAP
8412         if (t4_nnmtxq10g < 1)
8413                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8414
8415         if (t4_nnmtxq1g < 1)
8416                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8417
8418         if (t4_nnmrxq10g < 1)
8419                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8420
8421         if (t4_nnmrxq1g < 1)
8422                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8423 #endif
8424
8425         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8426                 t4_tmr_idx_10g = TMR_IDX_10G;
8427
8428         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8429                 t4_pktc_idx_10g = PKTC_IDX_10G;
8430
8431         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8432                 t4_tmr_idx_1g = TMR_IDX_1G;
8433
8434         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8435                 t4_pktc_idx_1g = PKTC_IDX_1G;
8436
8437         if (t4_qsize_txq < 128)
8438                 t4_qsize_txq = 128;
8439
8440         if (t4_qsize_rxq < 128)
8441                 t4_qsize_rxq = 128;
8442         while (t4_qsize_rxq & 7)
8443                 t4_qsize_rxq++;
8444
8445         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8446 }
8447
8448 static struct sx mlu;   /* mod load unload */
8449 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8450
8451 static int
8452 mod_event(module_t mod, int cmd, void *arg)
8453 {
8454         int rc = 0;
8455         static int loaded = 0;
8456
8457         switch (cmd) {
8458         case MOD_LOAD:
8459                 sx_xlock(&mlu);
8460                 if (loaded++ == 0) {
8461                         t4_sge_modload();
8462                         sx_init(&t4_list_lock, "T4/T5 adapters");
8463                         SLIST_INIT(&t4_list);
8464 #ifdef TCP_OFFLOAD
8465                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8466                         SLIST_INIT(&t4_uld_list);
8467 #endif
8468                         t4_tracer_modload();
8469                         tweak_tunables();
8470                 }
8471                 sx_xunlock(&mlu);
8472                 break;
8473
8474         case MOD_UNLOAD:
8475                 sx_xlock(&mlu);
8476                 if (--loaded == 0) {
8477                         int tries;
8478
8479                         sx_slock(&t4_list_lock);
8480                         if (!SLIST_EMPTY(&t4_list)) {
8481                                 rc = EBUSY;
8482                                 sx_sunlock(&t4_list_lock);
8483                                 goto done_unload;
8484                         }
8485 #ifdef TCP_OFFLOAD
8486                         sx_slock(&t4_uld_list_lock);
8487                         if (!SLIST_EMPTY(&t4_uld_list)) {
8488                                 rc = EBUSY;
8489                                 sx_sunlock(&t4_uld_list_lock);
8490                                 sx_sunlock(&t4_list_lock);
8491                                 goto done_unload;
8492                         }
8493 #endif
8494                         tries = 0;
8495                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8496                                 uprintf("%ju clusters with custom free routine "
8497                                     "still is use.\n", t4_sge_extfree_refs());
8498                                 pause("t4unload", 2 * hz);
8499                         }
8500 #ifdef TCP_OFFLOAD
8501                         sx_sunlock(&t4_uld_list_lock);
8502 #endif
8503                         sx_sunlock(&t4_list_lock);
8504
8505                         if (t4_sge_extfree_refs() == 0) {
8506                                 t4_tracer_modunload();
8507 #ifdef TCP_OFFLOAD
8508                                 sx_destroy(&t4_uld_list_lock);
8509 #endif
8510                                 sx_destroy(&t4_list_lock);
8511                                 t4_sge_modunload();
8512                                 loaded = 0;
8513                         } else {
8514                                 rc = EBUSY;
8515                                 loaded++;       /* undo earlier decrement */
8516                         }
8517                 }
8518 done_unload:
8519                 sx_xunlock(&mlu);
8520                 break;
8521         }
8522
8523         return (rc);
8524 }
8525
8526 static devclass_t t4_devclass, t5_devclass;
8527 static devclass_t cxgbe_devclass, cxl_devclass;
8528
8529 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8530 MODULE_VERSION(t4nex, 1);
8531 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8532
8533 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8534 MODULE_VERSION(t5nex, 1);
8535 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8536
8537 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8538 MODULE_VERSION(cxgbe, 1);
8539
8540 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8541 MODULE_VERSION(cxl, 1);