sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #if defined(__i386__) || defined(__amd64__)
  61 #include <vm/vm.h>
  62 #include <vm/pmap.h>
  63 #endif
  64
  65 #include "common/common.h"
  66 #include "common/t4_msg.h"
  67 #include "common/t4_regs.h"
  68 #include "common/t4_regs_values.h"
  69 #include "t4_ioctl.h"
  70 #include "t4_l2t.h"
  71 #include "t4_mp_ring.h"
  72
  73 /* T4 bus driver interface */
  74 static int t4_probe(device_t);
  75 static int t4_attach(device_t);
  76 static int t4_detach(device_t);
  77 static device_method_t t4_methods[] = {
  78         DEVMETHOD(device_probe,         t4_probe),
  79         DEVMETHOD(device_attach,        t4_attach),
  80         DEVMETHOD(device_detach,        t4_detach),
  81
  82         DEVMETHOD_END
  83 };
  84 static driver_t t4_driver = {
  85         "t4nex",
  86         t4_methods,
  87         sizeof(struct adapter)
  88 };
  89
  90
  91 /* T4 port (cxgbe) interface */
  92 static int cxgbe_probe(device_t);
  93 static int cxgbe_attach(device_t);
  94 static int cxgbe_detach(device_t);
  95 static device_method_t cxgbe_methods[] = {
  96         DEVMETHOD(device_probe,         cxgbe_probe),
  97         DEVMETHOD(device_attach,        cxgbe_attach),
  98         DEVMETHOD(device_detach,        cxgbe_detach),
  99         { 0, 0 }
 100 };
 101 static driver_t cxgbe_driver = {
 102         "cxgbe",
 103         cxgbe_methods,
 104         sizeof(struct port_info)
 105 };
 106
 107 static d_ioctl_t t4_ioctl;
 108 static d_open_t t4_open;
 109 static d_close_t t4_close;
 110
 111 static struct cdevsw t4_cdevsw = {
 112        .d_version = D_VERSION,
 113        .d_flags = 0,
 114        .d_open = t4_open,
 115        .d_close = t4_close,
 116        .d_ioctl = t4_ioctl,
 117        .d_name = "t4nex",
 118 };
 119
 120 /* T5 bus driver interface */
 121 static int t5_probe(device_t);
 122 static device_method_t t5_methods[] = {
 123         DEVMETHOD(device_probe,         t5_probe),
 124         DEVMETHOD(device_attach,        t4_attach),
 125         DEVMETHOD(device_detach,        t4_detach),
 126
 127         DEVMETHOD_END
 128 };
 129 static driver_t t5_driver = {
 130         "t5nex",
 131         t5_methods,
 132         sizeof(struct adapter)
 133 };
 134
 135
 136 /* T5 port (cxl) interface */
 137 static driver_t cxl_driver = {
 138         "cxl",
 139         cxgbe_methods,
 140         sizeof(struct port_info)
 141 };
 142
 143 static struct cdevsw t5_cdevsw = {
 144        .d_version = D_VERSION,
 145        .d_flags = 0,
 146        .d_open = t4_open,
 147        .d_close = t4_close,
 148        .d_ioctl = t4_ioctl,
 149        .d_name = "t5nex",
 150 };
 151
 152 /* ifnet + media interface */
 153 static void cxgbe_init(void *);
 154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 156 static void cxgbe_qflush(struct ifnet *);
 157 static int cxgbe_media_change(struct ifnet *);
 158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 159
 160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 161
 162 /*
 163  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 164  * then ADAPTER_LOCK, then t4_uld_list_lock.
 165  */
 166 static struct sx t4_list_lock;
 167 SLIST_HEAD(, adapter) t4_list;
 168 #ifdef TCP_OFFLOAD
 169 static struct sx t4_uld_list_lock;
 170 SLIST_HEAD(, uld_info) t4_uld_list;
 171 #endif
 172
 173 /*
 174  * Tunables.  See tweak_tunables() too.
 175  *
 176  * Each tunable is set to a default value here if it's known at compile-time.
 177  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 178  * provide a reasonable default when the driver is loaded.
 179  *
 180  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 181  * T5 are under hw.cxl.
 182  */
 183
 184 /*
 185  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 186  */
 187 #define NTXQ_10G 16
 188 static int t4_ntxq10g = -1;
 189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 190
 191 #define NRXQ_10G 8
 192 static int t4_nrxq10g = -1;
 193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 194
 195 #define NTXQ_1G 4
 196 static int t4_ntxq1g = -1;
 197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 198
 199 #define NRXQ_1G 2
 200 static int t4_nrxq1g = -1;
 201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 202
 203 static int t4_rsrv_noflowq = 0;
 204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 205
 206 #ifdef TCP_OFFLOAD
 207 #define NOFLDTXQ_10G 8
 208 static int t4_nofldtxq10g = -1;
 209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 210
 211 #define NOFLDRXQ_10G 2
 212 static int t4_nofldrxq10g = -1;
 213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 214
 215 #define NOFLDTXQ_1G 2
 216 static int t4_nofldtxq1g = -1;
 217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 218
 219 #define NOFLDRXQ_1G 1
 220 static int t4_nofldrxq1g = -1;
 221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 222 #endif
 223
 224 #ifdef DEV_NETMAP
 225 #define NNMTXQ_10G 2
 226 static int t4_nnmtxq10g = -1;
 227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 228
 229 #define NNMRXQ_10G 2
 230 static int t4_nnmrxq10g = -1;
 231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 232
 233 #define NNMTXQ_1G 1
 234 static int t4_nnmtxq1g = -1;
 235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 236
 237 #define NNMRXQ_1G 1
 238 static int t4_nnmrxq1g = -1;
 239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 240 #endif
 241
 242 /*
 243  * Holdoff parameters for 10G and 1G ports.
 244  */
 245 #define TMR_IDX_10G 1
 246 static int t4_tmr_idx_10g = TMR_IDX_10G;
 247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 248
 249 #define PKTC_IDX_10G (-1)
 250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 252
 253 #define TMR_IDX_1G 1
 254 static int t4_tmr_idx_1g = TMR_IDX_1G;
 255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 256
 257 #define PKTC_IDX_1G (-1)
 258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 260
 261 /*
 262  * Size (# of entries) of each tx and rx queue.
 263  */
 264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 266
 267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 269
 270 /*
 271  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 272  */
 273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 275
 276 /*
 277  * Configuration file.
 278  */
 279 #define DEFAULT_CF      "default"
 280 #define FLASH_CF        "flash"
 281 #define UWIRE_CF        "uwire"
 282 #define FPGA_CF         "fpga"
 283 static char t4_cfg_file[32] = DEFAULT_CF;
 284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 285
 286 /*
 287  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 288  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 289  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 290  *            mark or when signalled to do so, 0 to never emit PAUSE.
 291  */
 292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 294
 295 /*
 296  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 297  * encouraged respectively).
 298  */
 299 static unsigned int t4_fw_install = 1;
 300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 301
 302 /*
 303  * ASIC features that will be used.  Disable the ones you don't want so that the
 304  * chip resources aren't wasted on features that will not be used.
 305  */
 306 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 308
 309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 311
 312 static int t4_toecaps_allowed = -1;
 313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 314
 315 static int t4_rdmacaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 317
 318 static int t4_iscsicaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 320
 321 static int t4_fcoecaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 323
 324 static int t5_write_combine = 0;
 325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 326
 327 struct intrs_and_queues {
 328         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 329         uint16_t nirq;          /* Total # of vectors */
 330         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 331         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 332         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 333         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 334         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 335         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 336         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 337 #ifdef TCP_OFFLOAD
 338         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 339         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 340         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 341         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 342 #endif
 343 #ifdef DEV_NETMAP
 344         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 345         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 346         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 347         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 348 #endif
 349 };
 350
 351 struct filter_entry {
 352         uint32_t valid:1;       /* filter allocated and valid */
 353         uint32_t locked:1;      /* filter is administratively locked */
 354         uint32_t pending:1;     /* filter action is pending firmware reply */
 355         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 356         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 357
 358         struct t4_filter_specification fs;
 359 };
 360
 361 static int map_bars_0_and_4(struct adapter *);
 362 static int map_bar_2(struct adapter *);
 363 static void setup_memwin(struct adapter *);
 364 static int validate_mem_range(struct adapter *, uint32_t, int);
 365 static int fwmtype_to_hwmtype(int);
 366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 367     uint32_t *);
 368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 371     struct intrs_and_queues *);
 372 static int prep_firmware(struct adapter *);
 373 static int partition_resources(struct adapter *, const struct firmware *,
 374     const char *);
 375 static int get_params__pre_init(struct adapter *);
 376 static int get_params__post_init(struct adapter *);
 377 static int set_params__post_init(struct adapter *);
 378 static void t4_set_desc(struct adapter *);
 379 static void build_medialist(struct port_info *, struct ifmedia *);
 380 static int cxgbe_init_synchronized(struct port_info *);
 381 static int cxgbe_uninit_synchronized(struct port_info *);
 382 static int setup_intr_handlers(struct adapter *);
 383 static void quiesce_txq(struct adapter *, struct sge_txq *);
 384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 385 static void quiesce_iq(struct adapter *, struct sge_iq *);
 386 static void quiesce_fl(struct adapter *, struct sge_fl *);
 387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 388     driver_intr_t *, void *, char *);
 389 static int t4_free_irq(struct adapter *, struct irq *);
 390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 391     unsigned int);
 392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 394 static void cxgbe_tick(void *);
 395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 397     struct mbuf *);
 398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 400 static int t4_sysctls(struct adapter *);
 401 static int cxgbe_sysctls(struct port_info *);
 402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 413 #ifdef SBUF_DRAIN
 414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 439 #endif
 440 static uint32_t fconf_to_mode(uint32_t);
 441 static uint32_t mode_to_fconf(uint32_t);
 442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 443 static int get_filter_mode(struct adapter *, uint32_t *);
 444 static int set_filter_mode(struct adapter *, uint32_t);
 445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 446 static int get_filter(struct adapter *, struct t4_filter *);
 447 static int set_filter(struct adapter *, struct t4_filter *);
 448 static int del_filter(struct adapter *, struct t4_filter *);
 449 static void clear_filter(struct filter_entry *);
 450 static int set_filter_wr(struct adapter *, int);
 451 static int del_filter_wr(struct adapter *, int);
 452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 453 static int load_fw(struct adapter *, struct t4_data *);
 454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 458 #ifdef TCP_OFFLOAD
 459 static int toe_capability(struct port_info *, int);
 460 #endif
 461 static int mod_event(module_t, int, void *);
 462
 463 struct {
 464         uint16_t device;
 465         char *desc;
 466 } t4_pciids[] = {
 467         {0xa000, "Chelsio Terminator 4 FPGA"},
 468         {0x4400, "Chelsio T440-dbg"},
 469         {0x4401, "Chelsio T420-CR"},
 470         {0x4402, "Chelsio T422-CR"},
 471         {0x4403, "Chelsio T440-CR"},
 472         {0x4404, "Chelsio T420-BCH"},
 473         {0x4405, "Chelsio T440-BCH"},
 474         {0x4406, "Chelsio T440-CH"},
 475         {0x4407, "Chelsio T420-SO"},
 476         {0x4408, "Chelsio T420-CX"},
 477         {0x4409, "Chelsio T420-BT"},
 478         {0x440a, "Chelsio T404-BT"},
 479         {0x440e, "Chelsio T440-LP-CR"},
 480 }, t5_pciids[] = {
 481         {0xb000, "Chelsio Terminator 5 FPGA"},
 482         {0x5400, "Chelsio T580-dbg"},
 483         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 484         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 485         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 486         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 487         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 488         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 489         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 490         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 491         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 492         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 493         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 494         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 495         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 496 #ifdef notyet
 497         {0x5404,  "Chelsio T520-BCH"},
 498         {0x5405,  "Chelsio T540-BCH"},
 499         {0x5406,  "Chelsio T540-CH"},
 500         {0x5408,  "Chelsio T520-CX"},
 501         {0x540b,  "Chelsio B520-SR"},
 502         {0x540c,  "Chelsio B504-BT"},
 503         {0x540f,  "Chelsio Amsterdam"},
 504         {0x5413,  "Chelsio T580-CHR"},
 505 #endif
 506 };
 507
 508 #ifdef TCP_OFFLOAD
 509 /*
 510  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 511  * exactly the same for both rxq and ofld_rxq.
 512  */
 513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 515 #endif
 516
 517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 520
 521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 522
 523 static int
 524 t4_probe(device_t dev)
 525 {
 526         int i;
 527         uint16_t v = pci_get_vendor(dev);
 528         uint16_t d = pci_get_device(dev);
 529         uint8_t f = pci_get_function(dev);
 530
 531         if (v != PCI_VENDOR_ID_CHELSIO)
 532                 return (ENXIO);
 533
 534         /* Attach only to PF0 of the FPGA */
 535         if (d == 0xa000 && f != 0)
 536                 return (ENXIO);
 537
 538         for (i = 0; i < nitems(t4_pciids); i++) {
 539                 if (d == t4_pciids[i].device) {
 540                         device_set_desc(dev, t4_pciids[i].desc);
 541                         return (BUS_PROBE_DEFAULT);
 542                 }
 543         }
 544
 545         return (ENXIO);
 546 }
 547
 548 static int
 549 t5_probe(device_t dev)
 550 {
 551         int i;
 552         uint16_t v = pci_get_vendor(dev);
 553         uint16_t d = pci_get_device(dev);
 554         uint8_t f = pci_get_function(dev);
 555
 556         if (v != PCI_VENDOR_ID_CHELSIO)
 557                 return (ENXIO);
 558
 559         /* Attach only to PF0 of the FPGA */
 560         if (d == 0xb000 && f != 0)
 561                 return (ENXIO);
 562
 563         for (i = 0; i < nitems(t5_pciids); i++) {
 564                 if (d == t5_pciids[i].device) {
 565                         device_set_desc(dev, t5_pciids[i].desc);
 566                         return (BUS_PROBE_DEFAULT);
 567                 }
 568         }
 569
 570         return (ENXIO);
 571 }
 572
 573 static void
 574 t5_attribute_workaround(device_t dev)
 575 {
 576         device_t root_port;
 577         uint32_t v;
 578
 579         /*
 580          * The T5 chips do not properly echo the No Snoop and Relaxed
 581          * Ordering attributes when replying to a TLP from a Root
 582          * Port.  As a workaround, find the parent Root Port and
 583          * disable No Snoop and Relaxed Ordering.  Note that this
 584          * affects all devices under this root port.
 585          */
 586         root_port = pci_find_pcie_root_port(dev);
 587         if (root_port == NULL) {
 588                 device_printf(dev, "Unable to find parent root port\n");
 589                 return;
 590         }
 591
 592         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 593             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 594         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 595             0)
 596                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 597                     device_get_nameunit(root_port));
 598 }
 599
 600 static int
 601 t4_attach(device_t dev)
 602 {
 603         struct adapter *sc;
 604         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 605         struct intrs_and_queues iaq;
 606         struct sge *s;
 607 #ifdef TCP_OFFLOAD
 608         int ofld_rqidx, ofld_tqidx;
 609 #endif
 610 #ifdef DEV_NETMAP
 611         int nm_rqidx, nm_tqidx;
 612 #endif
 613
 614         sc = device_get_softc(dev);
 615         sc->dev = dev;
 616         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 617
 618         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 619                 t5_attribute_workaround(dev);
 620         pci_enable_busmaster(dev);
 621         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 622                 uint32_t v;
 623
 624                 pci_set_max_read_req(dev, 4096);
 625                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 626                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 627                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 628
 629                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 630         }
 631
 632         sc->traceq = -1;
 633         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 634         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 635             device_get_nameunit(dev));
 636
 637         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 638             device_get_nameunit(dev));
 639         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 640         sx_xlock(&t4_list_lock);
 641         SLIST_INSERT_HEAD(&t4_list, sc, link);
 642         sx_xunlock(&t4_list_lock);
 643
 644         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 645         TAILQ_INIT(&sc->sfl);
 646         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 647
 648         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 649
 650         rc = map_bars_0_and_4(sc);
 651         if (rc != 0)
 652                 goto done; /* error message displayed already */
 653
 654         /*
 655          * This is the real PF# to which we're attaching.  Works from within PCI
 656          * passthrough environments too, where pci_get_function() could return a
 657          * different PF# depending on the passthrough configuration.  We need to
 658          * use the real PF# in all our communication with the firmware.
 659          */
 660         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 661         sc->mbox = sc->pf;
 662
 663         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 664         sc->an_handler = an_not_handled;
 665         for (i = 0; i < nitems(sc->cpl_handler); i++)
 666                 sc->cpl_handler[i] = cpl_not_handled;
 667         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 668                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 669         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 670         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 671         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 672         t4_init_sge_cpl_handlers(sc);
 673
 674         /* Prepare the adapter for operation */
 675         rc = -t4_prep_adapter(sc);
 676         if (rc != 0) {
 677                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 678                 goto done;
 679         }
 680
 681         /*
 682          * Do this really early, with the memory windows set up even before the
 683          * character device.  The userland tool's register i/o and mem read
 684          * will work even in "recovery mode".
 685          */
 686         setup_memwin(sc);
 687         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 688             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 689             device_get_nameunit(dev));
 690         if (sc->cdev == NULL)
 691                 device_printf(dev, "failed to create nexus char device.\n");
 692         else
 693                 sc->cdev->si_drv1 = sc;
 694
 695         /* Go no further if recovery mode has been requested. */
 696         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 697                 device_printf(dev, "recovery mode.\n");
 698                 goto done;
 699         }
 700
 701 #if defined(__i386__)
 702         if ((cpu_feature & CPUID_CX8) == 0) {
 703                 device_printf(dev, "64 bit atomics not available.\n");
 704                 rc = ENOTSUP;
 705                 goto done;
 706         }
 707 #endif
 708
 709         /* Prepare the firmware for operation */
 710         rc = prep_firmware(sc);
 711         if (rc != 0)
 712                 goto done; /* error message displayed already */
 713
 714         rc = get_params__post_init(sc);
 715         if (rc != 0)
 716                 goto done; /* error message displayed already */
 717
 718         rc = set_params__post_init(sc);
 719         if (rc != 0)
 720                 goto done; /* error message displayed already */
 721
 722         rc = map_bar_2(sc);
 723         if (rc != 0)
 724                 goto done; /* error message displayed already */
 725
 726         rc = t4_create_dma_tag(sc);
 727         if (rc != 0)
 728                 goto done; /* error message displayed already */
 729
 730         /*
 731          * First pass over all the ports - allocate VIs and initialize some
 732          * basic parameters like mac address, port type, etc.  We also figure
 733          * out whether a port is 10G or 1G and use that information when
 734          * calculating how many interrupts to attempt to allocate.
 735          */
 736         n10g = n1g = 0;
 737         for_each_port(sc, i) {
 738                 struct port_info *pi;
 739
 740                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 741                 sc->port[i] = pi;
 742
 743                 /* These must be set before t4_port_init */
 744                 pi->adapter = sc;
 745                 pi->port_id = i;
 746
 747                 /* Allocate the vi and initialize parameters like mac addr */
 748                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 749                 if (rc != 0) {
 750                         device_printf(dev, "unable to initialize port %d: %d\n",
 751                             i, rc);
 752                         free(pi, M_CXGBE);
 753                         sc->port[i] = NULL;
 754                         goto done;
 755                 }
 756
 757                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 758                 pi->link_cfg.requested_fc |= t4_pause_settings;
 759                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 760                 pi->link_cfg.fc |= t4_pause_settings;
 761
 762                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 763                 if (rc != 0) {
 764                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 765                         free(pi, M_CXGBE);
 766                         sc->port[i] = NULL;
 767                         goto done;
 768                 }
 769
 770                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 771                     device_get_nameunit(dev), i);
 772                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 773                 sc->chan_map[pi->tx_chan] = i;
 774
 775                 if (is_10G_port(pi) || is_40G_port(pi)) {
 776                         n10g++;
 777                         pi->tmr_idx = t4_tmr_idx_10g;
 778                         pi->pktc_idx = t4_pktc_idx_10g;
 779                 } else {
 780                         n1g++;
 781                         pi->tmr_idx = t4_tmr_idx_1g;
 782                         pi->pktc_idx = t4_pktc_idx_1g;
 783                 }
 784
 785                 pi->xact_addr_filt = -1;
 786                 pi->linkdnrc = -1;
 787
 788                 pi->qsize_rxq = t4_qsize_rxq;
 789                 pi->qsize_txq = t4_qsize_txq;
 790
 791                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 792                 if (pi->dev == NULL) {
 793                         device_printf(dev,
 794                             "failed to add device for port %d.\n", i);
 795                         rc = ENXIO;
 796                         goto done;
 797                 }
 798                 device_set_softc(pi->dev, pi);
 799         }
 800
 801         /*
 802          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 803          */
 804         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 805         if (rc != 0)
 806                 goto done; /* error message displayed already */
 807
 808         sc->intr_type = iaq.intr_type;
 809         sc->intr_count = iaq.nirq;
 810
 811         s = &sc->sge;
 812         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 813         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 814         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 815         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 816         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 817 #ifdef TCP_OFFLOAD
 818         if (is_offload(sc)) {
 819                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 820                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 821                 s->neq += s->nofldtxq + s->nofldrxq;
 822                 s->niq += s->nofldrxq;
 823
 824                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 825                     M_CXGBE, M_ZERO | M_WAITOK);
 826                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 827                     M_CXGBE, M_ZERO | M_WAITOK);
 828         }
 829 #endif
 830 #ifdef DEV_NETMAP
 831         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 832         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 833         s->neq += s->nnmtxq + s->nnmrxq;
 834         s->niq += s->nnmrxq;
 835
 836         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 837             M_CXGBE, M_ZERO | M_WAITOK);
 838         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 839             M_CXGBE, M_ZERO | M_WAITOK);
 840 #endif
 841
 842         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 843             M_ZERO | M_WAITOK);
 844         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 845             M_ZERO | M_WAITOK);
 846         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 847             M_ZERO | M_WAITOK);
 848         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 849             M_ZERO | M_WAITOK);
 850         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 851             M_ZERO | M_WAITOK);
 852
 853         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 854             M_ZERO | M_WAITOK);
 855
 856         t4_init_l2t(sc, M_WAITOK);
 857
 858         /*
 859          * Second pass over the ports.  This time we know the number of rx and
 860          * tx queues that each port should get.
 861          */
 862         rqidx = tqidx = 0;
 863 #ifdef TCP_OFFLOAD
 864         ofld_rqidx = ofld_tqidx = 0;
 865 #endif
 866 #ifdef DEV_NETMAP
 867         nm_rqidx = nm_tqidx = 0;
 868 #endif
 869         for_each_port(sc, i) {
 870                 struct port_info *pi = sc->port[i];
 871
 872                 if (pi == NULL)
 873                         continue;
 874
 875                 pi->first_rxq = rqidx;
 876                 pi->first_txq = tqidx;
 877                 if (is_10G_port(pi) || is_40G_port(pi)) {
 878                         pi->flags |= iaq.intr_flags_10g;
 879                         pi->nrxq = iaq.nrxq10g;
 880                         pi->ntxq = iaq.ntxq10g;
 881                 } else {
 882                         pi->flags |= iaq.intr_flags_1g;
 883                         pi->nrxq = iaq.nrxq1g;
 884                         pi->ntxq = iaq.ntxq1g;
 885                 }
 886
 887                 if (pi->ntxq > 1)
 888                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 889                 else
 890                         pi->rsrv_noflowq = 0;
 891
 892                 rqidx += pi->nrxq;
 893                 tqidx += pi->ntxq;
 894 #ifdef TCP_OFFLOAD
 895                 if (is_offload(sc)) {
 896                         pi->first_ofld_rxq = ofld_rqidx;
 897                         pi->first_ofld_txq = ofld_tqidx;
 898                         if (is_10G_port(pi) || is_40G_port(pi)) {
 899                                 pi->nofldrxq = iaq.nofldrxq10g;
 900                                 pi->nofldtxq = iaq.nofldtxq10g;
 901                         } else {
 902                                 pi->nofldrxq = iaq.nofldrxq1g;
 903                                 pi->nofldtxq = iaq.nofldtxq1g;
 904                         }
 905                         ofld_rqidx += pi->nofldrxq;
 906                         ofld_tqidx += pi->nofldtxq;
 907                 }
 908 #endif
 909 #ifdef DEV_NETMAP
 910                 pi->first_nm_rxq = nm_rqidx;
 911                 pi->first_nm_txq = nm_tqidx;
 912                 if (is_10G_port(pi) || is_40G_port(pi)) {
 913                         pi->nnmrxq = iaq.nnmrxq10g;
 914                         pi->nnmtxq = iaq.nnmtxq10g;
 915                 } else {
 916                         pi->nnmrxq = iaq.nnmrxq1g;
 917                         pi->nnmtxq = iaq.nnmtxq1g;
 918                 }
 919                 nm_rqidx += pi->nnmrxq;
 920                 nm_tqidx += pi->nnmtxq;
 921 #endif
 922         }
 923
 924         rc = setup_intr_handlers(sc);
 925         if (rc != 0) {
 926                 device_printf(dev,
 927                     "failed to setup interrupt handlers: %d\n", rc);
 928                 goto done;
 929         }
 930
 931         rc = bus_generic_attach(dev);
 932         if (rc != 0) {
 933                 device_printf(dev,
 934                     "failed to attach all child ports: %d\n", rc);
 935                 goto done;
 936         }
 937
 938         device_printf(dev,
 939             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 940             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 941             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 942             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 943             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 944
 945         t4_set_desc(sc);
 946
 947 done:
 948         if (rc != 0 && sc->cdev) {
 949                 /* cdev was created and so cxgbetool works; recover that way. */
 950                 device_printf(dev,
 951                     "error during attach, adapter is now in recovery mode.\n");
 952                 rc = 0;
 953         }
 954
 955         if (rc != 0)
 956                 t4_detach(dev);
 957         else
 958                 t4_sysctls(sc);
 959
 960         return (rc);
 961 }
 962
 963 /*
 964  * Idempotent
 965  */
 966 static int
 967 t4_detach(device_t dev)
 968 {
 969         struct adapter *sc;
 970         struct port_info *pi;
 971         int i, rc;
 972
 973         sc = device_get_softc(dev);
 974
 975         if (sc->flags & FULL_INIT_DONE)
 976                 t4_intr_disable(sc);
 977
 978         if (sc->cdev) {
 979                 destroy_dev(sc->cdev);
 980                 sc->cdev = NULL;
 981         }
 982
 983         rc = bus_generic_detach(dev);
 984         if (rc) {
 985                 device_printf(dev,
 986                     "failed to detach child devices: %d\n", rc);
 987                 return (rc);
 988         }
 989
 990         for (i = 0; i < sc->intr_count; i++)
 991                 t4_free_irq(sc, &sc->irq[i]);
 992
 993         for (i = 0; i < MAX_NPORTS; i++) {
 994                 pi = sc->port[i];
 995                 if (pi) {
 996                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 997                         if (pi->dev)
 998                                 device_delete_child(dev, pi->dev);
 999
1000                         mtx_destroy(&pi->pi_lock);
1001                         free(pi, M_CXGBE);
1002                 }
1003         }
1004
1005         if (sc->flags & FULL_INIT_DONE)
1006                 adapter_full_uninit(sc);
1007
1008         if (sc->flags & FW_OK)
1009                 t4_fw_bye(sc, sc->mbox);
1010
1011         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1012                 pci_release_msi(dev);
1013
1014         if (sc->regs_res)
1015                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1016                     sc->regs_res);
1017
1018         if (sc->udbs_res)
1019                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1020                     sc->udbs_res);
1021
1022         if (sc->msix_res)
1023                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1024                     sc->msix_res);
1025
1026         if (sc->l2t)
1027                 t4_free_l2t(sc->l2t);
1028
1029 #ifdef TCP_OFFLOAD
1030         free(sc->sge.ofld_rxq, M_CXGBE);
1031         free(sc->sge.ofld_txq, M_CXGBE);
1032 #endif
1033 #ifdef DEV_NETMAP
1034         free(sc->sge.nm_rxq, M_CXGBE);
1035         free(sc->sge.nm_txq, M_CXGBE);
1036 #endif
1037         free(sc->irq, M_CXGBE);
1038         free(sc->sge.rxq, M_CXGBE);
1039         free(sc->sge.txq, M_CXGBE);
1040         free(sc->sge.ctrlq, M_CXGBE);
1041         free(sc->sge.iqmap, M_CXGBE);
1042         free(sc->sge.eqmap, M_CXGBE);
1043         free(sc->tids.ftid_tab, M_CXGBE);
1044         t4_destroy_dma_tag(sc);
1045         if (mtx_initialized(&sc->sc_lock)) {
1046                 sx_xlock(&t4_list_lock);
1047                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1048                 sx_xunlock(&t4_list_lock);
1049                 mtx_destroy(&sc->sc_lock);
1050         }
1051
1052         if (mtx_initialized(&sc->tids.ftid_lock))
1053                 mtx_destroy(&sc->tids.ftid_lock);
1054         if (mtx_initialized(&sc->sfl_lock))
1055                 mtx_destroy(&sc->sfl_lock);
1056         if (mtx_initialized(&sc->ifp_lock))
1057                 mtx_destroy(&sc->ifp_lock);
1058         if (mtx_initialized(&sc->regwin_lock))
1059                 mtx_destroy(&sc->regwin_lock);
1060
1061         bzero(sc, sizeof(*sc));
1062
1063         return (0);
1064 }
1065
1066 static int
1067 cxgbe_probe(device_t dev)
1068 {
1069         char buf[128];
1070         struct port_info *pi = device_get_softc(dev);
1071
1072         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1073         device_set_desc_copy(dev, buf);
1074
1075         return (BUS_PROBE_DEFAULT);
1076 }
1077
1078 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1079     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1080     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1081 #define T4_CAP_ENABLE (T4_CAP)
1082
1083 static int
1084 cxgbe_attach(device_t dev)
1085 {
1086         struct port_info *pi = device_get_softc(dev);
1087         struct ifnet *ifp;
1088         char *s;
1089         int n, o;
1090
1091         /* Allocate an ifnet and set it up */
1092         ifp = if_alloc(IFT_ETHER);
1093         if (ifp == NULL) {
1094                 device_printf(dev, "Cannot allocate ifnet\n");
1095                 return (ENOMEM);
1096         }
1097         pi->ifp = ifp;
1098         ifp->if_softc = pi;
1099
1100         callout_init(&pi->tick, CALLOUT_MPSAFE);
1101
1102         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1103         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1104
1105         ifp->if_init = cxgbe_init;
1106         ifp->if_ioctl = cxgbe_ioctl;
1107         ifp->if_transmit = cxgbe_transmit;
1108         ifp->if_qflush = cxgbe_qflush;
1109
1110         ifp->if_capabilities = T4_CAP;
1111 #ifdef TCP_OFFLOAD
1112         if (is_offload(pi->adapter))
1113                 ifp->if_capabilities |= IFCAP_TOE;
1114 #endif
1115         ifp->if_capenable = T4_CAP_ENABLE;
1116         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1117             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1118
1119         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1120         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1121         ifp->if_hw_tsomaxsegsize = 65536;
1122
1123         /* Initialize ifmedia for this port */
1124         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1125             cxgbe_media_status);
1126         build_medialist(pi, &pi->media);
1127
1128         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1129             EVENTHANDLER_PRI_ANY);
1130
1131         ether_ifattach(ifp, pi->hw_addr);
1132
1133         n = 128;
1134         s = malloc(n, M_CXGBE, M_WAITOK);
1135         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1136         MPASS(n > o);
1137 #ifdef TCP_OFFLOAD
1138         if (is_offload(pi->adapter)) {
1139                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1140                     pi->nofldtxq, pi->nofldrxq);
1141                 MPASS(n > o);
1142         }
1143 #endif
1144 #ifdef DEV_NETMAP
1145         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1146             pi->nnmrxq);
1147         MPASS(n > o);
1148 #endif
1149         device_printf(dev, "%s\n", s);
1150         free(s, M_CXGBE);
1151
1152 #ifdef DEV_NETMAP
1153         /* nm_media handled here to keep implementation private to this file */
1154         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1155             cxgbe_media_status);
1156         build_medialist(pi, &pi->nm_media);
1157         create_netmap_ifnet(pi);        /* logs errors it something fails */
1158 #endif
1159         cxgbe_sysctls(pi);
1160
1161         return (0);
1162 }
1163
1164 static int
1165 cxgbe_detach(device_t dev)
1166 {
1167         struct port_info *pi = device_get_softc(dev);
1168         struct adapter *sc = pi->adapter;
1169         struct ifnet *ifp = pi->ifp;
1170
1171         /* Tell if_ioctl and if_init that the port is going away */
1172         ADAPTER_LOCK(sc);
1173         SET_DOOMED(pi);
1174         wakeup(&sc->flags);
1175         while (IS_BUSY(sc))
1176                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1177         SET_BUSY(sc);
1178 #ifdef INVARIANTS
1179         sc->last_op = "t4detach";
1180         sc->last_op_thr = curthread;
1181         sc->last_op_flags = 0;
1182 #endif
1183         ADAPTER_UNLOCK(sc);
1184
1185         if (pi->flags & HAS_TRACEQ) {
1186                 sc->traceq = -1;        /* cloner should not create ifnet */
1187                 t4_tracer_port_detach(sc);
1188         }
1189
1190         if (pi->vlan_c)
1191                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1192
1193         PORT_LOCK(pi);
1194         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1195         callout_stop(&pi->tick);
1196         PORT_UNLOCK(pi);
1197         callout_drain(&pi->tick);
1198
1199         /* Let detach proceed even if these fail. */
1200         cxgbe_uninit_synchronized(pi);
1201         port_full_uninit(pi);
1202
1203         ifmedia_removeall(&pi->media);
1204         ether_ifdetach(pi->ifp);
1205         if_free(pi->ifp);
1206
1207 #ifdef DEV_NETMAP
1208         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1209         destroy_netmap_ifnet(pi);
1210 #endif
1211
1212         ADAPTER_LOCK(sc);
1213         CLR_BUSY(sc);
1214         wakeup(&sc->flags);
1215         ADAPTER_UNLOCK(sc);
1216
1217         return (0);
1218 }
1219
1220 static void
1221 cxgbe_init(void *arg)
1222 {
1223         struct port_info *pi = arg;
1224         struct adapter *sc = pi->adapter;
1225
1226         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1227                 return;
1228         cxgbe_init_synchronized(pi);
1229         end_synchronized_op(sc, 0);
1230 }
1231
1232 static int
1233 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1234 {
1235         int rc = 0, mtu, flags, can_sleep;
1236         struct port_info *pi = ifp->if_softc;
1237         struct adapter *sc = pi->adapter;
1238         struct ifreq *ifr = (struct ifreq *)data;
1239         uint32_t mask;
1240
1241         switch (cmd) {
1242         case SIOCSIFMTU:
1243                 mtu = ifr->ifr_mtu;
1244                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1245                         return (EINVAL);
1246
1247                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1248                 if (rc)
1249                         return (rc);
1250                 ifp->if_mtu = mtu;
1251                 if (pi->flags & PORT_INIT_DONE) {
1252                         t4_update_fl_bufsize(ifp);
1253                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1254                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1255                 }
1256                 end_synchronized_op(sc, 0);
1257                 break;
1258
1259         case SIOCSIFFLAGS:
1260                 can_sleep = 0;
1261 redo_sifflags:
1262                 rc = begin_synchronized_op(sc, pi,
1263                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1264                 if (rc)
1265                         return (rc);
1266
1267                 if (ifp->if_flags & IFF_UP) {
1268                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1269                                 flags = pi->if_flags;
1270                                 if ((ifp->if_flags ^ flags) &
1271                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1272                                         if (can_sleep == 1) {
1273                                                 end_synchronized_op(sc, 0);
1274                                                 can_sleep = 0;
1275                                                 goto redo_sifflags;
1276                                         }
1277                                         rc = update_mac_settings(ifp,
1278                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1279                                 }
1280                         } else {
1281                                 if (can_sleep == 0) {
1282                                         end_synchronized_op(sc, LOCK_HELD);
1283                                         can_sleep = 1;
1284                                         goto redo_sifflags;
1285                                 }
1286                                 rc = cxgbe_init_synchronized(pi);
1287                         }
1288                         pi->if_flags = ifp->if_flags;
1289                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1290                         if (can_sleep == 0) {
1291                                 end_synchronized_op(sc, LOCK_HELD);
1292                                 can_sleep = 1;
1293                                 goto redo_sifflags;
1294                         }
1295                         rc = cxgbe_uninit_synchronized(pi);
1296                 }
1297                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1298                 break;
1299
1300         case SIOCADDMULTI:
1301         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1302                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1303                 if (rc)
1304                         return (rc);
1305                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1306                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1307                 end_synchronized_op(sc, LOCK_HELD);
1308                 break;
1309
1310         case SIOCSIFCAP:
1311                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1312                 if (rc)
1313                         return (rc);
1314
1315                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1316                 if (mask & IFCAP_TXCSUM) {
1317                         ifp->if_capenable ^= IFCAP_TXCSUM;
1318                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1319
1320                         if (IFCAP_TSO4 & ifp->if_capenable &&
1321                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1322                                 ifp->if_capenable &= ~IFCAP_TSO4;
1323                                 if_printf(ifp,
1324                                     "tso4 disabled due to -txcsum.\n");
1325                         }
1326                 }
1327                 if (mask & IFCAP_TXCSUM_IPV6) {
1328                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1329                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1330
1331                         if (IFCAP_TSO6 & ifp->if_capenable &&
1332                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1333                                 ifp->if_capenable &= ~IFCAP_TSO6;
1334                                 if_printf(ifp,
1335                                     "tso6 disabled due to -txcsum6.\n");
1336                         }
1337                 }
1338                 if (mask & IFCAP_RXCSUM)
1339                         ifp->if_capenable ^= IFCAP_RXCSUM;
1340                 if (mask & IFCAP_RXCSUM_IPV6)
1341                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1342
1343                 /*
1344                  * Note that we leave CSUM_TSO alone (it is always set).  The
1345                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1346                  * sending a TSO request our way, so it's sufficient to toggle
1347                  * IFCAP_TSOx only.
1348                  */
1349                 if (mask & IFCAP_TSO4) {
1350                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1351                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1352                                 if_printf(ifp, "enable txcsum first.\n");
1353                                 rc = EAGAIN;
1354                                 goto fail;
1355                         }
1356                         ifp->if_capenable ^= IFCAP_TSO4;
1357                 }
1358                 if (mask & IFCAP_TSO6) {
1359                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1360                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1361                                 if_printf(ifp, "enable txcsum6 first.\n");
1362                                 rc = EAGAIN;
1363                                 goto fail;
1364                         }
1365                         ifp->if_capenable ^= IFCAP_TSO6;
1366                 }
1367                 if (mask & IFCAP_LRO) {
1368 #if defined(INET) || defined(INET6)
1369                         int i;
1370                         struct sge_rxq *rxq;
1371
1372                         ifp->if_capenable ^= IFCAP_LRO;
1373                         for_each_rxq(pi, i, rxq) {
1374                                 if (ifp->if_capenable & IFCAP_LRO)
1375                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1376                                 else
1377                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1378                         }
1379 #endif
1380                 }
1381 #ifdef TCP_OFFLOAD
1382                 if (mask & IFCAP_TOE) {
1383                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1384
1385                         rc = toe_capability(pi, enable);
1386                         if (rc != 0)
1387                                 goto fail;
1388
1389                         ifp->if_capenable ^= mask;
1390                 }
1391 #endif
1392                 if (mask & IFCAP_VLAN_HWTAGGING) {
1393                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1394                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1395                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1396                 }
1397                 if (mask & IFCAP_VLAN_MTU) {
1398                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1399
1400                         /* Need to find out how to disable auto-mtu-inflation */
1401                 }
1402                 if (mask & IFCAP_VLAN_HWTSO)
1403                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1404                 if (mask & IFCAP_VLAN_HWCSUM)
1405                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1406
1407 #ifdef VLAN_CAPABILITIES
1408                 VLAN_CAPABILITIES(ifp);
1409 #endif
1410 fail:
1411                 end_synchronized_op(sc, 0);
1412                 break;
1413
1414         case SIOCSIFMEDIA:
1415         case SIOCGIFMEDIA:
1416                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1417                 break;
1418
1419         case SIOCGI2C: {
1420                 struct ifi2creq i2c;
1421
1422                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1423                 if (rc != 0)
1424                         break;
1425                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1426                         rc = EPERM;
1427                         break;
1428                 }
1429                 if (i2c.len > sizeof(i2c.data)) {
1430                         rc = EINVAL;
1431                         break;
1432                 }
1433                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1434                 if (rc)
1435                         return (rc);
1436                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1437                     i2c.offset, i2c.len, &i2c.data[0]);
1438                 end_synchronized_op(sc, 0);
1439                 if (rc == 0)
1440                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1441                 break;
1442         }
1443
1444         default:
1445                 rc = ether_ioctl(ifp, cmd, data);
1446         }
1447
1448         return (rc);
1449 }
1450
1451 static int
1452 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1453 {
1454         struct port_info *pi = ifp->if_softc;
1455         struct adapter *sc = pi->adapter;
1456         struct sge_txq *txq;
1457         void *items[1];
1458         int rc;
1459
1460         M_ASSERTPKTHDR(m);
1461         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1462
1463         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1464                 m_freem(m);
1465                 return (ENETDOWN);
1466         }
1467
1468         rc = parse_pkt(&m);
1469         if (__predict_false(rc != 0)) {
1470                 MPASS(m == NULL);                       /* was freed already */
1471                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1472                 return (rc);
1473         }
1474
1475         /* Select a txq. */
1476         txq = &sc->sge.txq[pi->first_txq];
1477         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1478                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1479                     pi->rsrv_noflowq);
1480
1481         items[0] = m;
1482         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1483         if (__predict_false(rc != 0))
1484                 m_freem(m);
1485
1486         return (rc);
1487 }
1488
1489 static void
1490 cxgbe_qflush(struct ifnet *ifp)
1491 {
1492         struct port_info *pi = ifp->if_softc;
1493         struct sge_txq *txq;
1494         int i;
1495
1496         /* queues do not exist if !PORT_INIT_DONE. */
1497         if (pi->flags & PORT_INIT_DONE) {
1498                 for_each_txq(pi, i, txq) {
1499                         TXQ_LOCK(txq);
1500                         txq->eq.flags &= ~EQ_ENABLED;
1501                         TXQ_UNLOCK(txq);
1502                         while (!mp_ring_is_idle(txq->r)) {
1503                                 mp_ring_check_drainage(txq->r, 0);
1504                                 pause("qflush", 1);
1505                         }
1506                 }
1507         }
1508         if_qflush(ifp);
1509 }
1510
1511 static int
1512 cxgbe_media_change(struct ifnet *ifp)
1513 {
1514         struct port_info *pi = ifp->if_softc;
1515
1516         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1517
1518         return (EOPNOTSUPP);
1519 }
1520
1521 static void
1522 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1523 {
1524         struct port_info *pi = ifp->if_softc;
1525         struct ifmedia *media = NULL;
1526         struct ifmedia_entry *cur;
1527         int speed = pi->link_cfg.speed;
1528 #ifdef INVARIANTS
1529         int data = (pi->port_type << 8) | pi->mod_type;
1530 #endif
1531
1532         if (ifp == pi->ifp)
1533                 media = &pi->media;
1534 #ifdef DEV_NETMAP
1535         else if (ifp == pi->nm_ifp)
1536                 media = &pi->nm_media;
1537 #endif
1538         MPASS(media != NULL);
1539
1540         cur = media->ifm_cur;
1541         MPASS(cur->ifm_data == data);
1542
1543         ifmr->ifm_status = IFM_AVALID;
1544         if (!pi->link_cfg.link_ok)
1545                 return;
1546
1547         ifmr->ifm_status |= IFM_ACTIVE;
1548
1549         /* active and current will differ iff current media is autoselect. */
1550         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1551                 return;
1552
1553         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1554         if (speed == SPEED_10000)
1555                 ifmr->ifm_active |= IFM_10G_T;
1556         else if (speed == SPEED_1000)
1557                 ifmr->ifm_active |= IFM_1000_T;
1558         else if (speed == SPEED_100)
1559                 ifmr->ifm_active |= IFM_100_TX;
1560         else if (speed == SPEED_10)
1561                 ifmr->ifm_active |= IFM_10_T;
1562         else
1563                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1564                             speed));
1565 }
1566
1567 void
1568 t4_fatal_err(struct adapter *sc)
1569 {
1570         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1571         t4_intr_disable(sc);
1572         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1573             device_get_nameunit(sc->dev));
1574 }
1575
1576 static int
1577 map_bars_0_and_4(struct adapter *sc)
1578 {
1579         sc->regs_rid = PCIR_BAR(0);
1580         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1581             &sc->regs_rid, RF_ACTIVE);
1582         if (sc->regs_res == NULL) {
1583                 device_printf(sc->dev, "cannot map registers.\n");
1584                 return (ENXIO);
1585         }
1586         sc->bt = rman_get_bustag(sc->regs_res);
1587         sc->bh = rman_get_bushandle(sc->regs_res);
1588         sc->mmio_len = rman_get_size(sc->regs_res);
1589         setbit(&sc->doorbells, DOORBELL_KDB);
1590
1591         sc->msix_rid = PCIR_BAR(4);
1592         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1593             &sc->msix_rid, RF_ACTIVE);
1594         if (sc->msix_res == NULL) {
1595                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1596                 return (ENXIO);
1597         }
1598
1599         return (0);
1600 }
1601
1602 static int
1603 map_bar_2(struct adapter *sc)
1604 {
1605
1606         /*
1607          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1608          * to map it if RDMA is disabled.
1609          */
1610         if (is_t4(sc) && sc->rdmacaps == 0)
1611                 return (0);
1612
1613         sc->udbs_rid = PCIR_BAR(2);
1614         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1615             &sc->udbs_rid, RF_ACTIVE);
1616         if (sc->udbs_res == NULL) {
1617                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1618                 return (ENXIO);
1619         }
1620         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1621
1622         if (is_t5(sc)) {
1623                 setbit(&sc->doorbells, DOORBELL_UDB);
1624 #if defined(__i386__) || defined(__amd64__)
1625                 if (t5_write_combine) {
1626                         int rc;
1627
1628                         /*
1629                          * Enable write combining on BAR2.  This is the
1630                          * userspace doorbell BAR and is split into 128B
1631                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1632                          * with an egress queue.  The first 64B has the doorbell
1633                          * and the second 64B can be used to submit a tx work
1634                          * request with an implicit doorbell.
1635                          */
1636
1637                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1638                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1639                         if (rc == 0) {
1640                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1641                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1642                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1643                         } else {
1644                                 device_printf(sc->dev,
1645                                     "couldn't enable write combining: %d\n",
1646                                     rc);
1647                         }
1648
1649                         t4_write_reg(sc, A_SGE_STAT_CFG,
1650                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1651                 }
1652 #endif
1653         }
1654
1655         return (0);
1656 }
1657
1658 static const struct memwin t4_memwin[] = {
1659         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1660         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1661         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1662 };
1663
1664 static const struct memwin t5_memwin[] = {
1665         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1666         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1667         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1668 };
1669
1670 static void
1671 setup_memwin(struct adapter *sc)
1672 {
1673         const struct memwin *mw;
1674         int i, n;
1675         uint32_t bar0;
1676
1677         if (is_t4(sc)) {
1678                 /*
1679                  * Read low 32b of bar0 indirectly via the hardware backdoor
1680                  * mechanism.  Works from within PCI passthrough environments
1681                  * too, where rman_get_start() can return a different value.  We
1682                  * need to program the T4 memory window decoders with the actual
1683                  * addresses that will be coming across the PCIe link.
1684                  */
1685                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1686                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1687
1688                 mw = &t4_memwin[0];
1689                 n = nitems(t4_memwin);
1690         } else {
1691                 /* T5 uses the relative offset inside the PCIe BAR */
1692                 bar0 = 0;
1693
1694                 mw = &t5_memwin[0];
1695                 n = nitems(t5_memwin);
1696         }
1697
1698         for (i = 0; i < n; i++, mw++) {
1699                 t4_write_reg(sc,
1700                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1701                     (mw->base + bar0) | V_BIR(0) |
1702                     V_WINDOW(ilog2(mw->aperture) - 10));
1703         }
1704
1705         /* flush */
1706         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1707 }
1708
1709 /*
1710  * Verify that the memory range specified by the addr/len pair is valid and lies
1711  * entirely within a single region (EDCx or MCx).
1712  */
1713 static int
1714 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1715 {
1716         uint32_t em, addr_len, maddr, mlen;
1717
1718         /* Memory can only be accessed in naturally aligned 4 byte units */
1719         if (addr & 3 || len & 3 || len == 0)
1720                 return (EINVAL);
1721
1722         /* Enabled memories */
1723         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1724         if (em & F_EDRAM0_ENABLE) {
1725                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1726                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1727                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1728                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1729                     addr + len <= maddr + mlen)
1730                         return (0);
1731         }
1732         if (em & F_EDRAM1_ENABLE) {
1733                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1734                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1735                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1736                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1737                     addr + len <= maddr + mlen)
1738                         return (0);
1739         }
1740         if (em & F_EXT_MEM_ENABLE) {
1741                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1742                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1743                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1744                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1745                     addr + len <= maddr + mlen)
1746                         return (0);
1747         }
1748         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1749                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1750                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1751                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1752                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1753                     addr + len <= maddr + mlen)
1754                         return (0);
1755         }
1756
1757         return (EFAULT);
1758 }
1759
1760 static int
1761 fwmtype_to_hwmtype(int mtype)
1762 {
1763
1764         switch (mtype) {
1765         case FW_MEMTYPE_EDC0:
1766                 return (MEM_EDC0);
1767         case FW_MEMTYPE_EDC1:
1768                 return (MEM_EDC1);
1769         case FW_MEMTYPE_EXTMEM:
1770                 return (MEM_MC0);
1771         case FW_MEMTYPE_EXTMEM1:
1772                 return (MEM_MC1);
1773         default:
1774                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1775         }
1776 }
1777
1778 /*
1779  * Verify that the memory range specified by the memtype/offset/len pair is
1780  * valid and lies entirely within the memtype specified.  The global address of
1781  * the start of the range is returned in addr.
1782  */
1783 static int
1784 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1785     uint32_t *addr)
1786 {
1787         uint32_t em, addr_len, maddr, mlen;
1788
1789         /* Memory can only be accessed in naturally aligned 4 byte units */
1790         if (off & 3 || len & 3 || len == 0)
1791                 return (EINVAL);
1792
1793         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1794         switch (fwmtype_to_hwmtype(mtype)) {
1795         case MEM_EDC0:
1796                 if (!(em & F_EDRAM0_ENABLE))
1797                         return (EINVAL);
1798                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1799                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1800                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1801                 break;
1802         case MEM_EDC1:
1803                 if (!(em & F_EDRAM1_ENABLE))
1804                         return (EINVAL);
1805                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1806                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1807                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1808                 break;
1809         case MEM_MC:
1810                 if (!(em & F_EXT_MEM_ENABLE))
1811                         return (EINVAL);
1812                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1813                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1814                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1815                 break;
1816         case MEM_MC1:
1817                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1818                         return (EINVAL);
1819                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1820                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1821                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1822                 break;
1823         default:
1824                 return (EINVAL);
1825         }
1826
1827         if (mlen > 0 && off < mlen && off + len <= mlen) {
1828                 *addr = maddr + off;    /* global address */
1829                 return (0);
1830         }
1831
1832         return (EFAULT);
1833 }
1834
1835 static void
1836 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1837 {
1838         const struct memwin *mw;
1839
1840         if (is_t4(sc)) {
1841                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1842                     ("%s: incorrect memwin# (%d)", __func__, win));
1843                 mw = &t4_memwin[win];
1844         } else {
1845                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1846                     ("%s: incorrect memwin# (%d)", __func__, win));
1847                 mw = &t5_memwin[win];
1848         }
1849
1850         if (base != NULL)
1851                 *base = mw->base;
1852         if (aperture != NULL)
1853                 *aperture = mw->aperture;
1854 }
1855
1856 /*
1857  * Positions the memory window such that it can be used to access the specified
1858  * address in the chip's address space.  The return value is the offset of addr
1859  * from the start of the window.
1860  */
1861 static uint32_t
1862 position_memwin(struct adapter *sc, int n, uint32_t addr)
1863 {
1864         uint32_t start, pf;
1865         uint32_t reg;
1866
1867         KASSERT(n >= 0 && n <= 3,
1868             ("%s: invalid window %d.", __func__, n));
1869         KASSERT((addr & 3) == 0,
1870             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1871
1872         if (is_t4(sc)) {
1873                 pf = 0;
1874                 start = addr & ~0xf;    /* start must be 16B aligned */
1875         } else {
1876                 pf = V_PFNUM(sc->pf);
1877                 start = addr & ~0x7f;   /* start must be 128B aligned */
1878         }
1879         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1880
1881         t4_write_reg(sc, reg, start | pf);
1882         t4_read_reg(sc, reg);
1883
1884         return (addr - start);
1885 }
1886
1887 static int
1888 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1889     struct intrs_and_queues *iaq)
1890 {
1891         int rc, itype, navail, nrxq10g, nrxq1g, n;
1892         int nofldrxq10g = 0, nofldrxq1g = 0;
1893         int nnmrxq10g = 0, nnmrxq1g = 0;
1894
1895         bzero(iaq, sizeof(*iaq));
1896
1897         iaq->ntxq10g = t4_ntxq10g;
1898         iaq->ntxq1g = t4_ntxq1g;
1899         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1900         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1901         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1902 #ifdef TCP_OFFLOAD
1903         if (is_offload(sc)) {
1904                 iaq->nofldtxq10g = t4_nofldtxq10g;
1905                 iaq->nofldtxq1g = t4_nofldtxq1g;
1906                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1907                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1908         }
1909 #endif
1910 #ifdef DEV_NETMAP
1911         iaq->nnmtxq10g = t4_nnmtxq10g;
1912         iaq->nnmtxq1g = t4_nnmtxq1g;
1913         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1914         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1915 #endif
1916
1917         for (itype = INTR_MSIX; itype; itype >>= 1) {
1918
1919                 if ((itype & t4_intr_types) == 0)
1920                         continue;       /* not allowed */
1921
1922                 if (itype == INTR_MSIX)
1923                         navail = pci_msix_count(sc->dev);
1924                 else if (itype == INTR_MSI)
1925                         navail = pci_msi_count(sc->dev);
1926                 else
1927                         navail = 1;
1928 restart:
1929                 if (navail == 0)
1930                         continue;
1931
1932                 iaq->intr_type = itype;
1933                 iaq->intr_flags_10g = 0;
1934                 iaq->intr_flags_1g = 0;
1935
1936                 /*
1937                  * Best option: an interrupt vector for errors, one for the
1938                  * firmware event queue, and one for every rxq (NIC, TOE, and
1939                  * netmap).
1940                  */
1941                 iaq->nirq = T4_EXTRA_INTR;
1942                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1943                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1944                 if (iaq->nirq <= navail &&
1945                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1946                         iaq->intr_flags_10g = INTR_ALL;
1947                         iaq->intr_flags_1g = INTR_ALL;
1948                         goto allocate;
1949                 }
1950
1951                 /*
1952                  * Second best option: a vector for errors, one for the firmware
1953                  * event queue, and vectors for either all the NIC rx queues or
1954                  * all the TOE rx queues.  The queues that don't get vectors
1955                  * will forward their interrupts to those that do.
1956                  *
1957                  * Note: netmap rx queues cannot be created early and so they
1958                  * can't be setup to receive forwarded interrupts for others.
1959                  */
1960                 iaq->nirq = T4_EXTRA_INTR;
1961                 if (nrxq10g >= nofldrxq10g) {
1962                         iaq->intr_flags_10g = INTR_RXQ;
1963                         iaq->nirq += n10g * nrxq10g;
1964 #ifdef DEV_NETMAP
1965                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1966 #endif
1967                 } else {
1968                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1969                         iaq->nirq += n10g * nofldrxq10g;
1970 #ifdef DEV_NETMAP
1971                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1972 #endif
1973                 }
1974                 if (nrxq1g >= nofldrxq1g) {
1975                         iaq->intr_flags_1g = INTR_RXQ;
1976                         iaq->nirq += n1g * nrxq1g;
1977 #ifdef DEV_NETMAP
1978                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1979 #endif
1980                 } else {
1981                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1982                         iaq->nirq += n1g * nofldrxq1g;
1983 #ifdef DEV_NETMAP
1984                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1985 #endif
1986                 }
1987                 if (iaq->nirq <= navail &&
1988                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1989                         goto allocate;
1990
1991                 /*
1992                  * Next best option: an interrupt vector for errors, one for the
1993                  * firmware event queue, and at least one per port.  At this
1994                  * point we know we'll have to downsize nrxq and/or nofldrxq
1995                  * and/or nnmrxq to fit what's available to us.
1996                  */
1997                 iaq->nirq = T4_EXTRA_INTR;
1998                 iaq->nirq += n10g + n1g;
1999                 if (iaq->nirq <= navail) {
2000                         int leftover = navail - iaq->nirq;
2001
2002                         if (n10g > 0) {
2003                                 int target = max(nrxq10g, nofldrxq10g);
2004
2005                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2006                                     INTR_RXQ : INTR_OFLD_RXQ;
2007
2008                                 n = 1;
2009                                 while (n < target && leftover >= n10g) {
2010                                         leftover -= n10g;
2011                                         iaq->nirq += n10g;
2012                                         n++;
2013                                 }
2014                                 iaq->nrxq10g = min(n, nrxq10g);
2015 #ifdef TCP_OFFLOAD
2016                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2017 #endif
2018 #ifdef DEV_NETMAP
2019                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2020 #endif
2021                         }
2022
2023                         if (n1g > 0) {
2024                                 int target = max(nrxq1g, nofldrxq1g);
2025
2026                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2027                                     INTR_RXQ : INTR_OFLD_RXQ;
2028
2029                                 n = 1;
2030                                 while (n < target && leftover >= n1g) {
2031                                         leftover -= n1g;
2032                                         iaq->nirq += n1g;
2033                                         n++;
2034                                 }
2035                                 iaq->nrxq1g = min(n, nrxq1g);
2036 #ifdef TCP_OFFLOAD
2037                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2038 #endif
2039 #ifdef DEV_NETMAP
2040                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2041 #endif
2042                         }
2043
2044                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2045                                 goto allocate;
2046                 }
2047
2048                 /*
2049                  * Least desirable option: one interrupt vector for everything.
2050                  */
2051                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2052                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2053 #ifdef TCP_OFFLOAD
2054                 if (is_offload(sc))
2055                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2056 #endif
2057 #ifdef DEV_NETMAP
2058                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2059 #endif
2060
2061 allocate:
2062                 navail = iaq->nirq;
2063                 rc = 0;
2064                 if (itype == INTR_MSIX)
2065                         rc = pci_alloc_msix(sc->dev, &navail);
2066                 else if (itype == INTR_MSI)
2067                         rc = pci_alloc_msi(sc->dev, &navail);
2068
2069                 if (rc == 0) {
2070                         if (navail == iaq->nirq)
2071                                 return (0);
2072
2073                         /*
2074                          * Didn't get the number requested.  Use whatever number
2075                          * the kernel is willing to allocate (it's in navail).
2076                          */
2077                         device_printf(sc->dev, "fewer vectors than requested, "
2078                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2079                             itype, iaq->nirq, navail);
2080                         pci_release_msi(sc->dev);
2081                         goto restart;
2082                 }
2083
2084                 device_printf(sc->dev,
2085                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2086                     itype, rc, iaq->nirq, navail);
2087         }
2088
2089         device_printf(sc->dev,
2090             "failed to find a usable interrupt type.  "
2091             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2092             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2093
2094         return (ENXIO);
2095 }
2096
2097 #define FW_VERSION(chip) ( \
2098     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2099     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2100     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2101     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2102 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2103
2104 struct fw_info {
2105         uint8_t chip;
2106         char *kld_name;
2107         char *fw_mod_name;
2108         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2109 } fw_info[] = {
2110         {
2111                 .chip = CHELSIO_T4,
2112                 .kld_name = "t4fw_cfg",
2113                 .fw_mod_name = "t4fw",
2114                 .fw_hdr = {
2115                         .chip = FW_HDR_CHIP_T4,
2116                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2117                         .intfver_nic = FW_INTFVER(T4, NIC),
2118                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2119                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2120                         .intfver_ri = FW_INTFVER(T4, RI),
2121                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2122                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2123                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2124                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2125                 },
2126         }, {
2127                 .chip = CHELSIO_T5,
2128                 .kld_name = "t5fw_cfg",
2129                 .fw_mod_name = "t5fw",
2130                 .fw_hdr = {
2131                         .chip = FW_HDR_CHIP_T5,
2132                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2133                         .intfver_nic = FW_INTFVER(T5, NIC),
2134                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2135                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2136                         .intfver_ri = FW_INTFVER(T5, RI),
2137                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2138                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2139                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2140                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2141                 },
2142         }
2143 };
2144
2145 static struct fw_info *
2146 find_fw_info(int chip)
2147 {
2148         int i;
2149
2150         for (i = 0; i < nitems(fw_info); i++) {
2151                 if (fw_info[i].chip == chip)
2152                         return (&fw_info[i]);
2153         }
2154         return (NULL);
2155 }
2156
2157 /*
2158  * Is the given firmware API compatible with the one the driver was compiled
2159  * with?
2160  */
2161 static int
2162 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2163 {
2164
2165         /* short circuit if it's the exact same firmware version */
2166         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2167                 return (1);
2168
2169         /*
2170          * XXX: Is this too conservative?  Perhaps I should limit this to the
2171          * features that are supported in the driver.
2172          */
2173 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2174         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2175             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2176             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2177                 return (1);
2178 #undef SAME_INTF
2179
2180         return (0);
2181 }
2182
2183 /*
2184  * The firmware in the KLD is usable, but should it be installed?  This routine
2185  * explains itself in detail if it indicates the KLD firmware should be
2186  * installed.
2187  */
2188 static int
2189 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2190 {
2191         const char *reason;
2192
2193         if (!card_fw_usable) {
2194                 reason = "incompatible or unusable";
2195                 goto install;
2196         }
2197
2198         if (k > c) {
2199                 reason = "older than the version bundled with this driver";
2200                 goto install;
2201         }
2202
2203         if (t4_fw_install == 2 && k != c) {
2204                 reason = "different than the version bundled with this driver";
2205                 goto install;
2206         }
2207
2208         return (0);
2209
2210 install:
2211         if (t4_fw_install == 0) {
2212                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2213                     "but the driver is prohibited from installing a different "
2214                     "firmware on the card.\n",
2215                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2216                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2217
2218                 return (0);
2219         }
2220
2221         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2222             "installing firmware %u.%u.%u.%u on card.\n",
2223             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2224             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2225             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2226             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2227
2228         return (1);
2229 }
2230 /*
2231  * Establish contact with the firmware and determine if we are the master driver
2232  * or not, and whether we are responsible for chip initialization.
2233  */
2234 static int
2235 prep_firmware(struct adapter *sc)
2236 {
2237         const struct firmware *fw = NULL, *default_cfg;
2238         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2239         enum dev_state state;
2240         struct fw_info *fw_info;
2241         struct fw_hdr *card_fw;         /* fw on the card */
2242         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2243         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2244                                            against */
2245
2246         /* Contact firmware. */
2247         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2248         if (rc < 0 || state == DEV_STATE_ERR) {
2249                 rc = -rc;
2250                 device_printf(sc->dev,
2251                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2252                 return (rc);
2253         }
2254         pf = rc;
2255         if (pf == sc->mbox)
2256                 sc->flags |= MASTER_PF;
2257         else if (state == DEV_STATE_UNINIT) {
2258                 /*
2259                  * We didn't get to be the master so we definitely won't be
2260                  * configuring the chip.  It's a bug if someone else hasn't
2261                  * configured it already.
2262                  */
2263                 device_printf(sc->dev, "couldn't be master(%d), "
2264                     "device not already initialized either(%d).\n", rc, state);
2265                 return (EDOOFUS);
2266         }
2267
2268         /* This is the firmware whose headers the driver was compiled against */
2269         fw_info = find_fw_info(chip_id(sc));
2270         if (fw_info == NULL) {
2271                 device_printf(sc->dev,
2272                     "unable to look up firmware information for chip %d.\n",
2273                     chip_id(sc));
2274                 return (EINVAL);
2275         }
2276         drv_fw = &fw_info->fw_hdr;
2277
2278         /*
2279          * The firmware KLD contains many modules.  The KLD name is also the
2280          * name of the module that contains the default config file.
2281          */
2282         default_cfg = firmware_get(fw_info->kld_name);
2283
2284         /* Read the header of the firmware on the card */
2285         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2286         rc = -t4_read_flash(sc, FLASH_FW_START,
2287             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2288         if (rc == 0)
2289                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2290         else {
2291                 device_printf(sc->dev,
2292                     "Unable to read card's firmware header: %d\n", rc);
2293                 card_fw_usable = 0;
2294         }
2295
2296         /* This is the firmware in the KLD */
2297         fw = firmware_get(fw_info->fw_mod_name);
2298         if (fw != NULL) {
2299                 kld_fw = (const void *)fw->data;
2300                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2301         } else {
2302                 kld_fw = NULL;
2303                 kld_fw_usable = 0;
2304         }
2305
2306         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2307             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2308                 /*
2309                  * Common case: the firmware on the card is an exact match and
2310                  * the KLD is an exact match too, or the KLD is
2311                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2312                  * here -- use cxgbetool loadfw if you want to reinstall the
2313                  * same firmware as the one on the card.
2314                  */
2315         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2316             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2317             be32toh(card_fw->fw_ver))) {
2318
2319                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2320                 if (rc != 0) {
2321                         device_printf(sc->dev,
2322                             "failed to install firmware: %d\n", rc);
2323                         goto done;
2324                 }
2325
2326                 /* Installed successfully, update the cached header too. */
2327                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2328                 card_fw_usable = 1;
2329                 need_fw_reset = 0;      /* already reset as part of load_fw */
2330         }
2331
2332         if (!card_fw_usable) {
2333                 uint32_t d, c, k;
2334
2335                 d = ntohl(drv_fw->fw_ver);
2336                 c = ntohl(card_fw->fw_ver);
2337                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2338
2339                 device_printf(sc->dev, "Cannot find a usable firmware: "
2340                     "fw_install %d, chip state %d, "
2341                     "driver compiled with %d.%d.%d.%d, "
2342                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2343                     t4_fw_install, state,
2344                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2345                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2346                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2347                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2348                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2349                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2350                 rc = EINVAL;
2351                 goto done;
2352         }
2353
2354         /* We're using whatever's on the card and it's known to be good. */
2355         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2356         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2357             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2358             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2359             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2360             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2361         t4_get_tp_version(sc, &sc->params.tp_vers);
2362
2363         /* Reset device */
2364         if (need_fw_reset &&
2365             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2366                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2367                 if (rc != ETIMEDOUT && rc != EIO)
2368                         t4_fw_bye(sc, sc->mbox);
2369                 goto done;
2370         }
2371         sc->flags |= FW_OK;
2372
2373         rc = get_params__pre_init(sc);
2374         if (rc != 0)
2375                 goto done; /* error message displayed already */
2376
2377         /* Partition adapter resources as specified in the config file. */
2378         if (state == DEV_STATE_UNINIT) {
2379
2380                 KASSERT(sc->flags & MASTER_PF,
2381                     ("%s: trying to change chip settings when not master.",
2382                     __func__));
2383
2384                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2385                 if (rc != 0)
2386                         goto done;      /* error message displayed already */
2387
2388                 t4_tweak_chip_settings(sc);
2389
2390                 /* get basic stuff going */
2391                 rc = -t4_fw_initialize(sc, sc->mbox);
2392                 if (rc != 0) {
2393                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2394                         goto done;
2395                 }
2396         } else {
2397                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2398                 sc->cfcsum = 0;
2399         }
2400
2401 done:
2402         free(card_fw, M_CXGBE);
2403         if (fw != NULL)
2404                 firmware_put(fw, FIRMWARE_UNLOAD);
2405         if (default_cfg != NULL)
2406                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2407
2408         return (rc);
2409 }
2410
2411 #define FW_PARAM_DEV(param) \
2412         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2413          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2414 #define FW_PARAM_PFVF(param) \
2415         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2416          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2417
2418 /*
2419  * Partition chip resources for use between various PFs, VFs, etc.
2420  */
2421 static int
2422 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2423     const char *name_prefix)
2424 {
2425         const struct firmware *cfg = NULL;
2426         int rc = 0;
2427         struct fw_caps_config_cmd caps;
2428         uint32_t mtype, moff, finicsum, cfcsum;
2429
2430         /*
2431          * Figure out what configuration file to use.  Pick the default config
2432          * file for the card if the user hasn't specified one explicitly.
2433          */
2434         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2435         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2436                 /* Card specific overrides go here. */
2437                 if (pci_get_device(sc->dev) == 0x440a)
2438                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2439                 if (is_fpga(sc))
2440                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2441         }
2442
2443         /*
2444          * We need to load another module if the profile is anything except
2445          * "default" or "flash".
2446          */
2447         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2448             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2449                 char s[32];
2450
2451                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2452                 cfg = firmware_get(s);
2453                 if (cfg == NULL) {
2454                         if (default_cfg != NULL) {
2455                                 device_printf(sc->dev,
2456                                     "unable to load module \"%s\" for "
2457                                     "configuration profile \"%s\", will use "
2458                                     "the default config file instead.\n",
2459                                     s, sc->cfg_file);
2460                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2461                                     "%s", DEFAULT_CF);
2462                         } else {
2463                                 device_printf(sc->dev,
2464                                     "unable to load module \"%s\" for "
2465                                     "configuration profile \"%s\", will use "
2466                                     "the config file on the card's flash "
2467                                     "instead.\n", s, sc->cfg_file);
2468                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2469                                     "%s", FLASH_CF);
2470                         }
2471                 }
2472         }
2473
2474         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2475             default_cfg == NULL) {
2476                 device_printf(sc->dev,
2477                     "default config file not available, will use the config "
2478                     "file on the card's flash instead.\n");
2479                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2480         }
2481
2482         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2483                 u_int cflen, i, n;
2484                 const uint32_t *cfdata;
2485                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2486
2487                 KASSERT(cfg != NULL || default_cfg != NULL,
2488                     ("%s: no config to upload", __func__));
2489
2490                 /*
2491                  * Ask the firmware where it wants us to upload the config file.
2492                  */
2493                 param = FW_PARAM_DEV(CF);
2494                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2495                 if (rc != 0) {
2496                         /* No support for config file?  Shouldn't happen. */
2497                         device_printf(sc->dev,
2498                             "failed to query config file location: %d.\n", rc);
2499                         goto done;
2500                 }
2501                 mtype = G_FW_PARAMS_PARAM_Y(val);
2502                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2503
2504                 /*
2505                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2506                  * useless stuffing/comments at the end of the config file so
2507                  * it's ok to simply throw away the last remaining bytes when
2508                  * the config file is not an exact multiple of 4.  This also
2509                  * helps with the validate_mt_off_len check.
2510                  */
2511                 if (cfg != NULL) {
2512                         cflen = cfg->datasize & ~3;
2513                         cfdata = cfg->data;
2514                 } else {
2515                         cflen = default_cfg->datasize & ~3;
2516                         cfdata = default_cfg->data;
2517                 }
2518
2519                 if (cflen > FLASH_CFG_MAX_SIZE) {
2520                         device_printf(sc->dev,
2521                             "config file too long (%d, max allowed is %d).  "
2522                             "Will try to use the config on the card, if any.\n",
2523                             cflen, FLASH_CFG_MAX_SIZE);
2524                         goto use_config_on_flash;
2525                 }
2526
2527                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2528                 if (rc != 0) {
2529                         device_printf(sc->dev,
2530                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2531                             "Will try to use the config on the card, if any.\n",
2532                             __func__, mtype, moff, cflen, rc);
2533                         goto use_config_on_flash;
2534                 }
2535
2536                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2537                 while (cflen) {
2538                         off = position_memwin(sc, 2, addr);
2539                         n = min(cflen, mw_aperture - off);
2540                         for (i = 0; i < n; i += 4)
2541                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2542                         cflen -= n;
2543                         addr += n;
2544                 }
2545         } else {
2546 use_config_on_flash:
2547                 mtype = FW_MEMTYPE_FLASH;
2548                 moff = t4_flash_cfg_addr(sc);
2549         }
2550
2551         bzero(&caps, sizeof(caps));
2552         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2553             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2554         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2555             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2556             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2557         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2558         if (rc != 0) {
2559                 device_printf(sc->dev,
2560                     "failed to pre-process config file: %d "
2561                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2562                 goto done;
2563         }
2564
2565         finicsum = be32toh(caps.finicsum);
2566         cfcsum = be32toh(caps.cfcsum);
2567         if (finicsum != cfcsum) {
2568                 device_printf(sc->dev,
2569                     "WARNING: config file checksum mismatch: %08x %08x\n",
2570                     finicsum, cfcsum);
2571         }
2572         sc->cfcsum = cfcsum;
2573
2574 #define LIMIT_CAPS(x) do { \
2575         caps.x &= htobe16(t4_##x##_allowed); \
2576 } while (0)
2577
2578         /*
2579          * Let the firmware know what features will (not) be used so it can tune
2580          * things accordingly.
2581          */
2582         LIMIT_CAPS(linkcaps);
2583         LIMIT_CAPS(niccaps);
2584         LIMIT_CAPS(toecaps);
2585         LIMIT_CAPS(rdmacaps);
2586         LIMIT_CAPS(iscsicaps);
2587         LIMIT_CAPS(fcoecaps);
2588 #undef LIMIT_CAPS
2589
2590         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2591             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2592         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2593         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2594         if (rc != 0) {
2595                 device_printf(sc->dev,
2596                     "failed to process config file: %d.\n", rc);
2597         }
2598 done:
2599         if (cfg != NULL)
2600                 firmware_put(cfg, FIRMWARE_UNLOAD);
2601         return (rc);
2602 }
2603
2604 /*
2605  * Retrieve parameters that are needed (or nice to have) very early.
2606  */
2607 static int
2608 get_params__pre_init(struct adapter *sc)
2609 {
2610         int rc;
2611         uint32_t param[2], val[2];
2612         struct fw_devlog_cmd cmd;
2613         struct devlog_params *dlog = &sc->params.devlog;
2614
2615         param[0] = FW_PARAM_DEV(PORTVEC);
2616         param[1] = FW_PARAM_DEV(CCLK);
2617         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2618         if (rc != 0) {
2619                 device_printf(sc->dev,
2620                     "failed to query parameters (pre_init): %d.\n", rc);
2621                 return (rc);
2622         }
2623
2624         sc->params.portvec = val[0];
2625         sc->params.nports = bitcount32(val[0]);
2626         sc->params.vpd.cclk = val[1];
2627
2628         /* Read device log parameters. */
2629         bzero(&cmd, sizeof(cmd));
2630         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2631             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2632         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2633         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2634         if (rc != 0) {
2635                 device_printf(sc->dev,
2636                     "failed to get devlog parameters: %d.\n", rc);
2637                 bzero(dlog, sizeof (*dlog));
2638                 rc = 0; /* devlog isn't critical for device operation */
2639         } else {
2640                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2641                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2642                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2643                 dlog->size = be32toh(cmd.memsize_devlog);
2644         }
2645
2646         return (rc);
2647 }
2648
2649 /*
2650  * Retrieve various parameters that are of interest to the driver.  The device
2651  * has been initialized by the firmware at this point.
2652  */
2653 static int
2654 get_params__post_init(struct adapter *sc)
2655 {
2656         int rc;
2657         uint32_t param[7], val[7];
2658         struct fw_caps_config_cmd caps;
2659
2660         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2661         param[1] = FW_PARAM_PFVF(EQ_START);
2662         param[2] = FW_PARAM_PFVF(FILTER_START);
2663         param[3] = FW_PARAM_PFVF(FILTER_END);
2664         param[4] = FW_PARAM_PFVF(L2T_START);
2665         param[5] = FW_PARAM_PFVF(L2T_END);
2666         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2667         if (rc != 0) {
2668                 device_printf(sc->dev,
2669                     "failed to query parameters (post_init): %d.\n", rc);
2670                 return (rc);
2671         }
2672
2673         sc->sge.iq_start = val[0];
2674         sc->sge.eq_start = val[1];
2675         sc->tids.ftid_base = val[2];
2676         sc->tids.nftids = val[3] - val[2] + 1;
2677         sc->params.ftid_min = val[2];
2678         sc->params.ftid_max = val[3];
2679         sc->vres.l2t.start = val[4];
2680         sc->vres.l2t.size = val[5] - val[4] + 1;
2681         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2682             ("%s: L2 table size (%u) larger than expected (%u)",
2683             __func__, sc->vres.l2t.size, L2T_SIZE));
2684
2685         /* get capabilites */
2686         bzero(&caps, sizeof(caps));
2687         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2688             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2689         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2690         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2691         if (rc != 0) {
2692                 device_printf(sc->dev,
2693                     "failed to get card capabilities: %d.\n", rc);
2694                 return (rc);
2695         }
2696
2697 #define READ_CAPS(x) do { \
2698         sc->x = htobe16(caps.x); \
2699 } while (0)
2700         READ_CAPS(linkcaps);
2701         READ_CAPS(niccaps);
2702         READ_CAPS(toecaps);
2703         READ_CAPS(rdmacaps);
2704         READ_CAPS(iscsicaps);
2705         READ_CAPS(fcoecaps);
2706
2707         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2708                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2709                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2710                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2711                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2712                 if (rc != 0) {
2713                         device_printf(sc->dev,
2714                             "failed to query NIC parameters: %d.\n", rc);
2715                         return (rc);
2716                 }
2717                 sc->tids.etid_base = val[0];
2718                 sc->params.etid_min = val[0];
2719                 sc->tids.netids = val[1] - val[0] + 1;
2720                 sc->params.netids = sc->tids.netids;
2721                 sc->params.eo_wr_cred = val[2];
2722                 sc->params.ethoffload = 1;
2723         }
2724
2725         if (sc->toecaps) {
2726                 /* query offload-related parameters */
2727                 param[0] = FW_PARAM_DEV(NTID);
2728                 param[1] = FW_PARAM_PFVF(SERVER_START);
2729                 param[2] = FW_PARAM_PFVF(SERVER_END);
2730                 param[3] = FW_PARAM_PFVF(TDDP_START);
2731                 param[4] = FW_PARAM_PFVF(TDDP_END);
2732                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2733                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2734                 if (rc != 0) {
2735                         device_printf(sc->dev,
2736                             "failed to query TOE parameters: %d.\n", rc);
2737                         return (rc);
2738                 }
2739                 sc->tids.ntids = val[0];
2740                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2741                 sc->tids.stid_base = val[1];
2742                 sc->tids.nstids = val[2] - val[1] + 1;
2743                 sc->vres.ddp.start = val[3];
2744                 sc->vres.ddp.size = val[4] - val[3] + 1;
2745                 sc->params.ofldq_wr_cred = val[5];
2746                 sc->params.offload = 1;
2747         }
2748         if (sc->rdmacaps) {
2749                 param[0] = FW_PARAM_PFVF(STAG_START);
2750                 param[1] = FW_PARAM_PFVF(STAG_END);
2751                 param[2] = FW_PARAM_PFVF(RQ_START);
2752                 param[3] = FW_PARAM_PFVF(RQ_END);
2753                 param[4] = FW_PARAM_PFVF(PBL_START);
2754                 param[5] = FW_PARAM_PFVF(PBL_END);
2755                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2756                 if (rc != 0) {
2757                         device_printf(sc->dev,
2758                             "failed to query RDMA parameters(1): %d.\n", rc);
2759                         return (rc);
2760                 }
2761                 sc->vres.stag.start = val[0];
2762                 sc->vres.stag.size = val[1] - val[0] + 1;
2763                 sc->vres.rq.start = val[2];
2764                 sc->vres.rq.size = val[3] - val[2] + 1;
2765                 sc->vres.pbl.start = val[4];
2766                 sc->vres.pbl.size = val[5] - val[4] + 1;
2767
2768                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2769                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2770                 param[2] = FW_PARAM_PFVF(CQ_START);
2771                 param[3] = FW_PARAM_PFVF(CQ_END);
2772                 param[4] = FW_PARAM_PFVF(OCQ_START);
2773                 param[5] = FW_PARAM_PFVF(OCQ_END);
2774                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2775                 if (rc != 0) {
2776                         device_printf(sc->dev,
2777                             "failed to query RDMA parameters(2): %d.\n", rc);
2778                         return (rc);
2779                 }
2780                 sc->vres.qp.start = val[0];
2781                 sc->vres.qp.size = val[1] - val[0] + 1;
2782                 sc->vres.cq.start = val[2];
2783                 sc->vres.cq.size = val[3] - val[2] + 1;
2784                 sc->vres.ocq.start = val[4];
2785                 sc->vres.ocq.size = val[5] - val[4] + 1;
2786         }
2787         if (sc->iscsicaps) {
2788                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2789                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2790                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2791                 if (rc != 0) {
2792                         device_printf(sc->dev,
2793                             "failed to query iSCSI parameters: %d.\n", rc);
2794                         return (rc);
2795                 }
2796                 sc->vres.iscsi.start = val[0];
2797                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2798         }
2799
2800         /*
2801          * We've got the params we wanted to query via the firmware.  Now grab
2802          * some others directly from the chip.
2803          */
2804         rc = t4_read_chip_settings(sc);
2805
2806         return (rc);
2807 }
2808
2809 static int
2810 set_params__post_init(struct adapter *sc)
2811 {
2812         uint32_t param, val;
2813
2814         /* ask for encapsulated CPLs */
2815         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2816         val = 1;
2817         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2818
2819         return (0);
2820 }
2821
2822 #undef FW_PARAM_PFVF
2823 #undef FW_PARAM_DEV
2824
2825 static void
2826 t4_set_desc(struct adapter *sc)
2827 {
2828         char buf[128];
2829         struct adapter_params *p = &sc->params;
2830
2831         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2832             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2833             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2834
2835         device_set_desc_copy(sc->dev, buf);
2836 }
2837
2838 static void
2839 build_medialist(struct port_info *pi, struct ifmedia *media)
2840 {
2841         int data, m;
2842
2843         PORT_LOCK(pi);
2844
2845         ifmedia_removeall(media);
2846
2847         m = IFM_ETHER | IFM_FDX;
2848         data = (pi->port_type << 8) | pi->mod_type;
2849
2850         switch(pi->port_type) {
2851         case FW_PORT_TYPE_BT_XFI:
2852         case FW_PORT_TYPE_BT_XAUI:
2853                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2854                 /* fall through */
2855
2856         case FW_PORT_TYPE_BT_SGMII:
2857                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2858                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2859                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2860                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2861                 break;
2862
2863         case FW_PORT_TYPE_CX4:
2864                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2865                 ifmedia_set(media, m | IFM_10G_CX4);
2866                 break;
2867
2868         case FW_PORT_TYPE_QSFP_10G:
2869         case FW_PORT_TYPE_SFP:
2870         case FW_PORT_TYPE_FIBER_XFI:
2871         case FW_PORT_TYPE_FIBER_XAUI:
2872                 switch (pi->mod_type) {
2873
2874                 case FW_PORT_MOD_TYPE_LR:
2875                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2876                         ifmedia_set(media, m | IFM_10G_LR);
2877                         break;
2878
2879                 case FW_PORT_MOD_TYPE_SR:
2880                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2881                         ifmedia_set(media, m | IFM_10G_SR);
2882                         break;
2883
2884                 case FW_PORT_MOD_TYPE_LRM:
2885                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2886                         ifmedia_set(media, m | IFM_10G_LRM);
2887                         break;
2888
2889                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2890                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2891                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2892                         ifmedia_set(media, m | IFM_10G_TWINAX);
2893                         break;
2894
2895                 case FW_PORT_MOD_TYPE_NONE:
2896                         m &= ~IFM_FDX;
2897                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2898                         ifmedia_set(media, m | IFM_NONE);
2899                         break;
2900
2901                 case FW_PORT_MOD_TYPE_NA:
2902                 case FW_PORT_MOD_TYPE_ER:
2903                 default:
2904                         device_printf(pi->dev,
2905                             "unknown port_type (%d), mod_type (%d)\n",
2906                             pi->port_type, pi->mod_type);
2907                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2908                         ifmedia_set(media, m | IFM_UNKNOWN);
2909                         break;
2910                 }
2911                 break;
2912
2913         case FW_PORT_TYPE_QSFP:
2914                 switch (pi->mod_type) {
2915
2916                 case FW_PORT_MOD_TYPE_LR:
2917                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2918                         ifmedia_set(media, m | IFM_40G_LR4);
2919                         break;
2920
2921                 case FW_PORT_MOD_TYPE_SR:
2922                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2923                         ifmedia_set(media, m | IFM_40G_SR4);
2924                         break;
2925
2926                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2927                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2928                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2929                         ifmedia_set(media, m | IFM_40G_CR4);
2930                         break;
2931
2932                 case FW_PORT_MOD_TYPE_NONE:
2933                         m &= ~IFM_FDX;
2934                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2935                         ifmedia_set(media, m | IFM_NONE);
2936                         break;
2937
2938                 default:
2939                         device_printf(pi->dev,
2940                             "unknown port_type (%d), mod_type (%d)\n",
2941                             pi->port_type, pi->mod_type);
2942                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2943                         ifmedia_set(media, m | IFM_UNKNOWN);
2944                         break;
2945                 }
2946                 break;
2947
2948         default:
2949                 device_printf(pi->dev,
2950                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2951                     pi->mod_type);
2952                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2953                 ifmedia_set(media, m | IFM_UNKNOWN);
2954                 break;
2955         }
2956
2957         PORT_UNLOCK(pi);
2958 }
2959
2960 #define FW_MAC_EXACT_CHUNK      7
2961
2962 /*
2963  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2964  * indicates which parameters should be programmed (the rest are left alone).
2965  */
2966 int
2967 update_mac_settings(struct ifnet *ifp, int flags)
2968 {
2969         int rc = 0;
2970         struct port_info *pi = ifp->if_softc;
2971         struct adapter *sc = pi->adapter;
2972         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2973         uint16_t viid = 0xffff;
2974         int16_t *xact_addr_filt = NULL;
2975
2976         ASSERT_SYNCHRONIZED_OP(sc);
2977         KASSERT(flags, ("%s: not told what to update.", __func__));
2978
2979         if (ifp == pi->ifp) {
2980                 viid = pi->viid;
2981                 xact_addr_filt = &pi->xact_addr_filt;
2982         }
2983 #ifdef DEV_NETMAP
2984         else if (ifp == pi->nm_ifp) {
2985                 viid = pi->nm_viid;
2986                 xact_addr_filt = &pi->nm_xact_addr_filt;
2987         }
2988 #endif
2989         if (flags & XGMAC_MTU)
2990                 mtu = ifp->if_mtu;
2991
2992         if (flags & XGMAC_PROMISC)
2993                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2994
2995         if (flags & XGMAC_ALLMULTI)
2996                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2997
2998         if (flags & XGMAC_VLANEX)
2999                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3000
3001         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3002                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3003                     1, vlanex, false);
3004                 if (rc) {
3005                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3006                             rc);
3007                         return (rc);
3008                 }
3009         }
3010
3011         if (flags & XGMAC_UCADDR) {
3012                 uint8_t ucaddr[ETHER_ADDR_LEN];
3013
3014                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3015                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3016                     true, true);
3017                 if (rc < 0) {
3018                         rc = -rc;
3019                         if_printf(ifp, "change_mac failed: %d\n", rc);
3020                         return (rc);
3021                 } else {
3022                         *xact_addr_filt = rc;
3023                         rc = 0;
3024                 }
3025         }
3026
3027         if (flags & XGMAC_MCADDRS) {
3028                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3029                 int del = 1;
3030                 uint64_t hash = 0;
3031                 struct ifmultiaddr *ifma;
3032                 int i = 0, j;
3033
3034                 if_maddr_rlock(ifp);
3035                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3036                         if (ifma->ifma_addr->sa_family != AF_LINK)
3037                                 continue;
3038                         mcaddr[i] =
3039                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3040                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3041                         i++;
3042
3043                         if (i == FW_MAC_EXACT_CHUNK) {
3044                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3045                                     i, mcaddr, NULL, &hash, 0);
3046                                 if (rc < 0) {
3047                                         rc = -rc;
3048                                         for (j = 0; j < i; j++) {
3049                                                 if_printf(ifp,
3050                                                     "failed to add mc address"
3051                                                     " %02x:%02x:%02x:"
3052                                                     "%02x:%02x:%02x rc=%d\n",
3053                                                     mcaddr[j][0], mcaddr[j][1],
3054                                                     mcaddr[j][2], mcaddr[j][3],
3055                                                     mcaddr[j][4], mcaddr[j][5],
3056                                                     rc);
3057                                         }
3058                                         goto mcfail;
3059                                 }
3060                                 del = 0;
3061                                 i = 0;
3062                         }
3063                 }
3064                 if (i > 0) {
3065                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3066                             mcaddr, NULL, &hash, 0);
3067                         if (rc < 0) {
3068                                 rc = -rc;
3069                                 for (j = 0; j < i; j++) {
3070                                         if_printf(ifp,
3071                                             "failed to add mc address"
3072                                             " %02x:%02x:%02x:"
3073                                             "%02x:%02x:%02x rc=%d\n",
3074                                             mcaddr[j][0], mcaddr[j][1],
3075                                             mcaddr[j][2], mcaddr[j][3],
3076                                             mcaddr[j][4], mcaddr[j][5],
3077                                             rc);
3078                                 }
3079                                 goto mcfail;
3080                         }
3081                 }
3082
3083                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3084                 if (rc != 0)
3085                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3086 mcfail:
3087                 if_maddr_runlock(ifp);
3088         }
3089
3090         return (rc);
3091 }
3092
3093 /*
3094  * {begin|end}_synchronized_op must be called from the same thread.
3095  */
3096 int
3097 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3098     char *wmesg)
3099 {
3100         int rc, pri;
3101
3102 #ifdef WITNESS
3103         /* the caller thinks it's ok to sleep, but is it really? */
3104         if (flags & SLEEP_OK)
3105                 pause("t4slptst", 1);
3106 #endif
3107
3108         if (INTR_OK)
3109                 pri = PCATCH;
3110         else
3111                 pri = 0;
3112
3113         ADAPTER_LOCK(sc);
3114         for (;;) {
3115
3116                 if (pi && IS_DOOMED(pi)) {
3117                         rc = ENXIO;
3118                         goto done;
3119                 }
3120
3121                 if (!IS_BUSY(sc)) {
3122                         rc = 0;
3123                         break;
3124                 }
3125
3126                 if (!(flags & SLEEP_OK)) {
3127                         rc = EBUSY;
3128                         goto done;
3129                 }
3130
3131                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3132                         rc = EINTR;
3133                         goto done;
3134                 }
3135         }
3136
3137         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3138         SET_BUSY(sc);
3139 #ifdef INVARIANTS
3140         sc->last_op = wmesg;
3141         sc->last_op_thr = curthread;
3142         sc->last_op_flags = flags;
3143 #endif
3144
3145 done:
3146         if (!(flags & HOLD_LOCK) || rc)
3147                 ADAPTER_UNLOCK(sc);
3148
3149         return (rc);
3150 }
3151
3152 /*
3153  * {begin|end}_synchronized_op must be called from the same thread.
3154  */
3155 void
3156 end_synchronized_op(struct adapter *sc, int flags)
3157 {
3158
3159         if (flags & LOCK_HELD)
3160                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3161         else
3162                 ADAPTER_LOCK(sc);
3163
3164         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3165         CLR_BUSY(sc);
3166         wakeup(&sc->flags);
3167         ADAPTER_UNLOCK(sc);
3168 }
3169
3170 static int
3171 cxgbe_init_synchronized(struct port_info *pi)
3172 {
3173         struct adapter *sc = pi->adapter;
3174         struct ifnet *ifp = pi->ifp;
3175         int rc = 0, i;
3176         struct sge_txq *txq;
3177
3178         ASSERT_SYNCHRONIZED_OP(sc);
3179
3180         if (isset(&sc->open_device_map, pi->port_id)) {
3181                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3182                     ("mismatch between open_device_map and if_drv_flags"));
3183                 return (0);     /* already running */
3184         }
3185
3186         if (!(sc->flags & FULL_INIT_DONE) &&
3187             ((rc = adapter_full_init(sc)) != 0))
3188                 return (rc);    /* error message displayed already */
3189
3190         if (!(pi->flags & PORT_INIT_DONE) &&
3191             ((rc = port_full_init(pi)) != 0))
3192                 return (rc); /* error message displayed already */
3193
3194         rc = update_mac_settings(ifp, XGMAC_ALL);
3195         if (rc)
3196                 goto done;      /* error message displayed already */
3197
3198         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3199         if (rc != 0) {
3200                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3201                 goto done;
3202         }
3203
3204         /*
3205          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3206          * if this changes.
3207          */
3208
3209         for_each_txq(pi, i, txq) {
3210                 TXQ_LOCK(txq);
3211                 txq->eq.flags |= EQ_ENABLED;
3212                 TXQ_UNLOCK(txq);
3213         }
3214
3215         /*
3216          * The first iq of the first port to come up is used for tracing.
3217          */
3218         if (sc->traceq < 0) {
3219                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3220                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3221                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3222                     V_QUEUENUMBER(sc->traceq));
3223                 pi->flags |= HAS_TRACEQ;
3224         }
3225
3226         /* all ok */
3227         setbit(&sc->open_device_map, pi->port_id);
3228         PORT_LOCK(pi);
3229         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3230         PORT_UNLOCK(pi);
3231
3232         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3233 done:
3234         if (rc != 0)
3235                 cxgbe_uninit_synchronized(pi);
3236
3237         return (rc);
3238 }
3239
3240 /*
3241  * Idempotent.
3242  */
3243 static int
3244 cxgbe_uninit_synchronized(struct port_info *pi)
3245 {
3246         struct adapter *sc = pi->adapter;
3247         struct ifnet *ifp = pi->ifp;
3248         int rc, i;
3249         struct sge_txq *txq;
3250
3251         ASSERT_SYNCHRONIZED_OP(sc);
3252
3253         /*
3254          * Disable the VI so that all its data in either direction is discarded
3255          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3256          * tick) intact as the TP can deliver negative advice or data that it's
3257          * holding in its RAM (for an offloaded connection) even after the VI is
3258          * disabled.
3259          */
3260         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3261         if (rc) {
3262                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3263                 return (rc);
3264         }
3265
3266         for_each_txq(pi, i, txq) {
3267                 TXQ_LOCK(txq);
3268                 txq->eq.flags &= ~EQ_ENABLED;
3269                 TXQ_UNLOCK(txq);
3270         }
3271
3272         clrbit(&sc->open_device_map, pi->port_id);
3273         PORT_LOCK(pi);
3274         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3275         PORT_UNLOCK(pi);
3276
3277         pi->link_cfg.link_ok = 0;
3278         pi->link_cfg.speed = 0;
3279         pi->linkdnrc = -1;
3280         t4_os_link_changed(sc, pi->port_id, 0, -1);
3281
3282         return (0);
3283 }
3284
3285 /*
3286  * It is ok for this function to fail midway and return right away.  t4_detach
3287  * will walk the entire sc->irq list and clean up whatever is valid.
3288  */
3289 static int
3290 setup_intr_handlers(struct adapter *sc)
3291 {
3292         int rc, rid, p, q;
3293         char s[8];
3294         struct irq *irq;
3295         struct port_info *pi;
3296         struct sge_rxq *rxq;
3297 #ifdef TCP_OFFLOAD
3298         struct sge_ofld_rxq *ofld_rxq;
3299 #endif
3300 #ifdef DEV_NETMAP
3301         struct sge_nm_rxq *nm_rxq;
3302 #endif
3303
3304         /*
3305          * Setup interrupts.
3306          */
3307         irq = &sc->irq[0];
3308         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3309         if (sc->intr_count == 1)
3310                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3311
3312         /* Multiple interrupts. */
3313         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3314             ("%s: too few intr.", __func__));
3315
3316         /* The first one is always error intr */
3317         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3318         if (rc != 0)
3319                 return (rc);
3320         irq++;
3321         rid++;
3322
3323         /* The second one is always the firmware event queue */
3324         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3325         if (rc != 0)
3326                 return (rc);
3327         irq++;
3328         rid++;
3329
3330         for_each_port(sc, p) {
3331                 pi = sc->port[p];
3332
3333                 if (pi->flags & INTR_RXQ) {
3334                         for_each_rxq(pi, q, rxq) {
3335                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3336                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3337                                     s);
3338                                 if (rc != 0)
3339                                         return (rc);
3340                                 irq++;
3341                                 rid++;
3342                         }
3343                 }
3344 #ifdef TCP_OFFLOAD
3345                 if (pi->flags & INTR_OFLD_RXQ) {
3346                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3347                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3348                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3349                                     ofld_rxq, s);
3350                                 if (rc != 0)
3351                                         return (rc);
3352                                 irq++;
3353                                 rid++;
3354                         }
3355                 }
3356 #endif
3357 #ifdef DEV_NETMAP
3358                 if (pi->flags & INTR_NM_RXQ) {
3359                         for_each_nm_rxq(pi, q, nm_rxq) {
3360                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3361                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3362                                     nm_rxq, s);
3363                                 if (rc != 0)
3364                                         return (rc);
3365                                 irq++;
3366                                 rid++;
3367                         }
3368                 }
3369 #endif
3370         }
3371         MPASS(irq == &sc->irq[sc->intr_count]);
3372
3373         return (0);
3374 }
3375
3376 int
3377 adapter_full_init(struct adapter *sc)
3378 {
3379         int rc, i;
3380
3381         ASSERT_SYNCHRONIZED_OP(sc);
3382         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3383         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3384             ("%s: FULL_INIT_DONE already", __func__));
3385
3386         /*
3387          * queues that belong to the adapter (not any particular port).
3388          */
3389         rc = t4_setup_adapter_queues(sc);
3390         if (rc != 0)
3391                 goto done;
3392
3393         for (i = 0; i < nitems(sc->tq); i++) {
3394                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3395                     taskqueue_thread_enqueue, &sc->tq[i]);
3396                 if (sc->tq[i] == NULL) {
3397                         device_printf(sc->dev,
3398                             "failed to allocate task queue %d\n", i);
3399                         rc = ENOMEM;
3400                         goto done;
3401                 }
3402                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3403                     device_get_nameunit(sc->dev), i);
3404         }
3405
3406         t4_intr_enable(sc);
3407         sc->flags |= FULL_INIT_DONE;
3408 done:
3409         if (rc != 0)
3410                 adapter_full_uninit(sc);
3411
3412         return (rc);
3413 }
3414
3415 int
3416 adapter_full_uninit(struct adapter *sc)
3417 {
3418         int i;
3419
3420         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3421
3422         t4_teardown_adapter_queues(sc);
3423
3424         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3425                 taskqueue_free(sc->tq[i]);
3426                 sc->tq[i] = NULL;
3427         }
3428
3429         sc->flags &= ~FULL_INIT_DONE;
3430
3431         return (0);
3432 }
3433
3434 int
3435 port_full_init(struct port_info *pi)
3436 {
3437         struct adapter *sc = pi->adapter;
3438         struct ifnet *ifp = pi->ifp;
3439         uint16_t *rss;
3440         struct sge_rxq *rxq;
3441         int rc, i, j;
3442
3443         ASSERT_SYNCHRONIZED_OP(sc);
3444         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3445             ("%s: PORT_INIT_DONE already", __func__));
3446
3447         sysctl_ctx_init(&pi->ctx);
3448         pi->flags |= PORT_SYSCTL_CTX;
3449
3450         /*
3451          * Allocate tx/rx/fl queues for this port.
3452          */
3453         rc = t4_setup_port_queues(pi);
3454         if (rc != 0)
3455                 goto done;      /* error message displayed already */
3456
3457         /*
3458          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3459          */
3460         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3461         for (i = 0; i < pi->rss_size;) {
3462                 for_each_rxq(pi, j, rxq) {
3463                         rss[i++] = rxq->iq.abs_id;
3464                         if (i == pi->rss_size)
3465                                 break;
3466                 }
3467         }
3468
3469         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3470             pi->rss_size);
3471         if (rc != 0) {
3472                 if_printf(ifp, "rss_config failed: %d\n", rc);
3473                 goto done;
3474         }
3475
3476         pi->rss = rss;
3477         pi->flags |= PORT_INIT_DONE;
3478 done:
3479         if (rc != 0)
3480                 port_full_uninit(pi);
3481
3482         return (rc);
3483 }
3484
3485 /*
3486  * Idempotent.
3487  */
3488 int
3489 port_full_uninit(struct port_info *pi)
3490 {
3491         struct adapter *sc = pi->adapter;
3492         int i;
3493         struct sge_rxq *rxq;
3494         struct sge_txq *txq;
3495 #ifdef TCP_OFFLOAD
3496         struct sge_ofld_rxq *ofld_rxq;
3497         struct sge_wrq *ofld_txq;
3498 #endif
3499
3500         if (pi->flags & PORT_INIT_DONE) {
3501
3502                 /* Need to quiesce queues.  */
3503
3504                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3505
3506                 for_each_txq(pi, i, txq) {
3507                         quiesce_txq(sc, txq);
3508                 }
3509
3510 #ifdef TCP_OFFLOAD
3511                 for_each_ofld_txq(pi, i, ofld_txq) {
3512                         quiesce_wrq(sc, ofld_txq);
3513                 }
3514 #endif
3515
3516                 for_each_rxq(pi, i, rxq) {
3517                         quiesce_iq(sc, &rxq->iq);
3518                         quiesce_fl(sc, &rxq->fl);
3519                 }
3520
3521 #ifdef TCP_OFFLOAD
3522                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3523                         quiesce_iq(sc, &ofld_rxq->iq);
3524                         quiesce_fl(sc, &ofld_rxq->fl);
3525                 }
3526 #endif
3527                 free(pi->rss, M_CXGBE);
3528         }
3529
3530         t4_teardown_port_queues(pi);
3531         pi->flags &= ~PORT_INIT_DONE;
3532
3533         return (0);
3534 }
3535
3536 static void
3537 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3538 {
3539         struct sge_eq *eq = &txq->eq;
3540         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3541
3542         (void) sc;      /* unused */
3543
3544 #ifdef INVARIANTS
3545         TXQ_LOCK(txq);
3546         MPASS((eq->flags & EQ_ENABLED) == 0);
3547         TXQ_UNLOCK(txq);
3548 #endif
3549
3550         /* Wait for the mp_ring to empty. */
3551         while (!mp_ring_is_idle(txq->r)) {
3552                 mp_ring_check_drainage(txq->r, 0);
3553                 pause("rquiesce", 1);
3554         }
3555
3556         /* Then wait for the hardware to finish. */
3557         while (spg->cidx != htobe16(eq->pidx))
3558                 pause("equiesce", 1);
3559
3560         /* Finally, wait for the driver to reclaim all descriptors. */
3561         while (eq->cidx != eq->pidx)
3562                 pause("dquiesce", 1);
3563 }
3564
3565 static void
3566 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3567 {
3568
3569         /* XXXTX */
3570 }
3571
3572 static void
3573 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3574 {
3575         (void) sc;      /* unused */
3576
3577         /* Synchronize with the interrupt handler */
3578         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3579                 pause("iqfree", 1);
3580 }
3581
3582 static void
3583 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3584 {
3585         mtx_lock(&sc->sfl_lock);
3586         FL_LOCK(fl);
3587         fl->flags |= FL_DOOMED;
3588         FL_UNLOCK(fl);
3589         mtx_unlock(&sc->sfl_lock);
3590
3591         callout_drain(&sc->sfl_callout);
3592         KASSERT((fl->flags & FL_STARVING) == 0,
3593             ("%s: still starving", __func__));
3594 }
3595
3596 static int
3597 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3598     driver_intr_t *handler, void *arg, char *name)
3599 {
3600         int rc;
3601
3602         irq->rid = rid;
3603         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3604             RF_SHAREABLE | RF_ACTIVE);
3605         if (irq->res == NULL) {
3606                 device_printf(sc->dev,
3607                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3608                 return (ENOMEM);
3609         }
3610
3611         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3612             NULL, handler, arg, &irq->tag);
3613         if (rc != 0) {
3614                 device_printf(sc->dev,
3615                     "failed to setup interrupt for rid %d, name %s: %d\n",
3616                     rid, name, rc);
3617         } else if (name)
3618                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3619
3620         return (rc);
3621 }
3622
3623 static int
3624 t4_free_irq(struct adapter *sc, struct irq *irq)
3625 {
3626         if (irq->tag)
3627                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3628         if (irq->res)
3629                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3630
3631         bzero(irq, sizeof(*irq));
3632
3633         return (0);
3634 }
3635
3636 static void
3637 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3638     unsigned int end)
3639 {
3640         uint32_t *p = (uint32_t *)(buf + start);
3641
3642         for ( ; start <= end; start += sizeof(uint32_t))
3643                 *p++ = t4_read_reg(sc, start);
3644 }
3645
3646 static void
3647 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3648 {
3649         int i, n;
3650         const unsigned int *reg_ranges;
3651         static const unsigned int t4_reg_ranges[] = {
3652                 0x1008, 0x1108,
3653                 0x1180, 0x11b4,
3654                 0x11fc, 0x123c,
3655                 0x1300, 0x173c,
3656                 0x1800, 0x18fc,
3657                 0x3000, 0x30d8,
3658                 0x30e0, 0x5924,
3659                 0x5960, 0x59d4,
3660                 0x5a00, 0x5af8,
3661                 0x6000, 0x6098,
3662                 0x6100, 0x6150,
3663                 0x6200, 0x6208,
3664                 0x6240, 0x6248,
3665                 0x6280, 0x6338,
3666                 0x6370, 0x638c,
3667                 0x6400, 0x643c,
3668                 0x6500, 0x6524,
3669                 0x6a00, 0x6a38,
3670                 0x6a60, 0x6a78,
3671                 0x6b00, 0x6b84,
3672                 0x6bf0, 0x6c84,
3673                 0x6cf0, 0x6d84,
3674                 0x6df0, 0x6e84,
3675                 0x6ef0, 0x6f84,
3676                 0x6ff0, 0x7084,
3677                 0x70f0, 0x7184,
3678                 0x71f0, 0x7284,
3679                 0x72f0, 0x7384,
3680                 0x73f0, 0x7450,
3681                 0x7500, 0x7530,
3682                 0x7600, 0x761c,
3683                 0x7680, 0x76cc,
3684                 0x7700, 0x7798,
3685                 0x77c0, 0x77fc,
3686                 0x7900, 0x79fc,
3687                 0x7b00, 0x7c38,
3688                 0x7d00, 0x7efc,
3689                 0x8dc0, 0x8e1c,
3690                 0x8e30, 0x8e78,
3691                 0x8ea0, 0x8f6c,
3692                 0x8fc0, 0x9074,
3693                 0x90fc, 0x90fc,
3694                 0x9400, 0x9458,
3695                 0x9600, 0x96bc,
3696                 0x9800, 0x9808,
3697                 0x9820, 0x983c,
3698                 0x9850, 0x9864,
3699                 0x9c00, 0x9c6c,
3700                 0x9c80, 0x9cec,
3701                 0x9d00, 0x9d6c,
3702                 0x9d80, 0x9dec,
3703                 0x9e00, 0x9e6c,
3704                 0x9e80, 0x9eec,
3705                 0x9f00, 0x9f6c,
3706                 0x9f80, 0x9fec,
3707                 0xd004, 0xd03c,
3708                 0xdfc0, 0xdfe0,
3709                 0xe000, 0xea7c,
3710                 0xf000, 0x11110,
3711                 0x11118, 0x11190,
3712                 0x19040, 0x1906c,
3713                 0x19078, 0x19080,
3714                 0x1908c, 0x19124,
3715                 0x19150, 0x191b0,
3716                 0x191d0, 0x191e8,
3717                 0x19238, 0x1924c,
3718                 0x193f8, 0x19474,
3719                 0x19490, 0x194f8,
3720                 0x19800, 0x19f30,
3721                 0x1a000, 0x1a06c,
3722                 0x1a0b0, 0x1a120,
3723                 0x1a128, 0x1a138,
3724                 0x1a190, 0x1a1c4,
3725                 0x1a1fc, 0x1a1fc,
3726                 0x1e040, 0x1e04c,
3727                 0x1e284, 0x1e28c,
3728                 0x1e2c0, 0x1e2c0,
3729                 0x1e2e0, 0x1e2e0,
3730                 0x1e300, 0x1e384,
3731                 0x1e3c0, 0x1e3c8,
3732                 0x1e440, 0x1e44c,
3733                 0x1e684, 0x1e68c,
3734                 0x1e6c0, 0x1e6c0,
3735                 0x1e6e0, 0x1e6e0,
3736                 0x1e700, 0x1e784,
3737                 0x1e7c0, 0x1e7c8,
3738                 0x1e840, 0x1e84c,
3739                 0x1ea84, 0x1ea8c,
3740                 0x1eac0, 0x1eac0,
3741                 0x1eae0, 0x1eae0,
3742                 0x1eb00, 0x1eb84,
3743                 0x1ebc0, 0x1ebc8,
3744                 0x1ec40, 0x1ec4c,
3745                 0x1ee84, 0x1ee8c,
3746                 0x1eec0, 0x1eec0,
3747                 0x1eee0, 0x1eee0,
3748                 0x1ef00, 0x1ef84,
3749                 0x1efc0, 0x1efc8,
3750                 0x1f040, 0x1f04c,
3751                 0x1f284, 0x1f28c,
3752                 0x1f2c0, 0x1f2c0,
3753                 0x1f2e0, 0x1f2e0,
3754                 0x1f300, 0x1f384,
3755                 0x1f3c0, 0x1f3c8,
3756                 0x1f440, 0x1f44c,
3757                 0x1f684, 0x1f68c,
3758                 0x1f6c0, 0x1f6c0,
3759                 0x1f6e0, 0x1f6e0,
3760                 0x1f700, 0x1f784,
3761                 0x1f7c0, 0x1f7c8,
3762                 0x1f840, 0x1f84c,
3763                 0x1fa84, 0x1fa8c,
3764                 0x1fac0, 0x1fac0,
3765                 0x1fae0, 0x1fae0,
3766                 0x1fb00, 0x1fb84,
3767                 0x1fbc0, 0x1fbc8,
3768                 0x1fc40, 0x1fc4c,
3769                 0x1fe84, 0x1fe8c,
3770                 0x1fec0, 0x1fec0,
3771                 0x1fee0, 0x1fee0,
3772                 0x1ff00, 0x1ff84,
3773                 0x1ffc0, 0x1ffc8,
3774                 0x20000, 0x2002c,
3775                 0x20100, 0x2013c,
3776                 0x20190, 0x201c8,
3777                 0x20200, 0x20318,
3778                 0x20400, 0x20528,
3779                 0x20540, 0x20614,
3780                 0x21000, 0x21040,
3781                 0x2104c, 0x21060,
3782                 0x210c0, 0x210ec,
3783                 0x21200, 0x21268,
3784                 0x21270, 0x21284,
3785                 0x212fc, 0x21388,
3786                 0x21400, 0x21404,
3787                 0x21500, 0x21518,
3788                 0x2152c, 0x2153c,
3789                 0x21550, 0x21554,
3790                 0x21600, 0x21600,
3791                 0x21608, 0x21628,
3792                 0x21630, 0x2163c,
3793                 0x21700, 0x2171c,
3794                 0x21780, 0x2178c,
3795                 0x21800, 0x21c38,
3796                 0x21c80, 0x21d7c,
3797                 0x21e00, 0x21e04,
3798                 0x22000, 0x2202c,
3799                 0x22100, 0x2213c,
3800                 0x22190, 0x221c8,
3801                 0x22200, 0x22318,
3802                 0x22400, 0x22528,
3803                 0x22540, 0x22614,
3804                 0x23000, 0x23040,
3805                 0x2304c, 0x23060,
3806                 0x230c0, 0x230ec,
3807                 0x23200, 0x23268,
3808                 0x23270, 0x23284,
3809                 0x232fc, 0x23388,
3810                 0x23400, 0x23404,
3811                 0x23500, 0x23518,
3812                 0x2352c, 0x2353c,
3813                 0x23550, 0x23554,
3814                 0x23600, 0x23600,
3815                 0x23608, 0x23628,
3816                 0x23630, 0x2363c,
3817                 0x23700, 0x2371c,
3818                 0x23780, 0x2378c,
3819                 0x23800, 0x23c38,
3820                 0x23c80, 0x23d7c,
3821                 0x23e00, 0x23e04,
3822                 0x24000, 0x2402c,
3823                 0x24100, 0x2413c,
3824                 0x24190, 0x241c8,
3825                 0x24200, 0x24318,
3826                 0x24400, 0x24528,
3827                 0x24540, 0x24614,
3828                 0x25000, 0x25040,
3829                 0x2504c, 0x25060,
3830                 0x250c0, 0x250ec,
3831                 0x25200, 0x25268,
3832                 0x25270, 0x25284,
3833                 0x252fc, 0x25388,
3834                 0x25400, 0x25404,
3835                 0x25500, 0x25518,
3836                 0x2552c, 0x2553c,
3837                 0x25550, 0x25554,
3838                 0x25600, 0x25600,
3839                 0x25608, 0x25628,
3840                 0x25630, 0x2563c,
3841                 0x25700, 0x2571c,
3842                 0x25780, 0x2578c,
3843                 0x25800, 0x25c38,
3844                 0x25c80, 0x25d7c,
3845                 0x25e00, 0x25e04,
3846                 0x26000, 0x2602c,
3847                 0x26100, 0x2613c,
3848                 0x26190, 0x261c8,
3849                 0x26200, 0x26318,
3850                 0x26400, 0x26528,
3851                 0x26540, 0x26614,
3852                 0x27000, 0x27040,
3853                 0x2704c, 0x27060,
3854                 0x270c0, 0x270ec,
3855                 0x27200, 0x27268,
3856                 0x27270, 0x27284,
3857                 0x272fc, 0x27388,
3858                 0x27400, 0x27404,
3859                 0x27500, 0x27518,
3860                 0x2752c, 0x2753c,
3861                 0x27550, 0x27554,
3862                 0x27600, 0x27600,
3863                 0x27608, 0x27628,
3864                 0x27630, 0x2763c,
3865                 0x27700, 0x2771c,
3866                 0x27780, 0x2778c,
3867                 0x27800, 0x27c38,
3868                 0x27c80, 0x27d7c,
3869                 0x27e00, 0x27e04
3870         };
3871         static const unsigned int t5_reg_ranges[] = {
3872                 0x1008, 0x1148,
3873                 0x1180, 0x11b4,
3874                 0x11fc, 0x123c,
3875                 0x1280, 0x173c,
3876                 0x1800, 0x18fc,
3877                 0x3000, 0x3028,
3878                 0x3060, 0x30d8,
3879                 0x30e0, 0x30fc,
3880                 0x3140, 0x357c,
3881                 0x35a8, 0x35cc,
3882                 0x35ec, 0x35ec,
3883                 0x3600, 0x5624,
3884                 0x56cc, 0x575c,
3885                 0x580c, 0x5814,
3886                 0x5890, 0x58bc,
3887                 0x5940, 0x59dc,
3888                 0x59fc, 0x5a18,
3889                 0x5a60, 0x5a9c,
3890                 0x5b94, 0x5bfc,
3891                 0x6000, 0x6040,
3892                 0x6058, 0x614c,
3893                 0x7700, 0x7798,
3894                 0x77c0, 0x78fc,
3895                 0x7b00, 0x7c54,
3896                 0x7d00, 0x7efc,
3897                 0x8dc0, 0x8de0,
3898                 0x8df8, 0x8e84,
3899                 0x8ea0, 0x8f84,
3900                 0x8fc0, 0x90f8,
3901                 0x9400, 0x9470,
3902                 0x9600, 0x96f4,
3903                 0x9800, 0x9808,
3904                 0x9820, 0x983c,
3905                 0x9850, 0x9864,
3906                 0x9c00, 0x9c6c,
3907                 0x9c80, 0x9cec,
3908                 0x9d00, 0x9d6c,
3909                 0x9d80, 0x9dec,
3910                 0x9e00, 0x9e6c,
3911                 0x9e80, 0x9eec,
3912                 0x9f00, 0x9f6c,
3913                 0x9f80, 0xa020,
3914                 0xd004, 0xd03c,
3915                 0xdfc0, 0xdfe0,
3916                 0xe000, 0x11088,
3917                 0x1109c, 0x11110,
3918                 0x11118, 0x1117c,
3919                 0x11190, 0x11204,
3920                 0x19040, 0x1906c,
3921                 0x19078, 0x19080,
3922                 0x1908c, 0x19124,
3923                 0x19150, 0x191b0,
3924                 0x191d0, 0x191e8,
3925                 0x19238, 0x19290,
3926                 0x193f8, 0x19474,
3927                 0x19490, 0x194cc,
3928                 0x194f0, 0x194f8,
3929                 0x19c00, 0x19c60,
3930                 0x19c94, 0x19e10,
3931                 0x19e50, 0x19f34,
3932                 0x19f40, 0x19f50,
3933                 0x19f90, 0x19fe4,
3934                 0x1a000, 0x1a06c,
3935                 0x1a0b0, 0x1a120,
3936                 0x1a128, 0x1a138,
3937                 0x1a190, 0x1a1c4,
3938                 0x1a1fc, 0x1a1fc,
3939                 0x1e008, 0x1e00c,
3940                 0x1e040, 0x1e04c,
3941                 0x1e284, 0x1e290,
3942                 0x1e2c0, 0x1e2c0,
3943                 0x1e2e0, 0x1e2e0,
3944                 0x1e300, 0x1e384,
3945                 0x1e3c0, 0x1e3c8,
3946                 0x1e408, 0x1e40c,
3947                 0x1e440, 0x1e44c,
3948                 0x1e684, 0x1e690,
3949                 0x1e6c0, 0x1e6c0,
3950                 0x1e6e0, 0x1e6e0,
3951                 0x1e700, 0x1e784,
3952                 0x1e7c0, 0x1e7c8,
3953                 0x1e808, 0x1e80c,
3954                 0x1e840, 0x1e84c,
3955                 0x1ea84, 0x1ea90,
3956                 0x1eac0, 0x1eac0,
3957                 0x1eae0, 0x1eae0,
3958                 0x1eb00, 0x1eb84,
3959                 0x1ebc0, 0x1ebc8,
3960                 0x1ec08, 0x1ec0c,
3961                 0x1ec40, 0x1ec4c,
3962                 0x1ee84, 0x1ee90,
3963                 0x1eec0, 0x1eec0,
3964                 0x1eee0, 0x1eee0,
3965                 0x1ef00, 0x1ef84,
3966                 0x1efc0, 0x1efc8,
3967                 0x1f008, 0x1f00c,
3968                 0x1f040, 0x1f04c,
3969                 0x1f284, 0x1f290,
3970                 0x1f2c0, 0x1f2c0,
3971                 0x1f2e0, 0x1f2e0,
3972                 0x1f300, 0x1f384,
3973                 0x1f3c0, 0x1f3c8,
3974                 0x1f408, 0x1f40c,
3975                 0x1f440, 0x1f44c,
3976                 0x1f684, 0x1f690,
3977                 0x1f6c0, 0x1f6c0,
3978                 0x1f6e0, 0x1f6e0,
3979                 0x1f700, 0x1f784,
3980                 0x1f7c0, 0x1f7c8,
3981                 0x1f808, 0x1f80c,
3982                 0x1f840, 0x1f84c,
3983                 0x1fa84, 0x1fa90,
3984                 0x1fac0, 0x1fac0,
3985                 0x1fae0, 0x1fae0,
3986                 0x1fb00, 0x1fb84,
3987                 0x1fbc0, 0x1fbc8,
3988                 0x1fc08, 0x1fc0c,
3989                 0x1fc40, 0x1fc4c,
3990                 0x1fe84, 0x1fe90,
3991                 0x1fec0, 0x1fec0,
3992                 0x1fee0, 0x1fee0,
3993                 0x1ff00, 0x1ff84,
3994                 0x1ffc0, 0x1ffc8,
3995                 0x30000, 0x30030,
3996                 0x30100, 0x30144,
3997                 0x30190, 0x301d0,
3998                 0x30200, 0x30318,
3999                 0x30400, 0x3052c,
4000                 0x30540, 0x3061c,
4001                 0x30800, 0x30834,
4002                 0x308c0, 0x30908,
4003                 0x30910, 0x309ac,
4004                 0x30a00, 0x30a2c,
4005                 0x30a44, 0x30a50,
4006                 0x30a74, 0x30c24,
4007                 0x30d00, 0x30d00,
4008                 0x30d08, 0x30d14,
4009                 0x30d1c, 0x30d20,
4010                 0x30d3c, 0x30d50,
4011                 0x31200, 0x3120c,
4012                 0x31220, 0x31220,
4013                 0x31240, 0x31240,
4014                 0x31600, 0x3160c,
4015                 0x31a00, 0x31a1c,
4016                 0x31e00, 0x31e20,
4017                 0x31e38, 0x31e3c,
4018                 0x31e80, 0x31e80,
4019                 0x31e88, 0x31ea8,
4020                 0x31eb0, 0x31eb4,
4021                 0x31ec8, 0x31ed4,
4022                 0x31fb8, 0x32004,
4023                 0x32200, 0x32200,
4024                 0x32208, 0x32240,
4025                 0x32248, 0x32280,
4026                 0x32288, 0x322c0,
4027                 0x322c8, 0x322fc,
4028                 0x32600, 0x32630,
4029                 0x32a00, 0x32abc,
4030                 0x32b00, 0x32b70,
4031                 0x33000, 0x33048,
4032                 0x33060, 0x3309c,
4033                 0x330f0, 0x33148,
4034                 0x33160, 0x3319c,
4035                 0x331f0, 0x332e4,
4036                 0x332f8, 0x333e4,
4037                 0x333f8, 0x33448,
4038                 0x33460, 0x3349c,
4039                 0x334f0, 0x33548,
4040                 0x33560, 0x3359c,
4041                 0x335f0, 0x336e4,
4042                 0x336f8, 0x337e4,
4043                 0x337f8, 0x337fc,
4044                 0x33814, 0x33814,
4045                 0x3382c, 0x3382c,
4046                 0x33880, 0x3388c,
4047                 0x338e8, 0x338ec,
4048                 0x33900, 0x33948,
4049                 0x33960, 0x3399c,
4050                 0x339f0, 0x33ae4,
4051                 0x33af8, 0x33b10,
4052                 0x33b28, 0x33b28,
4053                 0x33b3c, 0x33b50,
4054                 0x33bf0, 0x33c10,
4055                 0x33c28, 0x33c28,
4056                 0x33c3c, 0x33c50,
4057                 0x33cf0, 0x33cfc,
4058                 0x34000, 0x34030,
4059                 0x34100, 0x34144,
4060                 0x34190, 0x341d0,
4061                 0x34200, 0x34318,
4062                 0x34400, 0x3452c,
4063                 0x34540, 0x3461c,
4064                 0x34800, 0x34834,
4065                 0x348c0, 0x34908,
4066                 0x34910, 0x349ac,
4067                 0x34a00, 0x34a2c,
4068                 0x34a44, 0x34a50,
4069                 0x34a74, 0x34c24,
4070                 0x34d00, 0x34d00,
4071                 0x34d08, 0x34d14,
4072                 0x34d1c, 0x34d20,
4073                 0x34d3c, 0x34d50,
4074                 0x35200, 0x3520c,
4075                 0x35220, 0x35220,
4076                 0x35240, 0x35240,
4077                 0x35600, 0x3560c,
4078                 0x35a00, 0x35a1c,
4079                 0x35e00, 0x35e20,
4080                 0x35e38, 0x35e3c,
4081                 0x35e80, 0x35e80,
4082                 0x35e88, 0x35ea8,
4083                 0x35eb0, 0x35eb4,
4084                 0x35ec8, 0x35ed4,
4085                 0x35fb8, 0x36004,
4086                 0x36200, 0x36200,
4087                 0x36208, 0x36240,
4088                 0x36248, 0x36280,
4089                 0x36288, 0x362c0,
4090                 0x362c8, 0x362fc,
4091                 0x36600, 0x36630,
4092                 0x36a00, 0x36abc,
4093                 0x36b00, 0x36b70,
4094                 0x37000, 0x37048,
4095                 0x37060, 0x3709c,
4096                 0x370f0, 0x37148,
4097                 0x37160, 0x3719c,
4098                 0x371f0, 0x372e4,
4099                 0x372f8, 0x373e4,
4100                 0x373f8, 0x37448,
4101                 0x37460, 0x3749c,
4102                 0x374f0, 0x37548,
4103                 0x37560, 0x3759c,
4104                 0x375f0, 0x376e4,
4105                 0x376f8, 0x377e4,
4106                 0x377f8, 0x377fc,
4107                 0x37814, 0x37814,
4108                 0x3782c, 0x3782c,
4109                 0x37880, 0x3788c,
4110                 0x378e8, 0x378ec,
4111                 0x37900, 0x37948,
4112                 0x37960, 0x3799c,
4113                 0x379f0, 0x37ae4,
4114                 0x37af8, 0x37b10,
4115                 0x37b28, 0x37b28,
4116                 0x37b3c, 0x37b50,
4117                 0x37bf0, 0x37c10,
4118                 0x37c28, 0x37c28,
4119                 0x37c3c, 0x37c50,
4120                 0x37cf0, 0x37cfc,
4121                 0x38000, 0x38030,
4122                 0x38100, 0x38144,
4123                 0x38190, 0x381d0,
4124                 0x38200, 0x38318,
4125                 0x38400, 0x3852c,
4126                 0x38540, 0x3861c,
4127                 0x38800, 0x38834,
4128                 0x388c0, 0x38908,
4129                 0x38910, 0x389ac,
4130                 0x38a00, 0x38a2c,
4131                 0x38a44, 0x38a50,
4132                 0x38a74, 0x38c24,
4133                 0x38d00, 0x38d00,
4134                 0x38d08, 0x38d14,
4135                 0x38d1c, 0x38d20,
4136                 0x38d3c, 0x38d50,
4137                 0x39200, 0x3920c,
4138                 0x39220, 0x39220,
4139                 0x39240, 0x39240,
4140                 0x39600, 0x3960c,
4141                 0x39a00, 0x39a1c,
4142                 0x39e00, 0x39e20,
4143                 0x39e38, 0x39e3c,
4144                 0x39e80, 0x39e80,
4145                 0x39e88, 0x39ea8,
4146                 0x39eb0, 0x39eb4,
4147                 0x39ec8, 0x39ed4,
4148                 0x39fb8, 0x3a004,
4149                 0x3a200, 0x3a200,
4150                 0x3a208, 0x3a240,
4151                 0x3a248, 0x3a280,
4152                 0x3a288, 0x3a2c0,
4153                 0x3a2c8, 0x3a2fc,
4154                 0x3a600, 0x3a630,
4155                 0x3aa00, 0x3aabc,
4156                 0x3ab00, 0x3ab70,
4157                 0x3b000, 0x3b048,
4158                 0x3b060, 0x3b09c,
4159                 0x3b0f0, 0x3b148,
4160                 0x3b160, 0x3b19c,
4161                 0x3b1f0, 0x3b2e4,
4162                 0x3b2f8, 0x3b3e4,
4163                 0x3b3f8, 0x3b448,
4164                 0x3b460, 0x3b49c,
4165                 0x3b4f0, 0x3b548,
4166                 0x3b560, 0x3b59c,
4167                 0x3b5f0, 0x3b6e4,
4168                 0x3b6f8, 0x3b7e4,
4169                 0x3b7f8, 0x3b7fc,
4170                 0x3b814, 0x3b814,
4171                 0x3b82c, 0x3b82c,
4172                 0x3b880, 0x3b88c,
4173                 0x3b8e8, 0x3b8ec,
4174                 0x3b900, 0x3b948,
4175                 0x3b960, 0x3b99c,
4176                 0x3b9f0, 0x3bae4,
4177                 0x3baf8, 0x3bb10,
4178                 0x3bb28, 0x3bb28,
4179                 0x3bb3c, 0x3bb50,
4180                 0x3bbf0, 0x3bc10,
4181                 0x3bc28, 0x3bc28,
4182                 0x3bc3c, 0x3bc50,
4183                 0x3bcf0, 0x3bcfc,
4184                 0x3c000, 0x3c030,
4185                 0x3c100, 0x3c144,
4186                 0x3c190, 0x3c1d0,
4187                 0x3c200, 0x3c318,
4188                 0x3c400, 0x3c52c,
4189                 0x3c540, 0x3c61c,
4190                 0x3c800, 0x3c834,
4191                 0x3c8c0, 0x3c908,
4192                 0x3c910, 0x3c9ac,
4193                 0x3ca00, 0x3ca2c,
4194                 0x3ca44, 0x3ca50,
4195                 0x3ca74, 0x3cc24,
4196                 0x3cd00, 0x3cd00,
4197                 0x3cd08, 0x3cd14,
4198                 0x3cd1c, 0x3cd20,
4199                 0x3cd3c, 0x3cd50,
4200                 0x3d200, 0x3d20c,
4201                 0x3d220, 0x3d220,
4202                 0x3d240, 0x3d240,
4203                 0x3d600, 0x3d60c,
4204                 0x3da00, 0x3da1c,
4205                 0x3de00, 0x3de20,
4206                 0x3de38, 0x3de3c,
4207                 0x3de80, 0x3de80,
4208                 0x3de88, 0x3dea8,
4209                 0x3deb0, 0x3deb4,
4210                 0x3dec8, 0x3ded4,
4211                 0x3dfb8, 0x3e004,
4212                 0x3e200, 0x3e200,
4213                 0x3e208, 0x3e240,
4214                 0x3e248, 0x3e280,
4215                 0x3e288, 0x3e2c0,
4216                 0x3e2c8, 0x3e2fc,
4217                 0x3e600, 0x3e630,
4218                 0x3ea00, 0x3eabc,
4219                 0x3eb00, 0x3eb70,
4220                 0x3f000, 0x3f048,
4221                 0x3f060, 0x3f09c,
4222                 0x3f0f0, 0x3f148,
4223                 0x3f160, 0x3f19c,
4224                 0x3f1f0, 0x3f2e4,
4225                 0x3f2f8, 0x3f3e4,
4226                 0x3f3f8, 0x3f448,
4227                 0x3f460, 0x3f49c,
4228                 0x3f4f0, 0x3f548,
4229                 0x3f560, 0x3f59c,
4230                 0x3f5f0, 0x3f6e4,
4231                 0x3f6f8, 0x3f7e4,
4232                 0x3f7f8, 0x3f7fc,
4233                 0x3f814, 0x3f814,
4234                 0x3f82c, 0x3f82c,
4235                 0x3f880, 0x3f88c,
4236                 0x3f8e8, 0x3f8ec,
4237                 0x3f900, 0x3f948,
4238                 0x3f960, 0x3f99c,
4239                 0x3f9f0, 0x3fae4,
4240                 0x3faf8, 0x3fb10,
4241                 0x3fb28, 0x3fb28,
4242                 0x3fb3c, 0x3fb50,
4243                 0x3fbf0, 0x3fc10,
4244                 0x3fc28, 0x3fc28,
4245                 0x3fc3c, 0x3fc50,
4246                 0x3fcf0, 0x3fcfc,
4247                 0x40000, 0x4000c,
4248                 0x40040, 0x40068,
4249                 0x4007c, 0x40144,
4250                 0x40180, 0x4018c,
4251                 0x40200, 0x40298,
4252                 0x402ac, 0x4033c,
4253                 0x403f8, 0x403fc,
4254                 0x41304, 0x413c4,
4255                 0x41400, 0x4141c,
4256                 0x41480, 0x414d0,
4257                 0x44000, 0x44078,
4258                 0x440c0, 0x44278,
4259                 0x442c0, 0x44478,
4260                 0x444c0, 0x44678,
4261                 0x446c0, 0x44878,
4262                 0x448c0, 0x449fc,
4263                 0x45000, 0x45068,
4264                 0x45080, 0x45084,
4265                 0x450a0, 0x450b0,
4266                 0x45200, 0x45268,
4267                 0x45280, 0x45284,
4268                 0x452a0, 0x452b0,
4269                 0x460c0, 0x460e4,
4270                 0x47000, 0x4708c,
4271                 0x47200, 0x47250,
4272                 0x47400, 0x47420,
4273                 0x47600, 0x47618,
4274                 0x47800, 0x47814,
4275                 0x48000, 0x4800c,
4276                 0x48040, 0x48068,
4277                 0x4807c, 0x48144,
4278                 0x48180, 0x4818c,
4279                 0x48200, 0x48298,
4280                 0x482ac, 0x4833c,
4281                 0x483f8, 0x483fc,
4282                 0x49304, 0x493c4,
4283                 0x49400, 0x4941c,
4284                 0x49480, 0x494d0,
4285                 0x4c000, 0x4c078,
4286                 0x4c0c0, 0x4c278,
4287                 0x4c2c0, 0x4c478,
4288                 0x4c4c0, 0x4c678,
4289                 0x4c6c0, 0x4c878,
4290                 0x4c8c0, 0x4c9fc,
4291                 0x4d000, 0x4d068,
4292                 0x4d080, 0x4d084,
4293                 0x4d0a0, 0x4d0b0,
4294                 0x4d200, 0x4d268,
4295                 0x4d280, 0x4d284,
4296                 0x4d2a0, 0x4d2b0,
4297                 0x4e0c0, 0x4e0e4,
4298                 0x4f000, 0x4f08c,
4299                 0x4f200, 0x4f250,
4300                 0x4f400, 0x4f420,
4301                 0x4f600, 0x4f618,
4302                 0x4f800, 0x4f814,
4303                 0x50000, 0x500cc,
4304                 0x50400, 0x50400,
4305                 0x50800, 0x508cc,
4306                 0x50c00, 0x50c00,
4307                 0x51000, 0x5101c,
4308                 0x51300, 0x51308,
4309         };
4310
4311         if (is_t4(sc)) {
4312                 reg_ranges = &t4_reg_ranges[0];
4313                 n = nitems(t4_reg_ranges);
4314         } else {
4315                 reg_ranges = &t5_reg_ranges[0];
4316                 n = nitems(t5_reg_ranges);
4317         }
4318
4319         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4320         for (i = 0; i < n; i += 2)
4321                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4322 }
4323
4324 static void
4325 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4326 {
4327         struct ifnet *ifp = pi->ifp;
4328         struct sge_txq *txq;
4329         int i, drops;
4330         struct port_stats *s = &pi->stats;
4331         struct timeval tv;
4332         const struct timeval interval = {0, 250000};    /* 250ms */
4333
4334         getmicrotime(&tv);
4335         timevalsub(&tv, &interval);
4336         if (timevalcmp(&tv, &pi->last_refreshed, <))
4337                 return;
4338
4339         t4_get_port_stats(sc, pi->tx_chan, s);
4340
4341         ifp->if_opackets = s->tx_frames - s->tx_pause;
4342         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4343         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4344         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4345         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4346         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4347         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4348             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4349             s->rx_trunc3;
4350         for (i = 0; i < NCHAN; i++) {
4351                 if (pi->rx_chan_map & (1 << i)) {
4352                         uint32_t v;
4353
4354                         mtx_lock(&sc->regwin_lock);
4355                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4356                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4357                         mtx_unlock(&sc->regwin_lock);
4358                         ifp->if_iqdrops += v;
4359                 }
4360         }
4361
4362         drops = s->tx_drop;
4363         for_each_txq(pi, i, txq)
4364                 drops += counter_u64_fetch(txq->r->drops);
4365         ifp->if_snd.ifq_drops = drops;
4366
4367         ifp->if_oerrors = s->tx_error_frames;
4368         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4369             s->rx_fcs_err + s->rx_len_err;
4370
4371         getmicrotime(&pi->last_refreshed);
4372 }
4373
4374 static void
4375 cxgbe_tick(void *arg)
4376 {
4377         struct port_info *pi = arg;
4378         struct adapter *sc = pi->adapter;
4379         struct ifnet *ifp = pi->ifp;
4380
4381         PORT_LOCK(pi);
4382         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4383                 PORT_UNLOCK(pi);
4384                 return; /* without scheduling another callout */
4385         }
4386
4387         cxgbe_refresh_stats(sc, pi);
4388
4389         callout_schedule(&pi->tick, hz);
4390         PORT_UNLOCK(pi);
4391 }
4392
4393 static void
4394 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4395 {
4396         struct ifnet *vlan;
4397
4398         if (arg != ifp || ifp->if_type != IFT_ETHER)
4399                 return;
4400
4401         vlan = VLAN_DEVAT(ifp, vid);
4402         VLAN_SETCOOKIE(vlan, ifp);
4403 }
4404
4405 static int
4406 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4407 {
4408
4409 #ifdef INVARIANTS
4410         panic("%s: opcode 0x%02x on iq %p with payload %p",
4411             __func__, rss->opcode, iq, m);
4412 #else
4413         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4414             __func__, rss->opcode, iq, m);
4415         m_freem(m);
4416 #endif
4417         return (EDOOFUS);
4418 }
4419
4420 int
4421 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4422 {
4423         uintptr_t *loc, new;
4424
4425         if (opcode >= nitems(sc->cpl_handler))
4426                 return (EINVAL);
4427
4428         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4429         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4430         atomic_store_rel_ptr(loc, new);
4431
4432         return (0);
4433 }
4434
4435 static int
4436 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4437 {
4438
4439 #ifdef INVARIANTS
4440         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4441 #else
4442         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4443             __func__, iq, ctrl);
4444 #endif
4445         return (EDOOFUS);
4446 }
4447
4448 int
4449 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4450 {
4451         uintptr_t *loc, new;
4452
4453         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4454         loc = (uintptr_t *) &sc->an_handler;
4455         atomic_store_rel_ptr(loc, new);
4456
4457         return (0);
4458 }
4459
4460 static int
4461 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4462 {
4463         const struct cpl_fw6_msg *cpl =
4464             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4465
4466 #ifdef INVARIANTS
4467         panic("%s: fw_msg type %d", __func__, cpl->type);
4468 #else
4469         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4470 #endif
4471         return (EDOOFUS);
4472 }
4473
4474 int
4475 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4476 {
4477         uintptr_t *loc, new;
4478
4479         if (type >= nitems(sc->fw_msg_handler))
4480                 return (EINVAL);
4481
4482         /*
4483          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4484          * handler dispatch table.  Reject any attempt to install a handler for
4485          * this subtype.
4486          */
4487         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4488                 return (EINVAL);
4489
4490         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4491         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4492         atomic_store_rel_ptr(loc, new);
4493
4494         return (0);
4495 }
4496
4497 static int
4498 t4_sysctls(struct adapter *sc)
4499 {
4500         struct sysctl_ctx_list *ctx;
4501         struct sysctl_oid *oid;
4502         struct sysctl_oid_list *children, *c0;
4503         static char *caps[] = {
4504                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4505                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4506                     "\6HASHFILTER\7ETHOFLD",
4507                 "\20\1TOE",                             /* caps[2] toecaps */
4508                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4509                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4510                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4511                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4512                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4513                     "\4PO_INITIAOR\5PO_TARGET"
4514         };
4515         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4516
4517         ctx = device_get_sysctl_ctx(sc->dev);
4518
4519         /*
4520          * dev.t4nex.X.
4521          */
4522         oid = device_get_sysctl_tree(sc->dev);
4523         c0 = children = SYSCTL_CHILDREN(oid);
4524
4525         sc->sc_do_rxcopy = 1;
4526         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4527             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4528
4529         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4530             sc->params.nports, "# of ports");
4531
4532         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4533             NULL, chip_rev(sc), "chip hardware revision");
4534
4535         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4536             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4537
4538         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4539             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4540
4541         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4542             sc->cfcsum, "config file checksum");
4543
4544         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4545             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4546             sysctl_bitfield, "A", "available doorbells");
4547
4548         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4549             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4550             sysctl_bitfield, "A", "available link capabilities");
4551
4552         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4553             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4554             sysctl_bitfield, "A", "available NIC capabilities");
4555
4556         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4557             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4558             sysctl_bitfield, "A", "available TCP offload capabilities");
4559
4560         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4561             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4562             sysctl_bitfield, "A", "available RDMA capabilities");
4563
4564         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4565             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4566             sysctl_bitfield, "A", "available iSCSI capabilities");
4567
4568         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4569             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4570             sysctl_bitfield, "A", "available FCoE capabilities");
4571
4572         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4573             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4574
4575         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4576             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4577             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4578             "interrupt holdoff timer values (us)");
4579
4580         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4581             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4582             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4583             "interrupt holdoff packet counter values");
4584
4585         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4586             NULL, sc->tids.nftids, "number of filters");
4587
4588         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4589             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4590             "chip temperature (in Celsius)");
4591
4592         t4_sge_sysctls(sc, ctx, children);
4593
4594         sc->lro_timeout = 100;
4595         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4596             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4597
4598         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4599             &sc->debug_flags, 0, "flags to enable runtime debugging");
4600
4601 #ifdef SBUF_DRAIN
4602         /*
4603          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4604          */
4605         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4606             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4607             "logs and miscellaneous information");
4608         children = SYSCTL_CHILDREN(oid);
4609
4610         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4611             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4612             sysctl_cctrl, "A", "congestion control");
4613
4614         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4615             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4616             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4620             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4624             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4628             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4629
4630         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4631             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4632             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4633
4634         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4635             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4636             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4637
4638         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4639             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4640             sysctl_cim_la, "A", "CIM logic analyzer");
4641
4642         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4643             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4644             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4645
4646         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4647             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4648             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4649
4650         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4651             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4652             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4653
4654         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4655             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4656             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4657
4658         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4659             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4660             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4661
4662         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4663             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4664             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4665
4666         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4667             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4668             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4669
4670         if (is_t5(sc)) {
4671                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4672                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4673                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4674
4675                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4676                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4677                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4678         }
4679
4680         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4681             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4682             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4683
4684         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4685             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4686             sysctl_cim_qcfg, "A", "CIM queue configuration");
4687
4688         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4689             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4690             sysctl_cpl_stats, "A", "CPL statistics");
4691
4692         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4693             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4695
4696         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4697             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4698             sysctl_devlog, "A", "firmware's device log");
4699
4700         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4701             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4702             sysctl_fcoe_stats, "A", "FCoE statistics");
4703
4704         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4705             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4706             sysctl_hw_sched, "A", "hardware scheduler ");
4707
4708         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4709             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4710             sysctl_l2t, "A", "hardware L2 table");
4711
4712         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4713             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4714             sysctl_lb_stats, "A", "loopback statistics");
4715
4716         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4717             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4718             sysctl_meminfo, "A", "memory regions");
4719
4720         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4721             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4722             sysctl_mps_tcam, "A", "MPS TCAM entries");
4723
4724         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4725             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4726             sysctl_path_mtus, "A", "path MTUs");
4727
4728         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4729             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4730             sysctl_pm_stats, "A", "PM statistics");
4731
4732         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4733             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4734             sysctl_rdma_stats, "A", "RDMA statistics");
4735
4736         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4737             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4738             sysctl_tcp_stats, "A", "TCP statistics");
4739
4740         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4741             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4742             sysctl_tids, "A", "TID information");
4743
4744         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4745             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4746             sysctl_tp_err_stats, "A", "TP error statistics");
4747
4748         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4749             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4750             sysctl_tp_la, "A", "TP logic analyzer");
4751
4752         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4753             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4754             sysctl_tx_rate, "A", "Tx rate");
4755
4756         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4757             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4758             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4759
4760         if (is_t5(sc)) {
4761                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4762                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4763                     sysctl_wcwr_stats, "A", "write combined work requests");
4764         }
4765 #endif
4766
4767 #ifdef TCP_OFFLOAD
4768         if (is_offload(sc)) {
4769                 /*
4770                  * dev.t4nex.X.toe.
4771                  */
4772                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4773                     NULL, "TOE parameters");
4774                 children = SYSCTL_CHILDREN(oid);
4775
4776                 sc->tt.sndbuf = 256 * 1024;
4777                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4778                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4779
4780                 sc->tt.ddp = 0;
4781                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4782                     &sc->tt.ddp, 0, "DDP allowed");
4783
4784                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4785                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4786                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4787
4788                 sc->tt.ddp_thres =
4789                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4790                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4791                     &sc->tt.ddp_thres, 0, "DDP threshold");
4792
4793                 sc->tt.rx_coalesce = 1;
4794                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4795                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4796
4797                 sc->tt.tx_align = 1;
4798                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4799                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4800         }
4801 #endif
4802
4803
4804         return (0);
4805 }
4806
4807 static int
4808 cxgbe_sysctls(struct port_info *pi)
4809 {
4810         struct sysctl_ctx_list *ctx;
4811         struct sysctl_oid *oid;
4812         struct sysctl_oid_list *children;
4813         struct adapter *sc = pi->adapter;
4814
4815         ctx = device_get_sysctl_ctx(pi->dev);
4816
4817         /*
4818          * dev.cxgbe.X.
4819          */
4820         oid = device_get_sysctl_tree(pi->dev);
4821         children = SYSCTL_CHILDREN(oid);
4822
4823         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4824            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4825         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4826                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4827                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4828                     "PHY temperature (in Celsius)");
4829                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4830                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4831                     "PHY firmware version");
4832         }
4833         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4834             &pi->nrxq, 0, "# of rx queues");
4835         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4836             &pi->ntxq, 0, "# of tx queues");
4837         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4838             &pi->first_rxq, 0, "index of first rx queue");
4839         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4840             &pi->first_txq, 0, "index of first tx queue");
4841         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4842             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4843             "Reserve queue 0 for non-flowid packets");
4844
4845 #ifdef TCP_OFFLOAD
4846         if (is_offload(sc)) {
4847                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4848                     &pi->nofldrxq, 0,
4849                     "# of rx queues for offloaded TCP connections");
4850                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4851                     &pi->nofldtxq, 0,
4852                     "# of tx queues for offloaded TCP connections");
4853                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4854                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4855                     "index of first TOE rx queue");
4856                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4857                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4858                     "index of first TOE tx queue");
4859         }
4860 #endif
4861 #ifdef DEV_NETMAP
4862         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4863             &pi->nnmrxq, 0, "# of rx queues for netmap");
4864         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4865             &pi->nnmtxq, 0, "# of tx queues for netmap");
4866         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4867             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4868             "index of first netmap rx queue");
4869         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4870             CTLFLAG_RD, &pi->first_nm_txq, 0,
4871             "index of first netmap tx queue");
4872 #endif
4873
4874         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4875             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4876             "holdoff timer index");
4877         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4878             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4879             "holdoff packet counter index");
4880
4881         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4882             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4883             "rx queue size");
4884         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4885             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4886             "tx queue size");
4887
4888         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4889             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4890             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4891
4892         /*
4893          * dev.cxgbe.X.stats.
4894          */
4895         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4896             NULL, "port statistics");
4897         children = SYSCTL_CHILDREN(oid);
4898         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4899             &pi->tx_parse_error, 0,
4900             "# of tx packets with invalid length or # of segments");
4901
4902 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4903         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4904             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4905             sysctl_handle_t4_reg64, "QU", desc)
4906
4907         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4909         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4911         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4913         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4915         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4917         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4919         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4920             "# of tx frames in this range",
4921             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4922         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4923             "# of tx frames in this range",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4925         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4926             "# of tx frames in this range",
4927             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4928         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4929             "# of tx frames in this range",
4930             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4931         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4932             "# of tx frames in this range",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4934         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4935             "# of tx frames in this range",
4936             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4937         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4938             "# of tx frames in this range",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4940         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4942         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4944         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4946         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4948         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4950         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4952         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4954         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4956         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4958         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4960
4961         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4962             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4963         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4964             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4965         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4966             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4967         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4968             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4969         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4971         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4972             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4973         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4974             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4975         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4976             "# of frames received with bad FCS",
4977             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4978         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4979             "# of frames received with length error",
4980             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4981         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4982             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4983         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4984             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4985         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4986             "# of rx frames in this range",
4987             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4988         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4989             "# of rx frames in this range",
4990             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4991         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4992             "# of rx frames in this range",
4993             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4994         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4995             "# of rx frames in this range",
4996             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4997         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4998             "# of rx frames in this range",
4999             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5000         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5001             "# of rx frames in this range",
5002             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5003         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5004             "# of rx frames in this range",
5005             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5006         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5007             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5008         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5009             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5010         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5011             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5012         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5013             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5014         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5015             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5016         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5017             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5018         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5019             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5020         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5021             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5022         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5023             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5024
5025 #undef SYSCTL_ADD_T4_REG64
5026
5027 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5028         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5029             &pi->stats.name, desc)
5030
5031         /* We get these from port_stats and they may be stale by upto 1s */
5032         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5033             "# drops due to buffer-group 0 overflows");
5034         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5035             "# drops due to buffer-group 1 overflows");
5036         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5037             "# drops due to buffer-group 2 overflows");
5038         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5039             "# drops due to buffer-group 3 overflows");
5040         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5041             "# of buffer-group 0 truncated packets");
5042         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5043             "# of buffer-group 1 truncated packets");
5044         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5045             "# of buffer-group 2 truncated packets");
5046         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5047             "# of buffer-group 3 truncated packets");
5048
5049 #undef SYSCTL_ADD_T4_PORTSTAT
5050
5051         return (0);
5052 }
5053
5054 static int
5055 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5056 {
5057         int rc, *i, space = 0;
5058         struct sbuf sb;
5059
5060         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5061         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5062                 if (space)
5063                         sbuf_printf(&sb, " ");
5064                 sbuf_printf(&sb, "%d", *i);
5065                 space = 1;
5066         }
5067         sbuf_finish(&sb);
5068         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5069         sbuf_delete(&sb);
5070         return (rc);
5071 }
5072
5073 static int
5074 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5075 {
5076         int rc;
5077         struct sbuf *sb;
5078
5079         rc = sysctl_wire_old_buffer(req, 0);
5080         if (rc != 0)
5081                 return(rc);
5082
5083         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5084         if (sb == NULL)
5085                 return (ENOMEM);
5086
5087         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5088         rc = sbuf_finish(sb);
5089         sbuf_delete(sb);
5090
5091         return (rc);
5092 }
5093
5094 static int
5095 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5096 {
5097         struct port_info *pi = arg1;
5098         int op = arg2;
5099         struct adapter *sc = pi->adapter;
5100         u_int v;
5101         int rc;
5102
5103         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5104         if (rc)
5105                 return (rc);
5106         /* XXX: magic numbers */
5107         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5108             &v);
5109         end_synchronized_op(sc, 0);
5110         if (rc)
5111                 return (rc);
5112         if (op == 0)
5113                 v /= 256;
5114
5115         rc = sysctl_handle_int(oidp, &v, 0, req);
5116         return (rc);
5117 }
5118
5119 static int
5120 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5121 {
5122         struct port_info *pi = arg1;
5123         int rc, val;
5124
5125         val = pi->rsrv_noflowq;
5126         rc = sysctl_handle_int(oidp, &val, 0, req);
5127         if (rc != 0 || req->newptr == NULL)
5128                 return (rc);
5129
5130         if ((val >= 1) && (pi->ntxq > 1))
5131                 pi->rsrv_noflowq = 1;
5132         else
5133                 pi->rsrv_noflowq = 0;
5134
5135         return (rc);
5136 }
5137
5138 static int
5139 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5140 {
5141         struct port_info *pi = arg1;
5142         struct adapter *sc = pi->adapter;
5143         int idx, rc, i;
5144         struct sge_rxq *rxq;
5145 #ifdef TCP_OFFLOAD
5146         struct sge_ofld_rxq *ofld_rxq;
5147 #endif
5148         uint8_t v;
5149
5150         idx = pi->tmr_idx;
5151
5152         rc = sysctl_handle_int(oidp, &idx, 0, req);
5153         if (rc != 0 || req->newptr == NULL)
5154                 return (rc);
5155
5156         if (idx < 0 || idx >= SGE_NTIMERS)
5157                 return (EINVAL);
5158
5159         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5160             "t4tmr");
5161         if (rc)
5162                 return (rc);
5163
5164         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5165         for_each_rxq(pi, i, rxq) {
5166 #ifdef atomic_store_rel_8
5167                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5168 #else
5169                 rxq->iq.intr_params = v;
5170 #endif
5171         }
5172 #ifdef TCP_OFFLOAD
5173         for_each_ofld_rxq(pi, i, ofld_rxq) {
5174 #ifdef atomic_store_rel_8
5175                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5176 #else
5177                 ofld_rxq->iq.intr_params = v;
5178 #endif
5179         }
5180 #endif
5181         pi->tmr_idx = idx;
5182
5183         end_synchronized_op(sc, LOCK_HELD);
5184         return (0);
5185 }
5186
5187 static int
5188 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5189 {
5190         struct port_info *pi = arg1;
5191         struct adapter *sc = pi->adapter;
5192         int idx, rc;
5193
5194         idx = pi->pktc_idx;
5195
5196         rc = sysctl_handle_int(oidp, &idx, 0, req);
5197         if (rc != 0 || req->newptr == NULL)
5198                 return (rc);
5199
5200         if (idx < -1 || idx >= SGE_NCOUNTERS)
5201                 return (EINVAL);
5202
5203         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5204             "t4pktc");
5205         if (rc)
5206                 return (rc);
5207
5208         if (pi->flags & PORT_INIT_DONE)
5209                 rc = EBUSY; /* cannot be changed once the queues are created */
5210         else
5211                 pi->pktc_idx = idx;
5212
5213         end_synchronized_op(sc, LOCK_HELD);
5214         return (rc);
5215 }
5216
5217 static int
5218 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5219 {
5220         struct port_info *pi = arg1;
5221         struct adapter *sc = pi->adapter;
5222         int qsize, rc;
5223
5224         qsize = pi->qsize_rxq;
5225
5226         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5227         if (rc != 0 || req->newptr == NULL)
5228                 return (rc);
5229
5230         if (qsize < 128 || (qsize & 7))
5231                 return (EINVAL);
5232
5233         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5234             "t4rxqs");
5235         if (rc)
5236                 return (rc);
5237
5238         if (pi->flags & PORT_INIT_DONE)
5239                 rc = EBUSY; /* cannot be changed once the queues are created */
5240         else
5241                 pi->qsize_rxq = qsize;
5242
5243         end_synchronized_op(sc, LOCK_HELD);
5244         return (rc);
5245 }
5246
5247 static int
5248 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5249 {
5250         struct port_info *pi = arg1;
5251         struct adapter *sc = pi->adapter;
5252         int qsize, rc;
5253
5254         qsize = pi->qsize_txq;
5255
5256         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5257         if (rc != 0 || req->newptr == NULL)
5258                 return (rc);
5259
5260         if (qsize < 128 || qsize > 65536)
5261                 return (EINVAL);
5262
5263         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5264             "t4txqs");
5265         if (rc)
5266                 return (rc);
5267
5268         if (pi->flags & PORT_INIT_DONE)
5269                 rc = EBUSY; /* cannot be changed once the queues are created */
5270         else
5271                 pi->qsize_txq = qsize;
5272
5273         end_synchronized_op(sc, LOCK_HELD);
5274         return (rc);
5275 }
5276
5277 static int
5278 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5279 {
5280         struct port_info *pi = arg1;
5281         struct adapter *sc = pi->adapter;
5282         struct link_config *lc = &pi->link_cfg;
5283         int rc;
5284
5285         if (req->newptr == NULL) {
5286                 struct sbuf *sb;
5287                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5288
5289                 rc = sysctl_wire_old_buffer(req, 0);
5290                 if (rc != 0)
5291                         return(rc);
5292
5293                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5294                 if (sb == NULL)
5295                         return (ENOMEM);
5296
5297                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5298                 rc = sbuf_finish(sb);
5299                 sbuf_delete(sb);
5300         } else {
5301                 char s[2];
5302                 int n;
5303
5304                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5305                 s[1] = 0;
5306
5307                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5308                 if (rc != 0)
5309                         return(rc);
5310
5311                 if (s[1] != 0)
5312                         return (EINVAL);
5313                 if (s[0] < '0' || s[0] > '9')
5314                         return (EINVAL);        /* not a number */
5315                 n = s[0] - '0';
5316                 if (n & ~(PAUSE_TX | PAUSE_RX))
5317                         return (EINVAL);        /* some other bit is set too */
5318
5319                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5320                 if (rc)
5321                         return (rc);
5322                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5323                         int link_ok = lc->link_ok;
5324
5325                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5326                         lc->requested_fc |= n;
5327                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5328                         lc->link_ok = link_ok;  /* restore */
5329                 }
5330                 end_synchronized_op(sc, 0);
5331         }
5332
5333         return (rc);
5334 }
5335
5336 static int
5337 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5338 {
5339         struct adapter *sc = arg1;
5340         int reg = arg2;
5341         uint64_t val;
5342
5343         val = t4_read_reg64(sc, reg);
5344
5345         return (sysctl_handle_64(oidp, &val, 0, req));
5346 }
5347
5348 static int
5349 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5350 {
5351         struct adapter *sc = arg1;
5352         int rc, t;
5353         uint32_t param, val;
5354
5355         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5356         if (rc)
5357                 return (rc);
5358         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5359             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5360             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5361         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5362         end_synchronized_op(sc, 0);
5363         if (rc)
5364                 return (rc);
5365
5366         /* unknown is returned as 0 but we display -1 in that case */
5367         t = val == 0 ? -1 : val;
5368
5369         rc = sysctl_handle_int(oidp, &t, 0, req);
5370         return (rc);
5371 }
5372
5373 #ifdef SBUF_DRAIN
5374 static int
5375 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5376 {
5377         struct adapter *sc = arg1;
5378         struct sbuf *sb;
5379         int rc, i;
5380         uint16_t incr[NMTUS][NCCTRL_WIN];
5381         static const char *dec_fac[] = {
5382                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5383                 "0.9375"
5384         };
5385
5386         rc = sysctl_wire_old_buffer(req, 0);
5387         if (rc != 0)
5388                 return (rc);
5389
5390         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5391         if (sb == NULL)
5392                 return (ENOMEM);
5393
5394         t4_read_cong_tbl(sc, incr);
5395
5396         for (i = 0; i < NCCTRL_WIN; ++i) {
5397                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5398                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5399                     incr[5][i], incr[6][i], incr[7][i]);
5400                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5401                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5402                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5403                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5404         }
5405
5406         rc = sbuf_finish(sb);
5407         sbuf_delete(sb);
5408
5409         return (rc);
5410 }
5411
5412 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5413         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5414         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5415         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5416 };
5417
5418 static int
5419 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5420 {
5421         struct adapter *sc = arg1;
5422         struct sbuf *sb;
5423         int rc, i, n, qid = arg2;
5424         uint32_t *buf, *p;
5425         char *qtype;
5426         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5427
5428         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5429             ("%s: bad qid %d\n", __func__, qid));
5430
5431         if (qid < CIM_NUM_IBQ) {
5432                 /* inbound queue */
5433                 qtype = "IBQ";
5434                 n = 4 * CIM_IBQ_SIZE;
5435                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5436                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5437         } else {
5438                 /* outbound queue */
5439                 qtype = "OBQ";
5440                 qid -= CIM_NUM_IBQ;
5441                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5442                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5443                 rc = t4_read_cim_obq(sc, qid, buf, n);
5444         }
5445
5446         if (rc < 0) {
5447                 rc = -rc;
5448                 goto done;
5449         }
5450         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5451
5452         rc = sysctl_wire_old_buffer(req, 0);
5453         if (rc != 0)
5454                 goto done;
5455
5456         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5457         if (sb == NULL) {
5458                 rc = ENOMEM;
5459                 goto done;
5460         }
5461
5462         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5463         for (i = 0, p = buf; i < n; i += 16, p += 4)
5464                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5465                     p[2], p[3]);
5466
5467         rc = sbuf_finish(sb);
5468         sbuf_delete(sb);
5469 done:
5470         free(buf, M_CXGBE);
5471         return (rc);
5472 }
5473
5474 static int
5475 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5476 {
5477         struct adapter *sc = arg1;
5478         u_int cfg;
5479         struct sbuf *sb;
5480         uint32_t *buf, *p;
5481         int rc;
5482
5483         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5484         if (rc != 0)
5485                 return (rc);
5486
5487         rc = sysctl_wire_old_buffer(req, 0);
5488         if (rc != 0)
5489                 return (rc);
5490
5491         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5492         if (sb == NULL)
5493                 return (ENOMEM);
5494
5495         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5496             M_ZERO | M_WAITOK);
5497
5498         rc = -t4_cim_read_la(sc, buf, NULL);
5499         if (rc != 0)
5500                 goto done;
5501
5502         sbuf_printf(sb, "Status   Data      PC%s",
5503             cfg & F_UPDBGLACAPTPCONLY ? "" :
5504             "     LS0Stat  LS0Addr             LS0Data");
5505
5506         KASSERT((sc->params.cim_la_size & 7) == 0,
5507             ("%s: p will walk off the end of buf", __func__));
5508
5509         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5510                 if (cfg & F_UPDBGLACAPTPCONLY) {
5511                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5512                             p[6], p[7]);
5513                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5514                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5515                             p[4] & 0xff, p[5] >> 8);
5516                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5517                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5518                             p[1] & 0xf, p[2] >> 4);
5519                 } else {
5520                         sbuf_printf(sb,
5521                             "\n  %02x   %x%07x %x%07x %08x %08x "
5522                             "%08x%08x%08x%08x",
5523                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5524                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5525                             p[6], p[7]);
5526                 }
5527         }
5528
5529         rc = sbuf_finish(sb);
5530         sbuf_delete(sb);
5531 done:
5532         free(buf, M_CXGBE);
5533         return (rc);
5534 }
5535
5536 static int
5537 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5538 {
5539         struct adapter *sc = arg1;
5540         u_int i;
5541         struct sbuf *sb;
5542         uint32_t *buf, *p;
5543         int rc;
5544
5545         rc = sysctl_wire_old_buffer(req, 0);
5546         if (rc != 0)
5547                 return (rc);
5548
5549         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5550         if (sb == NULL)
5551                 return (ENOMEM);
5552
5553         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5554             M_ZERO | M_WAITOK);
5555
5556         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5557         p = buf;
5558
5559         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5560                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5561                     p[1], p[0]);
5562         }
5563
5564         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5565         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5566                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5567                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5568                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5569                     (p[1] >> 2) | ((p[2] & 3) << 30),
5570                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5571                     p[0] & 1);
5572         }
5573
5574         rc = sbuf_finish(sb);
5575         sbuf_delete(sb);
5576         free(buf, M_CXGBE);
5577         return (rc);
5578 }
5579
5580 static int
5581 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5582 {
5583         struct adapter *sc = arg1;
5584         u_int i;
5585         struct sbuf *sb;
5586         uint32_t *buf, *p;
5587         int rc;
5588
5589         rc = sysctl_wire_old_buffer(req, 0);
5590         if (rc != 0)
5591                 return (rc);
5592
5593         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5594         if (sb == NULL)
5595                 return (ENOMEM);
5596
5597         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5598             M_ZERO | M_WAITOK);
5599
5600         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5601         p = buf;
5602
5603         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5604         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5605                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5606                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5607                     p[4], p[3], p[2], p[1], p[0]);
5608         }
5609
5610         sbuf_printf(sb, "\n\nCntl ID               Data");
5611         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5612                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5613                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5614         }
5615
5616         rc = sbuf_finish(sb);
5617         sbuf_delete(sb);
5618         free(buf, M_CXGBE);
5619         return (rc);
5620 }
5621
5622 static int
5623 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5624 {
5625         struct adapter *sc = arg1;
5626         struct sbuf *sb;
5627         int rc, i;
5628         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5629         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5630         uint16_t thres[CIM_NUM_IBQ];
5631         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5632         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5633         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5634
5635         if (is_t4(sc)) {
5636                 cim_num_obq = CIM_NUM_OBQ;
5637                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5638                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5639         } else {
5640                 cim_num_obq = CIM_NUM_OBQ_T5;
5641                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5642                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5643         }
5644         nq = CIM_NUM_IBQ + cim_num_obq;
5645
5646         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5647         if (rc == 0)
5648                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5649         if (rc != 0)
5650                 return (rc);
5651
5652         t4_read_cimq_cfg(sc, base, size, thres);
5653
5654         rc = sysctl_wire_old_buffer(req, 0);
5655         if (rc != 0)
5656                 return (rc);
5657
5658         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5659         if (sb == NULL)
5660                 return (ENOMEM);
5661
5662         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5663
5664         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5665                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5666                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5667                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5668                     G_QUEREMFLITS(p[2]) * 16);
5669         for ( ; i < nq; i++, p += 4, wr += 2)
5670                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5671                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5672                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5673                     G_QUEREMFLITS(p[2]) * 16);
5674
5675         rc = sbuf_finish(sb);
5676         sbuf_delete(sb);
5677
5678         return (rc);
5679 }
5680
5681 static int
5682 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5683 {
5684         struct adapter *sc = arg1;
5685         struct sbuf *sb;
5686         int rc;
5687         struct tp_cpl_stats stats;
5688
5689         rc = sysctl_wire_old_buffer(req, 0);
5690         if (rc != 0)
5691                 return (rc);
5692
5693         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5694         if (sb == NULL)
5695                 return (ENOMEM);
5696
5697         t4_tp_get_cpl_stats(sc, &stats);
5698
5699         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5700             "channel 3\n");
5701         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5702                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5703         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5704                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5705
5706         rc = sbuf_finish(sb);
5707         sbuf_delete(sb);
5708
5709         return (rc);
5710 }
5711
5712 static int
5713 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5714 {
5715         struct adapter *sc = arg1;
5716         struct sbuf *sb;
5717         int rc;
5718         struct tp_usm_stats stats;
5719
5720         rc = sysctl_wire_old_buffer(req, 0);
5721         if (rc != 0)
5722                 return(rc);
5723
5724         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5725         if (sb == NULL)
5726                 return (ENOMEM);
5727
5728         t4_get_usm_stats(sc, &stats);
5729
5730         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5731         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5732         sbuf_printf(sb, "Drops:  %u", stats.drops);
5733
5734         rc = sbuf_finish(sb);
5735         sbuf_delete(sb);
5736
5737         return (rc);
5738 }
5739
5740 const char *devlog_level_strings[] = {
5741         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5742         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5743         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5744         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5745         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5746         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5747 };
5748
5749 const char *devlog_facility_strings[] = {
5750         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5751         [FW_DEVLOG_FACILITY_CF]         = "CF",
5752         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5753         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5754         [FW_DEVLOG_FACILITY_RES]        = "RES",
5755         [FW_DEVLOG_FACILITY_HW]         = "HW",
5756         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5757         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5758         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5759         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5760         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5761         [FW_DEVLOG_FACILITY_VI]         = "VI",
5762         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5763         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5764         [FW_DEVLOG_FACILITY_TM]         = "TM",
5765         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5766         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5767         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5768         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5769         [FW_DEVLOG_FACILITY_RI]         = "RI",
5770         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5771         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5772         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5773         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5774 };
5775
5776 static int
5777 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5778 {
5779         struct adapter *sc = arg1;
5780         struct devlog_params *dparams = &sc->params.devlog;
5781         struct fw_devlog_e *buf, *e;
5782         int i, j, rc, nentries, first = 0, m;
5783         struct sbuf *sb;
5784         uint64_t ftstamp = UINT64_MAX;
5785
5786         if (dparams->start == 0) {
5787                 dparams->memtype = FW_MEMTYPE_EDC0;
5788                 dparams->start = 0x84000;
5789                 dparams->size = 32768;
5790         }
5791
5792         nentries = dparams->size / sizeof(struct fw_devlog_e);
5793
5794         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5795         if (buf == NULL)
5796                 return (ENOMEM);
5797
5798         m = fwmtype_to_hwmtype(dparams->memtype);
5799         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5800         if (rc != 0)
5801                 goto done;
5802
5803         for (i = 0; i < nentries; i++) {
5804                 e = &buf[i];
5805
5806                 if (e->timestamp == 0)
5807                         break;  /* end */
5808
5809                 e->timestamp = be64toh(e->timestamp);
5810                 e->seqno = be32toh(e->seqno);
5811                 for (j = 0; j < 8; j++)
5812                         e->params[j] = be32toh(e->params[j]);
5813
5814                 if (e->timestamp < ftstamp) {
5815                         ftstamp = e->timestamp;
5816                         first = i;
5817                 }
5818         }
5819
5820         if (buf[first].timestamp == 0)
5821                 goto done;      /* nothing in the log */
5822
5823         rc = sysctl_wire_old_buffer(req, 0);
5824         if (rc != 0)
5825                 goto done;
5826
5827         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5828         if (sb == NULL) {
5829                 rc = ENOMEM;
5830                 goto done;
5831         }
5832         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5833             "Seq#", "Tstamp", "Level", "Facility", "Message");
5834
5835         i = first;
5836         do {
5837                 e = &buf[i];
5838                 if (e->timestamp == 0)
5839                         break;  /* end */
5840
5841                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5842                     e->seqno, e->timestamp,
5843                     (e->level < nitems(devlog_level_strings) ?
5844                         devlog_level_strings[e->level] : "UNKNOWN"),
5845                     (e->facility < nitems(devlog_facility_strings) ?
5846                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5847                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5848                     e->params[2], e->params[3], e->params[4],
5849                     e->params[5], e->params[6], e->params[7]);
5850
5851                 if (++i == nentries)
5852                         i = 0;
5853         } while (i != first);
5854
5855         rc = sbuf_finish(sb);
5856         sbuf_delete(sb);
5857 done:
5858         free(buf, M_CXGBE);
5859         return (rc);
5860 }
5861
5862 static int
5863 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5864 {
5865         struct adapter *sc = arg1;
5866         struct sbuf *sb;
5867         int rc;
5868         struct tp_fcoe_stats stats[4];
5869
5870         rc = sysctl_wire_old_buffer(req, 0);
5871         if (rc != 0)
5872                 return (rc);
5873
5874         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5875         if (sb == NULL)
5876                 return (ENOMEM);
5877
5878         t4_get_fcoe_stats(sc, 0, &stats[0]);
5879         t4_get_fcoe_stats(sc, 1, &stats[1]);
5880         t4_get_fcoe_stats(sc, 2, &stats[2]);
5881         t4_get_fcoe_stats(sc, 3, &stats[3]);
5882
5883         sbuf_printf(sb, "                   channel 0        channel 1        "
5884             "channel 2        channel 3\n");
5885         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5886             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5887             stats[3].octetsDDP);
5888         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5889             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5890         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5891             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5892             stats[3].framesDrop);
5893
5894         rc = sbuf_finish(sb);
5895         sbuf_delete(sb);
5896
5897         return (rc);
5898 }
5899
5900 static int
5901 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5902 {
5903         struct adapter *sc = arg1;
5904         struct sbuf *sb;
5905         int rc, i;
5906         unsigned int map, kbps, ipg, mode;
5907         unsigned int pace_tab[NTX_SCHED];
5908
5909         rc = sysctl_wire_old_buffer(req, 0);
5910         if (rc != 0)
5911                 return (rc);
5912
5913         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5914         if (sb == NULL)
5915                 return (ENOMEM);
5916
5917         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5918         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5919         t4_read_pace_tbl(sc, pace_tab);
5920
5921         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5922             "Class IPG (0.1 ns)   Flow IPG (us)");
5923
5924         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5925                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5926                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5927                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5928                 if (kbps)
5929                         sbuf_printf(sb, "%9u     ", kbps);
5930                 else
5931                         sbuf_printf(sb, " disabled     ");
5932
5933                 if (ipg)
5934                         sbuf_printf(sb, "%13u        ", ipg);
5935                 else
5936                         sbuf_printf(sb, "     disabled        ");
5937
5938                 if (pace_tab[i])
5939                         sbuf_printf(sb, "%10u", pace_tab[i]);
5940                 else
5941                         sbuf_printf(sb, "  disabled");
5942         }
5943
5944         rc = sbuf_finish(sb);
5945         sbuf_delete(sb);
5946
5947         return (rc);
5948 }
5949
5950 static int
5951 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5952 {
5953         struct adapter *sc = arg1;
5954         struct sbuf *sb;
5955         int rc, i, j;
5956         uint64_t *p0, *p1;
5957         struct lb_port_stats s[2];
5958         static const char *stat_name[] = {
5959                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5960                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5961                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5962                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5963                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5964                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5965                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5966         };
5967
5968         rc = sysctl_wire_old_buffer(req, 0);
5969         if (rc != 0)
5970                 return (rc);
5971
5972         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5973         if (sb == NULL)
5974                 return (ENOMEM);
5975
5976         memset(s, 0, sizeof(s));
5977
5978         for (i = 0; i < 4; i += 2) {
5979                 t4_get_lb_stats(sc, i, &s[0]);
5980                 t4_get_lb_stats(sc, i + 1, &s[1]);
5981
5982                 p0 = &s[0].octets;
5983                 p1 = &s[1].octets;
5984                 sbuf_printf(sb, "%s                       Loopback %u"
5985                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5986
5987                 for (j = 0; j < nitems(stat_name); j++)
5988                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5989                                    *p0++, *p1++);
5990         }
5991
5992         rc = sbuf_finish(sb);
5993         sbuf_delete(sb);
5994
5995         return (rc);
5996 }
5997
5998 static int
5999 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6000 {
6001         int rc = 0;
6002         struct port_info *pi = arg1;
6003         struct sbuf *sb;
6004         static const char *linkdnreasons[] = {
6005                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6006                 "PHY overheated", "unknown", "rx los", "reserved7"
6007         };
6008
6009         rc = sysctl_wire_old_buffer(req, 0);
6010         if (rc != 0)
6011                 return(rc);
6012         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6013         if (sb == NULL)
6014                 return (ENOMEM);
6015
6016         if (pi->linkdnrc < 0)
6017                 sbuf_printf(sb, "n/a");
6018         else if (pi->linkdnrc < nitems(linkdnreasons))
6019                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6020         else
6021                 sbuf_printf(sb, "%d", pi->linkdnrc);
6022
6023         rc = sbuf_finish(sb);
6024         sbuf_delete(sb);
6025
6026         return (rc);
6027 }
6028
6029 struct mem_desc {
6030         unsigned int base;
6031         unsigned int limit;
6032         unsigned int idx;
6033 };
6034
6035 static int
6036 mem_desc_cmp(const void *a, const void *b)
6037 {
6038         return ((const struct mem_desc *)a)->base -
6039                ((const struct mem_desc *)b)->base;
6040 }
6041
6042 static void
6043 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6044     unsigned int to)
6045 {
6046         unsigned int size;
6047
6048         size = to - from + 1;
6049         if (size == 0)
6050                 return;
6051
6052         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6053         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6054 }
6055
6056 static int
6057 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6058 {
6059         struct adapter *sc = arg1;
6060         struct sbuf *sb;
6061         int rc, i, n;
6062         uint32_t lo, hi, used, alloc;
6063         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6064         static const char *region[] = {
6065                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6066                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6067                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6068                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6069                 "RQUDP region:", "PBL region:", "TXPBL region:",
6070                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6071                 "On-chip queues:"
6072         };
6073         struct mem_desc avail[4];
6074         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6075         struct mem_desc *md = mem;
6076
6077         rc = sysctl_wire_old_buffer(req, 0);
6078         if (rc != 0)
6079                 return (rc);
6080
6081         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6082         if (sb == NULL)
6083                 return (ENOMEM);
6084
6085         for (i = 0; i < nitems(mem); i++) {
6086                 mem[i].limit = 0;
6087                 mem[i].idx = i;
6088         }
6089
6090         /* Find and sort the populated memory ranges */
6091         i = 0;
6092         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6093         if (lo & F_EDRAM0_ENABLE) {
6094                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6095                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6096                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6097                 avail[i].idx = 0;
6098                 i++;
6099         }
6100         if (lo & F_EDRAM1_ENABLE) {
6101                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6102                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6103                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6104                 avail[i].idx = 1;
6105                 i++;
6106         }
6107         if (lo & F_EXT_MEM_ENABLE) {
6108                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6109                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6110                 avail[i].limit = avail[i].base +
6111                     (G_EXT_MEM_SIZE(hi) << 20);
6112                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6113                 i++;
6114         }
6115         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6116                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6117                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6118                 avail[i].limit = avail[i].base +
6119                     (G_EXT_MEM1_SIZE(hi) << 20);
6120                 avail[i].idx = 4;
6121                 i++;
6122         }
6123         if (!i)                                    /* no memory available */
6124                 return 0;
6125         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6126
6127         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6128         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6129         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6130         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6131         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6132         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6133         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6134         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6135         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6136
6137         /* the next few have explicit upper bounds */
6138         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6139         md->limit = md->base - 1 +
6140                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6141                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6142         md++;
6143
6144         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6145         md->limit = md->base - 1 +
6146                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6147                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6148         md++;
6149
6150         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6151                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6152                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6153                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6154         } else {
6155                 md->base = 0;
6156                 md->idx = nitems(region);  /* hide it */
6157         }
6158         md++;
6159
6160 #define ulp_region(reg) \
6161         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6162         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6163
6164         ulp_region(RX_ISCSI);
6165         ulp_region(RX_TDDP);
6166         ulp_region(TX_TPT);
6167         ulp_region(RX_STAG);
6168         ulp_region(RX_RQ);
6169         ulp_region(RX_RQUDP);
6170         ulp_region(RX_PBL);
6171         ulp_region(TX_PBL);
6172 #undef ulp_region
6173
6174         md->base = 0;
6175         md->idx = nitems(region);
6176         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6177                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6178                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6179                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6180         }
6181         md++;
6182
6183         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6184         md->limit = md->base + sc->tids.ntids - 1;
6185         md++;
6186         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6187         md->limit = md->base + sc->tids.ntids - 1;
6188         md++;
6189
6190         md->base = sc->vres.ocq.start;
6191         if (sc->vres.ocq.size)
6192                 md->limit = md->base + sc->vres.ocq.size - 1;
6193         else
6194                 md->idx = nitems(region);  /* hide it */
6195         md++;
6196
6197         /* add any address-space holes, there can be up to 3 */
6198         for (n = 0; n < i - 1; n++)
6199                 if (avail[n].limit < avail[n + 1].base)
6200                         (md++)->base = avail[n].limit;
6201         if (avail[n].limit)
6202                 (md++)->base = avail[n].limit;
6203
6204         n = md - mem;
6205         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6206
6207         for (lo = 0; lo < i; lo++)
6208                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6209                                 avail[lo].limit - 1);
6210
6211         sbuf_printf(sb, "\n");
6212         for (i = 0; i < n; i++) {
6213                 if (mem[i].idx >= nitems(region))
6214                         continue;                        /* skip holes */
6215                 if (!mem[i].limit)
6216                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6217                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6218                                 mem[i].limit);
6219         }
6220
6221         sbuf_printf(sb, "\n");
6222         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6223         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6224         mem_region_show(sb, "uP RAM:", lo, hi);
6225
6226         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6227         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6228         mem_region_show(sb, "uP Extmem2:", lo, hi);
6229
6230         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6231         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6232                    G_PMRXMAXPAGE(lo),
6233                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6234                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6235
6236         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6237         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6238         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6239                    G_PMTXMAXPAGE(lo),
6240                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6241                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6242         sbuf_printf(sb, "%u p-structs\n",
6243                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6244
6245         for (i = 0; i < 4; i++) {
6246                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6247                 if (is_t4(sc)) {
6248                         used = G_USED(lo);
6249                         alloc = G_ALLOC(lo);
6250                 } else {
6251                         used = G_T5_USED(lo);
6252                         alloc = G_T5_ALLOC(lo);
6253                 }
6254                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6255                            i, used, alloc);
6256         }
6257         for (i = 0; i < 4; i++) {
6258                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6259                 if (is_t4(sc)) {
6260                         used = G_USED(lo);
6261                         alloc = G_ALLOC(lo);
6262                 } else {
6263                         used = G_T5_USED(lo);
6264                         alloc = G_T5_ALLOC(lo);
6265                 }
6266                 sbuf_printf(sb,
6267                            "\nLoopback %d using %u pages out of %u allocated",
6268                            i, used, alloc);
6269         }
6270
6271         rc = sbuf_finish(sb);
6272         sbuf_delete(sb);
6273
6274         return (rc);
6275 }
6276
6277 static inline void
6278 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6279 {
6280         *mask = x | y;
6281         y = htobe64(y);
6282         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6283 }
6284
6285 static int
6286 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6287 {
6288         struct adapter *sc = arg1;
6289         struct sbuf *sb;
6290         int rc, i, n;
6291
6292         rc = sysctl_wire_old_buffer(req, 0);
6293         if (rc != 0)
6294                 return (rc);
6295
6296         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6297         if (sb == NULL)
6298                 return (ENOMEM);
6299
6300         sbuf_printf(sb,
6301             "Idx  Ethernet address     Mask     Vld Ports PF"
6302             "  VF              Replication             P0 P1 P2 P3  ML");
6303         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6304             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6305         for (i = 0; i < n; i++) {
6306                 uint64_t tcamx, tcamy, mask;
6307                 uint32_t cls_lo, cls_hi;
6308                 uint8_t addr[ETHER_ADDR_LEN];
6309
6310                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6311                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6312                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6313                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6314
6315                 if (tcamx & tcamy)
6316                         continue;
6317
6318                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6319                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6320                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6321                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6322                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6323                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6324                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6325
6326                 if (cls_lo & F_REPLICATE) {
6327                         struct fw_ldst_cmd ldst_cmd;
6328
6329                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6330                         ldst_cmd.op_to_addrspace =
6331                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6332                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6333                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6334                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6335                         ldst_cmd.u.mps.rplc.fid_idx =
6336                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6337                                 V_FW_LDST_CMD_IDX(i));
6338
6339                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6340                             "t4mps");
6341                         if (rc)
6342                                 break;
6343                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6344                             sizeof(ldst_cmd), &ldst_cmd);
6345                         end_synchronized_op(sc, 0);
6346
6347                         if (rc != 0) {
6348                                 sbuf_printf(sb,
6349                                     " ------------ error %3u ------------", rc);
6350                                 rc = 0;
6351                         } else {
6352                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6353                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6354                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6355                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6356                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6357                         }
6358                 } else
6359                         sbuf_printf(sb, "%36s", "");
6360
6361                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6362                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6363                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6364         }
6365
6366         if (rc)
6367                 (void) sbuf_finish(sb);
6368         else
6369                 rc = sbuf_finish(sb);
6370         sbuf_delete(sb);
6371
6372         return (rc);
6373 }
6374
6375 static int
6376 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6377 {
6378         struct adapter *sc = arg1;
6379         struct sbuf *sb;
6380         int rc;
6381         uint16_t mtus[NMTUS];
6382
6383         rc = sysctl_wire_old_buffer(req, 0);
6384         if (rc != 0)
6385                 return (rc);
6386
6387         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6388         if (sb == NULL)
6389                 return (ENOMEM);
6390
6391         t4_read_mtu_tbl(sc, mtus, NULL);
6392
6393         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6394             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6395             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6396             mtus[14], mtus[15]);
6397
6398         rc = sbuf_finish(sb);
6399         sbuf_delete(sb);
6400
6401         return (rc);
6402 }
6403
6404 static int
6405 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6406 {
6407         struct adapter *sc = arg1;
6408         struct sbuf *sb;
6409         int rc, i;
6410         uint32_t cnt[PM_NSTATS];
6411         uint64_t cyc[PM_NSTATS];
6412         static const char *rx_stats[] = {
6413                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6414         };
6415         static const char *tx_stats[] = {
6416                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6417         };
6418
6419         rc = sysctl_wire_old_buffer(req, 0);
6420         if (rc != 0)
6421                 return (rc);
6422
6423         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6424         if (sb == NULL)
6425                 return (ENOMEM);
6426
6427         t4_pmtx_get_stats(sc, cnt, cyc);
6428         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6429         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6430                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6431                     cyc[i]);
6432
6433         t4_pmrx_get_stats(sc, cnt, cyc);
6434         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6435         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6436                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6437                     cyc[i]);
6438
6439         rc = sbuf_finish(sb);
6440         sbuf_delete(sb);
6441
6442         return (rc);
6443 }
6444
6445 static int
6446 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6447 {
6448         struct adapter *sc = arg1;
6449         struct sbuf *sb;
6450         int rc;
6451         struct tp_rdma_stats stats;
6452
6453         rc = sysctl_wire_old_buffer(req, 0);
6454         if (rc != 0)
6455                 return (rc);
6456
6457         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6458         if (sb == NULL)
6459                 return (ENOMEM);
6460
6461         t4_tp_get_rdma_stats(sc, &stats);
6462         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6463         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6464
6465         rc = sbuf_finish(sb);
6466         sbuf_delete(sb);
6467
6468         return (rc);
6469 }
6470
6471 static int
6472 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6473 {
6474         struct adapter *sc = arg1;
6475         struct sbuf *sb;
6476         int rc;
6477         struct tp_tcp_stats v4, v6;
6478
6479         rc = sysctl_wire_old_buffer(req, 0);
6480         if (rc != 0)
6481                 return (rc);
6482
6483         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6484         if (sb == NULL)
6485                 return (ENOMEM);
6486
6487         t4_tp_get_tcp_stats(sc, &v4, &v6);
6488         sbuf_printf(sb,
6489             "                                IP                 IPv6\n");
6490         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6491             v4.tcpOutRsts, v6.tcpOutRsts);
6492         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6493             v4.tcpInSegs, v6.tcpInSegs);
6494         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6495             v4.tcpOutSegs, v6.tcpOutSegs);
6496         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6497             v4.tcpRetransSegs, v6.tcpRetransSegs);
6498
6499         rc = sbuf_finish(sb);
6500         sbuf_delete(sb);
6501
6502         return (rc);
6503 }
6504
6505 static int
6506 sysctl_tids(SYSCTL_HANDLER_ARGS)
6507 {
6508         struct adapter *sc = arg1;
6509         struct sbuf *sb;
6510         int rc;
6511         struct tid_info *t = &sc->tids;
6512
6513         rc = sysctl_wire_old_buffer(req, 0);
6514         if (rc != 0)
6515                 return (rc);
6516
6517         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6518         if (sb == NULL)
6519                 return (ENOMEM);
6520
6521         if (t->natids) {
6522                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6523                     t->atids_in_use);
6524         }
6525
6526         if (t->ntids) {
6527                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6528                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6529
6530                         if (b) {
6531                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6532                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6533                                     t->ntids - 1);
6534                         } else {
6535                                 sbuf_printf(sb, "TID range: %u-%u",
6536                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6537                                     t->ntids - 1);
6538                         }
6539                 } else
6540                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6541                 sbuf_printf(sb, ", in use: %u\n",
6542                     atomic_load_acq_int(&t->tids_in_use));
6543         }
6544
6545         if (t->nstids) {
6546                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6547                     t->stid_base + t->nstids - 1, t->stids_in_use);
6548         }
6549
6550         if (t->nftids) {
6551                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6552                     t->ftid_base + t->nftids - 1);
6553         }
6554
6555         if (t->netids) {
6556                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6557                     t->etid_base + t->netids - 1);
6558         }
6559
6560         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6561             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6562             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6563
6564         rc = sbuf_finish(sb);
6565         sbuf_delete(sb);
6566
6567         return (rc);
6568 }
6569
6570 static int
6571 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6572 {
6573         struct adapter *sc = arg1;
6574         struct sbuf *sb;
6575         int rc;
6576         struct tp_err_stats stats;
6577
6578         rc = sysctl_wire_old_buffer(req, 0);
6579         if (rc != 0)
6580                 return (rc);
6581
6582         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6583         if (sb == NULL)
6584                 return (ENOMEM);
6585
6586         t4_tp_get_err_stats(sc, &stats);
6587
6588         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6589                       "channel 3\n");
6590         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6591             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6592             stats.macInErrs[3]);
6593         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6594             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6595             stats.hdrInErrs[3]);
6596         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6597             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6598             stats.tcpInErrs[3]);
6599         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6600             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6601             stats.tcp6InErrs[3]);
6602         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6603             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6604             stats.tnlCongDrops[3]);
6605         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6606             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6607             stats.tnlTxDrops[3]);
6608         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6609             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6610             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6611         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6612             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6613             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6614         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6615             stats.ofldNoNeigh, stats.ofldCongDefer);
6616
6617         rc = sbuf_finish(sb);
6618         sbuf_delete(sb);
6619
6620         return (rc);
6621 }
6622
6623 struct field_desc {
6624         const char *name;
6625         u_int start;
6626         u_int width;
6627 };
6628
6629 static void
6630 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6631 {
6632         char buf[32];
6633         int line_size = 0;
6634
6635         while (f->name) {
6636                 uint64_t mask = (1ULL << f->width) - 1;
6637                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6638                     ((uintmax_t)v >> f->start) & mask);
6639
6640                 if (line_size + len >= 79) {
6641                         line_size = 8;
6642                         sbuf_printf(sb, "\n        ");
6643                 }
6644                 sbuf_printf(sb, "%s ", buf);
6645                 line_size += len + 1;
6646                 f++;
6647         }
6648         sbuf_printf(sb, "\n");
6649 }
6650
6651 static struct field_desc tp_la0[] = {
6652         { "RcfOpCodeOut", 60, 4 },
6653         { "State", 56, 4 },
6654         { "WcfState", 52, 4 },
6655         { "RcfOpcSrcOut", 50, 2 },
6656         { "CRxError", 49, 1 },
6657         { "ERxError", 48, 1 },
6658         { "SanityFailed", 47, 1 },
6659         { "SpuriousMsg", 46, 1 },
6660         { "FlushInputMsg", 45, 1 },
6661         { "FlushInputCpl", 44, 1 },
6662         { "RssUpBit", 43, 1 },
6663         { "RssFilterHit", 42, 1 },
6664         { "Tid", 32, 10 },
6665         { "InitTcb", 31, 1 },
6666         { "LineNumber", 24, 7 },
6667         { "Emsg", 23, 1 },
6668         { "EdataOut", 22, 1 },
6669         { "Cmsg", 21, 1 },
6670         { "CdataOut", 20, 1 },
6671         { "EreadPdu", 19, 1 },
6672         { "CreadPdu", 18, 1 },
6673         { "TunnelPkt", 17, 1 },
6674         { "RcfPeerFin", 16, 1 },
6675         { "RcfReasonOut", 12, 4 },
6676         { "TxCchannel", 10, 2 },
6677         { "RcfTxChannel", 8, 2 },
6678         { "RxEchannel", 6, 2 },
6679         { "RcfRxChannel", 5, 1 },
6680         { "RcfDataOutSrdy", 4, 1 },
6681         { "RxDvld", 3, 1 },
6682         { "RxOoDvld", 2, 1 },
6683         { "RxCongestion", 1, 1 },
6684         { "TxCongestion", 0, 1 },
6685         { NULL }
6686 };
6687
6688 static struct field_desc tp_la1[] = {
6689         { "CplCmdIn", 56, 8 },
6690         { "CplCmdOut", 48, 8 },
6691         { "ESynOut", 47, 1 },
6692         { "EAckOut", 46, 1 },
6693         { "EFinOut", 45, 1 },
6694         { "ERstOut", 44, 1 },
6695         { "SynIn", 43, 1 },
6696         { "AckIn", 42, 1 },
6697         { "FinIn", 41, 1 },
6698         { "RstIn", 40, 1 },
6699         { "DataIn", 39, 1 },
6700         { "DataInVld", 38, 1 },
6701         { "PadIn", 37, 1 },
6702         { "RxBufEmpty", 36, 1 },
6703         { "RxDdp", 35, 1 },
6704         { "RxFbCongestion", 34, 1 },
6705         { "TxFbCongestion", 33, 1 },
6706         { "TxPktSumSrdy", 32, 1 },
6707         { "RcfUlpType", 28, 4 },
6708         { "Eread", 27, 1 },
6709         { "Ebypass", 26, 1 },
6710         { "Esave", 25, 1 },
6711         { "Static0", 24, 1 },
6712         { "Cread", 23, 1 },
6713         { "Cbypass", 22, 1 },
6714         { "Csave", 21, 1 },
6715         { "CPktOut", 20, 1 },
6716         { "RxPagePoolFull", 18, 2 },
6717         { "RxLpbkPkt", 17, 1 },
6718         { "TxLpbkPkt", 16, 1 },
6719         { "RxVfValid", 15, 1 },
6720         { "SynLearned", 14, 1 },
6721         { "SetDelEntry", 13, 1 },
6722         { "SetInvEntry", 12, 1 },
6723         { "CpcmdDvld", 11, 1 },
6724         { "CpcmdSave", 10, 1 },
6725         { "RxPstructsFull", 8, 2 },
6726         { "EpcmdDvld", 7, 1 },
6727         { "EpcmdFlush", 6, 1 },
6728         { "EpcmdTrimPrefix", 5, 1 },
6729         { "EpcmdTrimPostfix", 4, 1 },
6730         { "ERssIp4Pkt", 3, 1 },
6731         { "ERssIp6Pkt", 2, 1 },
6732         { "ERssTcpUdpPkt", 1, 1 },
6733         { "ERssFceFipPkt", 0, 1 },
6734         { NULL }
6735 };
6736
6737 static struct field_desc tp_la2[] = {
6738         { "CplCmdIn", 56, 8 },
6739         { "MpsVfVld", 55, 1 },
6740         { "MpsPf", 52, 3 },
6741         { "MpsVf", 44, 8 },
6742         { "SynIn", 43, 1 },
6743         { "AckIn", 42, 1 },
6744         { "FinIn", 41, 1 },
6745         { "RstIn", 40, 1 },
6746         { "DataIn", 39, 1 },
6747         { "DataInVld", 38, 1 },
6748         { "PadIn", 37, 1 },
6749         { "RxBufEmpty", 36, 1 },
6750         { "RxDdp", 35, 1 },
6751         { "RxFbCongestion", 34, 1 },
6752         { "TxFbCongestion", 33, 1 },
6753         { "TxPktSumSrdy", 32, 1 },
6754         { "RcfUlpType", 28, 4 },
6755         { "Eread", 27, 1 },
6756         { "Ebypass", 26, 1 },
6757         { "Esave", 25, 1 },
6758         { "Static0", 24, 1 },
6759         { "Cread", 23, 1 },
6760         { "Cbypass", 22, 1 },
6761         { "Csave", 21, 1 },
6762         { "CPktOut", 20, 1 },
6763         { "RxPagePoolFull", 18, 2 },
6764         { "RxLpbkPkt", 17, 1 },
6765         { "TxLpbkPkt", 16, 1 },
6766         { "RxVfValid", 15, 1 },
6767         { "SynLearned", 14, 1 },
6768         { "SetDelEntry", 13, 1 },
6769         { "SetInvEntry", 12, 1 },
6770         { "CpcmdDvld", 11, 1 },
6771         { "CpcmdSave", 10, 1 },
6772         { "RxPstructsFull", 8, 2 },
6773         { "EpcmdDvld", 7, 1 },
6774         { "EpcmdFlush", 6, 1 },
6775         { "EpcmdTrimPrefix", 5, 1 },
6776         { "EpcmdTrimPostfix", 4, 1 },
6777         { "ERssIp4Pkt", 3, 1 },
6778         { "ERssIp6Pkt", 2, 1 },
6779         { "ERssTcpUdpPkt", 1, 1 },
6780         { "ERssFceFipPkt", 0, 1 },
6781         { NULL }
6782 };
6783
6784 static void
6785 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6786 {
6787
6788         field_desc_show(sb, *p, tp_la0);
6789 }
6790
6791 static void
6792 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6793 {
6794
6795         if (idx)
6796                 sbuf_printf(sb, "\n");
6797         field_desc_show(sb, p[0], tp_la0);
6798         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6799                 field_desc_show(sb, p[1], tp_la0);
6800 }
6801
6802 static void
6803 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6804 {
6805
6806         if (idx)
6807                 sbuf_printf(sb, "\n");
6808         field_desc_show(sb, p[0], tp_la0);
6809         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6810                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6811 }
6812
6813 static int
6814 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6815 {
6816         struct adapter *sc = arg1;
6817         struct sbuf *sb;
6818         uint64_t *buf, *p;
6819         int rc;
6820         u_int i, inc;
6821         void (*show_func)(struct sbuf *, uint64_t *, int);
6822
6823         rc = sysctl_wire_old_buffer(req, 0);
6824         if (rc != 0)
6825                 return (rc);
6826
6827         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6828         if (sb == NULL)
6829                 return (ENOMEM);
6830
6831         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6832
6833         t4_tp_read_la(sc, buf, NULL);
6834         p = buf;
6835
6836         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6837         case 2:
6838                 inc = 2;
6839                 show_func = tp_la_show2;
6840                 break;
6841         case 3:
6842                 inc = 2;
6843                 show_func = tp_la_show3;
6844                 break;
6845         default:
6846                 inc = 1;
6847                 show_func = tp_la_show;
6848         }
6849
6850         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6851                 (*show_func)(sb, p, i);
6852
6853         rc = sbuf_finish(sb);
6854         sbuf_delete(sb);
6855         free(buf, M_CXGBE);
6856         return (rc);
6857 }
6858
6859 static int
6860 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6861 {
6862         struct adapter *sc = arg1;
6863         struct sbuf *sb;
6864         int rc;
6865         u64 nrate[NCHAN], orate[NCHAN];
6866
6867         rc = sysctl_wire_old_buffer(req, 0);
6868         if (rc != 0)
6869                 return (rc);
6870
6871         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6872         if (sb == NULL)
6873                 return (ENOMEM);
6874
6875         t4_get_chan_txrate(sc, nrate, orate);
6876         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6877                  "channel 3\n");
6878         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6879             nrate[0], nrate[1], nrate[2], nrate[3]);
6880         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6881             orate[0], orate[1], orate[2], orate[3]);
6882
6883         rc = sbuf_finish(sb);
6884         sbuf_delete(sb);
6885
6886         return (rc);
6887 }
6888
6889 static int
6890 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6891 {
6892         struct adapter *sc = arg1;
6893         struct sbuf *sb;
6894         uint32_t *buf, *p;
6895         int rc, i;
6896
6897         rc = sysctl_wire_old_buffer(req, 0);
6898         if (rc != 0)
6899                 return (rc);
6900
6901         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6902         if (sb == NULL)
6903                 return (ENOMEM);
6904
6905         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6906             M_ZERO | M_WAITOK);
6907
6908         t4_ulprx_read_la(sc, buf);
6909         p = buf;
6910
6911         sbuf_printf(sb, "      Pcmd        Type   Message"
6912             "                Data");
6913         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6914                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6915                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6916         }
6917
6918         rc = sbuf_finish(sb);
6919         sbuf_delete(sb);
6920         free(buf, M_CXGBE);
6921         return (rc);
6922 }
6923
6924 static int
6925 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6926 {
6927         struct adapter *sc = arg1;
6928         struct sbuf *sb;
6929         int rc, v;
6930
6931         rc = sysctl_wire_old_buffer(req, 0);
6932         if (rc != 0)
6933                 return (rc);
6934
6935         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6936         if (sb == NULL)
6937                 return (ENOMEM);
6938
6939         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6940         if (G_STATSOURCE_T5(v) == 7) {
6941                 if (G_STATMODE(v) == 0) {
6942                         sbuf_printf(sb, "total %d, incomplete %d",
6943                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6944                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6945                 } else if (G_STATMODE(v) == 1) {
6946                         sbuf_printf(sb, "total %d, data overflow %d",
6947                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6948                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6949                 }
6950         }
6951         rc = sbuf_finish(sb);
6952         sbuf_delete(sb);
6953
6954         return (rc);
6955 }
6956 #endif
6957
6958 static uint32_t
6959 fconf_to_mode(uint32_t fconf)
6960 {
6961         uint32_t mode;
6962
6963         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6964             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6965
6966         if (fconf & F_FRAGMENTATION)
6967                 mode |= T4_FILTER_IP_FRAGMENT;
6968
6969         if (fconf & F_MPSHITTYPE)
6970                 mode |= T4_FILTER_MPS_HIT_TYPE;
6971
6972         if (fconf & F_MACMATCH)
6973                 mode |= T4_FILTER_MAC_IDX;
6974
6975         if (fconf & F_ETHERTYPE)
6976                 mode |= T4_FILTER_ETH_TYPE;
6977
6978         if (fconf & F_PROTOCOL)
6979                 mode |= T4_FILTER_IP_PROTO;
6980
6981         if (fconf & F_TOS)
6982                 mode |= T4_FILTER_IP_TOS;
6983
6984         if (fconf & F_VLAN)
6985                 mode |= T4_FILTER_VLAN;
6986
6987         if (fconf & F_VNIC_ID)
6988                 mode |= T4_FILTER_VNIC;
6989
6990         if (fconf & F_PORT)
6991                 mode |= T4_FILTER_PORT;
6992
6993         if (fconf & F_FCOE)
6994                 mode |= T4_FILTER_FCoE;
6995
6996         return (mode);
6997 }
6998
6999 static uint32_t
7000 mode_to_fconf(uint32_t mode)
7001 {
7002         uint32_t fconf = 0;
7003
7004         if (mode & T4_FILTER_IP_FRAGMENT)
7005                 fconf |= F_FRAGMENTATION;
7006
7007         if (mode & T4_FILTER_MPS_HIT_TYPE)
7008                 fconf |= F_MPSHITTYPE;
7009
7010         if (mode & T4_FILTER_MAC_IDX)
7011                 fconf |= F_MACMATCH;
7012
7013         if (mode & T4_FILTER_ETH_TYPE)
7014                 fconf |= F_ETHERTYPE;
7015
7016         if (mode & T4_FILTER_IP_PROTO)
7017                 fconf |= F_PROTOCOL;
7018
7019         if (mode & T4_FILTER_IP_TOS)
7020                 fconf |= F_TOS;
7021
7022         if (mode & T4_FILTER_VLAN)
7023                 fconf |= F_VLAN;
7024
7025         if (mode & T4_FILTER_VNIC)
7026                 fconf |= F_VNIC_ID;
7027
7028         if (mode & T4_FILTER_PORT)
7029                 fconf |= F_PORT;
7030
7031         if (mode & T4_FILTER_FCoE)
7032                 fconf |= F_FCOE;
7033
7034         return (fconf);
7035 }
7036
7037 static uint32_t
7038 fspec_to_fconf(struct t4_filter_specification *fs)
7039 {
7040         uint32_t fconf = 0;
7041
7042         if (fs->val.frag || fs->mask.frag)
7043                 fconf |= F_FRAGMENTATION;
7044
7045         if (fs->val.matchtype || fs->mask.matchtype)
7046                 fconf |= F_MPSHITTYPE;
7047
7048         if (fs->val.macidx || fs->mask.macidx)
7049                 fconf |= F_MACMATCH;
7050
7051         if (fs->val.ethtype || fs->mask.ethtype)
7052                 fconf |= F_ETHERTYPE;
7053
7054         if (fs->val.proto || fs->mask.proto)
7055                 fconf |= F_PROTOCOL;
7056
7057         if (fs->val.tos || fs->mask.tos)
7058                 fconf |= F_TOS;
7059
7060         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7061                 fconf |= F_VLAN;
7062
7063         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7064                 fconf |= F_VNIC_ID;
7065
7066         if (fs->val.iport || fs->mask.iport)
7067                 fconf |= F_PORT;
7068
7069         if (fs->val.fcoe || fs->mask.fcoe)
7070                 fconf |= F_FCOE;
7071
7072         return (fconf);
7073 }
7074
7075 static int
7076 get_filter_mode(struct adapter *sc, uint32_t *mode)
7077 {
7078         int rc;
7079         uint32_t fconf;
7080
7081         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7082             "t4getfm");
7083         if (rc)
7084                 return (rc);
7085
7086         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7087             A_TP_VLAN_PRI_MAP);
7088
7089         if (sc->params.tp.vlan_pri_map != fconf) {
7090                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7091                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7092                     fconf);
7093         }
7094
7095         *mode = fconf_to_mode(fconf);
7096
7097         end_synchronized_op(sc, LOCK_HELD);
7098         return (0);
7099 }
7100
7101 static int
7102 set_filter_mode(struct adapter *sc, uint32_t mode)
7103 {
7104         uint32_t fconf;
7105         int rc;
7106
7107         fconf = mode_to_fconf(mode);
7108
7109         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7110             "t4setfm");
7111         if (rc)
7112                 return (rc);
7113
7114         if (sc->tids.ftids_in_use > 0) {
7115                 rc = EBUSY;
7116                 goto done;
7117         }
7118
7119 #ifdef TCP_OFFLOAD
7120         if (uld_active(sc, ULD_TOM)) {
7121                 rc = EBUSY;
7122                 goto done;
7123         }
7124 #endif
7125
7126         rc = -t4_set_filter_mode(sc, fconf);
7127 done:
7128         end_synchronized_op(sc, LOCK_HELD);
7129         return (rc);
7130 }
7131
7132 static inline uint64_t
7133 get_filter_hits(struct adapter *sc, uint32_t fid)
7134 {
7135         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7136         uint64_t hits;
7137
7138         memwin_info(sc, 0, &mw_base, NULL);
7139         off = position_memwin(sc, 0,
7140             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7141         if (is_t4(sc)) {
7142                 hits = t4_read_reg64(sc, mw_base + off + 16);
7143                 hits = be64toh(hits);
7144         } else {
7145                 hits = t4_read_reg(sc, mw_base + off + 24);
7146                 hits = be32toh(hits);
7147         }
7148
7149         return (hits);
7150 }
7151
7152 static int
7153 get_filter(struct adapter *sc, struct t4_filter *t)
7154 {
7155         int i, rc, nfilters = sc->tids.nftids;
7156         struct filter_entry *f;
7157
7158         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7159             "t4getf");
7160         if (rc)
7161                 return (rc);
7162
7163         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7164             t->idx >= nfilters) {
7165                 t->idx = 0xffffffff;
7166                 goto done;
7167         }
7168
7169         f = &sc->tids.ftid_tab[t->idx];
7170         for (i = t->idx; i < nfilters; i++, f++) {
7171                 if (f->valid) {
7172                         t->idx = i;
7173                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7174                         t->smtidx = f->smtidx;
7175                         if (f->fs.hitcnts)
7176                                 t->hits = get_filter_hits(sc, t->idx);
7177                         else
7178                                 t->hits = UINT64_MAX;
7179                         t->fs = f->fs;
7180
7181                         goto done;
7182                 }
7183         }
7184
7185         t->idx = 0xffffffff;
7186 done:
7187         end_synchronized_op(sc, LOCK_HELD);
7188         return (0);
7189 }
7190
7191 static int
7192 set_filter(struct adapter *sc, struct t4_filter *t)
7193 {
7194         unsigned int nfilters, nports;
7195         struct filter_entry *f;
7196         int i, rc;
7197
7198         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7199         if (rc)
7200                 return (rc);
7201
7202         nfilters = sc->tids.nftids;
7203         nports = sc->params.nports;
7204
7205         if (nfilters == 0) {
7206                 rc = ENOTSUP;
7207                 goto done;
7208         }
7209
7210         if (!(sc->flags & FULL_INIT_DONE)) {
7211                 rc = EAGAIN;
7212                 goto done;
7213         }
7214
7215         if (t->idx >= nfilters) {
7216                 rc = EINVAL;
7217                 goto done;
7218         }
7219
7220         /* Validate against the global filter mode */
7221         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7222             sc->params.tp.vlan_pri_map) {
7223                 rc = E2BIG;
7224                 goto done;
7225         }
7226
7227         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7228                 rc = EINVAL;
7229                 goto done;
7230         }
7231
7232         if (t->fs.val.iport >= nports) {
7233                 rc = EINVAL;
7234                 goto done;
7235         }
7236
7237         /* Can't specify an iq if not steering to it */
7238         if (!t->fs.dirsteer && t->fs.iq) {
7239                 rc = EINVAL;
7240                 goto done;
7241         }
7242
7243         /* IPv6 filter idx must be 4 aligned */
7244         if (t->fs.type == 1 &&
7245             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7246                 rc = EINVAL;
7247                 goto done;
7248         }
7249
7250         if (sc->tids.ftid_tab == NULL) {
7251                 KASSERT(sc->tids.ftids_in_use == 0,
7252                     ("%s: no memory allocated but filters_in_use > 0",
7253                     __func__));
7254
7255                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7256                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7257                 if (sc->tids.ftid_tab == NULL) {
7258                         rc = ENOMEM;
7259                         goto done;
7260                 }
7261                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7262         }
7263
7264         for (i = 0; i < 4; i++) {
7265                 f = &sc->tids.ftid_tab[t->idx + i];
7266
7267                 if (f->pending || f->valid) {
7268                         rc = EBUSY;
7269                         goto done;
7270                 }
7271                 if (f->locked) {
7272                         rc = EPERM;
7273                         goto done;
7274                 }
7275
7276                 if (t->fs.type == 0)
7277                         break;
7278         }
7279
7280         f = &sc->tids.ftid_tab[t->idx];
7281         f->fs = t->fs;
7282
7283         rc = set_filter_wr(sc, t->idx);
7284 done:
7285         end_synchronized_op(sc, 0);
7286
7287         if (rc == 0) {
7288                 mtx_lock(&sc->tids.ftid_lock);
7289                 for (;;) {
7290                         if (f->pending == 0) {
7291                                 rc = f->valid ? 0 : EIO;
7292                                 break;
7293                         }
7294
7295                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7296                             PCATCH, "t4setfw", 0)) {
7297                                 rc = EINPROGRESS;
7298                                 break;
7299                         }
7300                 }
7301                 mtx_unlock(&sc->tids.ftid_lock);
7302         }
7303         return (rc);
7304 }
7305
7306 static int
7307 del_filter(struct adapter *sc, struct t4_filter *t)
7308 {
7309         unsigned int nfilters;
7310         struct filter_entry *f;
7311         int rc;
7312
7313         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7314         if (rc)
7315                 return (rc);
7316
7317         nfilters = sc->tids.nftids;
7318
7319         if (nfilters == 0) {
7320                 rc = ENOTSUP;
7321                 goto done;
7322         }
7323
7324         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7325             t->idx >= nfilters) {
7326                 rc = EINVAL;
7327                 goto done;
7328         }
7329
7330         if (!(sc->flags & FULL_INIT_DONE)) {
7331                 rc = EAGAIN;
7332                 goto done;
7333         }
7334
7335         f = &sc->tids.ftid_tab[t->idx];
7336
7337         if (f->pending) {
7338                 rc = EBUSY;
7339                 goto done;
7340         }
7341         if (f->locked) {
7342                 rc = EPERM;
7343                 goto done;
7344         }
7345
7346         if (f->valid) {
7347                 t->fs = f->fs;  /* extra info for the caller */
7348                 rc = del_filter_wr(sc, t->idx);
7349         }
7350
7351 done:
7352         end_synchronized_op(sc, 0);
7353
7354         if (rc == 0) {
7355                 mtx_lock(&sc->tids.ftid_lock);
7356                 for (;;) {
7357                         if (f->pending == 0) {
7358                                 rc = f->valid ? EIO : 0;
7359                                 break;
7360                         }
7361
7362                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7363                             PCATCH, "t4delfw", 0)) {
7364                                 rc = EINPROGRESS;
7365                                 break;
7366                         }
7367                 }
7368                 mtx_unlock(&sc->tids.ftid_lock);
7369         }
7370
7371         return (rc);
7372 }
7373
7374 static void
7375 clear_filter(struct filter_entry *f)
7376 {
7377         if (f->l2t)
7378                 t4_l2t_release(f->l2t);
7379
7380         bzero(f, sizeof (*f));
7381 }
7382
7383 static int
7384 set_filter_wr(struct adapter *sc, int fidx)
7385 {
7386         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7387         struct fw_filter_wr *fwr;
7388         unsigned int ftid;
7389         struct wrq_cookie cookie;
7390
7391         ASSERT_SYNCHRONIZED_OP(sc);
7392
7393         if (f->fs.newdmac || f->fs.newvlan) {
7394                 /* This filter needs an L2T entry; allocate one. */
7395                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7396                 if (f->l2t == NULL)
7397                         return (EAGAIN);
7398                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7399                     f->fs.dmac)) {
7400                         t4_l2t_release(f->l2t);
7401                         f->l2t = NULL;
7402                         return (ENOMEM);
7403                 }
7404         }
7405
7406         ftid = sc->tids.ftid_base + fidx;
7407
7408         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7409         if (fwr == NULL)
7410                 return (ENOMEM);
7411         bzero(fwr, sizeof(*fwr));
7412
7413         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7414         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7415         fwr->tid_to_iq =
7416             htobe32(V_FW_FILTER_WR_TID(ftid) |
7417                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7418                 V_FW_FILTER_WR_NOREPLY(0) |
7419                 V_FW_FILTER_WR_IQ(f->fs.iq));
7420         fwr->del_filter_to_l2tix =
7421             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7422                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7423                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7424                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7425                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7426                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7427                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7428                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7429                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7430                     f->fs.newvlan == VLAN_REWRITE) |
7431                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7432                     f->fs.newvlan == VLAN_REWRITE) |
7433                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7434                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7435                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7436                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7437         fwr->ethtype = htobe16(f->fs.val.ethtype);
7438         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7439         fwr->frag_to_ovlan_vldm =
7440             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7441                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7442                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7443                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7444                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7445                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7446         fwr->smac_sel = 0;
7447         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7448             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7449         fwr->maci_to_matchtypem =
7450             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7451                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7452                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7453                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7454                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7455                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7456                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7457                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7458         fwr->ptcl = f->fs.val.proto;
7459         fwr->ptclm = f->fs.mask.proto;
7460         fwr->ttyp = f->fs.val.tos;
7461         fwr->ttypm = f->fs.mask.tos;
7462         fwr->ivlan = htobe16(f->fs.val.vlan);
7463         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7464         fwr->ovlan = htobe16(f->fs.val.vnic);
7465         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7466         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7467         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7468         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7469         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7470         fwr->lp = htobe16(f->fs.val.dport);
7471         fwr->lpm = htobe16(f->fs.mask.dport);
7472         fwr->fp = htobe16(f->fs.val.sport);
7473         fwr->fpm = htobe16(f->fs.mask.sport);
7474         if (f->fs.newsmac)
7475                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7476
7477         f->pending = 1;
7478         sc->tids.ftids_in_use++;
7479
7480         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7481         return (0);
7482 }
7483
7484 static int
7485 del_filter_wr(struct adapter *sc, int fidx)
7486 {
7487         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7488         struct fw_filter_wr *fwr;
7489         unsigned int ftid;
7490         struct wrq_cookie cookie;
7491
7492         ftid = sc->tids.ftid_base + fidx;
7493
7494         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7495         if (fwr == NULL)
7496                 return (ENOMEM);
7497         bzero(fwr, sizeof (*fwr));
7498
7499         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7500
7501         f->pending = 1;
7502         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7503         return (0);
7504 }
7505
7506 int
7507 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7508 {
7509         struct adapter *sc = iq->adapter;
7510         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7511         unsigned int idx = GET_TID(rpl);
7512         unsigned int rc;
7513         struct filter_entry *f;
7514
7515         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7516             rss->opcode));
7517
7518         if (is_ftid(sc, idx)) {
7519
7520                 idx -= sc->tids.ftid_base;
7521                 f = &sc->tids.ftid_tab[idx];
7522                 rc = G_COOKIE(rpl->cookie);
7523
7524                 mtx_lock(&sc->tids.ftid_lock);
7525                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7526                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7527                             __func__, idx));
7528                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7529                         f->pending = 0;  /* asynchronous setup completed */
7530                         f->valid = 1;
7531                 } else {
7532                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7533                                 /* Add or delete failed, display an error */
7534                                 log(LOG_ERR,
7535                                     "filter %u setup failed with error %u\n",
7536                                     idx, rc);
7537                         }
7538
7539                         clear_filter(f);
7540                         sc->tids.ftids_in_use--;
7541                 }
7542                 wakeup(&sc->tids.ftid_tab);
7543                 mtx_unlock(&sc->tids.ftid_lock);
7544         }
7545
7546         return (0);
7547 }
7548
7549 static int
7550 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7551 {
7552         int rc;
7553
7554         if (cntxt->cid > M_CTXTQID)
7555                 return (EINVAL);
7556
7557         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7558             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7559                 return (EINVAL);
7560
7561         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7562         if (rc)
7563                 return (rc);
7564
7565         if (sc->flags & FW_OK) {
7566                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7567                     &cntxt->data[0]);
7568                 if (rc == 0)
7569                         goto done;
7570         }
7571
7572         /*
7573          * Read via firmware failed or wasn't even attempted.  Read directly via
7574          * the backdoor.
7575          */
7576         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7577 done:
7578         end_synchronized_op(sc, 0);
7579         return (rc);
7580 }
7581
7582 static int
7583 load_fw(struct adapter *sc, struct t4_data *fw)
7584 {
7585         int rc;
7586         uint8_t *fw_data;
7587
7588         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7589         if (rc)
7590                 return (rc);
7591
7592         if (sc->flags & FULL_INIT_DONE) {
7593                 rc = EBUSY;
7594                 goto done;
7595         }
7596
7597         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7598         if (fw_data == NULL) {
7599                 rc = ENOMEM;
7600                 goto done;
7601         }
7602
7603         rc = copyin(fw->data, fw_data, fw->len);
7604         if (rc == 0)
7605                 rc = -t4_load_fw(sc, fw_data, fw->len);
7606
7607         free(fw_data, M_CXGBE);
7608 done:
7609         end_synchronized_op(sc, 0);
7610         return (rc);
7611 }
7612
7613 static int
7614 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7615 {
7616         uint32_t addr, off, remaining, i, n;
7617         uint32_t *buf, *b;
7618         uint32_t mw_base, mw_aperture;
7619         int rc;
7620         uint8_t *dst;
7621
7622         rc = validate_mem_range(sc, mr->addr, mr->len);
7623         if (rc != 0)
7624                 return (rc);
7625
7626         memwin_info(sc, win, &mw_base, &mw_aperture);
7627         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7628         addr = mr->addr;
7629         remaining = mr->len;
7630         dst = (void *)mr->data;
7631
7632         while (remaining) {
7633                 off = position_memwin(sc, win, addr);
7634
7635                 /* number of bytes that we'll copy in the inner loop */
7636                 n = min(remaining, mw_aperture - off);
7637                 for (i = 0; i < n; i += 4)
7638                         *b++ = t4_read_reg(sc, mw_base + off + i);
7639
7640                 rc = copyout(buf, dst, n);
7641                 if (rc != 0)
7642                         break;
7643
7644                 b = buf;
7645                 dst += n;
7646                 remaining -= n;
7647                 addr += n;
7648         }
7649
7650         free(buf, M_CXGBE);
7651         return (rc);
7652 }
7653
7654 static int
7655 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7656 {
7657         int rc;
7658
7659         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7660                 return (EINVAL);
7661
7662         if (i2cd->len > sizeof(i2cd->data))
7663                 return (EFBIG);
7664
7665         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7666         if (rc)
7667                 return (rc);
7668         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7669             i2cd->offset, i2cd->len, &i2cd->data[0]);
7670         end_synchronized_op(sc, 0);
7671
7672         return (rc);
7673 }
7674
7675 static int
7676 in_range(int val, int lo, int hi)
7677 {
7678
7679         return (val < 0 || (val <= hi && val >= lo));
7680 }
7681
7682 static int
7683 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7684 {
7685         int fw_subcmd, fw_type, rc;
7686
7687         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7688         if (rc)
7689                 return (rc);
7690
7691         if (!(sc->flags & FULL_INIT_DONE)) {
7692                 rc = EAGAIN;
7693                 goto done;
7694         }
7695
7696         /*
7697          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7698          * sub-command and type are in common locations.)
7699          */
7700         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7701                 fw_subcmd = FW_SCHED_SC_CONFIG;
7702         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7703                 fw_subcmd = FW_SCHED_SC_PARAMS;
7704         else {
7705                 rc = EINVAL;
7706                 goto done;
7707         }
7708         if (p->type == SCHED_CLASS_TYPE_PACKET)
7709                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7710         else {
7711                 rc = EINVAL;
7712                 goto done;
7713         }
7714
7715         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7716                 /* Vet our parameters ..*/
7717                 if (p->u.config.minmax < 0) {
7718                         rc = EINVAL;
7719                         goto done;
7720                 }
7721
7722                 /* And pass the request to the firmware ...*/
7723                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7724                 goto done;
7725         }
7726
7727         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7728                 int fw_level;
7729                 int fw_mode;
7730                 int fw_rateunit;
7731                 int fw_ratemode;
7732
7733                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7734                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7735                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7736                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7737                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7738                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7739                 else {
7740                         rc = EINVAL;
7741                         goto done;
7742                 }
7743
7744                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7745                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7746                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7747                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7748                 else {
7749                         rc = EINVAL;
7750                         goto done;
7751                 }
7752
7753                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7754                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7755                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7756                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7757                 else {
7758                         rc = EINVAL;
7759                         goto done;
7760                 }
7761
7762                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7763                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7764                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7765                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7766                 else {
7767                         rc = EINVAL;
7768                         goto done;
7769                 }
7770
7771                 /* Vet our parameters ... */
7772                 if (!in_range(p->u.params.channel, 0, 3) ||
7773                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7774                     !in_range(p->u.params.minrate, 0, 10000000) ||
7775                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7776                     !in_range(p->u.params.weight, 0, 100)) {
7777                         rc = ERANGE;
7778                         goto done;
7779                 }
7780
7781                 /*
7782                  * Translate any unset parameters into the firmware's
7783                  * nomenclature and/or fail the call if the parameters
7784                  * are required ...
7785                  */
7786                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7787                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7788                         rc = EINVAL;
7789                         goto done;
7790                 }
7791                 if (p->u.params.minrate < 0)
7792                         p->u.params.minrate = 0;
7793                 if (p->u.params.maxrate < 0) {
7794                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7795                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7796                                 rc = EINVAL;
7797                                 goto done;
7798                         } else
7799                                 p->u.params.maxrate = 0;
7800                 }
7801                 if (p->u.params.weight < 0) {
7802                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7803                                 rc = EINVAL;
7804                                 goto done;
7805                         } else
7806                                 p->u.params.weight = 0;
7807                 }
7808                 if (p->u.params.pktsize < 0) {
7809                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7810                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7811                                 rc = EINVAL;
7812                                 goto done;
7813                         } else
7814                                 p->u.params.pktsize = 0;
7815                 }
7816
7817                 /* See what the firmware thinks of the request ... */
7818                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7819                     fw_rateunit, fw_ratemode, p->u.params.channel,
7820                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7821                     p->u.params.weight, p->u.params.pktsize, 1);
7822                 goto done;
7823         }
7824
7825         rc = EINVAL;
7826 done:
7827         end_synchronized_op(sc, 0);
7828         return (rc);
7829 }
7830
7831 static int
7832 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7833 {
7834         struct port_info *pi = NULL;
7835         struct sge_txq *txq;
7836         uint32_t fw_mnem, fw_queue, fw_class;
7837         int i, rc;
7838
7839         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7840         if (rc)
7841                 return (rc);
7842
7843         if (!(sc->flags & FULL_INIT_DONE)) {
7844                 rc = EAGAIN;
7845                 goto done;
7846         }
7847
7848         if (p->port >= sc->params.nports) {
7849                 rc = EINVAL;
7850                 goto done;
7851         }
7852
7853         pi = sc->port[p->port];
7854         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7855                 rc = EINVAL;
7856                 goto done;
7857         }
7858
7859         /*
7860          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7861          * Scheduling Class in this case).
7862          */
7863         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7864             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7865         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7866
7867         /*
7868          * If op.queue is non-negative, then we're only changing the scheduling
7869          * on a single specified TX queue.
7870          */
7871         if (p->queue >= 0) {
7872                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7873                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7874                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7875                     &fw_class);
7876                 goto done;
7877         }
7878
7879         /*
7880          * Change the scheduling on all the TX queues for the
7881          * interface.
7882          */
7883         for_each_txq(pi, i, txq) {
7884                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7885                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7886                     &fw_class);
7887                 if (rc)
7888                         goto done;
7889         }
7890
7891         rc = 0;
7892 done:
7893         end_synchronized_op(sc, 0);
7894         return (rc);
7895 }
7896
7897 int
7898 t4_os_find_pci_capability(struct adapter *sc, int cap)
7899 {
7900         int i;
7901
7902         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7903 }
7904
7905 int
7906 t4_os_pci_save_state(struct adapter *sc)
7907 {
7908         device_t dev;
7909         struct pci_devinfo *dinfo;
7910
7911         dev = sc->dev;
7912         dinfo = device_get_ivars(dev);
7913
7914         pci_cfg_save(dev, dinfo, 0);
7915         return (0);
7916 }
7917
7918 int
7919 t4_os_pci_restore_state(struct adapter *sc)
7920 {
7921         device_t dev;
7922         struct pci_devinfo *dinfo;
7923
7924         dev = sc->dev;
7925         dinfo = device_get_ivars(dev);
7926
7927         pci_cfg_restore(dev, dinfo);
7928         return (0);
7929 }
7930
7931 void
7932 t4_os_portmod_changed(const struct adapter *sc, int idx)
7933 {
7934         struct port_info *pi = sc->port[idx];
7935         static const char *mod_str[] = {
7936                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7937         };
7938
7939         build_medialist(pi, &pi->media);
7940 #ifdef DEV_NETMAP
7941         build_medialist(pi, &pi->nm_media);
7942 #endif
7943
7944         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7945                 if_printf(pi->ifp, "transceiver unplugged.\n");
7946         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7947                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7948         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7949                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7950         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7951                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7952                     mod_str[pi->mod_type]);
7953         } else {
7954                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7955                     pi->mod_type);
7956         }
7957 }
7958
7959 void
7960 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7961 {
7962         struct port_info *pi = sc->port[idx];
7963         struct ifnet *ifp = pi->ifp;
7964
7965         if (link_stat) {
7966                 pi->linkdnrc = -1;
7967                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7968                 if_link_state_change(ifp, LINK_STATE_UP);
7969         } else {
7970                 if (reason >= 0)
7971                         pi->linkdnrc = reason;
7972                 if_link_state_change(ifp, LINK_STATE_DOWN);
7973         }
7974 }
7975
7976 void
7977 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7978 {
7979         struct adapter *sc;
7980
7981         sx_slock(&t4_list_lock);
7982         SLIST_FOREACH(sc, &t4_list, link) {
7983                 /*
7984                  * func should not make any assumptions about what state sc is
7985                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7986                  */
7987                 func(sc, arg);
7988         }
7989         sx_sunlock(&t4_list_lock);
7990 }
7991
7992 static int
7993 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7994 {
7995        return (0);
7996 }
7997
7998 static int
7999 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8000 {
8001        return (0);
8002 }
8003
8004 static int
8005 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8006     struct thread *td)
8007 {
8008         int rc;
8009         struct adapter *sc = dev->si_drv1;
8010
8011         rc = priv_check(td, PRIV_DRIVER);
8012         if (rc != 0)
8013                 return (rc);
8014
8015         switch (cmd) {
8016         case CHELSIO_T4_GETREG: {
8017                 struct t4_reg *edata = (struct t4_reg *)data;
8018
8019                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8020                         return (EFAULT);
8021
8022                 if (edata->size == 4)
8023                         edata->val = t4_read_reg(sc, edata->addr);
8024                 else if (edata->size == 8)
8025                         edata->val = t4_read_reg64(sc, edata->addr);
8026                 else
8027                         return (EINVAL);
8028
8029                 break;
8030         }
8031         case CHELSIO_T4_SETREG: {
8032                 struct t4_reg *edata = (struct t4_reg *)data;
8033
8034                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8035                         return (EFAULT);
8036
8037                 if (edata->size == 4) {
8038                         if (edata->val & 0xffffffff00000000)
8039                                 return (EINVAL);
8040                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8041                 } else if (edata->size == 8)
8042                         t4_write_reg64(sc, edata->addr, edata->val);
8043                 else
8044                         return (EINVAL);
8045                 break;
8046         }
8047         case CHELSIO_T4_REGDUMP: {
8048                 struct t4_regdump *regs = (struct t4_regdump *)data;
8049                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8050                 uint8_t *buf;
8051
8052                 if (regs->len < reglen) {
8053                         regs->len = reglen; /* hint to the caller */
8054                         return (ENOBUFS);
8055                 }
8056
8057                 regs->len = reglen;
8058                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8059                 t4_get_regs(sc, regs, buf);
8060                 rc = copyout(buf, regs->data, reglen);
8061                 free(buf, M_CXGBE);
8062                 break;
8063         }
8064         case CHELSIO_T4_GET_FILTER_MODE:
8065                 rc = get_filter_mode(sc, (uint32_t *)data);
8066                 break;
8067         case CHELSIO_T4_SET_FILTER_MODE:
8068                 rc = set_filter_mode(sc, *(uint32_t *)data);
8069                 break;
8070         case CHELSIO_T4_GET_FILTER:
8071                 rc = get_filter(sc, (struct t4_filter *)data);
8072                 break;
8073         case CHELSIO_T4_SET_FILTER:
8074                 rc = set_filter(sc, (struct t4_filter *)data);
8075                 break;
8076         case CHELSIO_T4_DEL_FILTER:
8077                 rc = del_filter(sc, (struct t4_filter *)data);
8078                 break;
8079         case CHELSIO_T4_GET_SGE_CONTEXT:
8080                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8081                 break;
8082         case CHELSIO_T4_LOAD_FW:
8083                 rc = load_fw(sc, (struct t4_data *)data);
8084                 break;
8085         case CHELSIO_T4_GET_MEM:
8086                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8087                 break;
8088         case CHELSIO_T4_GET_I2C:
8089                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8090                 break;
8091         case CHELSIO_T4_CLEAR_STATS: {
8092                 int i;
8093                 u_int port_id = *(uint32_t *)data;
8094                 struct port_info *pi;
8095
8096                 if (port_id >= sc->params.nports)
8097                         return (EINVAL);
8098                 pi = sc->port[port_id];
8099
8100                 /* MAC stats */
8101                 t4_clr_port_stats(sc, pi->tx_chan);
8102                 pi->tx_parse_error = 0;
8103
8104                 if (pi->flags & PORT_INIT_DONE) {
8105                         struct sge_rxq *rxq;
8106                         struct sge_txq *txq;
8107                         struct sge_wrq *wrq;
8108
8109                         for_each_rxq(pi, i, rxq) {
8110 #if defined(INET) || defined(INET6)
8111                                 rxq->lro.lro_queued = 0;
8112                                 rxq->lro.lro_flushed = 0;
8113 #endif
8114                                 rxq->rxcsum = 0;
8115                                 rxq->vlan_extraction = 0;
8116                         }
8117
8118                         for_each_txq(pi, i, txq) {
8119                                 txq->txcsum = 0;
8120                                 txq->tso_wrs = 0;
8121                                 txq->vlan_insertion = 0;
8122                                 txq->imm_wrs = 0;
8123                                 txq->sgl_wrs = 0;
8124                                 txq->txpkt_wrs = 0;
8125                                 txq->txpkts0_wrs = 0;
8126                                 txq->txpkts1_wrs = 0;
8127                                 txq->txpkts0_pkts = 0;
8128                                 txq->txpkts1_pkts = 0;
8129                                 mp_ring_reset_stats(txq->r);
8130                         }
8131
8132 #ifdef TCP_OFFLOAD
8133                         /* nothing to clear for each ofld_rxq */
8134
8135                         for_each_ofld_txq(pi, i, wrq) {
8136                                 wrq->tx_wrs_direct = 0;
8137                                 wrq->tx_wrs_copied = 0;
8138                         }
8139 #endif
8140                         wrq = &sc->sge.ctrlq[pi->port_id];
8141                         wrq->tx_wrs_direct = 0;
8142                         wrq->tx_wrs_copied = 0;
8143                 }
8144                 break;
8145         }
8146         case CHELSIO_T4_SCHED_CLASS:
8147                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8148                 break;
8149         case CHELSIO_T4_SCHED_QUEUE:
8150                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8151                 break;
8152         case CHELSIO_T4_GET_TRACER:
8153                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8154                 break;
8155         case CHELSIO_T4_SET_TRACER:
8156                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8157                 break;
8158         default:
8159                 rc = EINVAL;
8160         }
8161
8162         return (rc);
8163 }
8164
8165 #ifdef TCP_OFFLOAD
8166 void
8167 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8168     const unsigned int *pgsz_order)
8169 {
8170         struct port_info *pi = ifp->if_softc;
8171         struct adapter *sc = pi->adapter;
8172
8173         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8174         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8175                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8176                 V_HPZ3(pgsz_order[3]));
8177 }
8178
8179 static int
8180 toe_capability(struct port_info *pi, int enable)
8181 {
8182         int rc;
8183         struct adapter *sc = pi->adapter;
8184
8185         ASSERT_SYNCHRONIZED_OP(sc);
8186
8187         if (!is_offload(sc))
8188                 return (ENODEV);
8189
8190         if (enable) {
8191                 /*
8192                  * We need the port's queues around so that we're able to send
8193                  * and receive CPLs to/from the TOE even if the ifnet for this
8194                  * port has never been UP'd administratively.
8195                  */
8196                 if (!(pi->flags & PORT_INIT_DONE)) {
8197                         rc = cxgbe_init_synchronized(pi);
8198                         if (rc)
8199                                 return (rc);
8200                 }
8201
8202                 if (isset(&sc->offload_map, pi->port_id))
8203                         return (0);
8204
8205                 if (!uld_active(sc, ULD_TOM)) {
8206                         rc = t4_activate_uld(sc, ULD_TOM);
8207                         if (rc == EAGAIN) {
8208                                 log(LOG_WARNING,
8209                                     "You must kldload t4_tom.ko before trying "
8210                                     "to enable TOE on a cxgbe interface.\n");
8211                         }
8212                         if (rc != 0)
8213                                 return (rc);
8214                         KASSERT(sc->tom_softc != NULL,
8215                             ("%s: TOM activated but softc NULL", __func__));
8216                         KASSERT(uld_active(sc, ULD_TOM),
8217                             ("%s: TOM activated but flag not set", __func__));
8218                 }
8219
8220                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8221                 if (!uld_active(sc, ULD_IWARP))
8222                         (void) t4_activate_uld(sc, ULD_IWARP);
8223                 if (!uld_active(sc, ULD_ISCSI))
8224                         (void) t4_activate_uld(sc, ULD_ISCSI);
8225
8226                 setbit(&sc->offload_map, pi->port_id);
8227         } else {
8228                 if (!isset(&sc->offload_map, pi->port_id))
8229                         return (0);
8230
8231                 KASSERT(uld_active(sc, ULD_TOM),
8232                     ("%s: TOM never initialized?", __func__));
8233                 clrbit(&sc->offload_map, pi->port_id);
8234         }
8235
8236         return (0);
8237 }
8238
8239 /*
8240  * Add an upper layer driver to the global list.
8241  */
8242 int
8243 t4_register_uld(struct uld_info *ui)
8244 {
8245         int rc = 0;
8246         struct uld_info *u;
8247
8248         sx_xlock(&t4_uld_list_lock);
8249         SLIST_FOREACH(u, &t4_uld_list, link) {
8250             if (u->uld_id == ui->uld_id) {
8251                     rc = EEXIST;
8252                     goto done;
8253             }
8254         }
8255
8256         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8257         ui->refcount = 0;
8258 done:
8259         sx_xunlock(&t4_uld_list_lock);
8260         return (rc);
8261 }
8262
8263 int
8264 t4_unregister_uld(struct uld_info *ui)
8265 {
8266         int rc = EINVAL;
8267         struct uld_info *u;
8268
8269         sx_xlock(&t4_uld_list_lock);
8270
8271         SLIST_FOREACH(u, &t4_uld_list, link) {
8272             if (u == ui) {
8273                     if (ui->refcount > 0) {
8274                             rc = EBUSY;
8275                             goto done;
8276                     }
8277
8278                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8279                     rc = 0;
8280                     goto done;
8281             }
8282         }
8283 done:
8284         sx_xunlock(&t4_uld_list_lock);
8285         return (rc);
8286 }
8287
8288 int
8289 t4_activate_uld(struct adapter *sc, int id)
8290 {
8291         int rc;
8292         struct uld_info *ui;
8293
8294         ASSERT_SYNCHRONIZED_OP(sc);
8295
8296         if (id < 0 || id > ULD_MAX)
8297                 return (EINVAL);
8298         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8299
8300         sx_slock(&t4_uld_list_lock);
8301
8302         SLIST_FOREACH(ui, &t4_uld_list, link) {
8303                 if (ui->uld_id == id) {
8304                         if (!(sc->flags & FULL_INIT_DONE)) {
8305                                 rc = adapter_full_init(sc);
8306                                 if (rc != 0)
8307                                         break;
8308                         }
8309
8310                         rc = ui->activate(sc);
8311                         if (rc == 0) {
8312                                 setbit(&sc->active_ulds, id);
8313                                 ui->refcount++;
8314                         }
8315                         break;
8316                 }
8317         }
8318
8319         sx_sunlock(&t4_uld_list_lock);
8320
8321         return (rc);
8322 }
8323
8324 int
8325 t4_deactivate_uld(struct adapter *sc, int id)
8326 {
8327         int rc;
8328         struct uld_info *ui;
8329
8330         ASSERT_SYNCHRONIZED_OP(sc);
8331
8332         if (id < 0 || id > ULD_MAX)
8333                 return (EINVAL);
8334         rc = ENXIO;
8335
8336         sx_slock(&t4_uld_list_lock);
8337
8338         SLIST_FOREACH(ui, &t4_uld_list, link) {
8339                 if (ui->uld_id == id) {
8340                         rc = ui->deactivate(sc);
8341                         if (rc == 0) {
8342                                 clrbit(&sc->active_ulds, id);
8343                                 ui->refcount--;
8344                         }
8345                         break;
8346                 }
8347         }
8348
8349         sx_sunlock(&t4_uld_list_lock);
8350
8351         return (rc);
8352 }
8353
8354 int
8355 uld_active(struct adapter *sc, int uld_id)
8356 {
8357
8358         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8359
8360         return (isset(&sc->active_ulds, uld_id));
8361 }
8362 #endif
8363
8364 /*
8365  * Come up with reasonable defaults for some of the tunables, provided they're
8366  * not set by the user (in which case we'll use the values as is).
8367  */
8368 static void
8369 tweak_tunables(void)
8370 {
8371         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8372
8373         if (t4_ntxq10g < 1)
8374                 t4_ntxq10g = min(nc, NTXQ_10G);
8375
8376         if (t4_ntxq1g < 1)
8377                 t4_ntxq1g = min(nc, NTXQ_1G);
8378
8379         if (t4_nrxq10g < 1)
8380                 t4_nrxq10g = min(nc, NRXQ_10G);
8381
8382         if (t4_nrxq1g < 1)
8383                 t4_nrxq1g = min(nc, NRXQ_1G);
8384
8385 #ifdef TCP_OFFLOAD
8386         if (t4_nofldtxq10g < 1)
8387                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8388
8389         if (t4_nofldtxq1g < 1)
8390                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8391
8392         if (t4_nofldrxq10g < 1)
8393                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8394
8395         if (t4_nofldrxq1g < 1)
8396                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8397
8398         if (t4_toecaps_allowed == -1)
8399                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8400 #else
8401         if (t4_toecaps_allowed == -1)
8402                 t4_toecaps_allowed = 0;
8403 #endif
8404
8405 #ifdef DEV_NETMAP
8406         if (t4_nnmtxq10g < 1)
8407                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8408
8409         if (t4_nnmtxq1g < 1)
8410                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8411
8412         if (t4_nnmrxq10g < 1)
8413                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8414
8415         if (t4_nnmrxq1g < 1)
8416                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8417 #endif
8418
8419         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8420                 t4_tmr_idx_10g = TMR_IDX_10G;
8421
8422         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8423                 t4_pktc_idx_10g = PKTC_IDX_10G;
8424
8425         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8426                 t4_tmr_idx_1g = TMR_IDX_1G;
8427
8428         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8429                 t4_pktc_idx_1g = PKTC_IDX_1G;
8430
8431         if (t4_qsize_txq < 128)
8432                 t4_qsize_txq = 128;
8433
8434         if (t4_qsize_rxq < 128)
8435                 t4_qsize_rxq = 128;
8436         while (t4_qsize_rxq & 7)
8437                 t4_qsize_rxq++;
8438
8439         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8440 }
8441
8442 static struct sx mlu;   /* mod load unload */
8443 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8444
8445 static int
8446 mod_event(module_t mod, int cmd, void *arg)
8447 {
8448         int rc = 0;
8449         static int loaded = 0;
8450
8451         switch (cmd) {
8452         case MOD_LOAD:
8453                 sx_xlock(&mlu);
8454                 if (loaded++ == 0) {
8455                         t4_sge_modload();
8456                         sx_init(&t4_list_lock, "T4/T5 adapters");
8457                         SLIST_INIT(&t4_list);
8458 #ifdef TCP_OFFLOAD
8459                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8460                         SLIST_INIT(&t4_uld_list);
8461 #endif
8462                         t4_tracer_modload();
8463                         tweak_tunables();
8464                 }
8465                 sx_xunlock(&mlu);
8466                 break;
8467
8468         case MOD_UNLOAD:
8469                 sx_xlock(&mlu);
8470                 if (--loaded == 0) {
8471                         int tries;
8472
8473                         sx_slock(&t4_list_lock);
8474                         if (!SLIST_EMPTY(&t4_list)) {
8475                                 rc = EBUSY;
8476                                 sx_sunlock(&t4_list_lock);
8477                                 goto done_unload;
8478                         }
8479 #ifdef TCP_OFFLOAD
8480                         sx_slock(&t4_uld_list_lock);
8481                         if (!SLIST_EMPTY(&t4_uld_list)) {
8482                                 rc = EBUSY;
8483                                 sx_sunlock(&t4_uld_list_lock);
8484                                 sx_sunlock(&t4_list_lock);
8485                                 goto done_unload;
8486                         }
8487 #endif
8488                         tries = 0;
8489                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8490                                 uprintf("%ju clusters with custom free routine "
8491                                     "still is use.\n", t4_sge_extfree_refs());
8492                                 pause("t4unload", 2 * hz);
8493                         }
8494 #ifdef TCP_OFFLOAD
8495                         sx_sunlock(&t4_uld_list_lock);
8496 #endif
8497                         sx_sunlock(&t4_list_lock);
8498
8499                         if (t4_sge_extfree_refs() == 0) {
8500                                 t4_tracer_modunload();
8501 #ifdef TCP_OFFLOAD
8502                                 sx_destroy(&t4_uld_list_lock);
8503 #endif
8504                                 sx_destroy(&t4_list_lock);
8505                                 t4_sge_modunload();
8506                                 loaded = 0;
8507                         } else {
8508                                 rc = EBUSY;
8509                                 loaded++;       /* undo earlier decrement */
8510                         }
8511                 }
8512 done_unload:
8513                 sx_xunlock(&mlu);
8514                 break;
8515         }
8516
8517         return (rc);
8518 }
8519
8520 static devclass_t t4_devclass, t5_devclass;
8521 static devclass_t cxgbe_devclass, cxl_devclass;
8522
8523 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8524 MODULE_VERSION(t4nex, 1);
8525 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8526
8527 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8528 MODULE_VERSION(t5nex, 1);
8529 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8530
8531 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8532 MODULE_VERSION(cxgbe, 1);
8533
8534 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8535 MODULE_VERSION(cxl, 1);