sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33 #include "opt_rss.h"
  34
  35 #include <sys/param.h>
  36 #include <sys/conf.h>
  37 #include <sys/priv.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bus.h>
  40 #include <sys/module.h>
  41 #include <sys/malloc.h>
  42 #include <sys/queue.h>
  43 #include <sys/taskqueue.h>
  44 #include <sys/pciio.h>
  45 #include <dev/pci/pcireg.h>
  46 #include <dev/pci/pcivar.h>
  47 #include <dev/pci/pci_private.h>
  48 #include <sys/firmware.h>
  49 #include <sys/sbuf.h>
  50 #include <sys/smp.h>
  51 #include <sys/socket.h>
  52 #include <sys/sockio.h>
  53 #include <sys/sysctl.h>
  54 #include <net/ethernet.h>
  55 #include <net/if.h>
  56 #include <net/if_types.h>
  57 #include <net/if_dl.h>
  58 #include <net/if_vlan_var.h>
  59 #ifdef RSS
  60 #include <net/rss_config.h>
  61 #endif
  62 #if defined(__i386__) || defined(__amd64__)
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65 #endif
  66
  67 #include "common/common.h"
  68 #include "common/t4_msg.h"
  69 #include "common/t4_regs.h"
  70 #include "common/t4_regs_values.h"
  71 #include "t4_ioctl.h"
  72 #include "t4_l2t.h"
  73 #include "t4_mp_ring.h"
  74
  75 /* T4 bus driver interface */
  76 static int t4_probe(device_t);
  77 static int t4_attach(device_t);
  78 static int t4_detach(device_t);
  79 static device_method_t t4_methods[] = {
  80         DEVMETHOD(device_probe,         t4_probe),
  81         DEVMETHOD(device_attach,        t4_attach),
  82         DEVMETHOD(device_detach,        t4_detach),
  83
  84         DEVMETHOD_END
  85 };
  86 static driver_t t4_driver = {
  87         "t4nex",
  88         t4_methods,
  89         sizeof(struct adapter)
  90 };
  91
  92
  93 /* T4 port (cxgbe) interface */
  94 static int cxgbe_probe(device_t);
  95 static int cxgbe_attach(device_t);
  96 static int cxgbe_detach(device_t);
  97 static device_method_t cxgbe_methods[] = {
  98         DEVMETHOD(device_probe,         cxgbe_probe),
  99         DEVMETHOD(device_attach,        cxgbe_attach),
 100         DEVMETHOD(device_detach,        cxgbe_detach),
 101         { 0, 0 }
 102 };
 103 static driver_t cxgbe_driver = {
 104         "cxgbe",
 105         cxgbe_methods,
 106         sizeof(struct port_info)
 107 };
 108
 109 static d_ioctl_t t4_ioctl;
 110 static d_open_t t4_open;
 111 static d_close_t t4_close;
 112
 113 static struct cdevsw t4_cdevsw = {
 114        .d_version = D_VERSION,
 115        .d_flags = 0,
 116        .d_open = t4_open,
 117        .d_close = t4_close,
 118        .d_ioctl = t4_ioctl,
 119        .d_name = "t4nex",
 120 };
 121
 122 /* T5 bus driver interface */
 123 static int t5_probe(device_t);
 124 static device_method_t t5_methods[] = {
 125         DEVMETHOD(device_probe,         t5_probe),
 126         DEVMETHOD(device_attach,        t4_attach),
 127         DEVMETHOD(device_detach,        t4_detach),
 128
 129         DEVMETHOD_END
 130 };
 131 static driver_t t5_driver = {
 132         "t5nex",
 133         t5_methods,
 134         sizeof(struct adapter)
 135 };
 136
 137
 138 /* T5 port (cxl) interface */
 139 static driver_t cxl_driver = {
 140         "cxl",
 141         cxgbe_methods,
 142         sizeof(struct port_info)
 143 };
 144
 145 static struct cdevsw t5_cdevsw = {
 146        .d_version = D_VERSION,
 147        .d_flags = 0,
 148        .d_open = t4_open,
 149        .d_close = t4_close,
 150        .d_ioctl = t4_ioctl,
 151        .d_name = "t5nex",
 152 };
 153
 154 /* ifnet + media interface */
 155 static void cxgbe_init(void *);
 156 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 157 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 158 static void cxgbe_qflush(struct ifnet *);
 159 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 160 static int cxgbe_media_change(struct ifnet *);
 161 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 162
 163 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 164
 165 /*
 166  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 167  * then ADAPTER_LOCK, then t4_uld_list_lock.
 168  */
 169 static struct sx t4_list_lock;
 170 SLIST_HEAD(, adapter) t4_list;
 171 #ifdef TCP_OFFLOAD
 172 static struct sx t4_uld_list_lock;
 173 SLIST_HEAD(, uld_info) t4_uld_list;
 174 #endif
 175
 176 /*
 177  * Tunables.  See tweak_tunables() too.
 178  *
 179  * Each tunable is set to a default value here if it's known at compile-time.
 180  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 181  * provide a reasonable default when the driver is loaded.
 182  *
 183  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 184  * T5 are under hw.cxl.
 185  */
 186
 187 /*
 188  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 189  */
 190 #define NTXQ_10G 16
 191 static int t4_ntxq10g = -1;
 192 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 193
 194 #define NRXQ_10G 8
 195 static int t4_nrxq10g = -1;
 196 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 197
 198 #define NTXQ_1G 4
 199 static int t4_ntxq1g = -1;
 200 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 201
 202 #define NRXQ_1G 2
 203 static int t4_nrxq1g = -1;
 204 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 205
 206 static int t4_rsrv_noflowq = 0;
 207 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 208
 209 #ifdef TCP_OFFLOAD
 210 #define NOFLDTXQ_10G 8
 211 static int t4_nofldtxq10g = -1;
 212 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 213
 214 #define NOFLDRXQ_10G 2
 215 static int t4_nofldrxq10g = -1;
 216 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 217
 218 #define NOFLDTXQ_1G 2
 219 static int t4_nofldtxq1g = -1;
 220 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 221
 222 #define NOFLDRXQ_1G 1
 223 static int t4_nofldrxq1g = -1;
 224 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 225 #endif
 226
 227 #ifdef DEV_NETMAP
 228 #define NNMTXQ_10G 2
 229 static int t4_nnmtxq10g = -1;
 230 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 231
 232 #define NNMRXQ_10G 2
 233 static int t4_nnmrxq10g = -1;
 234 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 235
 236 #define NNMTXQ_1G 1
 237 static int t4_nnmtxq1g = -1;
 238 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 239
 240 #define NNMRXQ_1G 1
 241 static int t4_nnmrxq1g = -1;
 242 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 243 #endif
 244
 245 /*
 246  * Holdoff parameters for 10G and 1G ports.
 247  */
 248 #define TMR_IDX_10G 1
 249 static int t4_tmr_idx_10g = TMR_IDX_10G;
 250 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 251
 252 #define PKTC_IDX_10G (-1)
 253 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 254 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 255
 256 #define TMR_IDX_1G 1
 257 static int t4_tmr_idx_1g = TMR_IDX_1G;
 258 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 259
 260 #define PKTC_IDX_1G (-1)
 261 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 262 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 263
 264 /*
 265  * Size (# of entries) of each tx and rx queue.
 266  */
 267 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 269
 270 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 271 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 272
 273 /*
 274  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 275  */
 276 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 277 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 278
 279 /*
 280  * Configuration file.
 281  */
 282 #define DEFAULT_CF      "default"
 283 #define FLASH_CF        "flash"
 284 #define UWIRE_CF        "uwire"
 285 #define FPGA_CF         "fpga"
 286 static char t4_cfg_file[32] = DEFAULT_CF;
 287 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 288
 289 /*
 290  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 291  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 292  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 293  *            mark or when signalled to do so, 0 to never emit PAUSE.
 294  */
 295 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 296 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 297
 298 /*
 299  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 300  * encouraged respectively).
 301  */
 302 static unsigned int t4_fw_install = 1;
 303 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 304
 305 /*
 306  * ASIC features that will be used.  Disable the ones you don't want so that the
 307  * chip resources aren't wasted on features that will not be used.
 308  */
 309 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 310 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 311
 312 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 313 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 314
 315 static int t4_toecaps_allowed = -1;
 316 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 317
 318 static int t4_rdmacaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 320
 321 static int t4_iscsicaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 323
 324 static int t4_fcoecaps_allowed = 0;
 325 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 326
 327 static int t5_write_combine = 0;
 328 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 329
 330 struct intrs_and_queues {
 331         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 332         uint16_t nirq;          /* Total # of vectors */
 333         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 334         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 335         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 336         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 337         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 338         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 339         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 340 #ifdef TCP_OFFLOAD
 341         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 342         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 343         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 344         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 345 #endif
 346 #ifdef DEV_NETMAP
 347         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 348         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 349         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 350         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 351 #endif
 352 };
 353
 354 struct filter_entry {
 355         uint32_t valid:1;       /* filter allocated and valid */
 356         uint32_t locked:1;      /* filter is administratively locked */
 357         uint32_t pending:1;     /* filter action is pending firmware reply */
 358         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 359         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 360
 361         struct t4_filter_specification fs;
 362 };
 363
 364 static int map_bars_0_and_4(struct adapter *);
 365 static int map_bar_2(struct adapter *);
 366 static void setup_memwin(struct adapter *);
 367 static int validate_mem_range(struct adapter *, uint32_t, int);
 368 static int fwmtype_to_hwmtype(int);
 369 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 370     uint32_t *);
 371 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 372 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 373 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 374     struct intrs_and_queues *);
 375 static int prep_firmware(struct adapter *);
 376 static int partition_resources(struct adapter *, const struct firmware *,
 377     const char *);
 378 static int get_params__pre_init(struct adapter *);
 379 static int get_params__post_init(struct adapter *);
 380 static int set_params__post_init(struct adapter *);
 381 static void t4_set_desc(struct adapter *);
 382 static void build_medialist(struct port_info *, struct ifmedia *);
 383 static int cxgbe_init_synchronized(struct port_info *);
 384 static int cxgbe_uninit_synchronized(struct port_info *);
 385 static int setup_intr_handlers(struct adapter *);
 386 static void quiesce_txq(struct adapter *, struct sge_txq *);
 387 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 388 static void quiesce_iq(struct adapter *, struct sge_iq *);
 389 static void quiesce_fl(struct adapter *, struct sge_fl *);
 390 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 391     driver_intr_t *, void *, char *);
 392 static int t4_free_irq(struct adapter *, struct irq *);
 393 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 394     unsigned int);
 395 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 396 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 397 static void cxgbe_tick(void *);
 398 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 399 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 400     struct mbuf *);
 401 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 402 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 403 static int t4_sysctls(struct adapter *);
 404 static int cxgbe_sysctls(struct port_info *);
 405 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 416 #ifdef SBUF_DRAIN
 417 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 439 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 440 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 441 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 442 #endif
 443 static uint32_t fconf_to_mode(uint32_t);
 444 static uint32_t mode_to_fconf(uint32_t);
 445 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 446 static int get_filter_mode(struct adapter *, uint32_t *);
 447 static int set_filter_mode(struct adapter *, uint32_t);
 448 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 449 static int get_filter(struct adapter *, struct t4_filter *);
 450 static int set_filter(struct adapter *, struct t4_filter *);
 451 static int del_filter(struct adapter *, struct t4_filter *);
 452 static void clear_filter(struct filter_entry *);
 453 static int set_filter_wr(struct adapter *, int);
 454 static int del_filter_wr(struct adapter *, int);
 455 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 456 static int load_fw(struct adapter *, struct t4_data *);
 457 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 458 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 459 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 460 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 461 #ifdef TCP_OFFLOAD
 462 static int toe_capability(struct port_info *, int);
 463 #endif
 464 static int mod_event(module_t, int, void *);
 465
 466 struct {
 467         uint16_t device;
 468         char *desc;
 469 } t4_pciids[] = {
 470         {0xa000, "Chelsio Terminator 4 FPGA"},
 471         {0x4400, "Chelsio T440-dbg"},
 472         {0x4401, "Chelsio T420-CR"},
 473         {0x4402, "Chelsio T422-CR"},
 474         {0x4403, "Chelsio T440-CR"},
 475         {0x4404, "Chelsio T420-BCH"},
 476         {0x4405, "Chelsio T440-BCH"},
 477         {0x4406, "Chelsio T440-CH"},
 478         {0x4407, "Chelsio T420-SO"},
 479         {0x4408, "Chelsio T420-CX"},
 480         {0x4409, "Chelsio T420-BT"},
 481         {0x440a, "Chelsio T404-BT"},
 482         {0x440e, "Chelsio T440-LP-CR"},
 483 }, t5_pciids[] = {
 484         {0xb000, "Chelsio Terminator 5 FPGA"},
 485         {0x5400, "Chelsio T580-dbg"},
 486         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 487         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 488         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 489         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 490         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 491         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 492         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 493         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 494         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 495         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 496         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 497         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 498         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 499 #ifdef notyet
 500         {0x5404,  "Chelsio T520-BCH"},
 501         {0x5405,  "Chelsio T540-BCH"},
 502         {0x5406,  "Chelsio T540-CH"},
 503         {0x5408,  "Chelsio T520-CX"},
 504         {0x540b,  "Chelsio B520-SR"},
 505         {0x540c,  "Chelsio B504-BT"},
 506         {0x540f,  "Chelsio Amsterdam"},
 507         {0x5413,  "Chelsio T580-CHR"},
 508 #endif
 509 };
 510
 511 #ifdef TCP_OFFLOAD
 512 /*
 513  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 514  * exactly the same for both rxq and ofld_rxq.
 515  */
 516 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 517 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 518 #endif
 519
 520 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 521 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 522 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 523
 524 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 525
 526 static int
 527 t4_probe(device_t dev)
 528 {
 529         int i;
 530         uint16_t v = pci_get_vendor(dev);
 531         uint16_t d = pci_get_device(dev);
 532         uint8_t f = pci_get_function(dev);
 533
 534         if (v != PCI_VENDOR_ID_CHELSIO)
 535                 return (ENXIO);
 536
 537         /* Attach only to PF0 of the FPGA */
 538         if (d == 0xa000 && f != 0)
 539                 return (ENXIO);
 540
 541         for (i = 0; i < nitems(t4_pciids); i++) {
 542                 if (d == t4_pciids[i].device) {
 543                         device_set_desc(dev, t4_pciids[i].desc);
 544                         return (BUS_PROBE_DEFAULT);
 545                 }
 546         }
 547
 548         return (ENXIO);
 549 }
 550
 551 static int
 552 t5_probe(device_t dev)
 553 {
 554         int i;
 555         uint16_t v = pci_get_vendor(dev);
 556         uint16_t d = pci_get_device(dev);
 557         uint8_t f = pci_get_function(dev);
 558
 559         if (v != PCI_VENDOR_ID_CHELSIO)
 560                 return (ENXIO);
 561
 562         /* Attach only to PF0 of the FPGA */
 563         if (d == 0xb000 && f != 0)
 564                 return (ENXIO);
 565
 566         for (i = 0; i < nitems(t5_pciids); i++) {
 567                 if (d == t5_pciids[i].device) {
 568                         device_set_desc(dev, t5_pciids[i].desc);
 569                         return (BUS_PROBE_DEFAULT);
 570                 }
 571         }
 572
 573         return (ENXIO);
 574 }
 575
 576 static void
 577 t5_attribute_workaround(device_t dev)
 578 {
 579         device_t root_port;
 580         uint32_t v;
 581
 582         /*
 583          * The T5 chips do not properly echo the No Snoop and Relaxed
 584          * Ordering attributes when replying to a TLP from a Root
 585          * Port.  As a workaround, find the parent Root Port and
 586          * disable No Snoop and Relaxed Ordering.  Note that this
 587          * affects all devices under this root port.
 588          */
 589         root_port = pci_find_pcie_root_port(dev);
 590         if (root_port == NULL) {
 591                 device_printf(dev, "Unable to find parent root port\n");
 592                 return;
 593         }
 594
 595         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 596             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 597         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 598             0)
 599                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 600                     device_get_nameunit(root_port));
 601 }
 602
 603 static int
 604 t4_attach(device_t dev)
 605 {
 606         struct adapter *sc;
 607         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 608         struct intrs_and_queues iaq;
 609         struct sge *s;
 610 #ifdef TCP_OFFLOAD
 611         int ofld_rqidx, ofld_tqidx;
 612 #endif
 613 #ifdef DEV_NETMAP
 614         int nm_rqidx, nm_tqidx;
 615 #endif
 616
 617         sc = device_get_softc(dev);
 618         sc->dev = dev;
 619         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 620
 621         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 622                 t5_attribute_workaround(dev);
 623         pci_enable_busmaster(dev);
 624         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 625                 uint32_t v;
 626
 627                 pci_set_max_read_req(dev, 4096);
 628                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 629                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 630                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 631
 632                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 633         }
 634
 635         sc->traceq = -1;
 636         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 637         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 638             device_get_nameunit(dev));
 639
 640         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 641             device_get_nameunit(dev));
 642         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 643         sx_xlock(&t4_list_lock);
 644         SLIST_INSERT_HEAD(&t4_list, sc, link);
 645         sx_xunlock(&t4_list_lock);
 646
 647         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 648         TAILQ_INIT(&sc->sfl);
 649         callout_init(&sc->sfl_callout, 1);
 650
 651         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 652
 653         rc = map_bars_0_and_4(sc);
 654         if (rc != 0)
 655                 goto done; /* error message displayed already */
 656
 657         /*
 658          * This is the real PF# to which we're attaching.  Works from within PCI
 659          * passthrough environments too, where pci_get_function() could return a
 660          * different PF# depending on the passthrough configuration.  We need to
 661          * use the real PF# in all our communication with the firmware.
 662          */
 663         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 664         sc->mbox = sc->pf;
 665
 666         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 667         sc->an_handler = an_not_handled;
 668         for (i = 0; i < nitems(sc->cpl_handler); i++)
 669                 sc->cpl_handler[i] = cpl_not_handled;
 670         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 671                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 672         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 673         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 674         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 675         t4_init_sge_cpl_handlers(sc);
 676
 677         /* Prepare the adapter for operation */
 678         rc = -t4_prep_adapter(sc);
 679         if (rc != 0) {
 680                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 681                 goto done;
 682         }
 683
 684         /*
 685          * Do this really early, with the memory windows set up even before the
 686          * character device.  The userland tool's register i/o and mem read
 687          * will work even in "recovery mode".
 688          */
 689         setup_memwin(sc);
 690         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 691             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 692             device_get_nameunit(dev));
 693         if (sc->cdev == NULL)
 694                 device_printf(dev, "failed to create nexus char device.\n");
 695         else
 696                 sc->cdev->si_drv1 = sc;
 697
 698         /* Go no further if recovery mode has been requested. */
 699         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 700                 device_printf(dev, "recovery mode.\n");
 701                 goto done;
 702         }
 703
 704 #if defined(__i386__)
 705         if ((cpu_feature & CPUID_CX8) == 0) {
 706                 device_printf(dev, "64 bit atomics not available.\n");
 707                 rc = ENOTSUP;
 708                 goto done;
 709         }
 710 #endif
 711
 712         /* Prepare the firmware for operation */
 713         rc = prep_firmware(sc);
 714         if (rc != 0)
 715                 goto done; /* error message displayed already */
 716
 717         rc = get_params__post_init(sc);
 718         if (rc != 0)
 719                 goto done; /* error message displayed already */
 720
 721         rc = set_params__post_init(sc);
 722         if (rc != 0)
 723                 goto done; /* error message displayed already */
 724
 725         rc = map_bar_2(sc);
 726         if (rc != 0)
 727                 goto done; /* error message displayed already */
 728
 729         rc = t4_create_dma_tag(sc);
 730         if (rc != 0)
 731                 goto done; /* error message displayed already */
 732
 733         /*
 734          * First pass over all the ports - allocate VIs and initialize some
 735          * basic parameters like mac address, port type, etc.  We also figure
 736          * out whether a port is 10G or 1G and use that information when
 737          * calculating how many interrupts to attempt to allocate.
 738          */
 739         n10g = n1g = 0;
 740         for_each_port(sc, i) {
 741                 struct port_info *pi;
 742
 743                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 744                 sc->port[i] = pi;
 745
 746                 /* These must be set before t4_port_init */
 747                 pi->adapter = sc;
 748                 pi->port_id = i;
 749
 750                 /* Allocate the vi and initialize parameters like mac addr */
 751                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 752                 if (rc != 0) {
 753                         device_printf(dev, "unable to initialize port %d: %d\n",
 754                             i, rc);
 755                         free(pi, M_CXGBE);
 756                         sc->port[i] = NULL;
 757                         goto done;
 758                 }
 759
 760                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 761                 pi->link_cfg.requested_fc |= t4_pause_settings;
 762                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 763                 pi->link_cfg.fc |= t4_pause_settings;
 764
 765                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 766                 if (rc != 0) {
 767                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 768                         free(pi, M_CXGBE);
 769                         sc->port[i] = NULL;
 770                         goto done;
 771                 }
 772
 773                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 774                     device_get_nameunit(dev), i);
 775                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 776                 sc->chan_map[pi->tx_chan] = i;
 777
 778                 if (is_10G_port(pi) || is_40G_port(pi)) {
 779                         n10g++;
 780                         pi->tmr_idx = t4_tmr_idx_10g;
 781                         pi->pktc_idx = t4_pktc_idx_10g;
 782                 } else {
 783                         n1g++;
 784                         pi->tmr_idx = t4_tmr_idx_1g;
 785                         pi->pktc_idx = t4_pktc_idx_1g;
 786                 }
 787
 788                 pi->xact_addr_filt = -1;
 789                 pi->linkdnrc = -1;
 790
 791                 pi->qsize_rxq = t4_qsize_rxq;
 792                 pi->qsize_txq = t4_qsize_txq;
 793
 794                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 795                 if (pi->dev == NULL) {
 796                         device_printf(dev,
 797                             "failed to add device for port %d.\n", i);
 798                         rc = ENXIO;
 799                         goto done;
 800                 }
 801                 device_set_softc(pi->dev, pi);
 802         }
 803
 804         /*
 805          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 806          */
 807         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 808         if (rc != 0)
 809                 goto done; /* error message displayed already */
 810
 811         sc->intr_type = iaq.intr_type;
 812         sc->intr_count = iaq.nirq;
 813
 814         s = &sc->sge;
 815         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 816         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 817         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 818         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 819         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 820 #ifdef TCP_OFFLOAD
 821         if (is_offload(sc)) {
 822                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 823                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 824                 s->neq += s->nofldtxq + s->nofldrxq;
 825                 s->niq += s->nofldrxq;
 826
 827                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 828                     M_CXGBE, M_ZERO | M_WAITOK);
 829                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 830                     M_CXGBE, M_ZERO | M_WAITOK);
 831         }
 832 #endif
 833 #ifdef DEV_NETMAP
 834         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 835         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 836         s->neq += s->nnmtxq + s->nnmrxq;
 837         s->niq += s->nnmrxq;
 838
 839         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 840             M_CXGBE, M_ZERO | M_WAITOK);
 841         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 842             M_CXGBE, M_ZERO | M_WAITOK);
 843 #endif
 844
 845         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 846             M_ZERO | M_WAITOK);
 847         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 848             M_ZERO | M_WAITOK);
 849         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 850             M_ZERO | M_WAITOK);
 851         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 852             M_ZERO | M_WAITOK);
 853         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 854             M_ZERO | M_WAITOK);
 855
 856         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 857             M_ZERO | M_WAITOK);
 858
 859         t4_init_l2t(sc, M_WAITOK);
 860
 861         /*
 862          * Second pass over the ports.  This time we know the number of rx and
 863          * tx queues that each port should get.
 864          */
 865         rqidx = tqidx = 0;
 866 #ifdef TCP_OFFLOAD
 867         ofld_rqidx = ofld_tqidx = 0;
 868 #endif
 869 #ifdef DEV_NETMAP
 870         nm_rqidx = nm_tqidx = 0;
 871 #endif
 872         for_each_port(sc, i) {
 873                 struct port_info *pi = sc->port[i];
 874
 875                 if (pi == NULL)
 876                         continue;
 877
 878                 pi->first_rxq = rqidx;
 879                 pi->first_txq = tqidx;
 880                 if (is_10G_port(pi) || is_40G_port(pi)) {
 881                         pi->flags |= iaq.intr_flags_10g;
 882                         pi->nrxq = iaq.nrxq10g;
 883                         pi->ntxq = iaq.ntxq10g;
 884                 } else {
 885                         pi->flags |= iaq.intr_flags_1g;
 886                         pi->nrxq = iaq.nrxq1g;
 887                         pi->ntxq = iaq.ntxq1g;
 888                 }
 889
 890                 if (pi->ntxq > 1)
 891                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 892                 else
 893                         pi->rsrv_noflowq = 0;
 894
 895                 rqidx += pi->nrxq;
 896                 tqidx += pi->ntxq;
 897 #ifdef TCP_OFFLOAD
 898                 if (is_offload(sc)) {
 899                         pi->first_ofld_rxq = ofld_rqidx;
 900                         pi->first_ofld_txq = ofld_tqidx;
 901                         if (is_10G_port(pi) || is_40G_port(pi)) {
 902                                 pi->nofldrxq = iaq.nofldrxq10g;
 903                                 pi->nofldtxq = iaq.nofldtxq10g;
 904                         } else {
 905                                 pi->nofldrxq = iaq.nofldrxq1g;
 906                                 pi->nofldtxq = iaq.nofldtxq1g;
 907                         }
 908                         ofld_rqidx += pi->nofldrxq;
 909                         ofld_tqidx += pi->nofldtxq;
 910                 }
 911 #endif
 912 #ifdef DEV_NETMAP
 913                 pi->first_nm_rxq = nm_rqidx;
 914                 pi->first_nm_txq = nm_tqidx;
 915                 if (is_10G_port(pi) || is_40G_port(pi)) {
 916                         pi->nnmrxq = iaq.nnmrxq10g;
 917                         pi->nnmtxq = iaq.nnmtxq10g;
 918                 } else {
 919                         pi->nnmrxq = iaq.nnmrxq1g;
 920                         pi->nnmtxq = iaq.nnmtxq1g;
 921                 }
 922                 nm_rqidx += pi->nnmrxq;
 923                 nm_tqidx += pi->nnmtxq;
 924 #endif
 925         }
 926
 927         rc = setup_intr_handlers(sc);
 928         if (rc != 0) {
 929                 device_printf(dev,
 930                     "failed to setup interrupt handlers: %d\n", rc);
 931                 goto done;
 932         }
 933
 934         rc = bus_generic_attach(dev);
 935         if (rc != 0) {
 936                 device_printf(dev,
 937                     "failed to attach all child ports: %d\n", rc);
 938                 goto done;
 939         }
 940
 941         device_printf(dev,
 942             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 943             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 944             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 945             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 946             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 947
 948         t4_set_desc(sc);
 949
 950 done:
 951         if (rc != 0 && sc->cdev) {
 952                 /* cdev was created and so cxgbetool works; recover that way. */
 953                 device_printf(dev,
 954                     "error during attach, adapter is now in recovery mode.\n");
 955                 rc = 0;
 956         }
 957
 958         if (rc != 0)
 959                 t4_detach(dev);
 960         else
 961                 t4_sysctls(sc);
 962
 963         return (rc);
 964 }
 965
 966 /*
 967  * Idempotent
 968  */
 969 static int
 970 t4_detach(device_t dev)
 971 {
 972         struct adapter *sc;
 973         struct port_info *pi;
 974         int i, rc;
 975
 976         sc = device_get_softc(dev);
 977
 978         if (sc->flags & FULL_INIT_DONE)
 979                 t4_intr_disable(sc);
 980
 981         if (sc->cdev) {
 982                 destroy_dev(sc->cdev);
 983                 sc->cdev = NULL;
 984         }
 985
 986         rc = bus_generic_detach(dev);
 987         if (rc) {
 988                 device_printf(dev,
 989                     "failed to detach child devices: %d\n", rc);
 990                 return (rc);
 991         }
 992
 993         for (i = 0; i < sc->intr_count; i++)
 994                 t4_free_irq(sc, &sc->irq[i]);
 995
 996         for (i = 0; i < MAX_NPORTS; i++) {
 997                 pi = sc->port[i];
 998                 if (pi) {
 999                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
1000                         if (pi->dev)
1001                                 device_delete_child(dev, pi->dev);
1002
1003                         mtx_destroy(&pi->pi_lock);
1004                         free(pi, M_CXGBE);
1005                 }
1006         }
1007
1008         if (sc->flags & FULL_INIT_DONE)
1009                 adapter_full_uninit(sc);
1010
1011         if (sc->flags & FW_OK)
1012                 t4_fw_bye(sc, sc->mbox);
1013
1014         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1015                 pci_release_msi(dev);
1016
1017         if (sc->regs_res)
1018                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1019                     sc->regs_res);
1020
1021         if (sc->udbs_res)
1022                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1023                     sc->udbs_res);
1024
1025         if (sc->msix_res)
1026                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1027                     sc->msix_res);
1028
1029         if (sc->l2t)
1030                 t4_free_l2t(sc->l2t);
1031
1032 #ifdef TCP_OFFLOAD
1033         free(sc->sge.ofld_rxq, M_CXGBE);
1034         free(sc->sge.ofld_txq, M_CXGBE);
1035 #endif
1036 #ifdef DEV_NETMAP
1037         free(sc->sge.nm_rxq, M_CXGBE);
1038         free(sc->sge.nm_txq, M_CXGBE);
1039 #endif
1040         free(sc->irq, M_CXGBE);
1041         free(sc->sge.rxq, M_CXGBE);
1042         free(sc->sge.txq, M_CXGBE);
1043         free(sc->sge.ctrlq, M_CXGBE);
1044         free(sc->sge.iqmap, M_CXGBE);
1045         free(sc->sge.eqmap, M_CXGBE);
1046         free(sc->tids.ftid_tab, M_CXGBE);
1047         t4_destroy_dma_tag(sc);
1048         if (mtx_initialized(&sc->sc_lock)) {
1049                 sx_xlock(&t4_list_lock);
1050                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1051                 sx_xunlock(&t4_list_lock);
1052                 mtx_destroy(&sc->sc_lock);
1053         }
1054
1055         if (mtx_initialized(&sc->tids.ftid_lock))
1056                 mtx_destroy(&sc->tids.ftid_lock);
1057         if (mtx_initialized(&sc->sfl_lock))
1058                 mtx_destroy(&sc->sfl_lock);
1059         if (mtx_initialized(&sc->ifp_lock))
1060                 mtx_destroy(&sc->ifp_lock);
1061         if (mtx_initialized(&sc->regwin_lock))
1062                 mtx_destroy(&sc->regwin_lock);
1063
1064         bzero(sc, sizeof(*sc));
1065
1066         return (0);
1067 }
1068
1069 static int
1070 cxgbe_probe(device_t dev)
1071 {
1072         char buf[128];
1073         struct port_info *pi = device_get_softc(dev);
1074
1075         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1076         device_set_desc_copy(dev, buf);
1077
1078         return (BUS_PROBE_DEFAULT);
1079 }
1080
1081 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1082     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1083     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1084 #define T4_CAP_ENABLE (T4_CAP)
1085
1086 static int
1087 cxgbe_attach(device_t dev)
1088 {
1089         struct port_info *pi = device_get_softc(dev);
1090         struct ifnet *ifp;
1091         char *s;
1092         int n, o;
1093
1094         /* Allocate an ifnet and set it up */
1095         ifp = if_alloc(IFT_ETHER);
1096         if (ifp == NULL) {
1097                 device_printf(dev, "Cannot allocate ifnet\n");
1098                 return (ENOMEM);
1099         }
1100         pi->ifp = ifp;
1101         ifp->if_softc = pi;
1102
1103         callout_init(&pi->tick, 1);
1104
1105         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1106         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1107
1108         ifp->if_init = cxgbe_init;
1109         ifp->if_ioctl = cxgbe_ioctl;
1110         ifp->if_transmit = cxgbe_transmit;
1111         ifp->if_qflush = cxgbe_qflush;
1112         ifp->if_get_counter = cxgbe_get_counter;
1113
1114         ifp->if_capabilities = T4_CAP;
1115 #ifdef TCP_OFFLOAD
1116         if (is_offload(pi->adapter))
1117                 ifp->if_capabilities |= IFCAP_TOE;
1118 #endif
1119         ifp->if_capenable = T4_CAP_ENABLE;
1120         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1121             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1122
1123         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1124         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1125         ifp->if_hw_tsomaxsegsize = 65536;
1126
1127         /* Initialize ifmedia for this port */
1128         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1129             cxgbe_media_status);
1130         build_medialist(pi, &pi->media);
1131
1132         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1133             EVENTHANDLER_PRI_ANY);
1134
1135         ether_ifattach(ifp, pi->hw_addr);
1136
1137         n = 128;
1138         s = malloc(n, M_CXGBE, M_WAITOK);
1139         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1140         MPASS(n > o);
1141 #ifdef TCP_OFFLOAD
1142         if (is_offload(pi->adapter)) {
1143                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1144                     pi->nofldtxq, pi->nofldrxq);
1145                 MPASS(n > o);
1146         }
1147 #endif
1148 #ifdef DEV_NETMAP
1149         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1150             pi->nnmrxq);
1151         MPASS(n > o);
1152 #endif
1153         device_printf(dev, "%s\n", s);
1154         free(s, M_CXGBE);
1155
1156 #ifdef DEV_NETMAP
1157         /* nm_media handled here to keep implementation private to this file */
1158         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1159             cxgbe_media_status);
1160         build_medialist(pi, &pi->nm_media);
1161         create_netmap_ifnet(pi);        /* logs errors it something fails */
1162 #endif
1163         cxgbe_sysctls(pi);
1164
1165         return (0);
1166 }
1167
1168 static int
1169 cxgbe_detach(device_t dev)
1170 {
1171         struct port_info *pi = device_get_softc(dev);
1172         struct adapter *sc = pi->adapter;
1173         struct ifnet *ifp = pi->ifp;
1174
1175         /* Tell if_ioctl and if_init that the port is going away */
1176         ADAPTER_LOCK(sc);
1177         SET_DOOMED(pi);
1178         wakeup(&sc->flags);
1179         while (IS_BUSY(sc))
1180                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1181         SET_BUSY(sc);
1182 #ifdef INVARIANTS
1183         sc->last_op = "t4detach";
1184         sc->last_op_thr = curthread;
1185         sc->last_op_flags = 0;
1186 #endif
1187         ADAPTER_UNLOCK(sc);
1188
1189         if (pi->flags & HAS_TRACEQ) {
1190                 sc->traceq = -1;        /* cloner should not create ifnet */
1191                 t4_tracer_port_detach(sc);
1192         }
1193
1194         if (pi->vlan_c)
1195                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1196
1197         PORT_LOCK(pi);
1198         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1199         callout_stop(&pi->tick);
1200         PORT_UNLOCK(pi);
1201         callout_drain(&pi->tick);
1202
1203         /* Let detach proceed even if these fail. */
1204         cxgbe_uninit_synchronized(pi);
1205         port_full_uninit(pi);
1206
1207         ifmedia_removeall(&pi->media);
1208         ether_ifdetach(pi->ifp);
1209         if_free(pi->ifp);
1210
1211 #ifdef DEV_NETMAP
1212         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1213         destroy_netmap_ifnet(pi);
1214 #endif
1215
1216         ADAPTER_LOCK(sc);
1217         CLR_BUSY(sc);
1218         wakeup(&sc->flags);
1219         ADAPTER_UNLOCK(sc);
1220
1221         return (0);
1222 }
1223
1224 static void
1225 cxgbe_init(void *arg)
1226 {
1227         struct port_info *pi = arg;
1228         struct adapter *sc = pi->adapter;
1229
1230         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1231                 return;
1232         cxgbe_init_synchronized(pi);
1233         end_synchronized_op(sc, 0);
1234 }
1235
1236 static int
1237 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1238 {
1239         int rc = 0, mtu, flags, can_sleep;
1240         struct port_info *pi = ifp->if_softc;
1241         struct adapter *sc = pi->adapter;
1242         struct ifreq *ifr = (struct ifreq *)data;
1243         uint32_t mask;
1244
1245         switch (cmd) {
1246         case SIOCSIFMTU:
1247                 mtu = ifr->ifr_mtu;
1248                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1249                         return (EINVAL);
1250
1251                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1252                 if (rc)
1253                         return (rc);
1254                 ifp->if_mtu = mtu;
1255                 if (pi->flags & PORT_INIT_DONE) {
1256                         t4_update_fl_bufsize(ifp);
1257                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1258                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1259                 }
1260                 end_synchronized_op(sc, 0);
1261                 break;
1262
1263         case SIOCSIFFLAGS:
1264                 can_sleep = 0;
1265 redo_sifflags:
1266                 rc = begin_synchronized_op(sc, pi,
1267                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1268                 if (rc)
1269                         return (rc);
1270
1271                 if (ifp->if_flags & IFF_UP) {
1272                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1273                                 flags = pi->if_flags;
1274                                 if ((ifp->if_flags ^ flags) &
1275                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1276                                         if (can_sleep == 1) {
1277                                                 end_synchronized_op(sc, 0);
1278                                                 can_sleep = 0;
1279                                                 goto redo_sifflags;
1280                                         }
1281                                         rc = update_mac_settings(ifp,
1282                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1283                                 }
1284                         } else {
1285                                 if (can_sleep == 0) {
1286                                         end_synchronized_op(sc, LOCK_HELD);
1287                                         can_sleep = 1;
1288                                         goto redo_sifflags;
1289                                 }
1290                                 rc = cxgbe_init_synchronized(pi);
1291                         }
1292                         pi->if_flags = ifp->if_flags;
1293                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1294                         if (can_sleep == 0) {
1295                                 end_synchronized_op(sc, LOCK_HELD);
1296                                 can_sleep = 1;
1297                                 goto redo_sifflags;
1298                         }
1299                         rc = cxgbe_uninit_synchronized(pi);
1300                 }
1301                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1302                 break;
1303
1304         case SIOCADDMULTI:
1305         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1306                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1307                 if (rc)
1308                         return (rc);
1309                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1310                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1311                 end_synchronized_op(sc, LOCK_HELD);
1312                 break;
1313
1314         case SIOCSIFCAP:
1315                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1316                 if (rc)
1317                         return (rc);
1318
1319                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1320                 if (mask & IFCAP_TXCSUM) {
1321                         ifp->if_capenable ^= IFCAP_TXCSUM;
1322                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1323
1324                         if (IFCAP_TSO4 & ifp->if_capenable &&
1325                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1326                                 ifp->if_capenable &= ~IFCAP_TSO4;
1327                                 if_printf(ifp,
1328                                     "tso4 disabled due to -txcsum.\n");
1329                         }
1330                 }
1331                 if (mask & IFCAP_TXCSUM_IPV6) {
1332                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1333                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1334
1335                         if (IFCAP_TSO6 & ifp->if_capenable &&
1336                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1337                                 ifp->if_capenable &= ~IFCAP_TSO6;
1338                                 if_printf(ifp,
1339                                     "tso6 disabled due to -txcsum6.\n");
1340                         }
1341                 }
1342                 if (mask & IFCAP_RXCSUM)
1343                         ifp->if_capenable ^= IFCAP_RXCSUM;
1344                 if (mask & IFCAP_RXCSUM_IPV6)
1345                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1346
1347                 /*
1348                  * Note that we leave CSUM_TSO alone (it is always set).  The
1349                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1350                  * sending a TSO request our way, so it's sufficient to toggle
1351                  * IFCAP_TSOx only.
1352                  */
1353                 if (mask & IFCAP_TSO4) {
1354                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1355                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1356                                 if_printf(ifp, "enable txcsum first.\n");
1357                                 rc = EAGAIN;
1358                                 goto fail;
1359                         }
1360                         ifp->if_capenable ^= IFCAP_TSO4;
1361                 }
1362                 if (mask & IFCAP_TSO6) {
1363                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1364                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1365                                 if_printf(ifp, "enable txcsum6 first.\n");
1366                                 rc = EAGAIN;
1367                                 goto fail;
1368                         }
1369                         ifp->if_capenable ^= IFCAP_TSO6;
1370                 }
1371                 if (mask & IFCAP_LRO) {
1372 #if defined(INET) || defined(INET6)
1373                         int i;
1374                         struct sge_rxq *rxq;
1375
1376                         ifp->if_capenable ^= IFCAP_LRO;
1377                         for_each_rxq(pi, i, rxq) {
1378                                 if (ifp->if_capenable & IFCAP_LRO)
1379                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1380                                 else
1381                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1382                         }
1383 #endif
1384                 }
1385 #ifdef TCP_OFFLOAD
1386                 if (mask & IFCAP_TOE) {
1387                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1388
1389                         rc = toe_capability(pi, enable);
1390                         if (rc != 0)
1391                                 goto fail;
1392
1393                         ifp->if_capenable ^= mask;
1394                 }
1395 #endif
1396                 if (mask & IFCAP_VLAN_HWTAGGING) {
1397                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1398                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1399                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1400                 }
1401                 if (mask & IFCAP_VLAN_MTU) {
1402                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1403
1404                         /* Need to find out how to disable auto-mtu-inflation */
1405                 }
1406                 if (mask & IFCAP_VLAN_HWTSO)
1407                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1408                 if (mask & IFCAP_VLAN_HWCSUM)
1409                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1410
1411 #ifdef VLAN_CAPABILITIES
1412                 VLAN_CAPABILITIES(ifp);
1413 #endif
1414 fail:
1415                 end_synchronized_op(sc, 0);
1416                 break;
1417
1418         case SIOCSIFMEDIA:
1419         case SIOCGIFMEDIA:
1420                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1421                 break;
1422
1423         case SIOCGI2C: {
1424                 struct ifi2creq i2c;
1425
1426                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1427                 if (rc != 0)
1428                         break;
1429                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1430                         rc = EPERM;
1431                         break;
1432                 }
1433                 if (i2c.len > sizeof(i2c.data)) {
1434                         rc = EINVAL;
1435                         break;
1436                 }
1437                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1438                 if (rc)
1439                         return (rc);
1440                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1441                     i2c.offset, i2c.len, &i2c.data[0]);
1442                 end_synchronized_op(sc, 0);
1443                 if (rc == 0)
1444                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1445                 break;
1446         }
1447
1448         default:
1449                 rc = ether_ioctl(ifp, cmd, data);
1450         }
1451
1452         return (rc);
1453 }
1454
1455 static int
1456 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1457 {
1458         struct port_info *pi = ifp->if_softc;
1459         struct adapter *sc = pi->adapter;
1460         struct sge_txq *txq;
1461         void *items[1];
1462         int rc;
1463
1464         M_ASSERTPKTHDR(m);
1465         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1466
1467         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1468                 m_freem(m);
1469                 return (ENETDOWN);
1470         }
1471
1472         rc = parse_pkt(&m);
1473         if (__predict_false(rc != 0)) {
1474                 MPASS(m == NULL);                       /* was freed already */
1475                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1476                 return (rc);
1477         }
1478
1479         /* Select a txq. */
1480         txq = &sc->sge.txq[pi->first_txq];
1481         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1482                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1483                     pi->rsrv_noflowq);
1484
1485         items[0] = m;
1486         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1487         if (__predict_false(rc != 0))
1488                 m_freem(m);
1489
1490         return (rc);
1491 }
1492
1493 static void
1494 cxgbe_qflush(struct ifnet *ifp)
1495 {
1496         struct port_info *pi = ifp->if_softc;
1497         struct sge_txq *txq;
1498         int i;
1499
1500         /* queues do not exist if !PORT_INIT_DONE. */
1501         if (pi->flags & PORT_INIT_DONE) {
1502                 for_each_txq(pi, i, txq) {
1503                         TXQ_LOCK(txq);
1504                         txq->eq.flags &= ~EQ_ENABLED;
1505                         TXQ_UNLOCK(txq);
1506                         while (!mp_ring_is_idle(txq->r)) {
1507                                 mp_ring_check_drainage(txq->r, 0);
1508                                 pause("qflush", 1);
1509                         }
1510                 }
1511         }
1512         if_qflush(ifp);
1513 }
1514
1515 static uint64_t
1516 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1517 {
1518         struct port_info *pi = ifp->if_softc;
1519         struct adapter *sc = pi->adapter;
1520         struct port_stats *s = &pi->stats;
1521
1522         cxgbe_refresh_stats(sc, pi);
1523
1524         switch (c) {
1525         case IFCOUNTER_IPACKETS:
1526                 return (s->rx_frames - s->rx_pause);
1527
1528         case IFCOUNTER_IERRORS:
1529                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1530                     s->rx_fcs_err + s->rx_len_err);
1531
1532         case IFCOUNTER_OPACKETS:
1533                 return (s->tx_frames - s->tx_pause);
1534
1535         case IFCOUNTER_OERRORS:
1536                 return (s->tx_error_frames);
1537
1538         case IFCOUNTER_IBYTES:
1539                 return (s->rx_octets - s->rx_pause * 64);
1540
1541         case IFCOUNTER_OBYTES:
1542                 return (s->tx_octets - s->tx_pause * 64);
1543
1544         case IFCOUNTER_IMCASTS:
1545                 return (s->rx_mcast_frames - s->rx_pause);
1546
1547         case IFCOUNTER_OMCASTS:
1548                 return (s->tx_mcast_frames - s->tx_pause);
1549
1550         case IFCOUNTER_IQDROPS:
1551                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1552                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1553                     s->rx_trunc3 + pi->tnl_cong_drops);
1554
1555         case IFCOUNTER_OQDROPS: {
1556                 uint64_t drops;
1557
1558                 drops = s->tx_drop;
1559                 if (pi->flags & PORT_INIT_DONE) {
1560                         int i;
1561                         struct sge_txq *txq;
1562
1563                         for_each_txq(pi, i, txq)
1564                                 drops += counter_u64_fetch(txq->r->drops);
1565                 }
1566
1567                 return (drops);
1568
1569         }
1570
1571         default:
1572                 return (if_get_counter_default(ifp, c));
1573         }
1574 }
1575
1576 static int
1577 cxgbe_media_change(struct ifnet *ifp)
1578 {
1579         struct port_info *pi = ifp->if_softc;
1580
1581         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1582
1583         return (EOPNOTSUPP);
1584 }
1585
1586 static void
1587 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1588 {
1589         struct port_info *pi = ifp->if_softc;
1590         struct ifmedia *media = NULL;
1591         struct ifmedia_entry *cur;
1592         int speed = pi->link_cfg.speed;
1593
1594         if (ifp == pi->ifp)
1595                 media = &pi->media;
1596 #ifdef DEV_NETMAP
1597         else if (ifp == pi->nm_ifp)
1598                 media = &pi->nm_media;
1599 #endif
1600         MPASS(media != NULL);
1601
1602         cur = media->ifm_cur;
1603
1604         ifmr->ifm_status = IFM_AVALID;
1605         if (!pi->link_cfg.link_ok)
1606                 return;
1607
1608         ifmr->ifm_status |= IFM_ACTIVE;
1609
1610         /* active and current will differ iff current media is autoselect. */
1611         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1612                 return;
1613
1614         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1615         if (speed == SPEED_10000)
1616                 ifmr->ifm_active |= IFM_10G_T;
1617         else if (speed == SPEED_1000)
1618                 ifmr->ifm_active |= IFM_1000_T;
1619         else if (speed == SPEED_100)
1620                 ifmr->ifm_active |= IFM_100_TX;
1621         else if (speed == SPEED_10)
1622                 ifmr->ifm_active |= IFM_10_T;
1623         else
1624                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1625                             speed));
1626 }
1627
1628 void
1629 t4_fatal_err(struct adapter *sc)
1630 {
1631         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1632         t4_intr_disable(sc);
1633         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1634             device_get_nameunit(sc->dev));
1635 }
1636
1637 static int
1638 map_bars_0_and_4(struct adapter *sc)
1639 {
1640         sc->regs_rid = PCIR_BAR(0);
1641         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1642             &sc->regs_rid, RF_ACTIVE);
1643         if (sc->regs_res == NULL) {
1644                 device_printf(sc->dev, "cannot map registers.\n");
1645                 return (ENXIO);
1646         }
1647         sc->bt = rman_get_bustag(sc->regs_res);
1648         sc->bh = rman_get_bushandle(sc->regs_res);
1649         sc->mmio_len = rman_get_size(sc->regs_res);
1650         setbit(&sc->doorbells, DOORBELL_KDB);
1651
1652         sc->msix_rid = PCIR_BAR(4);
1653         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1654             &sc->msix_rid, RF_ACTIVE);
1655         if (sc->msix_res == NULL) {
1656                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1657                 return (ENXIO);
1658         }
1659
1660         return (0);
1661 }
1662
1663 static int
1664 map_bar_2(struct adapter *sc)
1665 {
1666
1667         /*
1668          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1669          * to map it if RDMA is disabled.
1670          */
1671         if (is_t4(sc) && sc->rdmacaps == 0)
1672                 return (0);
1673
1674         sc->udbs_rid = PCIR_BAR(2);
1675         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1676             &sc->udbs_rid, RF_ACTIVE);
1677         if (sc->udbs_res == NULL) {
1678                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1679                 return (ENXIO);
1680         }
1681         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1682
1683         if (is_t5(sc)) {
1684                 setbit(&sc->doorbells, DOORBELL_UDB);
1685 #if defined(__i386__) || defined(__amd64__)
1686                 if (t5_write_combine) {
1687                         int rc;
1688
1689                         /*
1690                          * Enable write combining on BAR2.  This is the
1691                          * userspace doorbell BAR and is split into 128B
1692                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1693                          * with an egress queue.  The first 64B has the doorbell
1694                          * and the second 64B can be used to submit a tx work
1695                          * request with an implicit doorbell.
1696                          */
1697
1698                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1699                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1700                         if (rc == 0) {
1701                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1702                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1703                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1704                         } else {
1705                                 device_printf(sc->dev,
1706                                     "couldn't enable write combining: %d\n",
1707                                     rc);
1708                         }
1709
1710                         t4_write_reg(sc, A_SGE_STAT_CFG,
1711                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1712                 }
1713 #endif
1714         }
1715
1716         return (0);
1717 }
1718
1719 static const struct memwin t4_memwin[] = {
1720         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1721         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1722         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1723 };
1724
1725 static const struct memwin t5_memwin[] = {
1726         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1727         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1728         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1729 };
1730
1731 static void
1732 setup_memwin(struct adapter *sc)
1733 {
1734         const struct memwin *mw;
1735         int i, n;
1736         uint32_t bar0;
1737
1738         if (is_t4(sc)) {
1739                 /*
1740                  * Read low 32b of bar0 indirectly via the hardware backdoor
1741                  * mechanism.  Works from within PCI passthrough environments
1742                  * too, where rman_get_start() can return a different value.  We
1743                  * need to program the T4 memory window decoders with the actual
1744                  * addresses that will be coming across the PCIe link.
1745                  */
1746                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1747                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1748
1749                 mw = &t4_memwin[0];
1750                 n = nitems(t4_memwin);
1751         } else {
1752                 /* T5 uses the relative offset inside the PCIe BAR */
1753                 bar0 = 0;
1754
1755                 mw = &t5_memwin[0];
1756                 n = nitems(t5_memwin);
1757         }
1758
1759         for (i = 0; i < n; i++, mw++) {
1760                 t4_write_reg(sc,
1761                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1762                     (mw->base + bar0) | V_BIR(0) |
1763                     V_WINDOW(ilog2(mw->aperture) - 10));
1764         }
1765
1766         /* flush */
1767         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1768 }
1769
1770 /*
1771  * Verify that the memory range specified by the addr/len pair is valid and lies
1772  * entirely within a single region (EDCx or MCx).
1773  */
1774 static int
1775 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1776 {
1777         uint32_t em, addr_len, maddr, mlen;
1778
1779         /* Memory can only be accessed in naturally aligned 4 byte units */
1780         if (addr & 3 || len & 3 || len == 0)
1781                 return (EINVAL);
1782
1783         /* Enabled memories */
1784         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1785         if (em & F_EDRAM0_ENABLE) {
1786                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1787                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1788                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1789                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1790                     addr + len <= maddr + mlen)
1791                         return (0);
1792         }
1793         if (em & F_EDRAM1_ENABLE) {
1794                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1795                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1796                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1797                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1798                     addr + len <= maddr + mlen)
1799                         return (0);
1800         }
1801         if (em & F_EXT_MEM_ENABLE) {
1802                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1803                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1804                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1805                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1806                     addr + len <= maddr + mlen)
1807                         return (0);
1808         }
1809         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1810                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1811                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1812                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1813                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1814                     addr + len <= maddr + mlen)
1815                         return (0);
1816         }
1817
1818         return (EFAULT);
1819 }
1820
1821 static int
1822 fwmtype_to_hwmtype(int mtype)
1823 {
1824
1825         switch (mtype) {
1826         case FW_MEMTYPE_EDC0:
1827                 return (MEM_EDC0);
1828         case FW_MEMTYPE_EDC1:
1829                 return (MEM_EDC1);
1830         case FW_MEMTYPE_EXTMEM:
1831                 return (MEM_MC0);
1832         case FW_MEMTYPE_EXTMEM1:
1833                 return (MEM_MC1);
1834         default:
1835                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1836         }
1837 }
1838
1839 /*
1840  * Verify that the memory range specified by the memtype/offset/len pair is
1841  * valid and lies entirely within the memtype specified.  The global address of
1842  * the start of the range is returned in addr.
1843  */
1844 static int
1845 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1846     uint32_t *addr)
1847 {
1848         uint32_t em, addr_len, maddr, mlen;
1849
1850         /* Memory can only be accessed in naturally aligned 4 byte units */
1851         if (off & 3 || len & 3 || len == 0)
1852                 return (EINVAL);
1853
1854         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1855         switch (fwmtype_to_hwmtype(mtype)) {
1856         case MEM_EDC0:
1857                 if (!(em & F_EDRAM0_ENABLE))
1858                         return (EINVAL);
1859                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1860                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1861                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1862                 break;
1863         case MEM_EDC1:
1864                 if (!(em & F_EDRAM1_ENABLE))
1865                         return (EINVAL);
1866                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1867                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1868                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1869                 break;
1870         case MEM_MC:
1871                 if (!(em & F_EXT_MEM_ENABLE))
1872                         return (EINVAL);
1873                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1874                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1875                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1876                 break;
1877         case MEM_MC1:
1878                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1879                         return (EINVAL);
1880                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1881                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1882                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1883                 break;
1884         default:
1885                 return (EINVAL);
1886         }
1887
1888         if (mlen > 0 && off < mlen && off + len <= mlen) {
1889                 *addr = maddr + off;    /* global address */
1890                 return (0);
1891         }
1892
1893         return (EFAULT);
1894 }
1895
1896 static void
1897 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1898 {
1899         const struct memwin *mw;
1900
1901         if (is_t4(sc)) {
1902                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1903                     ("%s: incorrect memwin# (%d)", __func__, win));
1904                 mw = &t4_memwin[win];
1905         } else {
1906                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1907                     ("%s: incorrect memwin# (%d)", __func__, win));
1908                 mw = &t5_memwin[win];
1909         }
1910
1911         if (base != NULL)
1912                 *base = mw->base;
1913         if (aperture != NULL)
1914                 *aperture = mw->aperture;
1915 }
1916
1917 /*
1918  * Positions the memory window such that it can be used to access the specified
1919  * address in the chip's address space.  The return value is the offset of addr
1920  * from the start of the window.
1921  */
1922 static uint32_t
1923 position_memwin(struct adapter *sc, int n, uint32_t addr)
1924 {
1925         uint32_t start, pf;
1926         uint32_t reg;
1927
1928         KASSERT(n >= 0 && n <= 3,
1929             ("%s: invalid window %d.", __func__, n));
1930         KASSERT((addr & 3) == 0,
1931             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1932
1933         if (is_t4(sc)) {
1934                 pf = 0;
1935                 start = addr & ~0xf;    /* start must be 16B aligned */
1936         } else {
1937                 pf = V_PFNUM(sc->pf);
1938                 start = addr & ~0x7f;   /* start must be 128B aligned */
1939         }
1940         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1941
1942         t4_write_reg(sc, reg, start | pf);
1943         t4_read_reg(sc, reg);
1944
1945         return (addr - start);
1946 }
1947
1948 static int
1949 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1950     struct intrs_and_queues *iaq)
1951 {
1952         int rc, itype, navail, nrxq10g, nrxq1g, n;
1953         int nofldrxq10g = 0, nofldrxq1g = 0;
1954         int nnmrxq10g = 0, nnmrxq1g = 0;
1955
1956         bzero(iaq, sizeof(*iaq));
1957
1958         iaq->ntxq10g = t4_ntxq10g;
1959         iaq->ntxq1g = t4_ntxq1g;
1960         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1961         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1962         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1963 #ifdef TCP_OFFLOAD
1964         if (is_offload(sc)) {
1965                 iaq->nofldtxq10g = t4_nofldtxq10g;
1966                 iaq->nofldtxq1g = t4_nofldtxq1g;
1967                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1968                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1969         }
1970 #endif
1971 #ifdef DEV_NETMAP
1972         iaq->nnmtxq10g = t4_nnmtxq10g;
1973         iaq->nnmtxq1g = t4_nnmtxq1g;
1974         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1975         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1976 #endif
1977
1978         for (itype = INTR_MSIX; itype; itype >>= 1) {
1979
1980                 if ((itype & t4_intr_types) == 0)
1981                         continue;       /* not allowed */
1982
1983                 if (itype == INTR_MSIX)
1984                         navail = pci_msix_count(sc->dev);
1985                 else if (itype == INTR_MSI)
1986                         navail = pci_msi_count(sc->dev);
1987                 else
1988                         navail = 1;
1989 restart:
1990                 if (navail == 0)
1991                         continue;
1992
1993                 iaq->intr_type = itype;
1994                 iaq->intr_flags_10g = 0;
1995                 iaq->intr_flags_1g = 0;
1996
1997                 /*
1998                  * Best option: an interrupt vector for errors, one for the
1999                  * firmware event queue, and one for every rxq (NIC, TOE, and
2000                  * netmap).
2001                  */
2002                 iaq->nirq = T4_EXTRA_INTR;
2003                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
2004                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
2005                 if (iaq->nirq <= navail &&
2006                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
2007                         iaq->intr_flags_10g = INTR_ALL;
2008                         iaq->intr_flags_1g = INTR_ALL;
2009                         goto allocate;
2010                 }
2011
2012                 /*
2013                  * Second best option: a vector for errors, one for the firmware
2014                  * event queue, and vectors for either all the NIC rx queues or
2015                  * all the TOE rx queues.  The queues that don't get vectors
2016                  * will forward their interrupts to those that do.
2017                  *
2018                  * Note: netmap rx queues cannot be created early and so they
2019                  * can't be setup to receive forwarded interrupts for others.
2020                  */
2021                 iaq->nirq = T4_EXTRA_INTR;
2022                 if (nrxq10g >= nofldrxq10g) {
2023                         iaq->intr_flags_10g = INTR_RXQ;
2024                         iaq->nirq += n10g * nrxq10g;
2025 #ifdef DEV_NETMAP
2026                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2027 #endif
2028                 } else {
2029                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2030                         iaq->nirq += n10g * nofldrxq10g;
2031 #ifdef DEV_NETMAP
2032                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2033 #endif
2034                 }
2035                 if (nrxq1g >= nofldrxq1g) {
2036                         iaq->intr_flags_1g = INTR_RXQ;
2037                         iaq->nirq += n1g * nrxq1g;
2038 #ifdef DEV_NETMAP
2039                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2040 #endif
2041                 } else {
2042                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2043                         iaq->nirq += n1g * nofldrxq1g;
2044 #ifdef DEV_NETMAP
2045                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2046 #endif
2047                 }
2048                 if (iaq->nirq <= navail &&
2049                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2050                         goto allocate;
2051
2052                 /*
2053                  * Next best option: an interrupt vector for errors, one for the
2054                  * firmware event queue, and at least one per port.  At this
2055                  * point we know we'll have to downsize nrxq and/or nofldrxq
2056                  * and/or nnmrxq to fit what's available to us.
2057                  */
2058                 iaq->nirq = T4_EXTRA_INTR;
2059                 iaq->nirq += n10g + n1g;
2060                 if (iaq->nirq <= navail) {
2061                         int leftover = navail - iaq->nirq;
2062
2063                         if (n10g > 0) {
2064                                 int target = max(nrxq10g, nofldrxq10g);
2065
2066                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2067                                     INTR_RXQ : INTR_OFLD_RXQ;
2068
2069                                 n = 1;
2070                                 while (n < target && leftover >= n10g) {
2071                                         leftover -= n10g;
2072                                         iaq->nirq += n10g;
2073                                         n++;
2074                                 }
2075                                 iaq->nrxq10g = min(n, nrxq10g);
2076 #ifdef TCP_OFFLOAD
2077                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2078 #endif
2079 #ifdef DEV_NETMAP
2080                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2081 #endif
2082                         }
2083
2084                         if (n1g > 0) {
2085                                 int target = max(nrxq1g, nofldrxq1g);
2086
2087                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2088                                     INTR_RXQ : INTR_OFLD_RXQ;
2089
2090                                 n = 1;
2091                                 while (n < target && leftover >= n1g) {
2092                                         leftover -= n1g;
2093                                         iaq->nirq += n1g;
2094                                         n++;
2095                                 }
2096                                 iaq->nrxq1g = min(n, nrxq1g);
2097 #ifdef TCP_OFFLOAD
2098                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2099 #endif
2100 #ifdef DEV_NETMAP
2101                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2102 #endif
2103                         }
2104
2105                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2106                                 goto allocate;
2107                 }
2108
2109                 /*
2110                  * Least desirable option: one interrupt vector for everything.
2111                  */
2112                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2113                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2114 #ifdef TCP_OFFLOAD
2115                 if (is_offload(sc))
2116                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2117 #endif
2118 #ifdef DEV_NETMAP
2119                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2120 #endif
2121
2122 allocate:
2123                 navail = iaq->nirq;
2124                 rc = 0;
2125                 if (itype == INTR_MSIX)
2126                         rc = pci_alloc_msix(sc->dev, &navail);
2127                 else if (itype == INTR_MSI)
2128                         rc = pci_alloc_msi(sc->dev, &navail);
2129
2130                 if (rc == 0) {
2131                         if (navail == iaq->nirq)
2132                                 return (0);
2133
2134                         /*
2135                          * Didn't get the number requested.  Use whatever number
2136                          * the kernel is willing to allocate (it's in navail).
2137                          */
2138                         device_printf(sc->dev, "fewer vectors than requested, "
2139                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2140                             itype, iaq->nirq, navail);
2141                         pci_release_msi(sc->dev);
2142                         goto restart;
2143                 }
2144
2145                 device_printf(sc->dev,
2146                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2147                     itype, rc, iaq->nirq, navail);
2148         }
2149
2150         device_printf(sc->dev,
2151             "failed to find a usable interrupt type.  "
2152             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2153             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2154
2155         return (ENXIO);
2156 }
2157
2158 #define FW_VERSION(chip) ( \
2159     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2160     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2161     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2162     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2163 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2164
2165 struct fw_info {
2166         uint8_t chip;
2167         char *kld_name;
2168         char *fw_mod_name;
2169         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2170 } fw_info[] = {
2171         {
2172                 .chip = CHELSIO_T4,
2173                 .kld_name = "t4fw_cfg",
2174                 .fw_mod_name = "t4fw",
2175                 .fw_hdr = {
2176                         .chip = FW_HDR_CHIP_T4,
2177                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2178                         .intfver_nic = FW_INTFVER(T4, NIC),
2179                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2180                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2181                         .intfver_ri = FW_INTFVER(T4, RI),
2182                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2183                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2184                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2185                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2186                 },
2187         }, {
2188                 .chip = CHELSIO_T5,
2189                 .kld_name = "t5fw_cfg",
2190                 .fw_mod_name = "t5fw",
2191                 .fw_hdr = {
2192                         .chip = FW_HDR_CHIP_T5,
2193                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2194                         .intfver_nic = FW_INTFVER(T5, NIC),
2195                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2196                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2197                         .intfver_ri = FW_INTFVER(T5, RI),
2198                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2199                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2200                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2201                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2202                 },
2203         }
2204 };
2205
2206 static struct fw_info *
2207 find_fw_info(int chip)
2208 {
2209         int i;
2210
2211         for (i = 0; i < nitems(fw_info); i++) {
2212                 if (fw_info[i].chip == chip)
2213                         return (&fw_info[i]);
2214         }
2215         return (NULL);
2216 }
2217
2218 /*
2219  * Is the given firmware API compatible with the one the driver was compiled
2220  * with?
2221  */
2222 static int
2223 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2224 {
2225
2226         /* short circuit if it's the exact same firmware version */
2227         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2228                 return (1);
2229
2230         /*
2231          * XXX: Is this too conservative?  Perhaps I should limit this to the
2232          * features that are supported in the driver.
2233          */
2234 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2235         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2236             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2237             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2238                 return (1);
2239 #undef SAME_INTF
2240
2241         return (0);
2242 }
2243
2244 /*
2245  * The firmware in the KLD is usable, but should it be installed?  This routine
2246  * explains itself in detail if it indicates the KLD firmware should be
2247  * installed.
2248  */
2249 static int
2250 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2251 {
2252         const char *reason;
2253
2254         if (!card_fw_usable) {
2255                 reason = "incompatible or unusable";
2256                 goto install;
2257         }
2258
2259         if (k > c) {
2260                 reason = "older than the version bundled with this driver";
2261                 goto install;
2262         }
2263
2264         if (t4_fw_install == 2 && k != c) {
2265                 reason = "different than the version bundled with this driver";
2266                 goto install;
2267         }
2268
2269         return (0);
2270
2271 install:
2272         if (t4_fw_install == 0) {
2273                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2274                     "but the driver is prohibited from installing a different "
2275                     "firmware on the card.\n",
2276                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2277                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2278
2279                 return (0);
2280         }
2281
2282         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2283             "installing firmware %u.%u.%u.%u on card.\n",
2284             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2285             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2286             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2287             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2288
2289         return (1);
2290 }
2291 /*
2292  * Establish contact with the firmware and determine if we are the master driver
2293  * or not, and whether we are responsible for chip initialization.
2294  */
2295 static int
2296 prep_firmware(struct adapter *sc)
2297 {
2298         const struct firmware *fw = NULL, *default_cfg;
2299         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2300         enum dev_state state;
2301         struct fw_info *fw_info;
2302         struct fw_hdr *card_fw;         /* fw on the card */
2303         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2304         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2305                                            against */
2306
2307         /* Contact firmware. */
2308         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2309         if (rc < 0 || state == DEV_STATE_ERR) {
2310                 rc = -rc;
2311                 device_printf(sc->dev,
2312                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2313                 return (rc);
2314         }
2315         pf = rc;
2316         if (pf == sc->mbox)
2317                 sc->flags |= MASTER_PF;
2318         else if (state == DEV_STATE_UNINIT) {
2319                 /*
2320                  * We didn't get to be the master so we definitely won't be
2321                  * configuring the chip.  It's a bug if someone else hasn't
2322                  * configured it already.
2323                  */
2324                 device_printf(sc->dev, "couldn't be master(%d), "
2325                     "device not already initialized either(%d).\n", rc, state);
2326                 return (EDOOFUS);
2327         }
2328
2329         /* This is the firmware whose headers the driver was compiled against */
2330         fw_info = find_fw_info(chip_id(sc));
2331         if (fw_info == NULL) {
2332                 device_printf(sc->dev,
2333                     "unable to look up firmware information for chip %d.\n",
2334                     chip_id(sc));
2335                 return (EINVAL);
2336         }
2337         drv_fw = &fw_info->fw_hdr;
2338
2339         /*
2340          * The firmware KLD contains many modules.  The KLD name is also the
2341          * name of the module that contains the default config file.
2342          */
2343         default_cfg = firmware_get(fw_info->kld_name);
2344
2345         /* Read the header of the firmware on the card */
2346         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2347         rc = -t4_read_flash(sc, FLASH_FW_START,
2348             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2349         if (rc == 0)
2350                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2351         else {
2352                 device_printf(sc->dev,
2353                     "Unable to read card's firmware header: %d\n", rc);
2354                 card_fw_usable = 0;
2355         }
2356
2357         /* This is the firmware in the KLD */
2358         fw = firmware_get(fw_info->fw_mod_name);
2359         if (fw != NULL) {
2360                 kld_fw = (const void *)fw->data;
2361                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2362         } else {
2363                 kld_fw = NULL;
2364                 kld_fw_usable = 0;
2365         }
2366
2367         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2368             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2369                 /*
2370                  * Common case: the firmware on the card is an exact match and
2371                  * the KLD is an exact match too, or the KLD is
2372                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2373                  * here -- use cxgbetool loadfw if you want to reinstall the
2374                  * same firmware as the one on the card.
2375                  */
2376         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2377             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2378             be32toh(card_fw->fw_ver))) {
2379
2380                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2381                 if (rc != 0) {
2382                         device_printf(sc->dev,
2383                             "failed to install firmware: %d\n", rc);
2384                         goto done;
2385                 }
2386
2387                 /* Installed successfully, update the cached header too. */
2388                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2389                 card_fw_usable = 1;
2390                 need_fw_reset = 0;      /* already reset as part of load_fw */
2391         }
2392
2393         if (!card_fw_usable) {
2394                 uint32_t d, c, k;
2395
2396                 d = ntohl(drv_fw->fw_ver);
2397                 c = ntohl(card_fw->fw_ver);
2398                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2399
2400                 device_printf(sc->dev, "Cannot find a usable firmware: "
2401                     "fw_install %d, chip state %d, "
2402                     "driver compiled with %d.%d.%d.%d, "
2403                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2404                     t4_fw_install, state,
2405                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2406                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2407                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2408                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2409                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2410                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2411                 rc = EINVAL;
2412                 goto done;
2413         }
2414
2415         /* We're using whatever's on the card and it's known to be good. */
2416         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2417         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2418             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2419             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2420             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2421             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2422         t4_get_tp_version(sc, &sc->params.tp_vers);
2423
2424         /* Reset device */
2425         if (need_fw_reset &&
2426             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2427                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2428                 if (rc != ETIMEDOUT && rc != EIO)
2429                         t4_fw_bye(sc, sc->mbox);
2430                 goto done;
2431         }
2432         sc->flags |= FW_OK;
2433
2434         rc = get_params__pre_init(sc);
2435         if (rc != 0)
2436                 goto done; /* error message displayed already */
2437
2438         /* Partition adapter resources as specified in the config file. */
2439         if (state == DEV_STATE_UNINIT) {
2440
2441                 KASSERT(sc->flags & MASTER_PF,
2442                     ("%s: trying to change chip settings when not master.",
2443                     __func__));
2444
2445                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2446                 if (rc != 0)
2447                         goto done;      /* error message displayed already */
2448
2449                 t4_tweak_chip_settings(sc);
2450
2451                 /* get basic stuff going */
2452                 rc = -t4_fw_initialize(sc, sc->mbox);
2453                 if (rc != 0) {
2454                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2455                         goto done;
2456                 }
2457         } else {
2458                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2459                 sc->cfcsum = 0;
2460         }
2461
2462 done:
2463         free(card_fw, M_CXGBE);
2464         if (fw != NULL)
2465                 firmware_put(fw, FIRMWARE_UNLOAD);
2466         if (default_cfg != NULL)
2467                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2468
2469         return (rc);
2470 }
2471
2472 #define FW_PARAM_DEV(param) \
2473         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2474          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2475 #define FW_PARAM_PFVF(param) \
2476         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2477          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2478
2479 /*
2480  * Partition chip resources for use between various PFs, VFs, etc.
2481  */
2482 static int
2483 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2484     const char *name_prefix)
2485 {
2486         const struct firmware *cfg = NULL;
2487         int rc = 0;
2488         struct fw_caps_config_cmd caps;
2489         uint32_t mtype, moff, finicsum, cfcsum;
2490
2491         /*
2492          * Figure out what configuration file to use.  Pick the default config
2493          * file for the card if the user hasn't specified one explicitly.
2494          */
2495         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2496         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2497                 /* Card specific overrides go here. */
2498                 if (pci_get_device(sc->dev) == 0x440a)
2499                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2500                 if (is_fpga(sc))
2501                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2502         }
2503
2504         /*
2505          * We need to load another module if the profile is anything except
2506          * "default" or "flash".
2507          */
2508         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2509             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2510                 char s[32];
2511
2512                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2513                 cfg = firmware_get(s);
2514                 if (cfg == NULL) {
2515                         if (default_cfg != NULL) {
2516                                 device_printf(sc->dev,
2517                                     "unable to load module \"%s\" for "
2518                                     "configuration profile \"%s\", will use "
2519                                     "the default config file instead.\n",
2520                                     s, sc->cfg_file);
2521                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2522                                     "%s", DEFAULT_CF);
2523                         } else {
2524                                 device_printf(sc->dev,
2525                                     "unable to load module \"%s\" for "
2526                                     "configuration profile \"%s\", will use "
2527                                     "the config file on the card's flash "
2528                                     "instead.\n", s, sc->cfg_file);
2529                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2530                                     "%s", FLASH_CF);
2531                         }
2532                 }
2533         }
2534
2535         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2536             default_cfg == NULL) {
2537                 device_printf(sc->dev,
2538                     "default config file not available, will use the config "
2539                     "file on the card's flash instead.\n");
2540                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2541         }
2542
2543         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2544                 u_int cflen, i, n;
2545                 const uint32_t *cfdata;
2546                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2547
2548                 KASSERT(cfg != NULL || default_cfg != NULL,
2549                     ("%s: no config to upload", __func__));
2550
2551                 /*
2552                  * Ask the firmware where it wants us to upload the config file.
2553                  */
2554                 param = FW_PARAM_DEV(CF);
2555                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2556                 if (rc != 0) {
2557                         /* No support for config file?  Shouldn't happen. */
2558                         device_printf(sc->dev,
2559                             "failed to query config file location: %d.\n", rc);
2560                         goto done;
2561                 }
2562                 mtype = G_FW_PARAMS_PARAM_Y(val);
2563                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2564
2565                 /*
2566                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2567                  * useless stuffing/comments at the end of the config file so
2568                  * it's ok to simply throw away the last remaining bytes when
2569                  * the config file is not an exact multiple of 4.  This also
2570                  * helps with the validate_mt_off_len check.
2571                  */
2572                 if (cfg != NULL) {
2573                         cflen = cfg->datasize & ~3;
2574                         cfdata = cfg->data;
2575                 } else {
2576                         cflen = default_cfg->datasize & ~3;
2577                         cfdata = default_cfg->data;
2578                 }
2579
2580                 if (cflen > FLASH_CFG_MAX_SIZE) {
2581                         device_printf(sc->dev,
2582                             "config file too long (%d, max allowed is %d).  "
2583                             "Will try to use the config on the card, if any.\n",
2584                             cflen, FLASH_CFG_MAX_SIZE);
2585                         goto use_config_on_flash;
2586                 }
2587
2588                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2589                 if (rc != 0) {
2590                         device_printf(sc->dev,
2591                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2592                             "Will try to use the config on the card, if any.\n",
2593                             __func__, mtype, moff, cflen, rc);
2594                         goto use_config_on_flash;
2595                 }
2596
2597                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2598                 while (cflen) {
2599                         off = position_memwin(sc, 2, addr);
2600                         n = min(cflen, mw_aperture - off);
2601                         for (i = 0; i < n; i += 4)
2602                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2603                         cflen -= n;
2604                         addr += n;
2605                 }
2606         } else {
2607 use_config_on_flash:
2608                 mtype = FW_MEMTYPE_FLASH;
2609                 moff = t4_flash_cfg_addr(sc);
2610         }
2611
2612         bzero(&caps, sizeof(caps));
2613         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2614             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2615         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2616             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2617             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2618         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2619         if (rc != 0) {
2620                 device_printf(sc->dev,
2621                     "failed to pre-process config file: %d "
2622                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2623                 goto done;
2624         }
2625
2626         finicsum = be32toh(caps.finicsum);
2627         cfcsum = be32toh(caps.cfcsum);
2628         if (finicsum != cfcsum) {
2629                 device_printf(sc->dev,
2630                     "WARNING: config file checksum mismatch: %08x %08x\n",
2631                     finicsum, cfcsum);
2632         }
2633         sc->cfcsum = cfcsum;
2634
2635 #define LIMIT_CAPS(x) do { \
2636         caps.x &= htobe16(t4_##x##_allowed); \
2637 } while (0)
2638
2639         /*
2640          * Let the firmware know what features will (not) be used so it can tune
2641          * things accordingly.
2642          */
2643         LIMIT_CAPS(linkcaps);
2644         LIMIT_CAPS(niccaps);
2645         LIMIT_CAPS(toecaps);
2646         LIMIT_CAPS(rdmacaps);
2647         LIMIT_CAPS(iscsicaps);
2648         LIMIT_CAPS(fcoecaps);
2649 #undef LIMIT_CAPS
2650
2651         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2652             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2653         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2654         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2655         if (rc != 0) {
2656                 device_printf(sc->dev,
2657                     "failed to process config file: %d.\n", rc);
2658         }
2659 done:
2660         if (cfg != NULL)
2661                 firmware_put(cfg, FIRMWARE_UNLOAD);
2662         return (rc);
2663 }
2664
2665 /*
2666  * Retrieve parameters that are needed (or nice to have) very early.
2667  */
2668 static int
2669 get_params__pre_init(struct adapter *sc)
2670 {
2671         int rc;
2672         uint32_t param[2], val[2];
2673         struct fw_devlog_cmd cmd;
2674         struct devlog_params *dlog = &sc->params.devlog;
2675
2676         param[0] = FW_PARAM_DEV(PORTVEC);
2677         param[1] = FW_PARAM_DEV(CCLK);
2678         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2679         if (rc != 0) {
2680                 device_printf(sc->dev,
2681                     "failed to query parameters (pre_init): %d.\n", rc);
2682                 return (rc);
2683         }
2684
2685         sc->params.portvec = val[0];
2686         sc->params.nports = bitcount32(val[0]);
2687         sc->params.vpd.cclk = val[1];
2688
2689         /* Read device log parameters. */
2690         bzero(&cmd, sizeof(cmd));
2691         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2692             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2693         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2694         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2695         if (rc != 0) {
2696                 device_printf(sc->dev,
2697                     "failed to get devlog parameters: %d.\n", rc);
2698                 bzero(dlog, sizeof (*dlog));
2699                 rc = 0; /* devlog isn't critical for device operation */
2700         } else {
2701                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2702                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2703                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2704                 dlog->size = be32toh(cmd.memsize_devlog);
2705         }
2706
2707         return (rc);
2708 }
2709
2710 /*
2711  * Retrieve various parameters that are of interest to the driver.  The device
2712  * has been initialized by the firmware at this point.
2713  */
2714 static int
2715 get_params__post_init(struct adapter *sc)
2716 {
2717         int rc;
2718         uint32_t param[7], val[7];
2719         struct fw_caps_config_cmd caps;
2720
2721         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2722         param[1] = FW_PARAM_PFVF(EQ_START);
2723         param[2] = FW_PARAM_PFVF(FILTER_START);
2724         param[3] = FW_PARAM_PFVF(FILTER_END);
2725         param[4] = FW_PARAM_PFVF(L2T_START);
2726         param[5] = FW_PARAM_PFVF(L2T_END);
2727         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2728         if (rc != 0) {
2729                 device_printf(sc->dev,
2730                     "failed to query parameters (post_init): %d.\n", rc);
2731                 return (rc);
2732         }
2733
2734         sc->sge.iq_start = val[0];
2735         sc->sge.eq_start = val[1];
2736         sc->tids.ftid_base = val[2];
2737         sc->tids.nftids = val[3] - val[2] + 1;
2738         sc->params.ftid_min = val[2];
2739         sc->params.ftid_max = val[3];
2740         sc->vres.l2t.start = val[4];
2741         sc->vres.l2t.size = val[5] - val[4] + 1;
2742         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2743             ("%s: L2 table size (%u) larger than expected (%u)",
2744             __func__, sc->vres.l2t.size, L2T_SIZE));
2745
2746         /* get capabilites */
2747         bzero(&caps, sizeof(caps));
2748         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2749             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2750         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2751         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2752         if (rc != 0) {
2753                 device_printf(sc->dev,
2754                     "failed to get card capabilities: %d.\n", rc);
2755                 return (rc);
2756         }
2757
2758 #define READ_CAPS(x) do { \
2759         sc->x = htobe16(caps.x); \
2760 } while (0)
2761         READ_CAPS(linkcaps);
2762         READ_CAPS(niccaps);
2763         READ_CAPS(toecaps);
2764         READ_CAPS(rdmacaps);
2765         READ_CAPS(iscsicaps);
2766         READ_CAPS(fcoecaps);
2767
2768         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2769                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2770                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2771                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2772                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2773                 if (rc != 0) {
2774                         device_printf(sc->dev,
2775                             "failed to query NIC parameters: %d.\n", rc);
2776                         return (rc);
2777                 }
2778                 sc->tids.etid_base = val[0];
2779                 sc->params.etid_min = val[0];
2780                 sc->tids.netids = val[1] - val[0] + 1;
2781                 sc->params.netids = sc->tids.netids;
2782                 sc->params.eo_wr_cred = val[2];
2783                 sc->params.ethoffload = 1;
2784         }
2785
2786         if (sc->toecaps) {
2787                 /* query offload-related parameters */
2788                 param[0] = FW_PARAM_DEV(NTID);
2789                 param[1] = FW_PARAM_PFVF(SERVER_START);
2790                 param[2] = FW_PARAM_PFVF(SERVER_END);
2791                 param[3] = FW_PARAM_PFVF(TDDP_START);
2792                 param[4] = FW_PARAM_PFVF(TDDP_END);
2793                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2794                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2795                 if (rc != 0) {
2796                         device_printf(sc->dev,
2797                             "failed to query TOE parameters: %d.\n", rc);
2798                         return (rc);
2799                 }
2800                 sc->tids.ntids = val[0];
2801                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2802                 sc->tids.stid_base = val[1];
2803                 sc->tids.nstids = val[2] - val[1] + 1;
2804                 sc->vres.ddp.start = val[3];
2805                 sc->vres.ddp.size = val[4] - val[3] + 1;
2806                 sc->params.ofldq_wr_cred = val[5];
2807                 sc->params.offload = 1;
2808         }
2809         if (sc->rdmacaps) {
2810                 param[0] = FW_PARAM_PFVF(STAG_START);
2811                 param[1] = FW_PARAM_PFVF(STAG_END);
2812                 param[2] = FW_PARAM_PFVF(RQ_START);
2813                 param[3] = FW_PARAM_PFVF(RQ_END);
2814                 param[4] = FW_PARAM_PFVF(PBL_START);
2815                 param[5] = FW_PARAM_PFVF(PBL_END);
2816                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2817                 if (rc != 0) {
2818                         device_printf(sc->dev,
2819                             "failed to query RDMA parameters(1): %d.\n", rc);
2820                         return (rc);
2821                 }
2822                 sc->vres.stag.start = val[0];
2823                 sc->vres.stag.size = val[1] - val[0] + 1;
2824                 sc->vres.rq.start = val[2];
2825                 sc->vres.rq.size = val[3] - val[2] + 1;
2826                 sc->vres.pbl.start = val[4];
2827                 sc->vres.pbl.size = val[5] - val[4] + 1;
2828
2829                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2830                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2831                 param[2] = FW_PARAM_PFVF(CQ_START);
2832                 param[3] = FW_PARAM_PFVF(CQ_END);
2833                 param[4] = FW_PARAM_PFVF(OCQ_START);
2834                 param[5] = FW_PARAM_PFVF(OCQ_END);
2835                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2836                 if (rc != 0) {
2837                         device_printf(sc->dev,
2838                             "failed to query RDMA parameters(2): %d.\n", rc);
2839                         return (rc);
2840                 }
2841                 sc->vres.qp.start = val[0];
2842                 sc->vres.qp.size = val[1] - val[0] + 1;
2843                 sc->vres.cq.start = val[2];
2844                 sc->vres.cq.size = val[3] - val[2] + 1;
2845                 sc->vres.ocq.start = val[4];
2846                 sc->vres.ocq.size = val[5] - val[4] + 1;
2847         }
2848         if (sc->iscsicaps) {
2849                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2850                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2851                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2852                 if (rc != 0) {
2853                         device_printf(sc->dev,
2854                             "failed to query iSCSI parameters: %d.\n", rc);
2855                         return (rc);
2856                 }
2857                 sc->vres.iscsi.start = val[0];
2858                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2859         }
2860
2861         /*
2862          * We've got the params we wanted to query via the firmware.  Now grab
2863          * some others directly from the chip.
2864          */
2865         rc = t4_read_chip_settings(sc);
2866
2867         return (rc);
2868 }
2869
2870 static int
2871 set_params__post_init(struct adapter *sc)
2872 {
2873         uint32_t param, val;
2874
2875         /* ask for encapsulated CPLs */
2876         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2877         val = 1;
2878         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2879
2880         return (0);
2881 }
2882
2883 #undef FW_PARAM_PFVF
2884 #undef FW_PARAM_DEV
2885
2886 static void
2887 t4_set_desc(struct adapter *sc)
2888 {
2889         char buf[128];
2890         struct adapter_params *p = &sc->params;
2891
2892         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2893             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2894             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2895
2896         device_set_desc_copy(sc->dev, buf);
2897 }
2898
2899 static void
2900 build_medialist(struct port_info *pi, struct ifmedia *media)
2901 {
2902         int m;
2903
2904         PORT_LOCK(pi);
2905
2906         ifmedia_removeall(media);
2907
2908         m = IFM_ETHER | IFM_FDX;
2909
2910         switch(pi->port_type) {
2911         case FW_PORT_TYPE_BT_XFI:
2912         case FW_PORT_TYPE_BT_XAUI:
2913                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
2914                 /* fall through */
2915
2916         case FW_PORT_TYPE_BT_SGMII:
2917                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
2918                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
2919                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
2920                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2921                 break;
2922
2923         case FW_PORT_TYPE_CX4:
2924                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
2925                 ifmedia_set(media, m | IFM_10G_CX4);
2926                 break;
2927
2928         case FW_PORT_TYPE_QSFP_10G:
2929         case FW_PORT_TYPE_SFP:
2930         case FW_PORT_TYPE_FIBER_XFI:
2931         case FW_PORT_TYPE_FIBER_XAUI:
2932                 switch (pi->mod_type) {
2933
2934                 case FW_PORT_MOD_TYPE_LR:
2935                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
2936                         ifmedia_set(media, m | IFM_10G_LR);
2937                         break;
2938
2939                 case FW_PORT_MOD_TYPE_SR:
2940                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
2941                         ifmedia_set(media, m | IFM_10G_SR);
2942                         break;
2943
2944                 case FW_PORT_MOD_TYPE_LRM:
2945                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
2946                         ifmedia_set(media, m | IFM_10G_LRM);
2947                         break;
2948
2949                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2950                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2951                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
2952                         ifmedia_set(media, m | IFM_10G_TWINAX);
2953                         break;
2954
2955                 case FW_PORT_MOD_TYPE_NONE:
2956                         m &= ~IFM_FDX;
2957                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2958                         ifmedia_set(media, m | IFM_NONE);
2959                         break;
2960
2961                 case FW_PORT_MOD_TYPE_NA:
2962                 case FW_PORT_MOD_TYPE_ER:
2963                 default:
2964                         device_printf(pi->dev,
2965                             "unknown port_type (%d), mod_type (%d)\n",
2966                             pi->port_type, pi->mod_type);
2967                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2968                         ifmedia_set(media, m | IFM_UNKNOWN);
2969                         break;
2970                 }
2971                 break;
2972
2973         case FW_PORT_TYPE_QSFP:
2974                 switch (pi->mod_type) {
2975
2976                 case FW_PORT_MOD_TYPE_LR:
2977                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
2978                         ifmedia_set(media, m | IFM_40G_LR4);
2979                         break;
2980
2981                 case FW_PORT_MOD_TYPE_SR:
2982                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
2983                         ifmedia_set(media, m | IFM_40G_SR4);
2984                         break;
2985
2986                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2987                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2988                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
2989                         ifmedia_set(media, m | IFM_40G_CR4);
2990                         break;
2991
2992                 case FW_PORT_MOD_TYPE_NONE:
2993                         m &= ~IFM_FDX;
2994                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2995                         ifmedia_set(media, m | IFM_NONE);
2996                         break;
2997
2998                 default:
2999                         device_printf(pi->dev,
3000                             "unknown port_type (%d), mod_type (%d)\n",
3001                             pi->port_type, pi->mod_type);
3002                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3003                         ifmedia_set(media, m | IFM_UNKNOWN);
3004                         break;
3005                 }
3006                 break;
3007
3008         default:
3009                 device_printf(pi->dev,
3010                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3011                     pi->mod_type);
3012                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3013                 ifmedia_set(media, m | IFM_UNKNOWN);
3014                 break;
3015         }
3016
3017         PORT_UNLOCK(pi);
3018 }
3019
3020 #define FW_MAC_EXACT_CHUNK      7
3021
3022 /*
3023  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3024  * indicates which parameters should be programmed (the rest are left alone).
3025  */
3026 int
3027 update_mac_settings(struct ifnet *ifp, int flags)
3028 {
3029         int rc = 0;
3030         struct port_info *pi = ifp->if_softc;
3031         struct adapter *sc = pi->adapter;
3032         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3033         uint16_t viid = 0xffff;
3034         int16_t *xact_addr_filt = NULL;
3035
3036         ASSERT_SYNCHRONIZED_OP(sc);
3037         KASSERT(flags, ("%s: not told what to update.", __func__));
3038
3039         if (ifp == pi->ifp) {
3040                 viid = pi->viid;
3041                 xact_addr_filt = &pi->xact_addr_filt;
3042         }
3043 #ifdef DEV_NETMAP
3044         else if (ifp == pi->nm_ifp) {
3045                 viid = pi->nm_viid;
3046                 xact_addr_filt = &pi->nm_xact_addr_filt;
3047         }
3048 #endif
3049         if (flags & XGMAC_MTU)
3050                 mtu = ifp->if_mtu;
3051
3052         if (flags & XGMAC_PROMISC)
3053                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3054
3055         if (flags & XGMAC_ALLMULTI)
3056                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3057
3058         if (flags & XGMAC_VLANEX)
3059                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3060
3061         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3062                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3063                     1, vlanex, false);
3064                 if (rc) {
3065                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3066                             rc);
3067                         return (rc);
3068                 }
3069         }
3070
3071         if (flags & XGMAC_UCADDR) {
3072                 uint8_t ucaddr[ETHER_ADDR_LEN];
3073
3074                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3075                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3076                     true, true);
3077                 if (rc < 0) {
3078                         rc = -rc;
3079                         if_printf(ifp, "change_mac failed: %d\n", rc);
3080                         return (rc);
3081                 } else {
3082                         *xact_addr_filt = rc;
3083                         rc = 0;
3084                 }
3085         }
3086
3087         if (flags & XGMAC_MCADDRS) {
3088                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3089                 int del = 1;
3090                 uint64_t hash = 0;
3091                 struct ifmultiaddr *ifma;
3092                 int i = 0, j;
3093
3094                 if_maddr_rlock(ifp);
3095                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3096                         if (ifma->ifma_addr->sa_family != AF_LINK)
3097                                 continue;
3098                         mcaddr[i] =
3099                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3100                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3101                         i++;
3102
3103                         if (i == FW_MAC_EXACT_CHUNK) {
3104                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3105                                     i, mcaddr, NULL, &hash, 0);
3106                                 if (rc < 0) {
3107                                         rc = -rc;
3108                                         for (j = 0; j < i; j++) {
3109                                                 if_printf(ifp,
3110                                                     "failed to add mc address"
3111                                                     " %02x:%02x:%02x:"
3112                                                     "%02x:%02x:%02x rc=%d\n",
3113                                                     mcaddr[j][0], mcaddr[j][1],
3114                                                     mcaddr[j][2], mcaddr[j][3],
3115                                                     mcaddr[j][4], mcaddr[j][5],
3116                                                     rc);
3117                                         }
3118                                         goto mcfail;
3119                                 }
3120                                 del = 0;
3121                                 i = 0;
3122                         }
3123                 }
3124                 if (i > 0) {
3125                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3126                             mcaddr, NULL, &hash, 0);
3127                         if (rc < 0) {
3128                                 rc = -rc;
3129                                 for (j = 0; j < i; j++) {
3130                                         if_printf(ifp,
3131                                             "failed to add mc address"
3132                                             " %02x:%02x:%02x:"
3133                                             "%02x:%02x:%02x rc=%d\n",
3134                                             mcaddr[j][0], mcaddr[j][1],
3135                                             mcaddr[j][2], mcaddr[j][3],
3136                                             mcaddr[j][4], mcaddr[j][5],
3137                                             rc);
3138                                 }
3139                                 goto mcfail;
3140                         }
3141                 }
3142
3143                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3144                 if (rc != 0)
3145                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3146 mcfail:
3147                 if_maddr_runlock(ifp);
3148         }
3149
3150         return (rc);
3151 }
3152
3153 /*
3154  * {begin|end}_synchronized_op must be called from the same thread.
3155  */
3156 int
3157 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3158     char *wmesg)
3159 {
3160         int rc, pri;
3161
3162 #ifdef WITNESS
3163         /* the caller thinks it's ok to sleep, but is it really? */
3164         if (flags & SLEEP_OK)
3165                 pause("t4slptst", 1);
3166 #endif
3167
3168         if (INTR_OK)
3169                 pri = PCATCH;
3170         else
3171                 pri = 0;
3172
3173         ADAPTER_LOCK(sc);
3174         for (;;) {
3175
3176                 if (pi && IS_DOOMED(pi)) {
3177                         rc = ENXIO;
3178                         goto done;
3179                 }
3180
3181                 if (!IS_BUSY(sc)) {
3182                         rc = 0;
3183                         break;
3184                 }
3185
3186                 if (!(flags & SLEEP_OK)) {
3187                         rc = EBUSY;
3188                         goto done;
3189                 }
3190
3191                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3192                         rc = EINTR;
3193                         goto done;
3194                 }
3195         }
3196
3197         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3198         SET_BUSY(sc);
3199 #ifdef INVARIANTS
3200         sc->last_op = wmesg;
3201         sc->last_op_thr = curthread;
3202         sc->last_op_flags = flags;
3203 #endif
3204
3205 done:
3206         if (!(flags & HOLD_LOCK) || rc)
3207                 ADAPTER_UNLOCK(sc);
3208
3209         return (rc);
3210 }
3211
3212 /*
3213  * {begin|end}_synchronized_op must be called from the same thread.
3214  */
3215 void
3216 end_synchronized_op(struct adapter *sc, int flags)
3217 {
3218
3219         if (flags & LOCK_HELD)
3220                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3221         else
3222                 ADAPTER_LOCK(sc);
3223
3224         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3225         CLR_BUSY(sc);
3226         wakeup(&sc->flags);
3227         ADAPTER_UNLOCK(sc);
3228 }
3229
3230 static int
3231 cxgbe_init_synchronized(struct port_info *pi)
3232 {
3233         struct adapter *sc = pi->adapter;
3234         struct ifnet *ifp = pi->ifp;
3235         int rc = 0, i;
3236         struct sge_txq *txq;
3237
3238         ASSERT_SYNCHRONIZED_OP(sc);
3239
3240         if (isset(&sc->open_device_map, pi->port_id)) {
3241                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3242                     ("mismatch between open_device_map and if_drv_flags"));
3243                 return (0);     /* already running */
3244         }
3245
3246         if (!(sc->flags & FULL_INIT_DONE) &&
3247             ((rc = adapter_full_init(sc)) != 0))
3248                 return (rc);    /* error message displayed already */
3249
3250         if (!(pi->flags & PORT_INIT_DONE) &&
3251             ((rc = port_full_init(pi)) != 0))
3252                 return (rc); /* error message displayed already */
3253
3254         rc = update_mac_settings(ifp, XGMAC_ALL);
3255         if (rc)
3256                 goto done;      /* error message displayed already */
3257
3258         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3259         if (rc != 0) {
3260                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3261                 goto done;
3262         }
3263
3264         /*
3265          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3266          * if this changes.
3267          */
3268
3269         for_each_txq(pi, i, txq) {
3270                 TXQ_LOCK(txq);
3271                 txq->eq.flags |= EQ_ENABLED;
3272                 TXQ_UNLOCK(txq);
3273         }
3274
3275         /*
3276          * The first iq of the first port to come up is used for tracing.
3277          */
3278         if (sc->traceq < 0) {
3279                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3280                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3281                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3282                     V_QUEUENUMBER(sc->traceq));
3283                 pi->flags |= HAS_TRACEQ;
3284         }
3285
3286         /* all ok */
3287         setbit(&sc->open_device_map, pi->port_id);
3288         PORT_LOCK(pi);
3289         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3290         PORT_UNLOCK(pi);
3291
3292         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3293 done:
3294         if (rc != 0)
3295                 cxgbe_uninit_synchronized(pi);
3296
3297         return (rc);
3298 }
3299
3300 /*
3301  * Idempotent.
3302  */
3303 static int
3304 cxgbe_uninit_synchronized(struct port_info *pi)
3305 {
3306         struct adapter *sc = pi->adapter;
3307         struct ifnet *ifp = pi->ifp;
3308         int rc, i;
3309         struct sge_txq *txq;
3310
3311         ASSERT_SYNCHRONIZED_OP(sc);
3312
3313         if (!(pi->flags & PORT_INIT_DONE)) {
3314                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3315                     ("uninited port is running"));
3316                 return (0);
3317         }
3318
3319         /*
3320          * Disable the VI so that all its data in either direction is discarded
3321          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3322          * tick) intact as the TP can deliver negative advice or data that it's
3323          * holding in its RAM (for an offloaded connection) even after the VI is
3324          * disabled.
3325          */
3326         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3327         if (rc) {
3328                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3329                 return (rc);
3330         }
3331
3332         for_each_txq(pi, i, txq) {
3333                 TXQ_LOCK(txq);
3334                 txq->eq.flags &= ~EQ_ENABLED;
3335                 TXQ_UNLOCK(txq);
3336         }
3337
3338         clrbit(&sc->open_device_map, pi->port_id);
3339         PORT_LOCK(pi);
3340         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3341         PORT_UNLOCK(pi);
3342
3343         pi->link_cfg.link_ok = 0;
3344         pi->link_cfg.speed = 0;
3345         pi->linkdnrc = -1;
3346         t4_os_link_changed(sc, pi->port_id, 0, -1);
3347
3348         return (0);
3349 }
3350
3351 /*
3352  * It is ok for this function to fail midway and return right away.  t4_detach
3353  * will walk the entire sc->irq list and clean up whatever is valid.
3354  */
3355 static int
3356 setup_intr_handlers(struct adapter *sc)
3357 {
3358         int rc, rid, p, q;
3359         char s[8];
3360         struct irq *irq;
3361         struct port_info *pi;
3362         struct sge_rxq *rxq;
3363 #ifdef TCP_OFFLOAD
3364         struct sge_ofld_rxq *ofld_rxq;
3365 #endif
3366 #ifdef DEV_NETMAP
3367         struct sge_nm_rxq *nm_rxq;
3368 #endif
3369
3370         /*
3371          * Setup interrupts.
3372          */
3373         irq = &sc->irq[0];
3374         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3375         if (sc->intr_count == 1)
3376                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3377
3378         /* Multiple interrupts. */
3379         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3380             ("%s: too few intr.", __func__));
3381
3382         /* The first one is always error intr */
3383         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3384         if (rc != 0)
3385                 return (rc);
3386         irq++;
3387         rid++;
3388
3389         /* The second one is always the firmware event queue */
3390         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3391         if (rc != 0)
3392                 return (rc);
3393         irq++;
3394         rid++;
3395
3396         for_each_port(sc, p) {
3397                 pi = sc->port[p];
3398
3399                 if (pi->flags & INTR_RXQ) {
3400                         for_each_rxq(pi, q, rxq) {
3401                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3402                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3403                                     s);
3404                                 if (rc != 0)
3405                                         return (rc);
3406                                 irq++;
3407                                 rid++;
3408                         }
3409                 }
3410 #ifdef TCP_OFFLOAD
3411                 if (pi->flags & INTR_OFLD_RXQ) {
3412                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3413                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3414                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3415                                     ofld_rxq, s);
3416                                 if (rc != 0)
3417                                         return (rc);
3418                                 irq++;
3419                                 rid++;
3420                         }
3421                 }
3422 #endif
3423 #ifdef DEV_NETMAP
3424                 if (pi->flags & INTR_NM_RXQ) {
3425                         for_each_nm_rxq(pi, q, nm_rxq) {
3426                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3427                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3428                                     nm_rxq, s);
3429                                 if (rc != 0)
3430                                         return (rc);
3431                                 irq++;
3432                                 rid++;
3433                         }
3434                 }
3435 #endif
3436         }
3437         MPASS(irq == &sc->irq[sc->intr_count]);
3438
3439         return (0);
3440 }
3441
3442 int
3443 adapter_full_init(struct adapter *sc)
3444 {
3445         int rc, i;
3446
3447         ASSERT_SYNCHRONIZED_OP(sc);
3448         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3449         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3450             ("%s: FULL_INIT_DONE already", __func__));
3451
3452         /*
3453          * queues that belong to the adapter (not any particular port).
3454          */
3455         rc = t4_setup_adapter_queues(sc);
3456         if (rc != 0)
3457                 goto done;
3458
3459         for (i = 0; i < nitems(sc->tq); i++) {
3460                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3461                     taskqueue_thread_enqueue, &sc->tq[i]);
3462                 if (sc->tq[i] == NULL) {
3463                         device_printf(sc->dev,
3464                             "failed to allocate task queue %d\n", i);
3465                         rc = ENOMEM;
3466                         goto done;
3467                 }
3468                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3469                     device_get_nameunit(sc->dev), i);
3470         }
3471
3472         t4_intr_enable(sc);
3473         sc->flags |= FULL_INIT_DONE;
3474 done:
3475         if (rc != 0)
3476                 adapter_full_uninit(sc);
3477
3478         return (rc);
3479 }
3480
3481 int
3482 adapter_full_uninit(struct adapter *sc)
3483 {
3484         int i;
3485
3486         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3487
3488         t4_teardown_adapter_queues(sc);
3489
3490         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3491                 taskqueue_free(sc->tq[i]);
3492                 sc->tq[i] = NULL;
3493         }
3494
3495         sc->flags &= ~FULL_INIT_DONE;
3496
3497         return (0);
3498 }
3499
3500 #ifdef RSS
3501 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
3502     RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
3503     RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
3504     RSS_HASHTYPE_RSS_UDP_IPV6)
3505
3506 /* Translates kernel hash types to hardware. */
3507 static int
3508 hashconfig_to_hashen(int hashconfig)
3509 {
3510         int hashen = 0;
3511
3512         if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
3513                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
3514         if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
3515                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
3516         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
3517                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3518                     F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3519         }
3520         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
3521                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
3522                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3523         }
3524         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
3525                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
3526         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
3527                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
3528
3529         return (hashen);
3530 }
3531
3532 /* Translates hardware hash types to kernel. */
3533 static int
3534 hashen_to_hashconfig(int hashen)
3535 {
3536         int hashconfig = 0;
3537
3538         if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
3539                 /*
3540                  * If UDP hashing was enabled it must have been enabled for
3541                  * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
3542                  * enabling any 4-tuple hash is nonsense configuration.
3543                  */
3544                 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3545                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
3546
3547                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3548                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
3549                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3550                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
3551         }
3552         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
3553                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
3554         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
3555                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
3556         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
3557                 hashconfig |= RSS_HASHTYPE_RSS_IPV4;
3558         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
3559                 hashconfig |= RSS_HASHTYPE_RSS_IPV6;
3560
3561         return (hashconfig);
3562 }
3563 #endif
3564
3565 int
3566 port_full_init(struct port_info *pi)
3567 {
3568         struct adapter *sc = pi->adapter;
3569         struct ifnet *ifp = pi->ifp;
3570         uint16_t *rss;
3571         struct sge_rxq *rxq;
3572         int rc, i, j, hashen;
3573 #ifdef RSS
3574         int nbuckets = rss_getnumbuckets();
3575         int hashconfig = rss_gethashconfig();
3576         int extra;
3577         uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3578         uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3579 #endif
3580
3581         ASSERT_SYNCHRONIZED_OP(sc);
3582         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3583             ("%s: PORT_INIT_DONE already", __func__));
3584
3585         sysctl_ctx_init(&pi->ctx);
3586         pi->flags |= PORT_SYSCTL_CTX;
3587
3588         /*
3589          * Allocate tx/rx/fl queues for this port.
3590          */
3591         rc = t4_setup_port_queues(pi);
3592         if (rc != 0)
3593                 goto done;      /* error message displayed already */
3594
3595         /*
3596          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3597          */
3598         if (pi->nrxq > pi->rss_size) {
3599                 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
3600                     "some queues will never receive traffic.\n", pi->nrxq,
3601                     pi->rss_size);
3602         } else if (pi->rss_size % pi->nrxq) {
3603                 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
3604                     "expect uneven traffic distribution.\n", pi->nrxq,
3605                     pi->rss_size);
3606         }
3607 #ifdef RSS
3608         MPASS(RSS_KEYSIZE == 40);
3609         if (pi->nrxq != nbuckets) {
3610                 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
3611                     "performance will be impacted.\n", pi->nrxq, nbuckets);
3612         }
3613
3614         rss_getkey((void *)&raw_rss_key[0]);
3615         for (i = 0; i < nitems(rss_key); i++) {
3616                 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
3617         }
3618         t4_write_rss_key(sc, (void *)&rss_key[0], -1);
3619 #endif
3620         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3621         for (i = 0; i < pi->rss_size;) {
3622 #ifdef RSS
3623                 j = rss_get_indirection_to_bucket(i);
3624                 j %= pi->nrxq;
3625                 rxq = &sc->sge.rxq[pi->first_rxq + j];
3626                 rss[i++] = rxq->iq.abs_id;
3627 #else
3628                 for_each_rxq(pi, j, rxq) {
3629                         rss[i++] = rxq->iq.abs_id;
3630                         if (i == pi->rss_size)
3631                                 break;
3632                 }
3633 #endif
3634         }
3635
3636         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3637             pi->rss_size);
3638         if (rc != 0) {
3639                 if_printf(ifp, "rss_config failed: %d\n", rc);
3640                 goto done;
3641         }
3642
3643 #ifdef RSS
3644         hashen = hashconfig_to_hashen(hashconfig);
3645
3646         /*
3647          * We may have had to enable some hashes even though the global config
3648          * wants them disabled.  This is a potential problem that must be
3649          * reported to the user.
3650          */
3651         extra = hashen_to_hashconfig(hashen) ^ hashconfig;
3652
3653         /*
3654          * If we consider only the supported hash types, then the enabled hashes
3655          * are a superset of the requested hashes.  In other words, there cannot
3656          * be any supported hash that was requested but not enabled, but there
3657          * can be hashes that were not requested but had to be enabled.
3658          */
3659         extra &= SUPPORTED_RSS_HASHTYPES;
3660         MPASS((extra & hashconfig) == 0);
3661
3662         if (extra) {
3663                 if_printf(ifp,
3664                     "global RSS config (0x%x) cannot be accomodated.\n",
3665                     hashconfig);
3666         }
3667         if (extra & RSS_HASHTYPE_RSS_IPV4)
3668                 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
3669         if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
3670                 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
3671         if (extra & RSS_HASHTYPE_RSS_IPV6)
3672                 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
3673         if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
3674                 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
3675         if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
3676                 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
3677         if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
3678                 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
3679 #else
3680         hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
3681             F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
3682             F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
3683             F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
3684 #endif
3685         rc = -t4_config_vi_rss(sc, sc->mbox, pi->viid, hashen, rss[0]);
3686         if (rc != 0) {
3687                 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
3688                 goto done;
3689         }
3690
3691         pi->rss = rss;
3692         pi->flags |= PORT_INIT_DONE;
3693 done:
3694         if (rc != 0)
3695                 port_full_uninit(pi);
3696
3697         return (rc);
3698 }
3699
3700 /*
3701  * Idempotent.
3702  */
3703 int
3704 port_full_uninit(struct port_info *pi)
3705 {
3706         struct adapter *sc = pi->adapter;
3707         int i;
3708         struct sge_rxq *rxq;
3709         struct sge_txq *txq;
3710 #ifdef TCP_OFFLOAD
3711         struct sge_ofld_rxq *ofld_rxq;
3712         struct sge_wrq *ofld_txq;
3713 #endif
3714
3715         if (pi->flags & PORT_INIT_DONE) {
3716
3717                 /* Need to quiesce queues.  */
3718
3719                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3720
3721                 for_each_txq(pi, i, txq) {
3722                         quiesce_txq(sc, txq);
3723                 }
3724
3725 #ifdef TCP_OFFLOAD
3726                 for_each_ofld_txq(pi, i, ofld_txq) {
3727                         quiesce_wrq(sc, ofld_txq);
3728                 }
3729 #endif
3730
3731                 for_each_rxq(pi, i, rxq) {
3732                         quiesce_iq(sc, &rxq->iq);
3733                         quiesce_fl(sc, &rxq->fl);
3734                 }
3735
3736 #ifdef TCP_OFFLOAD
3737                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3738                         quiesce_iq(sc, &ofld_rxq->iq);
3739                         quiesce_fl(sc, &ofld_rxq->fl);
3740                 }
3741 #endif
3742                 free(pi->rss, M_CXGBE);
3743         }
3744
3745         t4_teardown_port_queues(pi);
3746         pi->flags &= ~PORT_INIT_DONE;
3747
3748         return (0);
3749 }
3750
3751 static void
3752 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3753 {
3754         struct sge_eq *eq = &txq->eq;
3755         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3756
3757         (void) sc;      /* unused */
3758
3759 #ifdef INVARIANTS
3760         TXQ_LOCK(txq);
3761         MPASS((eq->flags & EQ_ENABLED) == 0);
3762         TXQ_UNLOCK(txq);
3763 #endif
3764
3765         /* Wait for the mp_ring to empty. */
3766         while (!mp_ring_is_idle(txq->r)) {
3767                 mp_ring_check_drainage(txq->r, 0);
3768                 pause("rquiesce", 1);
3769         }
3770
3771         /* Then wait for the hardware to finish. */
3772         while (spg->cidx != htobe16(eq->pidx))
3773                 pause("equiesce", 1);
3774
3775         /* Finally, wait for the driver to reclaim all descriptors. */
3776         while (eq->cidx != eq->pidx)
3777                 pause("dquiesce", 1);
3778 }
3779
3780 static void
3781 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3782 {
3783
3784         /* XXXTX */
3785 }
3786
3787 static void
3788 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3789 {
3790         (void) sc;      /* unused */
3791
3792         /* Synchronize with the interrupt handler */
3793         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3794                 pause("iqfree", 1);
3795 }
3796
3797 static void
3798 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3799 {
3800         mtx_lock(&sc->sfl_lock);
3801         FL_LOCK(fl);
3802         fl->flags |= FL_DOOMED;
3803         FL_UNLOCK(fl);
3804         mtx_unlock(&sc->sfl_lock);
3805
3806         callout_drain(&sc->sfl_callout);
3807         KASSERT((fl->flags & FL_STARVING) == 0,
3808             ("%s: still starving", __func__));
3809 }
3810
3811 static int
3812 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3813     driver_intr_t *handler, void *arg, char *name)
3814 {
3815         int rc;
3816
3817         irq->rid = rid;
3818         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3819             RF_SHAREABLE | RF_ACTIVE);
3820         if (irq->res == NULL) {
3821                 device_printf(sc->dev,
3822                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3823                 return (ENOMEM);
3824         }
3825
3826         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3827             NULL, handler, arg, &irq->tag);
3828         if (rc != 0) {
3829                 device_printf(sc->dev,
3830                     "failed to setup interrupt for rid %d, name %s: %d\n",
3831                     rid, name, rc);
3832         } else if (name)
3833                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3834
3835         return (rc);
3836 }
3837
3838 static int
3839 t4_free_irq(struct adapter *sc, struct irq *irq)
3840 {
3841         if (irq->tag)
3842                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3843         if (irq->res)
3844                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3845
3846         bzero(irq, sizeof(*irq));
3847
3848         return (0);
3849 }
3850
3851 static void
3852 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3853     unsigned int end)
3854 {
3855         uint32_t *p = (uint32_t *)(buf + start);
3856
3857         for ( ; start <= end; start += sizeof(uint32_t))
3858                 *p++ = t4_read_reg(sc, start);
3859 }
3860
3861 static void
3862 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3863 {
3864         int i, n;
3865         const unsigned int *reg_ranges;
3866         static const unsigned int t4_reg_ranges[] = {
3867                 0x1008, 0x1108,
3868                 0x1180, 0x11b4,
3869                 0x11fc, 0x123c,
3870                 0x1300, 0x173c,
3871                 0x1800, 0x18fc,
3872                 0x3000, 0x30d8,
3873                 0x30e0, 0x5924,
3874                 0x5960, 0x59d4,
3875                 0x5a00, 0x5af8,
3876                 0x6000, 0x6098,
3877                 0x6100, 0x6150,
3878                 0x6200, 0x6208,
3879                 0x6240, 0x6248,
3880                 0x6280, 0x6338,
3881                 0x6370, 0x638c,
3882                 0x6400, 0x643c,
3883                 0x6500, 0x6524,
3884                 0x6a00, 0x6a38,
3885                 0x6a60, 0x6a78,
3886                 0x6b00, 0x6b84,
3887                 0x6bf0, 0x6c84,
3888                 0x6cf0, 0x6d84,
3889                 0x6df0, 0x6e84,
3890                 0x6ef0, 0x6f84,
3891                 0x6ff0, 0x7084,
3892                 0x70f0, 0x7184,
3893                 0x71f0, 0x7284,
3894                 0x72f0, 0x7384,
3895                 0x73f0, 0x7450,
3896                 0x7500, 0x7530,
3897                 0x7600, 0x761c,
3898                 0x7680, 0x76cc,
3899                 0x7700, 0x7798,
3900                 0x77c0, 0x77fc,
3901                 0x7900, 0x79fc,
3902                 0x7b00, 0x7c38,
3903                 0x7d00, 0x7efc,
3904                 0x8dc0, 0x8e1c,
3905                 0x8e30, 0x8e78,
3906                 0x8ea0, 0x8f6c,
3907                 0x8fc0, 0x9074,
3908                 0x90fc, 0x90fc,
3909                 0x9400, 0x9458,
3910                 0x9600, 0x96bc,
3911                 0x9800, 0x9808,
3912                 0x9820, 0x983c,
3913                 0x9850, 0x9864,
3914                 0x9c00, 0x9c6c,
3915                 0x9c80, 0x9cec,
3916                 0x9d00, 0x9d6c,
3917                 0x9d80, 0x9dec,
3918                 0x9e00, 0x9e6c,
3919                 0x9e80, 0x9eec,
3920                 0x9f00, 0x9f6c,
3921                 0x9f80, 0x9fec,
3922                 0xd004, 0xd03c,
3923                 0xdfc0, 0xdfe0,
3924                 0xe000, 0xea7c,
3925                 0xf000, 0x11110,
3926                 0x11118, 0x11190,
3927                 0x19040, 0x1906c,
3928                 0x19078, 0x19080,
3929                 0x1908c, 0x19124,
3930                 0x19150, 0x191b0,
3931                 0x191d0, 0x191e8,
3932                 0x19238, 0x1924c,
3933                 0x193f8, 0x19474,
3934                 0x19490, 0x194f8,
3935                 0x19800, 0x19f30,
3936                 0x1a000, 0x1a06c,
3937                 0x1a0b0, 0x1a120,
3938                 0x1a128, 0x1a138,
3939                 0x1a190, 0x1a1c4,
3940                 0x1a1fc, 0x1a1fc,
3941                 0x1e040, 0x1e04c,
3942                 0x1e284, 0x1e28c,
3943                 0x1e2c0, 0x1e2c0,
3944                 0x1e2e0, 0x1e2e0,
3945                 0x1e300, 0x1e384,
3946                 0x1e3c0, 0x1e3c8,
3947                 0x1e440, 0x1e44c,
3948                 0x1e684, 0x1e68c,
3949                 0x1e6c0, 0x1e6c0,
3950                 0x1e6e0, 0x1e6e0,
3951                 0x1e700, 0x1e784,
3952                 0x1e7c0, 0x1e7c8,
3953                 0x1e840, 0x1e84c,
3954                 0x1ea84, 0x1ea8c,
3955                 0x1eac0, 0x1eac0,
3956                 0x1eae0, 0x1eae0,
3957                 0x1eb00, 0x1eb84,
3958                 0x1ebc0, 0x1ebc8,
3959                 0x1ec40, 0x1ec4c,
3960                 0x1ee84, 0x1ee8c,
3961                 0x1eec0, 0x1eec0,
3962                 0x1eee0, 0x1eee0,
3963                 0x1ef00, 0x1ef84,
3964                 0x1efc0, 0x1efc8,
3965                 0x1f040, 0x1f04c,
3966                 0x1f284, 0x1f28c,
3967                 0x1f2c0, 0x1f2c0,
3968                 0x1f2e0, 0x1f2e0,
3969                 0x1f300, 0x1f384,
3970                 0x1f3c0, 0x1f3c8,
3971                 0x1f440, 0x1f44c,
3972                 0x1f684, 0x1f68c,
3973                 0x1f6c0, 0x1f6c0,
3974                 0x1f6e0, 0x1f6e0,
3975                 0x1f700, 0x1f784,
3976                 0x1f7c0, 0x1f7c8,
3977                 0x1f840, 0x1f84c,
3978                 0x1fa84, 0x1fa8c,
3979                 0x1fac0, 0x1fac0,
3980                 0x1fae0, 0x1fae0,
3981                 0x1fb00, 0x1fb84,
3982                 0x1fbc0, 0x1fbc8,
3983                 0x1fc40, 0x1fc4c,
3984                 0x1fe84, 0x1fe8c,
3985                 0x1fec0, 0x1fec0,
3986                 0x1fee0, 0x1fee0,
3987                 0x1ff00, 0x1ff84,
3988                 0x1ffc0, 0x1ffc8,
3989                 0x20000, 0x2002c,
3990                 0x20100, 0x2013c,
3991                 0x20190, 0x201c8,
3992                 0x20200, 0x20318,
3993                 0x20400, 0x20528,
3994                 0x20540, 0x20614,
3995                 0x21000, 0x21040,
3996                 0x2104c, 0x21060,
3997                 0x210c0, 0x210ec,
3998                 0x21200, 0x21268,
3999                 0x21270, 0x21284,
4000                 0x212fc, 0x21388,
4001                 0x21400, 0x21404,
4002                 0x21500, 0x21518,
4003                 0x2152c, 0x2153c,
4004                 0x21550, 0x21554,
4005                 0x21600, 0x21600,
4006                 0x21608, 0x21628,
4007                 0x21630, 0x2163c,
4008                 0x21700, 0x2171c,
4009                 0x21780, 0x2178c,
4010                 0x21800, 0x21c38,
4011                 0x21c80, 0x21d7c,
4012                 0x21e00, 0x21e04,
4013                 0x22000, 0x2202c,
4014                 0x22100, 0x2213c,
4015                 0x22190, 0x221c8,
4016                 0x22200, 0x22318,
4017                 0x22400, 0x22528,
4018                 0x22540, 0x22614,
4019                 0x23000, 0x23040,
4020                 0x2304c, 0x23060,
4021                 0x230c0, 0x230ec,
4022                 0x23200, 0x23268,
4023                 0x23270, 0x23284,
4024                 0x232fc, 0x23388,
4025                 0x23400, 0x23404,
4026                 0x23500, 0x23518,
4027                 0x2352c, 0x2353c,
4028                 0x23550, 0x23554,
4029                 0x23600, 0x23600,
4030                 0x23608, 0x23628,
4031                 0x23630, 0x2363c,
4032                 0x23700, 0x2371c,
4033                 0x23780, 0x2378c,
4034                 0x23800, 0x23c38,
4035                 0x23c80, 0x23d7c,
4036                 0x23e00, 0x23e04,
4037                 0x24000, 0x2402c,
4038                 0x24100, 0x2413c,
4039                 0x24190, 0x241c8,
4040                 0x24200, 0x24318,
4041                 0x24400, 0x24528,
4042                 0x24540, 0x24614,
4043                 0x25000, 0x25040,
4044                 0x2504c, 0x25060,
4045                 0x250c0, 0x250ec,
4046                 0x25200, 0x25268,
4047                 0x25270, 0x25284,
4048                 0x252fc, 0x25388,
4049                 0x25400, 0x25404,
4050                 0x25500, 0x25518,
4051                 0x2552c, 0x2553c,
4052                 0x25550, 0x25554,
4053                 0x25600, 0x25600,
4054                 0x25608, 0x25628,
4055                 0x25630, 0x2563c,
4056                 0x25700, 0x2571c,
4057                 0x25780, 0x2578c,
4058                 0x25800, 0x25c38,
4059                 0x25c80, 0x25d7c,
4060                 0x25e00, 0x25e04,
4061                 0x26000, 0x2602c,
4062                 0x26100, 0x2613c,
4063                 0x26190, 0x261c8,
4064                 0x26200, 0x26318,
4065                 0x26400, 0x26528,
4066                 0x26540, 0x26614,
4067                 0x27000, 0x27040,
4068                 0x2704c, 0x27060,
4069                 0x270c0, 0x270ec,
4070                 0x27200, 0x27268,
4071                 0x27270, 0x27284,
4072                 0x272fc, 0x27388,
4073                 0x27400, 0x27404,
4074                 0x27500, 0x27518,
4075                 0x2752c, 0x2753c,
4076                 0x27550, 0x27554,
4077                 0x27600, 0x27600,
4078                 0x27608, 0x27628,
4079                 0x27630, 0x2763c,
4080                 0x27700, 0x2771c,
4081                 0x27780, 0x2778c,
4082                 0x27800, 0x27c38,
4083                 0x27c80, 0x27d7c,
4084                 0x27e00, 0x27e04
4085         };
4086         static const unsigned int t5_reg_ranges[] = {
4087                 0x1008, 0x1148,
4088                 0x1180, 0x11b4,
4089                 0x11fc, 0x123c,
4090                 0x1280, 0x173c,
4091                 0x1800, 0x18fc,
4092                 0x3000, 0x3028,
4093                 0x3060, 0x30d8,
4094                 0x30e0, 0x30fc,
4095                 0x3140, 0x357c,
4096                 0x35a8, 0x35cc,
4097                 0x35ec, 0x35ec,
4098                 0x3600, 0x5624,
4099                 0x56cc, 0x575c,
4100                 0x580c, 0x5814,
4101                 0x5890, 0x58bc,
4102                 0x5940, 0x59dc,
4103                 0x59fc, 0x5a18,
4104                 0x5a60, 0x5a9c,
4105                 0x5b94, 0x5bfc,
4106                 0x6000, 0x6040,
4107                 0x6058, 0x614c,
4108                 0x7700, 0x7798,
4109                 0x77c0, 0x78fc,
4110                 0x7b00, 0x7c54,
4111                 0x7d00, 0x7efc,
4112                 0x8dc0, 0x8de0,
4113                 0x8df8, 0x8e84,
4114                 0x8ea0, 0x8f84,
4115                 0x8fc0, 0x90f8,
4116                 0x9400, 0x9470,
4117                 0x9600, 0x96f4,
4118                 0x9800, 0x9808,
4119                 0x9820, 0x983c,
4120                 0x9850, 0x9864,
4121                 0x9c00, 0x9c6c,
4122                 0x9c80, 0x9cec,
4123                 0x9d00, 0x9d6c,
4124                 0x9d80, 0x9dec,
4125                 0x9e00, 0x9e6c,
4126                 0x9e80, 0x9eec,
4127                 0x9f00, 0x9f6c,
4128                 0x9f80, 0xa020,
4129                 0xd004, 0xd03c,
4130                 0xdfc0, 0xdfe0,
4131                 0xe000, 0x11088,
4132                 0x1109c, 0x11110,
4133                 0x11118, 0x1117c,
4134                 0x11190, 0x11204,
4135                 0x19040, 0x1906c,
4136                 0x19078, 0x19080,
4137                 0x1908c, 0x19124,
4138                 0x19150, 0x191b0,
4139                 0x191d0, 0x191e8,
4140                 0x19238, 0x19290,
4141                 0x193f8, 0x19474,
4142                 0x19490, 0x194cc,
4143                 0x194f0, 0x194f8,
4144                 0x19c00, 0x19c60,
4145                 0x19c94, 0x19e10,
4146                 0x19e50, 0x19f34,
4147                 0x19f40, 0x19f50,
4148                 0x19f90, 0x19fe4,
4149                 0x1a000, 0x1a06c,
4150                 0x1a0b0, 0x1a120,
4151                 0x1a128, 0x1a138,
4152                 0x1a190, 0x1a1c4,
4153                 0x1a1fc, 0x1a1fc,
4154                 0x1e008, 0x1e00c,
4155                 0x1e040, 0x1e04c,
4156                 0x1e284, 0x1e290,
4157                 0x1e2c0, 0x1e2c0,
4158                 0x1e2e0, 0x1e2e0,
4159                 0x1e300, 0x1e384,
4160                 0x1e3c0, 0x1e3c8,
4161                 0x1e408, 0x1e40c,
4162                 0x1e440, 0x1e44c,
4163                 0x1e684, 0x1e690,
4164                 0x1e6c0, 0x1e6c0,
4165                 0x1e6e0, 0x1e6e0,
4166                 0x1e700, 0x1e784,
4167                 0x1e7c0, 0x1e7c8,
4168                 0x1e808, 0x1e80c,
4169                 0x1e840, 0x1e84c,
4170                 0x1ea84, 0x1ea90,
4171                 0x1eac0, 0x1eac0,
4172                 0x1eae0, 0x1eae0,
4173                 0x1eb00, 0x1eb84,
4174                 0x1ebc0, 0x1ebc8,
4175                 0x1ec08, 0x1ec0c,
4176                 0x1ec40, 0x1ec4c,
4177                 0x1ee84, 0x1ee90,
4178                 0x1eec0, 0x1eec0,
4179                 0x1eee0, 0x1eee0,
4180                 0x1ef00, 0x1ef84,
4181                 0x1efc0, 0x1efc8,
4182                 0x1f008, 0x1f00c,
4183                 0x1f040, 0x1f04c,
4184                 0x1f284, 0x1f290,
4185                 0x1f2c0, 0x1f2c0,
4186                 0x1f2e0, 0x1f2e0,
4187                 0x1f300, 0x1f384,
4188                 0x1f3c0, 0x1f3c8,
4189                 0x1f408, 0x1f40c,
4190                 0x1f440, 0x1f44c,
4191                 0x1f684, 0x1f690,
4192                 0x1f6c0, 0x1f6c0,
4193                 0x1f6e0, 0x1f6e0,
4194                 0x1f700, 0x1f784,
4195                 0x1f7c0, 0x1f7c8,
4196                 0x1f808, 0x1f80c,
4197                 0x1f840, 0x1f84c,
4198                 0x1fa84, 0x1fa90,
4199                 0x1fac0, 0x1fac0,
4200                 0x1fae0, 0x1fae0,
4201                 0x1fb00, 0x1fb84,
4202                 0x1fbc0, 0x1fbc8,
4203                 0x1fc08, 0x1fc0c,
4204                 0x1fc40, 0x1fc4c,
4205                 0x1fe84, 0x1fe90,
4206                 0x1fec0, 0x1fec0,
4207                 0x1fee0, 0x1fee0,
4208                 0x1ff00, 0x1ff84,
4209                 0x1ffc0, 0x1ffc8,
4210                 0x30000, 0x30030,
4211                 0x30100, 0x30144,
4212                 0x30190, 0x301d0,
4213                 0x30200, 0x30318,
4214                 0x30400, 0x3052c,
4215                 0x30540, 0x3061c,
4216                 0x30800, 0x30834,
4217                 0x308c0, 0x30908,
4218                 0x30910, 0x309ac,
4219                 0x30a00, 0x30a2c,
4220                 0x30a44, 0x30a50,
4221                 0x30a74, 0x30c24,
4222                 0x30d00, 0x30d00,
4223                 0x30d08, 0x30d14,
4224                 0x30d1c, 0x30d20,
4225                 0x30d3c, 0x30d50,
4226                 0x31200, 0x3120c,
4227                 0x31220, 0x31220,
4228                 0x31240, 0x31240,
4229                 0x31600, 0x3160c,
4230                 0x31a00, 0x31a1c,
4231                 0x31e00, 0x31e20,
4232                 0x31e38, 0x31e3c,
4233                 0x31e80, 0x31e80,
4234                 0x31e88, 0x31ea8,
4235                 0x31eb0, 0x31eb4,
4236                 0x31ec8, 0x31ed4,
4237                 0x31fb8, 0x32004,
4238                 0x32200, 0x32200,
4239                 0x32208, 0x32240,
4240                 0x32248, 0x32280,
4241                 0x32288, 0x322c0,
4242                 0x322c8, 0x322fc,
4243                 0x32600, 0x32630,
4244                 0x32a00, 0x32abc,
4245                 0x32b00, 0x32b70,
4246                 0x33000, 0x33048,
4247                 0x33060, 0x3309c,
4248                 0x330f0, 0x33148,
4249                 0x33160, 0x3319c,
4250                 0x331f0, 0x332e4,
4251                 0x332f8, 0x333e4,
4252                 0x333f8, 0x33448,
4253                 0x33460, 0x3349c,
4254                 0x334f0, 0x33548,
4255                 0x33560, 0x3359c,
4256                 0x335f0, 0x336e4,
4257                 0x336f8, 0x337e4,
4258                 0x337f8, 0x337fc,
4259                 0x33814, 0x33814,
4260                 0x3382c, 0x3382c,
4261                 0x33880, 0x3388c,
4262                 0x338e8, 0x338ec,
4263                 0x33900, 0x33948,
4264                 0x33960, 0x3399c,
4265                 0x339f0, 0x33ae4,
4266                 0x33af8, 0x33b10,
4267                 0x33b28, 0x33b28,
4268                 0x33b3c, 0x33b50,
4269                 0x33bf0, 0x33c10,
4270                 0x33c28, 0x33c28,
4271                 0x33c3c, 0x33c50,
4272                 0x33cf0, 0x33cfc,
4273                 0x34000, 0x34030,
4274                 0x34100, 0x34144,
4275                 0x34190, 0x341d0,
4276                 0x34200, 0x34318,
4277                 0x34400, 0x3452c,
4278                 0x34540, 0x3461c,
4279                 0x34800, 0x34834,
4280                 0x348c0, 0x34908,
4281                 0x34910, 0x349ac,
4282                 0x34a00, 0x34a2c,
4283                 0x34a44, 0x34a50,
4284                 0x34a74, 0x34c24,
4285                 0x34d00, 0x34d00,
4286                 0x34d08, 0x34d14,
4287                 0x34d1c, 0x34d20,
4288                 0x34d3c, 0x34d50,
4289                 0x35200, 0x3520c,
4290                 0x35220, 0x35220,
4291                 0x35240, 0x35240,
4292                 0x35600, 0x3560c,
4293                 0x35a00, 0x35a1c,
4294                 0x35e00, 0x35e20,
4295                 0x35e38, 0x35e3c,
4296                 0x35e80, 0x35e80,
4297                 0x35e88, 0x35ea8,
4298                 0x35eb0, 0x35eb4,
4299                 0x35ec8, 0x35ed4,
4300                 0x35fb8, 0x36004,
4301                 0x36200, 0x36200,
4302                 0x36208, 0x36240,
4303                 0x36248, 0x36280,
4304                 0x36288, 0x362c0,
4305                 0x362c8, 0x362fc,
4306                 0x36600, 0x36630,
4307                 0x36a00, 0x36abc,
4308                 0x36b00, 0x36b70,
4309                 0x37000, 0x37048,
4310                 0x37060, 0x3709c,
4311                 0x370f0, 0x37148,
4312                 0x37160, 0x3719c,
4313                 0x371f0, 0x372e4,
4314                 0x372f8, 0x373e4,
4315                 0x373f8, 0x37448,
4316                 0x37460, 0x3749c,
4317                 0x374f0, 0x37548,
4318                 0x37560, 0x3759c,
4319                 0x375f0, 0x376e4,
4320                 0x376f8, 0x377e4,
4321                 0x377f8, 0x377fc,
4322                 0x37814, 0x37814,
4323                 0x3782c, 0x3782c,
4324                 0x37880, 0x3788c,
4325                 0x378e8, 0x378ec,
4326                 0x37900, 0x37948,
4327                 0x37960, 0x3799c,
4328                 0x379f0, 0x37ae4,
4329                 0x37af8, 0x37b10,
4330                 0x37b28, 0x37b28,
4331                 0x37b3c, 0x37b50,
4332                 0x37bf0, 0x37c10,
4333                 0x37c28, 0x37c28,
4334                 0x37c3c, 0x37c50,
4335                 0x37cf0, 0x37cfc,
4336                 0x38000, 0x38030,
4337                 0x38100, 0x38144,
4338                 0x38190, 0x381d0,
4339                 0x38200, 0x38318,
4340                 0x38400, 0x3852c,
4341                 0x38540, 0x3861c,
4342                 0x38800, 0x38834,
4343                 0x388c0, 0x38908,
4344                 0x38910, 0x389ac,
4345                 0x38a00, 0x38a2c,
4346                 0x38a44, 0x38a50,
4347                 0x38a74, 0x38c24,
4348                 0x38d00, 0x38d00,
4349                 0x38d08, 0x38d14,
4350                 0x38d1c, 0x38d20,
4351                 0x38d3c, 0x38d50,
4352                 0x39200, 0x3920c,
4353                 0x39220, 0x39220,
4354                 0x39240, 0x39240,
4355                 0x39600, 0x3960c,
4356                 0x39a00, 0x39a1c,
4357                 0x39e00, 0x39e20,
4358                 0x39e38, 0x39e3c,
4359                 0x39e80, 0x39e80,
4360                 0x39e88, 0x39ea8,
4361                 0x39eb0, 0x39eb4,
4362                 0x39ec8, 0x39ed4,
4363                 0x39fb8, 0x3a004,
4364                 0x3a200, 0x3a200,
4365                 0x3a208, 0x3a240,
4366                 0x3a248, 0x3a280,
4367                 0x3a288, 0x3a2c0,
4368                 0x3a2c8, 0x3a2fc,
4369                 0x3a600, 0x3a630,
4370                 0x3aa00, 0x3aabc,
4371                 0x3ab00, 0x3ab70,
4372                 0x3b000, 0x3b048,
4373                 0x3b060, 0x3b09c,
4374                 0x3b0f0, 0x3b148,
4375                 0x3b160, 0x3b19c,
4376                 0x3b1f0, 0x3b2e4,
4377                 0x3b2f8, 0x3b3e4,
4378                 0x3b3f8, 0x3b448,
4379                 0x3b460, 0x3b49c,
4380                 0x3b4f0, 0x3b548,
4381                 0x3b560, 0x3b59c,
4382                 0x3b5f0, 0x3b6e4,
4383                 0x3b6f8, 0x3b7e4,
4384                 0x3b7f8, 0x3b7fc,
4385                 0x3b814, 0x3b814,
4386                 0x3b82c, 0x3b82c,
4387                 0x3b880, 0x3b88c,
4388                 0x3b8e8, 0x3b8ec,
4389                 0x3b900, 0x3b948,
4390                 0x3b960, 0x3b99c,
4391                 0x3b9f0, 0x3bae4,
4392                 0x3baf8, 0x3bb10,
4393                 0x3bb28, 0x3bb28,
4394                 0x3bb3c, 0x3bb50,
4395                 0x3bbf0, 0x3bc10,
4396                 0x3bc28, 0x3bc28,
4397                 0x3bc3c, 0x3bc50,
4398                 0x3bcf0, 0x3bcfc,
4399                 0x3c000, 0x3c030,
4400                 0x3c100, 0x3c144,
4401                 0x3c190, 0x3c1d0,
4402                 0x3c200, 0x3c318,
4403                 0x3c400, 0x3c52c,
4404                 0x3c540, 0x3c61c,
4405                 0x3c800, 0x3c834,
4406                 0x3c8c0, 0x3c908,
4407                 0x3c910, 0x3c9ac,
4408                 0x3ca00, 0x3ca2c,
4409                 0x3ca44, 0x3ca50,
4410                 0x3ca74, 0x3cc24,
4411                 0x3cd00, 0x3cd00,
4412                 0x3cd08, 0x3cd14,
4413                 0x3cd1c, 0x3cd20,
4414                 0x3cd3c, 0x3cd50,
4415                 0x3d200, 0x3d20c,
4416                 0x3d220, 0x3d220,
4417                 0x3d240, 0x3d240,
4418                 0x3d600, 0x3d60c,
4419                 0x3da00, 0x3da1c,
4420                 0x3de00, 0x3de20,
4421                 0x3de38, 0x3de3c,
4422                 0x3de80, 0x3de80,
4423                 0x3de88, 0x3dea8,
4424                 0x3deb0, 0x3deb4,
4425                 0x3dec8, 0x3ded4,
4426                 0x3dfb8, 0x3e004,
4427                 0x3e200, 0x3e200,
4428                 0x3e208, 0x3e240,
4429                 0x3e248, 0x3e280,
4430                 0x3e288, 0x3e2c0,
4431                 0x3e2c8, 0x3e2fc,
4432                 0x3e600, 0x3e630,
4433                 0x3ea00, 0x3eabc,
4434                 0x3eb00, 0x3eb70,
4435                 0x3f000, 0x3f048,
4436                 0x3f060, 0x3f09c,
4437                 0x3f0f0, 0x3f148,
4438                 0x3f160, 0x3f19c,
4439                 0x3f1f0, 0x3f2e4,
4440                 0x3f2f8, 0x3f3e4,
4441                 0x3f3f8, 0x3f448,
4442                 0x3f460, 0x3f49c,
4443                 0x3f4f0, 0x3f548,
4444                 0x3f560, 0x3f59c,
4445                 0x3f5f0, 0x3f6e4,
4446                 0x3f6f8, 0x3f7e4,
4447                 0x3f7f8, 0x3f7fc,
4448                 0x3f814, 0x3f814,
4449                 0x3f82c, 0x3f82c,
4450                 0x3f880, 0x3f88c,
4451                 0x3f8e8, 0x3f8ec,
4452                 0x3f900, 0x3f948,
4453                 0x3f960, 0x3f99c,
4454                 0x3f9f0, 0x3fae4,
4455                 0x3faf8, 0x3fb10,
4456                 0x3fb28, 0x3fb28,
4457                 0x3fb3c, 0x3fb50,
4458                 0x3fbf0, 0x3fc10,
4459                 0x3fc28, 0x3fc28,
4460                 0x3fc3c, 0x3fc50,
4461                 0x3fcf0, 0x3fcfc,
4462                 0x40000, 0x4000c,
4463                 0x40040, 0x40068,
4464                 0x4007c, 0x40144,
4465                 0x40180, 0x4018c,
4466                 0x40200, 0x40298,
4467                 0x402ac, 0x4033c,
4468                 0x403f8, 0x403fc,
4469                 0x41304, 0x413c4,
4470                 0x41400, 0x4141c,
4471                 0x41480, 0x414d0,
4472                 0x44000, 0x44078,
4473                 0x440c0, 0x44278,
4474                 0x442c0, 0x44478,
4475                 0x444c0, 0x44678,
4476                 0x446c0, 0x44878,
4477                 0x448c0, 0x449fc,
4478                 0x45000, 0x45068,
4479                 0x45080, 0x45084,
4480                 0x450a0, 0x450b0,
4481                 0x45200, 0x45268,
4482                 0x45280, 0x45284,
4483                 0x452a0, 0x452b0,
4484                 0x460c0, 0x460e4,
4485                 0x47000, 0x4708c,
4486                 0x47200, 0x47250,
4487                 0x47400, 0x47420,
4488                 0x47600, 0x47618,
4489                 0x47800, 0x47814,
4490                 0x48000, 0x4800c,
4491                 0x48040, 0x48068,
4492                 0x4807c, 0x48144,
4493                 0x48180, 0x4818c,
4494                 0x48200, 0x48298,
4495                 0x482ac, 0x4833c,
4496                 0x483f8, 0x483fc,
4497                 0x49304, 0x493c4,
4498                 0x49400, 0x4941c,
4499                 0x49480, 0x494d0,
4500                 0x4c000, 0x4c078,
4501                 0x4c0c0, 0x4c278,
4502                 0x4c2c0, 0x4c478,
4503                 0x4c4c0, 0x4c678,
4504                 0x4c6c0, 0x4c878,
4505                 0x4c8c0, 0x4c9fc,
4506                 0x4d000, 0x4d068,
4507                 0x4d080, 0x4d084,
4508                 0x4d0a0, 0x4d0b0,
4509                 0x4d200, 0x4d268,
4510                 0x4d280, 0x4d284,
4511                 0x4d2a0, 0x4d2b0,
4512                 0x4e0c0, 0x4e0e4,
4513                 0x4f000, 0x4f08c,
4514                 0x4f200, 0x4f250,
4515                 0x4f400, 0x4f420,
4516                 0x4f600, 0x4f618,
4517                 0x4f800, 0x4f814,
4518                 0x50000, 0x500cc,
4519                 0x50400, 0x50400,
4520                 0x50800, 0x508cc,
4521                 0x50c00, 0x50c00,
4522                 0x51000, 0x5101c,
4523                 0x51300, 0x51308,
4524         };
4525
4526         if (is_t4(sc)) {
4527                 reg_ranges = &t4_reg_ranges[0];
4528                 n = nitems(t4_reg_ranges);
4529         } else {
4530                 reg_ranges = &t5_reg_ranges[0];
4531                 n = nitems(t5_reg_ranges);
4532         }
4533
4534         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4535         for (i = 0; i < n; i += 2)
4536                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4537 }
4538
4539 static void
4540 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4541 {
4542         int i;
4543         u_int v, tnl_cong_drops;
4544         struct timeval tv;
4545         const struct timeval interval = {0, 250000};    /* 250ms */
4546
4547         getmicrotime(&tv);
4548         timevalsub(&tv, &interval);
4549         if (timevalcmp(&tv, &pi->last_refreshed, <))
4550                 return;
4551
4552         tnl_cong_drops = 0;
4553         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4554         for (i = 0; i < NCHAN; i++) {
4555                 if (pi->rx_chan_map & (1 << i)) {
4556                         mtx_lock(&sc->regwin_lock);
4557                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4558                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4559                         mtx_unlock(&sc->regwin_lock);
4560                         tnl_cong_drops += v;
4561                 }
4562         }
4563         pi->tnl_cong_drops = tnl_cong_drops;
4564         getmicrotime(&pi->last_refreshed);
4565 }
4566
4567 static void
4568 cxgbe_tick(void *arg)
4569 {
4570         struct port_info *pi = arg;
4571         struct adapter *sc = pi->adapter;
4572         struct ifnet *ifp = pi->ifp;
4573
4574         PORT_LOCK(pi);
4575         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4576                 PORT_UNLOCK(pi);
4577                 return; /* without scheduling another callout */
4578         }
4579
4580         cxgbe_refresh_stats(sc, pi);
4581
4582         callout_schedule(&pi->tick, hz);
4583         PORT_UNLOCK(pi);
4584 }
4585
4586 static void
4587 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4588 {
4589         struct ifnet *vlan;
4590
4591         if (arg != ifp || ifp->if_type != IFT_ETHER)
4592                 return;
4593
4594         vlan = VLAN_DEVAT(ifp, vid);
4595         VLAN_SETCOOKIE(vlan, ifp);
4596 }
4597
4598 static int
4599 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4600 {
4601
4602 #ifdef INVARIANTS
4603         panic("%s: opcode 0x%02x on iq %p with payload %p",
4604             __func__, rss->opcode, iq, m);
4605 #else
4606         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4607             __func__, rss->opcode, iq, m);
4608         m_freem(m);
4609 #endif
4610         return (EDOOFUS);
4611 }
4612
4613 int
4614 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4615 {
4616         uintptr_t *loc, new;
4617
4618         if (opcode >= nitems(sc->cpl_handler))
4619                 return (EINVAL);
4620
4621         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4622         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4623         atomic_store_rel_ptr(loc, new);
4624
4625         return (0);
4626 }
4627
4628 static int
4629 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4630 {
4631
4632 #ifdef INVARIANTS
4633         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4634 #else
4635         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4636             __func__, iq, ctrl);
4637 #endif
4638         return (EDOOFUS);
4639 }
4640
4641 int
4642 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4643 {
4644         uintptr_t *loc, new;
4645
4646         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4647         loc = (uintptr_t *) &sc->an_handler;
4648         atomic_store_rel_ptr(loc, new);
4649
4650         return (0);
4651 }
4652
4653 static int
4654 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4655 {
4656         const struct cpl_fw6_msg *cpl =
4657             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4658
4659 #ifdef INVARIANTS
4660         panic("%s: fw_msg type %d", __func__, cpl->type);
4661 #else
4662         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4663 #endif
4664         return (EDOOFUS);
4665 }
4666
4667 int
4668 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4669 {
4670         uintptr_t *loc, new;
4671
4672         if (type >= nitems(sc->fw_msg_handler))
4673                 return (EINVAL);
4674
4675         /*
4676          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4677          * handler dispatch table.  Reject any attempt to install a handler for
4678          * this subtype.
4679          */
4680         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4681                 return (EINVAL);
4682
4683         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4684         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4685         atomic_store_rel_ptr(loc, new);
4686
4687         return (0);
4688 }
4689
4690 static int
4691 t4_sysctls(struct adapter *sc)
4692 {
4693         struct sysctl_ctx_list *ctx;
4694         struct sysctl_oid *oid;
4695         struct sysctl_oid_list *children, *c0;
4696         static char *caps[] = {
4697                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4698                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4699                     "\6HASHFILTER\7ETHOFLD",
4700                 "\20\1TOE",                             /* caps[2] toecaps */
4701                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4702                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4703                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4704                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4705                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4706                     "\4PO_INITIAOR\5PO_TARGET"
4707         };
4708         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4709
4710         ctx = device_get_sysctl_ctx(sc->dev);
4711
4712         /*
4713          * dev.t4nex.X.
4714          */
4715         oid = device_get_sysctl_tree(sc->dev);
4716         c0 = children = SYSCTL_CHILDREN(oid);
4717
4718         sc->sc_do_rxcopy = 1;
4719         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4720             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4721
4722         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4723             sc->params.nports, "# of ports");
4724
4725         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4726             NULL, chip_rev(sc), "chip hardware revision");
4727
4728         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4729             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4730
4731         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4732             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4733
4734         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4735             sc->cfcsum, "config file checksum");
4736
4737         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4738             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4739             sysctl_bitfield, "A", "available doorbells");
4740
4741         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4742             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4743             sysctl_bitfield, "A", "available link capabilities");
4744
4745         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4746             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4747             sysctl_bitfield, "A", "available NIC capabilities");
4748
4749         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4750             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4751             sysctl_bitfield, "A", "available TCP offload capabilities");
4752
4753         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4754             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4755             sysctl_bitfield, "A", "available RDMA capabilities");
4756
4757         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4758             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4759             sysctl_bitfield, "A", "available iSCSI capabilities");
4760
4761         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4762             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4763             sysctl_bitfield, "A", "available FCoE capabilities");
4764
4765         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4766             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4767
4768         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4769             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4770             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4771             "interrupt holdoff timer values (us)");
4772
4773         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4774             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4775             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4776             "interrupt holdoff packet counter values");
4777
4778         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4779             NULL, sc->tids.nftids, "number of filters");
4780
4781         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4782             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4783             "chip temperature (in Celsius)");
4784
4785         t4_sge_sysctls(sc, ctx, children);
4786
4787         sc->lro_timeout = 100;
4788         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4789             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4790
4791         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4792             &sc->debug_flags, 0, "flags to enable runtime debugging");
4793
4794 #ifdef SBUF_DRAIN
4795         /*
4796          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4797          */
4798         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4799             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4800             "logs and miscellaneous information");
4801         children = SYSCTL_CHILDREN(oid);
4802
4803         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4804             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4805             sysctl_cctrl, "A", "congestion control");
4806
4807         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4808             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4809             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4810
4811         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4812             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4813             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4814
4815         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4816             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4817             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4818
4819         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4820             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4821             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4822
4823         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4824             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4825             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4826
4827         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4828             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4829             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4830
4831         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4832             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4833             sysctl_cim_la, "A", "CIM logic analyzer");
4834
4835         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4836             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4837             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4838
4839         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4840             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4841             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4842
4843         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4844             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4845             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4846
4847         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4848             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4849             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4850
4851         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4852             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4853             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4854
4855         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4856             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4857             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4858
4859         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4860             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4861             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4862
4863         if (is_t5(sc)) {
4864                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4865                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4866                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4867
4868                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4869                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4870                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4871         }
4872
4873         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4874             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4875             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4876
4877         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4878             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4879             sysctl_cim_qcfg, "A", "CIM queue configuration");
4880
4881         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4882             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4883             sysctl_cpl_stats, "A", "CPL statistics");
4884
4885         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4886             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4887             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4888
4889         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4890             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4891             sysctl_devlog, "A", "firmware's device log");
4892
4893         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4894             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4895             sysctl_fcoe_stats, "A", "FCoE statistics");
4896
4897         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4898             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4899             sysctl_hw_sched, "A", "hardware scheduler ");
4900
4901         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4902             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4903             sysctl_l2t, "A", "hardware L2 table");
4904
4905         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4906             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4907             sysctl_lb_stats, "A", "loopback statistics");
4908
4909         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4910             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4911             sysctl_meminfo, "A", "memory regions");
4912
4913         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4914             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4915             sysctl_mps_tcam, "A", "MPS TCAM entries");
4916
4917         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4918             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4919             sysctl_path_mtus, "A", "path MTUs");
4920
4921         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4922             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4923             sysctl_pm_stats, "A", "PM statistics");
4924
4925         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4926             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4927             sysctl_rdma_stats, "A", "RDMA statistics");
4928
4929         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4930             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4931             sysctl_tcp_stats, "A", "TCP statistics");
4932
4933         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4934             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4935             sysctl_tids, "A", "TID information");
4936
4937         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4938             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4939             sysctl_tp_err_stats, "A", "TP error statistics");
4940
4941         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4942             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4943             sysctl_tp_la, "A", "TP logic analyzer");
4944
4945         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4946             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4947             sysctl_tx_rate, "A", "Tx rate");
4948
4949         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4950             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4951             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4952
4953         if (is_t5(sc)) {
4954                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4955                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4956                     sysctl_wcwr_stats, "A", "write combined work requests");
4957         }
4958 #endif
4959
4960 #ifdef TCP_OFFLOAD
4961         if (is_offload(sc)) {
4962                 /*
4963                  * dev.t4nex.X.toe.
4964                  */
4965                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4966                     NULL, "TOE parameters");
4967                 children = SYSCTL_CHILDREN(oid);
4968
4969                 sc->tt.sndbuf = 256 * 1024;
4970                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4971                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4972
4973                 sc->tt.ddp = 0;
4974                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4975                     &sc->tt.ddp, 0, "DDP allowed");
4976
4977                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4978                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4979                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4980
4981                 sc->tt.ddp_thres =
4982                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4983                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4984                     &sc->tt.ddp_thres, 0, "DDP threshold");
4985
4986                 sc->tt.rx_coalesce = 1;
4987                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4988                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4989
4990                 sc->tt.tx_align = 1;
4991                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4992                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4993         }
4994 #endif
4995
4996
4997         return (0);
4998 }
4999
5000 static int
5001 cxgbe_sysctls(struct port_info *pi)
5002 {
5003         struct sysctl_ctx_list *ctx;
5004         struct sysctl_oid *oid;
5005         struct sysctl_oid_list *children;
5006         struct adapter *sc = pi->adapter;
5007
5008         ctx = device_get_sysctl_ctx(pi->dev);
5009
5010         /*
5011          * dev.cxgbe.X.
5012          */
5013         oid = device_get_sysctl_tree(pi->dev);
5014         children = SYSCTL_CHILDREN(oid);
5015
5016         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
5017            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
5018         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
5019                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
5020                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
5021                     "PHY temperature (in Celsius)");
5022                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
5023                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
5024                     "PHY firmware version");
5025         }
5026         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
5027             &pi->nrxq, 0, "# of rx queues");
5028         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
5029             &pi->ntxq, 0, "# of tx queues");
5030         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
5031             &pi->first_rxq, 0, "index of first rx queue");
5032         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
5033             &pi->first_txq, 0, "index of first tx queue");
5034         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
5035             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
5036             "Reserve queue 0 for non-flowid packets");
5037
5038 #ifdef TCP_OFFLOAD
5039         if (is_offload(sc)) {
5040                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
5041                     &pi->nofldrxq, 0,
5042                     "# of rx queues for offloaded TCP connections");
5043                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
5044                     &pi->nofldtxq, 0,
5045                     "# of tx queues for offloaded TCP connections");
5046                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
5047                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
5048                     "index of first TOE rx queue");
5049                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
5050                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
5051                     "index of first TOE tx queue");
5052         }
5053 #endif
5054 #ifdef DEV_NETMAP
5055         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
5056             &pi->nnmrxq, 0, "# of rx queues for netmap");
5057         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
5058             &pi->nnmtxq, 0, "# of tx queues for netmap");
5059         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
5060             CTLFLAG_RD, &pi->first_nm_rxq, 0,
5061             "index of first netmap rx queue");
5062         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
5063             CTLFLAG_RD, &pi->first_nm_txq, 0,
5064             "index of first netmap tx queue");
5065 #endif
5066
5067         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
5068             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
5069             "holdoff timer index");
5070         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
5071             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
5072             "holdoff packet counter index");
5073
5074         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
5075             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
5076             "rx queue size");
5077         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
5078             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
5079             "tx queue size");
5080
5081         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
5082             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
5083             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
5084
5085         /*
5086          * dev.cxgbe.X.stats.
5087          */
5088         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
5089             NULL, "port statistics");
5090         children = SYSCTL_CHILDREN(oid);
5091         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
5092             &pi->tx_parse_error, 0,
5093             "# of tx packets with invalid length or # of segments");
5094
5095 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
5096         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
5097             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
5098             sysctl_handle_t4_reg64, "QU", desc)
5099
5100         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
5101             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
5102         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
5103             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
5104         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
5105             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
5106         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
5107             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
5108         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
5109             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
5110         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
5111             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
5112         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
5113             "# of tx frames in this range",
5114             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
5115         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
5116             "# of tx frames in this range",
5117             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
5118         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
5119             "# of tx frames in this range",
5120             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
5121         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
5122             "# of tx frames in this range",
5123             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
5124         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
5125             "# of tx frames in this range",
5126             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
5127         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
5128             "# of tx frames in this range",
5129             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
5130         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
5131             "# of tx frames in this range",
5132             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
5133         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
5134             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
5135         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
5136             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
5137         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
5138             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
5139         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
5140             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
5141         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
5142             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
5143         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
5144             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
5145         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
5146             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
5147         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
5148             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
5149         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
5150             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
5151         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
5152             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
5153
5154         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
5155             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
5156         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
5157             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
5158         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
5159             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
5160         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
5161             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
5162         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
5163             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
5164         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
5165             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
5166         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
5167             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
5168         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
5169             "# of frames received with bad FCS",
5170             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
5171         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5172             "# of frames received with length error",
5173             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5174         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5175             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5176         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5177             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5178         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5179             "# of rx frames in this range",
5180             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5181         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5182             "# of rx frames in this range",
5183             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5184         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5185             "# of rx frames in this range",
5186             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5187         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5188             "# of rx frames in this range",
5189             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5190         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5191             "# of rx frames in this range",
5192             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5193         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5194             "# of rx frames in this range",
5195             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5196         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5197             "# of rx frames in this range",
5198             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5199         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5200             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5201         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5202             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5203         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5204             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5205         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5206             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5207         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5208             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5209         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5210             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5211         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5212             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5213         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5214             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5215         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5216             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5217
5218 #undef SYSCTL_ADD_T4_REG64
5219
5220 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5221         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5222             &pi->stats.name, desc)
5223
5224         /* We get these from port_stats and they may be stale by upto 1s */
5225         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5226             "# drops due to buffer-group 0 overflows");
5227         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5228             "# drops due to buffer-group 1 overflows");
5229         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5230             "# drops due to buffer-group 2 overflows");
5231         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5232             "# drops due to buffer-group 3 overflows");
5233         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5234             "# of buffer-group 0 truncated packets");
5235         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5236             "# of buffer-group 1 truncated packets");
5237         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5238             "# of buffer-group 2 truncated packets");
5239         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5240             "# of buffer-group 3 truncated packets");
5241
5242 #undef SYSCTL_ADD_T4_PORTSTAT
5243
5244         return (0);
5245 }
5246
5247 static int
5248 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5249 {
5250         int rc, *i, space = 0;
5251         struct sbuf sb;
5252
5253         sbuf_new_for_sysctl(&sb, NULL, 64, req);
5254         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5255                 if (space)
5256                         sbuf_printf(&sb, " ");
5257                 sbuf_printf(&sb, "%d", *i);
5258                 space = 1;
5259         }
5260         rc = sbuf_finish(&sb);
5261         sbuf_delete(&sb);
5262         return (rc);
5263 }
5264
5265 static int
5266 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5267 {
5268         int rc;
5269         struct sbuf *sb;
5270
5271         rc = sysctl_wire_old_buffer(req, 0);
5272         if (rc != 0)
5273                 return(rc);
5274
5275         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5276         if (sb == NULL)
5277                 return (ENOMEM);
5278
5279         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5280         rc = sbuf_finish(sb);
5281         sbuf_delete(sb);
5282
5283         return (rc);
5284 }
5285
5286 static int
5287 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5288 {
5289         struct port_info *pi = arg1;
5290         int op = arg2;
5291         struct adapter *sc = pi->adapter;
5292         u_int v;
5293         int rc;
5294
5295         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5296         if (rc)
5297                 return (rc);
5298         /* XXX: magic numbers */
5299         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5300             &v);
5301         end_synchronized_op(sc, 0);
5302         if (rc)
5303                 return (rc);
5304         if (op == 0)
5305                 v /= 256;
5306
5307         rc = sysctl_handle_int(oidp, &v, 0, req);
5308         return (rc);
5309 }
5310
5311 static int
5312 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5313 {
5314         struct port_info *pi = arg1;
5315         int rc, val;
5316
5317         val = pi->rsrv_noflowq;
5318         rc = sysctl_handle_int(oidp, &val, 0, req);
5319         if (rc != 0 || req->newptr == NULL)
5320                 return (rc);
5321
5322         if ((val >= 1) && (pi->ntxq > 1))
5323                 pi->rsrv_noflowq = 1;
5324         else
5325                 pi->rsrv_noflowq = 0;
5326
5327         return (rc);
5328 }
5329
5330 static int
5331 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5332 {
5333         struct port_info *pi = arg1;
5334         struct adapter *sc = pi->adapter;
5335         int idx, rc, i;
5336         struct sge_rxq *rxq;
5337 #ifdef TCP_OFFLOAD
5338         struct sge_ofld_rxq *ofld_rxq;
5339 #endif
5340         uint8_t v;
5341
5342         idx = pi->tmr_idx;
5343
5344         rc = sysctl_handle_int(oidp, &idx, 0, req);
5345         if (rc != 0 || req->newptr == NULL)
5346                 return (rc);
5347
5348         if (idx < 0 || idx >= SGE_NTIMERS)
5349                 return (EINVAL);
5350
5351         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5352             "t4tmr");
5353         if (rc)
5354                 return (rc);
5355
5356         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5357         for_each_rxq(pi, i, rxq) {
5358 #ifdef atomic_store_rel_8
5359                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5360 #else
5361                 rxq->iq.intr_params = v;
5362 #endif
5363         }
5364 #ifdef TCP_OFFLOAD
5365         for_each_ofld_rxq(pi, i, ofld_rxq) {
5366 #ifdef atomic_store_rel_8
5367                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5368 #else
5369                 ofld_rxq->iq.intr_params = v;
5370 #endif
5371         }
5372 #endif
5373         pi->tmr_idx = idx;
5374
5375         end_synchronized_op(sc, LOCK_HELD);
5376         return (0);
5377 }
5378
5379 static int
5380 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5381 {
5382         struct port_info *pi = arg1;
5383         struct adapter *sc = pi->adapter;
5384         int idx, rc;
5385
5386         idx = pi->pktc_idx;
5387
5388         rc = sysctl_handle_int(oidp, &idx, 0, req);
5389         if (rc != 0 || req->newptr == NULL)
5390                 return (rc);
5391
5392         if (idx < -1 || idx >= SGE_NCOUNTERS)
5393                 return (EINVAL);
5394
5395         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5396             "t4pktc");
5397         if (rc)
5398                 return (rc);
5399
5400         if (pi->flags & PORT_INIT_DONE)
5401                 rc = EBUSY; /* cannot be changed once the queues are created */
5402         else
5403                 pi->pktc_idx = idx;
5404
5405         end_synchronized_op(sc, LOCK_HELD);
5406         return (rc);
5407 }
5408
5409 static int
5410 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5411 {
5412         struct port_info *pi = arg1;
5413         struct adapter *sc = pi->adapter;
5414         int qsize, rc;
5415
5416         qsize = pi->qsize_rxq;
5417
5418         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5419         if (rc != 0 || req->newptr == NULL)
5420                 return (rc);
5421
5422         if (qsize < 128 || (qsize & 7))
5423                 return (EINVAL);
5424
5425         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5426             "t4rxqs");
5427         if (rc)
5428                 return (rc);
5429
5430         if (pi->flags & PORT_INIT_DONE)
5431                 rc = EBUSY; /* cannot be changed once the queues are created */
5432         else
5433                 pi->qsize_rxq = qsize;
5434
5435         end_synchronized_op(sc, LOCK_HELD);
5436         return (rc);
5437 }
5438
5439 static int
5440 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5441 {
5442         struct port_info *pi = arg1;
5443         struct adapter *sc = pi->adapter;
5444         int qsize, rc;
5445
5446         qsize = pi->qsize_txq;
5447
5448         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5449         if (rc != 0 || req->newptr == NULL)
5450                 return (rc);
5451
5452         if (qsize < 128 || qsize > 65536)
5453                 return (EINVAL);
5454
5455         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5456             "t4txqs");
5457         if (rc)
5458                 return (rc);
5459
5460         if (pi->flags & PORT_INIT_DONE)
5461                 rc = EBUSY; /* cannot be changed once the queues are created */
5462         else
5463                 pi->qsize_txq = qsize;
5464
5465         end_synchronized_op(sc, LOCK_HELD);
5466         return (rc);
5467 }
5468
5469 static int
5470 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5471 {
5472         struct port_info *pi = arg1;
5473         struct adapter *sc = pi->adapter;
5474         struct link_config *lc = &pi->link_cfg;
5475         int rc;
5476
5477         if (req->newptr == NULL) {
5478                 struct sbuf *sb;
5479                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5480
5481                 rc = sysctl_wire_old_buffer(req, 0);
5482                 if (rc != 0)
5483                         return(rc);
5484
5485                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5486                 if (sb == NULL)
5487                         return (ENOMEM);
5488
5489                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5490                 rc = sbuf_finish(sb);
5491                 sbuf_delete(sb);
5492         } else {
5493                 char s[2];
5494                 int n;
5495
5496                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5497                 s[1] = 0;
5498
5499                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5500                 if (rc != 0)
5501                         return(rc);
5502
5503                 if (s[1] != 0)
5504                         return (EINVAL);
5505                 if (s[0] < '0' || s[0] > '9')
5506                         return (EINVAL);        /* not a number */
5507                 n = s[0] - '0';
5508                 if (n & ~(PAUSE_TX | PAUSE_RX))
5509                         return (EINVAL);        /* some other bit is set too */
5510
5511                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5512                 if (rc)
5513                         return (rc);
5514                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5515                         int link_ok = lc->link_ok;
5516
5517                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5518                         lc->requested_fc |= n;
5519                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5520                         lc->link_ok = link_ok;  /* restore */
5521                 }
5522                 end_synchronized_op(sc, 0);
5523         }
5524
5525         return (rc);
5526 }
5527
5528 static int
5529 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5530 {
5531         struct adapter *sc = arg1;
5532         int reg = arg2;
5533         uint64_t val;
5534
5535         val = t4_read_reg64(sc, reg);
5536
5537         return (sysctl_handle_64(oidp, &val, 0, req));
5538 }
5539
5540 static int
5541 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5542 {
5543         struct adapter *sc = arg1;
5544         int rc, t;
5545         uint32_t param, val;
5546
5547         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5548         if (rc)
5549                 return (rc);
5550         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5551             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5552             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5553         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5554         end_synchronized_op(sc, 0);
5555         if (rc)
5556                 return (rc);
5557
5558         /* unknown is returned as 0 but we display -1 in that case */
5559         t = val == 0 ? -1 : val;
5560
5561         rc = sysctl_handle_int(oidp, &t, 0, req);
5562         return (rc);
5563 }
5564
5565 #ifdef SBUF_DRAIN
5566 static int
5567 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5568 {
5569         struct adapter *sc = arg1;
5570         struct sbuf *sb;
5571         int rc, i;
5572         uint16_t incr[NMTUS][NCCTRL_WIN];
5573         static const char *dec_fac[] = {
5574                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5575                 "0.9375"
5576         };
5577
5578         rc = sysctl_wire_old_buffer(req, 0);
5579         if (rc != 0)
5580                 return (rc);
5581
5582         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5583         if (sb == NULL)
5584                 return (ENOMEM);
5585
5586         t4_read_cong_tbl(sc, incr);
5587
5588         for (i = 0; i < NCCTRL_WIN; ++i) {
5589                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5590                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5591                     incr[5][i], incr[6][i], incr[7][i]);
5592                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5593                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5594                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5595                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5596         }
5597
5598         rc = sbuf_finish(sb);
5599         sbuf_delete(sb);
5600
5601         return (rc);
5602 }
5603
5604 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5605         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5606         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5607         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5608 };
5609
5610 static int
5611 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5612 {
5613         struct adapter *sc = arg1;
5614         struct sbuf *sb;
5615         int rc, i, n, qid = arg2;
5616         uint32_t *buf, *p;
5617         char *qtype;
5618         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5619
5620         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5621             ("%s: bad qid %d\n", __func__, qid));
5622
5623         if (qid < CIM_NUM_IBQ) {
5624                 /* inbound queue */
5625                 qtype = "IBQ";
5626                 n = 4 * CIM_IBQ_SIZE;
5627                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5628                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5629         } else {
5630                 /* outbound queue */
5631                 qtype = "OBQ";
5632                 qid -= CIM_NUM_IBQ;
5633                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5634                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5635                 rc = t4_read_cim_obq(sc, qid, buf, n);
5636         }
5637
5638         if (rc < 0) {
5639                 rc = -rc;
5640                 goto done;
5641         }
5642         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5643
5644         rc = sysctl_wire_old_buffer(req, 0);
5645         if (rc != 0)
5646                 goto done;
5647
5648         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5649         if (sb == NULL) {
5650                 rc = ENOMEM;
5651                 goto done;
5652         }
5653
5654         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5655         for (i = 0, p = buf; i < n; i += 16, p += 4)
5656                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5657                     p[2], p[3]);
5658
5659         rc = sbuf_finish(sb);
5660         sbuf_delete(sb);
5661 done:
5662         free(buf, M_CXGBE);
5663         return (rc);
5664 }
5665
5666 static int
5667 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5668 {
5669         struct adapter *sc = arg1;
5670         u_int cfg;
5671         struct sbuf *sb;
5672         uint32_t *buf, *p;
5673         int rc;
5674
5675         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5676         if (rc != 0)
5677                 return (rc);
5678
5679         rc = sysctl_wire_old_buffer(req, 0);
5680         if (rc != 0)
5681                 return (rc);
5682
5683         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5684         if (sb == NULL)
5685                 return (ENOMEM);
5686
5687         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5688             M_ZERO | M_WAITOK);
5689
5690         rc = -t4_cim_read_la(sc, buf, NULL);
5691         if (rc != 0)
5692                 goto done;
5693
5694         sbuf_printf(sb, "Status   Data      PC%s",
5695             cfg & F_UPDBGLACAPTPCONLY ? "" :
5696             "     LS0Stat  LS0Addr             LS0Data");
5697
5698         KASSERT((sc->params.cim_la_size & 7) == 0,
5699             ("%s: p will walk off the end of buf", __func__));
5700
5701         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5702                 if (cfg & F_UPDBGLACAPTPCONLY) {
5703                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5704                             p[6], p[7]);
5705                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5706                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5707                             p[4] & 0xff, p[5] >> 8);
5708                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5709                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5710                             p[1] & 0xf, p[2] >> 4);
5711                 } else {
5712                         sbuf_printf(sb,
5713                             "\n  %02x   %x%07x %x%07x %08x %08x "
5714                             "%08x%08x%08x%08x",
5715                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5716                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5717                             p[6], p[7]);
5718                 }
5719         }
5720
5721         rc = sbuf_finish(sb);
5722         sbuf_delete(sb);
5723 done:
5724         free(buf, M_CXGBE);
5725         return (rc);
5726 }
5727
5728 static int
5729 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5730 {
5731         struct adapter *sc = arg1;
5732         u_int i;
5733         struct sbuf *sb;
5734         uint32_t *buf, *p;
5735         int rc;
5736
5737         rc = sysctl_wire_old_buffer(req, 0);
5738         if (rc != 0)
5739                 return (rc);
5740
5741         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5742         if (sb == NULL)
5743                 return (ENOMEM);
5744
5745         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5746             M_ZERO | M_WAITOK);
5747
5748         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5749         p = buf;
5750
5751         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5752                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5753                     p[1], p[0]);
5754         }
5755
5756         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5757         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5758                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5759                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5760                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5761                     (p[1] >> 2) | ((p[2] & 3) << 30),
5762                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5763                     p[0] & 1);
5764         }
5765
5766         rc = sbuf_finish(sb);
5767         sbuf_delete(sb);
5768         free(buf, M_CXGBE);
5769         return (rc);
5770 }
5771
5772 static int
5773 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5774 {
5775         struct adapter *sc = arg1;
5776         u_int i;
5777         struct sbuf *sb;
5778         uint32_t *buf, *p;
5779         int rc;
5780
5781         rc = sysctl_wire_old_buffer(req, 0);
5782         if (rc != 0)
5783                 return (rc);
5784
5785         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5786         if (sb == NULL)
5787                 return (ENOMEM);
5788
5789         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5790             M_ZERO | M_WAITOK);
5791
5792         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5793         p = buf;
5794
5795         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5796         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5797                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5798                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5799                     p[4], p[3], p[2], p[1], p[0]);
5800         }
5801
5802         sbuf_printf(sb, "\n\nCntl ID               Data");
5803         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5804                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5805                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5806         }
5807
5808         rc = sbuf_finish(sb);
5809         sbuf_delete(sb);
5810         free(buf, M_CXGBE);
5811         return (rc);
5812 }
5813
5814 static int
5815 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5816 {
5817         struct adapter *sc = arg1;
5818         struct sbuf *sb;
5819         int rc, i;
5820         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5821         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5822         uint16_t thres[CIM_NUM_IBQ];
5823         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5824         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5825         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5826
5827         if (is_t4(sc)) {
5828                 cim_num_obq = CIM_NUM_OBQ;
5829                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5830                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5831         } else {
5832                 cim_num_obq = CIM_NUM_OBQ_T5;
5833                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5834                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5835         }
5836         nq = CIM_NUM_IBQ + cim_num_obq;
5837
5838         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5839         if (rc == 0)
5840                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5841         if (rc != 0)
5842                 return (rc);
5843
5844         t4_read_cimq_cfg(sc, base, size, thres);
5845
5846         rc = sysctl_wire_old_buffer(req, 0);
5847         if (rc != 0)
5848                 return (rc);
5849
5850         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5851         if (sb == NULL)
5852                 return (ENOMEM);
5853
5854         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5855
5856         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5857                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5858                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5859                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5860                     G_QUEREMFLITS(p[2]) * 16);
5861         for ( ; i < nq; i++, p += 4, wr += 2)
5862                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5863                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5864                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5865                     G_QUEREMFLITS(p[2]) * 16);
5866
5867         rc = sbuf_finish(sb);
5868         sbuf_delete(sb);
5869
5870         return (rc);
5871 }
5872
5873 static int
5874 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5875 {
5876         struct adapter *sc = arg1;
5877         struct sbuf *sb;
5878         int rc;
5879         struct tp_cpl_stats stats;
5880
5881         rc = sysctl_wire_old_buffer(req, 0);
5882         if (rc != 0)
5883                 return (rc);
5884
5885         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5886         if (sb == NULL)
5887                 return (ENOMEM);
5888
5889         t4_tp_get_cpl_stats(sc, &stats);
5890
5891         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5892             "channel 3\n");
5893         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5894                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5895         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5896                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5897
5898         rc = sbuf_finish(sb);
5899         sbuf_delete(sb);
5900
5901         return (rc);
5902 }
5903
5904 static int
5905 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5906 {
5907         struct adapter *sc = arg1;
5908         struct sbuf *sb;
5909         int rc;
5910         struct tp_usm_stats stats;
5911
5912         rc = sysctl_wire_old_buffer(req, 0);
5913         if (rc != 0)
5914                 return(rc);
5915
5916         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5917         if (sb == NULL)
5918                 return (ENOMEM);
5919
5920         t4_get_usm_stats(sc, &stats);
5921
5922         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5923         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5924         sbuf_printf(sb, "Drops:  %u", stats.drops);
5925
5926         rc = sbuf_finish(sb);
5927         sbuf_delete(sb);
5928
5929         return (rc);
5930 }
5931
5932 const char *devlog_level_strings[] = {
5933         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5934         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5935         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5936         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5937         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5938         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5939 };
5940
5941 const char *devlog_facility_strings[] = {
5942         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5943         [FW_DEVLOG_FACILITY_CF]         = "CF",
5944         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5945         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5946         [FW_DEVLOG_FACILITY_RES]        = "RES",
5947         [FW_DEVLOG_FACILITY_HW]         = "HW",
5948         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5949         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5950         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5951         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5952         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5953         [FW_DEVLOG_FACILITY_VI]         = "VI",
5954         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5955         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5956         [FW_DEVLOG_FACILITY_TM]         = "TM",
5957         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5958         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5959         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5960         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5961         [FW_DEVLOG_FACILITY_RI]         = "RI",
5962         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5963         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5964         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5965         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5966 };
5967
5968 static int
5969 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5970 {
5971         struct adapter *sc = arg1;
5972         struct devlog_params *dparams = &sc->params.devlog;
5973         struct fw_devlog_e *buf, *e;
5974         int i, j, rc, nentries, first = 0, m;
5975         struct sbuf *sb;
5976         uint64_t ftstamp = UINT64_MAX;
5977
5978         if (dparams->start == 0) {
5979                 dparams->memtype = FW_MEMTYPE_EDC0;
5980                 dparams->start = 0x84000;
5981                 dparams->size = 32768;
5982         }
5983
5984         nentries = dparams->size / sizeof(struct fw_devlog_e);
5985
5986         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5987         if (buf == NULL)
5988                 return (ENOMEM);
5989
5990         m = fwmtype_to_hwmtype(dparams->memtype);
5991         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5992         if (rc != 0)
5993                 goto done;
5994
5995         for (i = 0; i < nentries; i++) {
5996                 e = &buf[i];
5997
5998                 if (e->timestamp == 0)
5999                         break;  /* end */
6000
6001                 e->timestamp = be64toh(e->timestamp);
6002                 e->seqno = be32toh(e->seqno);
6003                 for (j = 0; j < 8; j++)
6004                         e->params[j] = be32toh(e->params[j]);
6005
6006                 if (e->timestamp < ftstamp) {
6007                         ftstamp = e->timestamp;
6008                         first = i;
6009                 }
6010         }
6011
6012         if (buf[first].timestamp == 0)
6013                 goto done;      /* nothing in the log */
6014
6015         rc = sysctl_wire_old_buffer(req, 0);
6016         if (rc != 0)
6017                 goto done;
6018
6019         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6020         if (sb == NULL) {
6021                 rc = ENOMEM;
6022                 goto done;
6023         }
6024         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
6025             "Seq#", "Tstamp", "Level", "Facility", "Message");
6026
6027         i = first;
6028         do {
6029                 e = &buf[i];
6030                 if (e->timestamp == 0)
6031                         break;  /* end */
6032
6033                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
6034                     e->seqno, e->timestamp,
6035                     (e->level < nitems(devlog_level_strings) ?
6036                         devlog_level_strings[e->level] : "UNKNOWN"),
6037                     (e->facility < nitems(devlog_facility_strings) ?
6038                         devlog_facility_strings[e->facility] : "UNKNOWN"));
6039                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
6040                     e->params[2], e->params[3], e->params[4],
6041                     e->params[5], e->params[6], e->params[7]);
6042
6043                 if (++i == nentries)
6044                         i = 0;
6045         } while (i != first);
6046
6047         rc = sbuf_finish(sb);
6048         sbuf_delete(sb);
6049 done:
6050         free(buf, M_CXGBE);
6051         return (rc);
6052 }
6053
6054 static int
6055 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
6056 {
6057         struct adapter *sc = arg1;
6058         struct sbuf *sb;
6059         int rc;
6060         struct tp_fcoe_stats stats[4];
6061
6062         rc = sysctl_wire_old_buffer(req, 0);
6063         if (rc != 0)
6064                 return (rc);
6065
6066         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6067         if (sb == NULL)
6068                 return (ENOMEM);
6069
6070         t4_get_fcoe_stats(sc, 0, &stats[0]);
6071         t4_get_fcoe_stats(sc, 1, &stats[1]);
6072         t4_get_fcoe_stats(sc, 2, &stats[2]);
6073         t4_get_fcoe_stats(sc, 3, &stats[3]);
6074
6075         sbuf_printf(sb, "                   channel 0        channel 1        "
6076             "channel 2        channel 3\n");
6077         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
6078             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
6079             stats[3].octetsDDP);
6080         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
6081             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
6082         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
6083             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
6084             stats[3].framesDrop);
6085
6086         rc = sbuf_finish(sb);
6087         sbuf_delete(sb);
6088
6089         return (rc);
6090 }
6091
6092 static int
6093 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
6094 {
6095         struct adapter *sc = arg1;
6096         struct sbuf *sb;
6097         int rc, i;
6098         unsigned int map, kbps, ipg, mode;
6099         unsigned int pace_tab[NTX_SCHED];
6100
6101         rc = sysctl_wire_old_buffer(req, 0);
6102         if (rc != 0)
6103                 return (rc);
6104
6105         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6106         if (sb == NULL)
6107                 return (ENOMEM);
6108
6109         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
6110         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
6111         t4_read_pace_tbl(sc, pace_tab);
6112
6113         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
6114             "Class IPG (0.1 ns)   Flow IPG (us)");
6115
6116         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
6117                 t4_get_tx_sched(sc, i, &kbps, &ipg);
6118                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
6119                     (mode & (1 << i)) ? "flow" : "class", map & 3);
6120                 if (kbps)
6121                         sbuf_printf(sb, "%9u     ", kbps);
6122                 else
6123                         sbuf_printf(sb, " disabled     ");
6124
6125                 if (ipg)
6126                         sbuf_printf(sb, "%13u        ", ipg);
6127                 else
6128                         sbuf_printf(sb, "     disabled        ");
6129
6130                 if (pace_tab[i])
6131                         sbuf_printf(sb, "%10u", pace_tab[i]);
6132                 else
6133                         sbuf_printf(sb, "  disabled");
6134         }
6135
6136         rc = sbuf_finish(sb);
6137         sbuf_delete(sb);
6138
6139         return (rc);
6140 }
6141
6142 static int
6143 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
6144 {
6145         struct adapter *sc = arg1;
6146         struct sbuf *sb;
6147         int rc, i, j;
6148         uint64_t *p0, *p1;
6149         struct lb_port_stats s[2];
6150         static const char *stat_name[] = {
6151                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
6152                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
6153                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
6154                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
6155                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
6156                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
6157                 "BG2FramesTrunc:", "BG3FramesTrunc:"
6158         };
6159
6160         rc = sysctl_wire_old_buffer(req, 0);
6161         if (rc != 0)
6162                 return (rc);
6163
6164         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6165         if (sb == NULL)
6166                 return (ENOMEM);
6167
6168         memset(s, 0, sizeof(s));
6169
6170         for (i = 0; i < 4; i += 2) {
6171                 t4_get_lb_stats(sc, i, &s[0]);
6172                 t4_get_lb_stats(sc, i + 1, &s[1]);
6173
6174                 p0 = &s[0].octets;
6175                 p1 = &s[1].octets;
6176                 sbuf_printf(sb, "%s                       Loopback %u"
6177                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6178
6179                 for (j = 0; j < nitems(stat_name); j++)
6180                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6181                                    *p0++, *p1++);
6182         }
6183
6184         rc = sbuf_finish(sb);
6185         sbuf_delete(sb);
6186
6187         return (rc);
6188 }
6189
6190 static int
6191 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6192 {
6193         int rc = 0;
6194         struct port_info *pi = arg1;
6195         struct sbuf *sb;
6196         static const char *linkdnreasons[] = {
6197                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6198                 "PHY overheated", "unknown", "rx los", "reserved7"
6199         };
6200
6201         rc = sysctl_wire_old_buffer(req, 0);
6202         if (rc != 0)
6203                 return(rc);
6204         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6205         if (sb == NULL)
6206                 return (ENOMEM);
6207
6208         if (pi->linkdnrc < 0)
6209                 sbuf_printf(sb, "n/a");
6210         else if (pi->linkdnrc < nitems(linkdnreasons))
6211                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6212         else
6213                 sbuf_printf(sb, "%d", pi->linkdnrc);
6214
6215         rc = sbuf_finish(sb);
6216         sbuf_delete(sb);
6217
6218         return (rc);
6219 }
6220
6221 struct mem_desc {
6222         unsigned int base;
6223         unsigned int limit;
6224         unsigned int idx;
6225 };
6226
6227 static int
6228 mem_desc_cmp(const void *a, const void *b)
6229 {
6230         return ((const struct mem_desc *)a)->base -
6231                ((const struct mem_desc *)b)->base;
6232 }
6233
6234 static void
6235 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6236     unsigned int to)
6237 {
6238         unsigned int size;
6239
6240         size = to - from + 1;
6241         if (size == 0)
6242                 return;
6243
6244         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6245         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6246 }
6247
6248 static int
6249 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6250 {
6251         struct adapter *sc = arg1;
6252         struct sbuf *sb;
6253         int rc, i, n;
6254         uint32_t lo, hi, used, alloc;
6255         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6256         static const char *region[] = {
6257                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6258                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6259                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6260                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6261                 "RQUDP region:", "PBL region:", "TXPBL region:",
6262                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6263                 "On-chip queues:"
6264         };
6265         struct mem_desc avail[4];
6266         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6267         struct mem_desc *md = mem;
6268
6269         rc = sysctl_wire_old_buffer(req, 0);
6270         if (rc != 0)
6271                 return (rc);
6272
6273         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6274         if (sb == NULL)
6275                 return (ENOMEM);
6276
6277         for (i = 0; i < nitems(mem); i++) {
6278                 mem[i].limit = 0;
6279                 mem[i].idx = i;
6280         }
6281
6282         /* Find and sort the populated memory ranges */
6283         i = 0;
6284         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6285         if (lo & F_EDRAM0_ENABLE) {
6286                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6287                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6288                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6289                 avail[i].idx = 0;
6290                 i++;
6291         }
6292         if (lo & F_EDRAM1_ENABLE) {
6293                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6294                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6295                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6296                 avail[i].idx = 1;
6297                 i++;
6298         }
6299         if (lo & F_EXT_MEM_ENABLE) {
6300                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6301                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6302                 avail[i].limit = avail[i].base +
6303                     (G_EXT_MEM_SIZE(hi) << 20);
6304                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6305                 i++;
6306         }
6307         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6308                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6309                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6310                 avail[i].limit = avail[i].base +
6311                     (G_EXT_MEM1_SIZE(hi) << 20);
6312                 avail[i].idx = 4;
6313                 i++;
6314         }
6315         if (!i)                                    /* no memory available */
6316                 return 0;
6317         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6318
6319         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6320         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6321         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6322         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6323         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6324         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6325         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6326         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6327         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6328
6329         /* the next few have explicit upper bounds */
6330         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6331         md->limit = md->base - 1 +
6332                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6333                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6334         md++;
6335
6336         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6337         md->limit = md->base - 1 +
6338                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6339                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6340         md++;
6341
6342         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6343                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6344                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6345                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6346         } else {
6347                 md->base = 0;
6348                 md->idx = nitems(region);  /* hide it */
6349         }
6350         md++;
6351
6352 #define ulp_region(reg) \
6353         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6354         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6355
6356         ulp_region(RX_ISCSI);
6357         ulp_region(RX_TDDP);
6358         ulp_region(TX_TPT);
6359         ulp_region(RX_STAG);
6360         ulp_region(RX_RQ);
6361         ulp_region(RX_RQUDP);
6362         ulp_region(RX_PBL);
6363         ulp_region(TX_PBL);
6364 #undef ulp_region
6365
6366         md->base = 0;
6367         md->idx = nitems(region);
6368         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6369                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6370                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6371                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6372         }
6373         md++;
6374
6375         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6376         md->limit = md->base + sc->tids.ntids - 1;
6377         md++;
6378         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6379         md->limit = md->base + sc->tids.ntids - 1;
6380         md++;
6381
6382         md->base = sc->vres.ocq.start;
6383         if (sc->vres.ocq.size)
6384                 md->limit = md->base + sc->vres.ocq.size - 1;
6385         else
6386                 md->idx = nitems(region);  /* hide it */
6387         md++;
6388
6389         /* add any address-space holes, there can be up to 3 */
6390         for (n = 0; n < i - 1; n++)
6391                 if (avail[n].limit < avail[n + 1].base)
6392                         (md++)->base = avail[n].limit;
6393         if (avail[n].limit)
6394                 (md++)->base = avail[n].limit;
6395
6396         n = md - mem;
6397         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6398
6399         for (lo = 0; lo < i; lo++)
6400                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6401                                 avail[lo].limit - 1);
6402
6403         sbuf_printf(sb, "\n");
6404         for (i = 0; i < n; i++) {
6405                 if (mem[i].idx >= nitems(region))
6406                         continue;                        /* skip holes */
6407                 if (!mem[i].limit)
6408                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6409                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6410                                 mem[i].limit);
6411         }
6412
6413         sbuf_printf(sb, "\n");
6414         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6415         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6416         mem_region_show(sb, "uP RAM:", lo, hi);
6417
6418         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6419         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6420         mem_region_show(sb, "uP Extmem2:", lo, hi);
6421
6422         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6423         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6424                    G_PMRXMAXPAGE(lo),
6425                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6426                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6427
6428         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6429         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6430         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6431                    G_PMTXMAXPAGE(lo),
6432                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6433                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6434         sbuf_printf(sb, "%u p-structs\n",
6435                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6436
6437         for (i = 0; i < 4; i++) {
6438                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6439                 if (is_t4(sc)) {
6440                         used = G_USED(lo);
6441                         alloc = G_ALLOC(lo);
6442                 } else {
6443                         used = G_T5_USED(lo);
6444                         alloc = G_T5_ALLOC(lo);
6445                 }
6446                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6447                            i, used, alloc);
6448         }
6449         for (i = 0; i < 4; i++) {
6450                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6451                 if (is_t4(sc)) {
6452                         used = G_USED(lo);
6453                         alloc = G_ALLOC(lo);
6454                 } else {
6455                         used = G_T5_USED(lo);
6456                         alloc = G_T5_ALLOC(lo);
6457                 }
6458                 sbuf_printf(sb,
6459                            "\nLoopback %d using %u pages out of %u allocated",
6460                            i, used, alloc);
6461         }
6462
6463         rc = sbuf_finish(sb);
6464         sbuf_delete(sb);
6465
6466         return (rc);
6467 }
6468
6469 static inline void
6470 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6471 {
6472         *mask = x | y;
6473         y = htobe64(y);
6474         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6475 }
6476
6477 static int
6478 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6479 {
6480         struct adapter *sc = arg1;
6481         struct sbuf *sb;
6482         int rc, i, n;
6483
6484         rc = sysctl_wire_old_buffer(req, 0);
6485         if (rc != 0)
6486                 return (rc);
6487
6488         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6489         if (sb == NULL)
6490                 return (ENOMEM);
6491
6492         sbuf_printf(sb,
6493             "Idx  Ethernet address     Mask     Vld Ports PF"
6494             "  VF              Replication             P0 P1 P2 P3  ML");
6495         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6496             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6497         for (i = 0; i < n; i++) {
6498                 uint64_t tcamx, tcamy, mask;
6499                 uint32_t cls_lo, cls_hi;
6500                 uint8_t addr[ETHER_ADDR_LEN];
6501
6502                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6503                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6504                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6505                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6506
6507                 if (tcamx & tcamy)
6508                         continue;
6509
6510                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6511                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6512                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6513                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6514                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6515                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6516                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6517
6518                 if (cls_lo & F_REPLICATE) {
6519                         struct fw_ldst_cmd ldst_cmd;
6520
6521                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6522                         ldst_cmd.op_to_addrspace =
6523                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6524                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6525                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6526                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6527                         ldst_cmd.u.mps.rplc.fid_idx =
6528                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6529                                 V_FW_LDST_CMD_IDX(i));
6530
6531                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6532                             "t4mps");
6533                         if (rc)
6534                                 break;
6535                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6536                             sizeof(ldst_cmd), &ldst_cmd);
6537                         end_synchronized_op(sc, 0);
6538
6539                         if (rc != 0) {
6540                                 sbuf_printf(sb,
6541                                     " ------------ error %3u ------------", rc);
6542                                 rc = 0;
6543                         } else {
6544                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6545                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6546                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6547                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6548                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6549                         }
6550                 } else
6551                         sbuf_printf(sb, "%36s", "");
6552
6553                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6554                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6555                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6556         }
6557
6558         if (rc)
6559                 (void) sbuf_finish(sb);
6560         else
6561                 rc = sbuf_finish(sb);
6562         sbuf_delete(sb);
6563
6564         return (rc);
6565 }
6566
6567 static int
6568 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6569 {
6570         struct adapter *sc = arg1;
6571         struct sbuf *sb;
6572         int rc;
6573         uint16_t mtus[NMTUS];
6574
6575         rc = sysctl_wire_old_buffer(req, 0);
6576         if (rc != 0)
6577                 return (rc);
6578
6579         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6580         if (sb == NULL)
6581                 return (ENOMEM);
6582
6583         t4_read_mtu_tbl(sc, mtus, NULL);
6584
6585         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6586             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6587             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6588             mtus[14], mtus[15]);
6589
6590         rc = sbuf_finish(sb);
6591         sbuf_delete(sb);
6592
6593         return (rc);
6594 }
6595
6596 static int
6597 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6598 {
6599         struct adapter *sc = arg1;
6600         struct sbuf *sb;
6601         int rc, i;
6602         uint32_t cnt[PM_NSTATS];
6603         uint64_t cyc[PM_NSTATS];
6604         static const char *rx_stats[] = {
6605                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6606         };
6607         static const char *tx_stats[] = {
6608                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6609         };
6610
6611         rc = sysctl_wire_old_buffer(req, 0);
6612         if (rc != 0)
6613                 return (rc);
6614
6615         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6616         if (sb == NULL)
6617                 return (ENOMEM);
6618
6619         t4_pmtx_get_stats(sc, cnt, cyc);
6620         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6621         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6622                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6623                     cyc[i]);
6624
6625         t4_pmrx_get_stats(sc, cnt, cyc);
6626         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6627         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6628                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6629                     cyc[i]);
6630
6631         rc = sbuf_finish(sb);
6632         sbuf_delete(sb);
6633
6634         return (rc);
6635 }
6636
6637 static int
6638 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6639 {
6640         struct adapter *sc = arg1;
6641         struct sbuf *sb;
6642         int rc;
6643         struct tp_rdma_stats stats;
6644
6645         rc = sysctl_wire_old_buffer(req, 0);
6646         if (rc != 0)
6647                 return (rc);
6648
6649         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6650         if (sb == NULL)
6651                 return (ENOMEM);
6652
6653         t4_tp_get_rdma_stats(sc, &stats);
6654         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6655         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6656
6657         rc = sbuf_finish(sb);
6658         sbuf_delete(sb);
6659
6660         return (rc);
6661 }
6662
6663 static int
6664 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6665 {
6666         struct adapter *sc = arg1;
6667         struct sbuf *sb;
6668         int rc;
6669         struct tp_tcp_stats v4, v6;
6670
6671         rc = sysctl_wire_old_buffer(req, 0);
6672         if (rc != 0)
6673                 return (rc);
6674
6675         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6676         if (sb == NULL)
6677                 return (ENOMEM);
6678
6679         t4_tp_get_tcp_stats(sc, &v4, &v6);
6680         sbuf_printf(sb,
6681             "                                IP                 IPv6\n");
6682         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6683             v4.tcpOutRsts, v6.tcpOutRsts);
6684         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6685             v4.tcpInSegs, v6.tcpInSegs);
6686         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6687             v4.tcpOutSegs, v6.tcpOutSegs);
6688         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6689             v4.tcpRetransSegs, v6.tcpRetransSegs);
6690
6691         rc = sbuf_finish(sb);
6692         sbuf_delete(sb);
6693
6694         return (rc);
6695 }
6696
6697 static int
6698 sysctl_tids(SYSCTL_HANDLER_ARGS)
6699 {
6700         struct adapter *sc = arg1;
6701         struct sbuf *sb;
6702         int rc;
6703         struct tid_info *t = &sc->tids;
6704
6705         rc = sysctl_wire_old_buffer(req, 0);
6706         if (rc != 0)
6707                 return (rc);
6708
6709         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6710         if (sb == NULL)
6711                 return (ENOMEM);
6712
6713         if (t->natids) {
6714                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6715                     t->atids_in_use);
6716         }
6717
6718         if (t->ntids) {
6719                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6720                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6721
6722                         if (b) {
6723                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6724                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6725                                     t->ntids - 1);
6726                         } else {
6727                                 sbuf_printf(sb, "TID range: %u-%u",
6728                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6729                                     t->ntids - 1);
6730                         }
6731                 } else
6732                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6733                 sbuf_printf(sb, ", in use: %u\n",
6734                     atomic_load_acq_int(&t->tids_in_use));
6735         }
6736
6737         if (t->nstids) {
6738                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6739                     t->stid_base + t->nstids - 1, t->stids_in_use);
6740         }
6741
6742         if (t->nftids) {
6743                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6744                     t->ftid_base + t->nftids - 1);
6745         }
6746
6747         if (t->netids) {
6748                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6749                     t->etid_base + t->netids - 1);
6750         }
6751
6752         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6753             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6754             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6755
6756         rc = sbuf_finish(sb);
6757         sbuf_delete(sb);
6758
6759         return (rc);
6760 }
6761
6762 static int
6763 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6764 {
6765         struct adapter *sc = arg1;
6766         struct sbuf *sb;
6767         int rc;
6768         struct tp_err_stats stats;
6769
6770         rc = sysctl_wire_old_buffer(req, 0);
6771         if (rc != 0)
6772                 return (rc);
6773
6774         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6775         if (sb == NULL)
6776                 return (ENOMEM);
6777
6778         t4_tp_get_err_stats(sc, &stats);
6779
6780         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6781                       "channel 3\n");
6782         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6783             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6784             stats.macInErrs[3]);
6785         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6786             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6787             stats.hdrInErrs[3]);
6788         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6789             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6790             stats.tcpInErrs[3]);
6791         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6792             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6793             stats.tcp6InErrs[3]);
6794         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6795             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6796             stats.tnlCongDrops[3]);
6797         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6798             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6799             stats.tnlTxDrops[3]);
6800         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6801             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6802             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6803         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6804             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6805             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6806         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6807             stats.ofldNoNeigh, stats.ofldCongDefer);
6808
6809         rc = sbuf_finish(sb);
6810         sbuf_delete(sb);
6811
6812         return (rc);
6813 }
6814
6815 struct field_desc {
6816         const char *name;
6817         u_int start;
6818         u_int width;
6819 };
6820
6821 static void
6822 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6823 {
6824         char buf[32];
6825         int line_size = 0;
6826
6827         while (f->name) {
6828                 uint64_t mask = (1ULL << f->width) - 1;
6829                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6830                     ((uintmax_t)v >> f->start) & mask);
6831
6832                 if (line_size + len >= 79) {
6833                         line_size = 8;
6834                         sbuf_printf(sb, "\n        ");
6835                 }
6836                 sbuf_printf(sb, "%s ", buf);
6837                 line_size += len + 1;
6838                 f++;
6839         }
6840         sbuf_printf(sb, "\n");
6841 }
6842
6843 static struct field_desc tp_la0[] = {
6844         { "RcfOpCodeOut", 60, 4 },
6845         { "State", 56, 4 },
6846         { "WcfState", 52, 4 },
6847         { "RcfOpcSrcOut", 50, 2 },
6848         { "CRxError", 49, 1 },
6849         { "ERxError", 48, 1 },
6850         { "SanityFailed", 47, 1 },
6851         { "SpuriousMsg", 46, 1 },
6852         { "FlushInputMsg", 45, 1 },
6853         { "FlushInputCpl", 44, 1 },
6854         { "RssUpBit", 43, 1 },
6855         { "RssFilterHit", 42, 1 },
6856         { "Tid", 32, 10 },
6857         { "InitTcb", 31, 1 },
6858         { "LineNumber", 24, 7 },
6859         { "Emsg", 23, 1 },
6860         { "EdataOut", 22, 1 },
6861         { "Cmsg", 21, 1 },
6862         { "CdataOut", 20, 1 },
6863         { "EreadPdu", 19, 1 },
6864         { "CreadPdu", 18, 1 },
6865         { "TunnelPkt", 17, 1 },
6866         { "RcfPeerFin", 16, 1 },
6867         { "RcfReasonOut", 12, 4 },
6868         { "TxCchannel", 10, 2 },
6869         { "RcfTxChannel", 8, 2 },
6870         { "RxEchannel", 6, 2 },
6871         { "RcfRxChannel", 5, 1 },
6872         { "RcfDataOutSrdy", 4, 1 },
6873         { "RxDvld", 3, 1 },
6874         { "RxOoDvld", 2, 1 },
6875         { "RxCongestion", 1, 1 },
6876         { "TxCongestion", 0, 1 },
6877         { NULL }
6878 };
6879
6880 static struct field_desc tp_la1[] = {
6881         { "CplCmdIn", 56, 8 },
6882         { "CplCmdOut", 48, 8 },
6883         { "ESynOut", 47, 1 },
6884         { "EAckOut", 46, 1 },
6885         { "EFinOut", 45, 1 },
6886         { "ERstOut", 44, 1 },
6887         { "SynIn", 43, 1 },
6888         { "AckIn", 42, 1 },
6889         { "FinIn", 41, 1 },
6890         { "RstIn", 40, 1 },
6891         { "DataIn", 39, 1 },
6892         { "DataInVld", 38, 1 },
6893         { "PadIn", 37, 1 },
6894         { "RxBufEmpty", 36, 1 },
6895         { "RxDdp", 35, 1 },
6896         { "RxFbCongestion", 34, 1 },
6897         { "TxFbCongestion", 33, 1 },
6898         { "TxPktSumSrdy", 32, 1 },
6899         { "RcfUlpType", 28, 4 },
6900         { "Eread", 27, 1 },
6901         { "Ebypass", 26, 1 },
6902         { "Esave", 25, 1 },
6903         { "Static0", 24, 1 },
6904         { "Cread", 23, 1 },
6905         { "Cbypass", 22, 1 },
6906         { "Csave", 21, 1 },
6907         { "CPktOut", 20, 1 },
6908         { "RxPagePoolFull", 18, 2 },
6909         { "RxLpbkPkt", 17, 1 },
6910         { "TxLpbkPkt", 16, 1 },
6911         { "RxVfValid", 15, 1 },
6912         { "SynLearned", 14, 1 },
6913         { "SetDelEntry", 13, 1 },
6914         { "SetInvEntry", 12, 1 },
6915         { "CpcmdDvld", 11, 1 },
6916         { "CpcmdSave", 10, 1 },
6917         { "RxPstructsFull", 8, 2 },
6918         { "EpcmdDvld", 7, 1 },
6919         { "EpcmdFlush", 6, 1 },
6920         { "EpcmdTrimPrefix", 5, 1 },
6921         { "EpcmdTrimPostfix", 4, 1 },
6922         { "ERssIp4Pkt", 3, 1 },
6923         { "ERssIp6Pkt", 2, 1 },
6924         { "ERssTcpUdpPkt", 1, 1 },
6925         { "ERssFceFipPkt", 0, 1 },
6926         { NULL }
6927 };
6928
6929 static struct field_desc tp_la2[] = {
6930         { "CplCmdIn", 56, 8 },
6931         { "MpsVfVld", 55, 1 },
6932         { "MpsPf", 52, 3 },
6933         { "MpsVf", 44, 8 },
6934         { "SynIn", 43, 1 },
6935         { "AckIn", 42, 1 },
6936         { "FinIn", 41, 1 },
6937         { "RstIn", 40, 1 },
6938         { "DataIn", 39, 1 },
6939         { "DataInVld", 38, 1 },
6940         { "PadIn", 37, 1 },
6941         { "RxBufEmpty", 36, 1 },
6942         { "RxDdp", 35, 1 },
6943         { "RxFbCongestion", 34, 1 },
6944         { "TxFbCongestion", 33, 1 },
6945         { "TxPktSumSrdy", 32, 1 },
6946         { "RcfUlpType", 28, 4 },
6947         { "Eread", 27, 1 },
6948         { "Ebypass", 26, 1 },
6949         { "Esave", 25, 1 },
6950         { "Static0", 24, 1 },
6951         { "Cread", 23, 1 },
6952         { "Cbypass", 22, 1 },
6953         { "Csave", 21, 1 },
6954         { "CPktOut", 20, 1 },
6955         { "RxPagePoolFull", 18, 2 },
6956         { "RxLpbkPkt", 17, 1 },
6957         { "TxLpbkPkt", 16, 1 },
6958         { "RxVfValid", 15, 1 },
6959         { "SynLearned", 14, 1 },
6960         { "SetDelEntry", 13, 1 },
6961         { "SetInvEntry", 12, 1 },
6962         { "CpcmdDvld", 11, 1 },
6963         { "CpcmdSave", 10, 1 },
6964         { "RxPstructsFull", 8, 2 },
6965         { "EpcmdDvld", 7, 1 },
6966         { "EpcmdFlush", 6, 1 },
6967         { "EpcmdTrimPrefix", 5, 1 },
6968         { "EpcmdTrimPostfix", 4, 1 },
6969         { "ERssIp4Pkt", 3, 1 },
6970         { "ERssIp6Pkt", 2, 1 },
6971         { "ERssTcpUdpPkt", 1, 1 },
6972         { "ERssFceFipPkt", 0, 1 },
6973         { NULL }
6974 };
6975
6976 static void
6977 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6978 {
6979
6980         field_desc_show(sb, *p, tp_la0);
6981 }
6982
6983 static void
6984 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6985 {
6986
6987         if (idx)
6988                 sbuf_printf(sb, "\n");
6989         field_desc_show(sb, p[0], tp_la0);
6990         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6991                 field_desc_show(sb, p[1], tp_la0);
6992 }
6993
6994 static void
6995 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6996 {
6997
6998         if (idx)
6999                 sbuf_printf(sb, "\n");
7000         field_desc_show(sb, p[0], tp_la0);
7001         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7002                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
7003 }
7004
7005 static int
7006 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
7007 {
7008         struct adapter *sc = arg1;
7009         struct sbuf *sb;
7010         uint64_t *buf, *p;
7011         int rc;
7012         u_int i, inc;
7013         void (*show_func)(struct sbuf *, uint64_t *, int);
7014
7015         rc = sysctl_wire_old_buffer(req, 0);
7016         if (rc != 0)
7017                 return (rc);
7018
7019         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7020         if (sb == NULL)
7021                 return (ENOMEM);
7022
7023         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
7024
7025         t4_tp_read_la(sc, buf, NULL);
7026         p = buf;
7027
7028         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
7029         case 2:
7030                 inc = 2;
7031                 show_func = tp_la_show2;
7032                 break;
7033         case 3:
7034                 inc = 2;
7035                 show_func = tp_la_show3;
7036                 break;
7037         default:
7038                 inc = 1;
7039                 show_func = tp_la_show;
7040         }
7041
7042         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
7043                 (*show_func)(sb, p, i);
7044
7045         rc = sbuf_finish(sb);
7046         sbuf_delete(sb);
7047         free(buf, M_CXGBE);
7048         return (rc);
7049 }
7050
7051 static int
7052 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
7053 {
7054         struct adapter *sc = arg1;
7055         struct sbuf *sb;
7056         int rc;
7057         u64 nrate[NCHAN], orate[NCHAN];
7058
7059         rc = sysctl_wire_old_buffer(req, 0);
7060         if (rc != 0)
7061                 return (rc);
7062
7063         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7064         if (sb == NULL)
7065                 return (ENOMEM);
7066
7067         t4_get_chan_txrate(sc, nrate, orate);
7068         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
7069                  "channel 3\n");
7070         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
7071             nrate[0], nrate[1], nrate[2], nrate[3]);
7072         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
7073             orate[0], orate[1], orate[2], orate[3]);
7074
7075         rc = sbuf_finish(sb);
7076         sbuf_delete(sb);
7077
7078         return (rc);
7079 }
7080
7081 static int
7082 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
7083 {
7084         struct adapter *sc = arg1;
7085         struct sbuf *sb;
7086         uint32_t *buf, *p;
7087         int rc, i;
7088
7089         rc = sysctl_wire_old_buffer(req, 0);
7090         if (rc != 0)
7091                 return (rc);
7092
7093         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7094         if (sb == NULL)
7095                 return (ENOMEM);
7096
7097         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
7098             M_ZERO | M_WAITOK);
7099
7100         t4_ulprx_read_la(sc, buf);
7101         p = buf;
7102
7103         sbuf_printf(sb, "      Pcmd        Type   Message"
7104             "                Data");
7105         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
7106                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
7107                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
7108         }
7109
7110         rc = sbuf_finish(sb);
7111         sbuf_delete(sb);
7112         free(buf, M_CXGBE);
7113         return (rc);
7114 }
7115
7116 static int
7117 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
7118 {
7119         struct adapter *sc = arg1;
7120         struct sbuf *sb;
7121         int rc, v;
7122
7123         rc = sysctl_wire_old_buffer(req, 0);
7124         if (rc != 0)
7125                 return (rc);
7126
7127         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7128         if (sb == NULL)
7129                 return (ENOMEM);
7130
7131         v = t4_read_reg(sc, A_SGE_STAT_CFG);
7132         if (G_STATSOURCE_T5(v) == 7) {
7133                 if (G_STATMODE(v) == 0) {
7134                         sbuf_printf(sb, "total %d, incomplete %d",
7135                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7136                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7137                 } else if (G_STATMODE(v) == 1) {
7138                         sbuf_printf(sb, "total %d, data overflow %d",
7139                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7140                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7141                 }
7142         }
7143         rc = sbuf_finish(sb);
7144         sbuf_delete(sb);
7145
7146         return (rc);
7147 }
7148 #endif
7149
7150 static uint32_t
7151 fconf_to_mode(uint32_t fconf)
7152 {
7153         uint32_t mode;
7154
7155         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
7156             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
7157
7158         if (fconf & F_FRAGMENTATION)
7159                 mode |= T4_FILTER_IP_FRAGMENT;
7160
7161         if (fconf & F_MPSHITTYPE)
7162                 mode |= T4_FILTER_MPS_HIT_TYPE;
7163
7164         if (fconf & F_MACMATCH)
7165                 mode |= T4_FILTER_MAC_IDX;
7166
7167         if (fconf & F_ETHERTYPE)
7168                 mode |= T4_FILTER_ETH_TYPE;
7169
7170         if (fconf & F_PROTOCOL)
7171                 mode |= T4_FILTER_IP_PROTO;
7172
7173         if (fconf & F_TOS)
7174                 mode |= T4_FILTER_IP_TOS;
7175
7176         if (fconf & F_VLAN)
7177                 mode |= T4_FILTER_VLAN;
7178
7179         if (fconf & F_VNIC_ID)
7180                 mode |= T4_FILTER_VNIC;
7181
7182         if (fconf & F_PORT)
7183                 mode |= T4_FILTER_PORT;
7184
7185         if (fconf & F_FCOE)
7186                 mode |= T4_FILTER_FCoE;
7187
7188         return (mode);
7189 }
7190
7191 static uint32_t
7192 mode_to_fconf(uint32_t mode)
7193 {
7194         uint32_t fconf = 0;
7195
7196         if (mode & T4_FILTER_IP_FRAGMENT)
7197                 fconf |= F_FRAGMENTATION;
7198
7199         if (mode & T4_FILTER_MPS_HIT_TYPE)
7200                 fconf |= F_MPSHITTYPE;
7201
7202         if (mode & T4_FILTER_MAC_IDX)
7203                 fconf |= F_MACMATCH;
7204
7205         if (mode & T4_FILTER_ETH_TYPE)
7206                 fconf |= F_ETHERTYPE;
7207
7208         if (mode & T4_FILTER_IP_PROTO)
7209                 fconf |= F_PROTOCOL;
7210
7211         if (mode & T4_FILTER_IP_TOS)
7212                 fconf |= F_TOS;
7213
7214         if (mode & T4_FILTER_VLAN)
7215                 fconf |= F_VLAN;
7216
7217         if (mode & T4_FILTER_VNIC)
7218                 fconf |= F_VNIC_ID;
7219
7220         if (mode & T4_FILTER_PORT)
7221                 fconf |= F_PORT;
7222
7223         if (mode & T4_FILTER_FCoE)
7224                 fconf |= F_FCOE;
7225
7226         return (fconf);
7227 }
7228
7229 static uint32_t
7230 fspec_to_fconf(struct t4_filter_specification *fs)
7231 {
7232         uint32_t fconf = 0;
7233
7234         if (fs->val.frag || fs->mask.frag)
7235                 fconf |= F_FRAGMENTATION;
7236
7237         if (fs->val.matchtype || fs->mask.matchtype)
7238                 fconf |= F_MPSHITTYPE;
7239
7240         if (fs->val.macidx || fs->mask.macidx)
7241                 fconf |= F_MACMATCH;
7242
7243         if (fs->val.ethtype || fs->mask.ethtype)
7244                 fconf |= F_ETHERTYPE;
7245
7246         if (fs->val.proto || fs->mask.proto)
7247                 fconf |= F_PROTOCOL;
7248
7249         if (fs->val.tos || fs->mask.tos)
7250                 fconf |= F_TOS;
7251
7252         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7253                 fconf |= F_VLAN;
7254
7255         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7256                 fconf |= F_VNIC_ID;
7257
7258         if (fs->val.iport || fs->mask.iport)
7259                 fconf |= F_PORT;
7260
7261         if (fs->val.fcoe || fs->mask.fcoe)
7262                 fconf |= F_FCOE;
7263
7264         return (fconf);
7265 }
7266
7267 static int
7268 get_filter_mode(struct adapter *sc, uint32_t *mode)
7269 {
7270         int rc;
7271         uint32_t fconf;
7272
7273         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7274             "t4getfm");
7275         if (rc)
7276                 return (rc);
7277
7278         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7279             A_TP_VLAN_PRI_MAP);
7280
7281         if (sc->params.tp.vlan_pri_map != fconf) {
7282                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7283                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7284                     fconf);
7285         }
7286
7287         *mode = fconf_to_mode(fconf);
7288
7289         end_synchronized_op(sc, LOCK_HELD);
7290         return (0);
7291 }
7292
7293 static int
7294 set_filter_mode(struct adapter *sc, uint32_t mode)
7295 {
7296         uint32_t fconf;
7297         int rc;
7298
7299         fconf = mode_to_fconf(mode);
7300
7301         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7302             "t4setfm");
7303         if (rc)
7304                 return (rc);
7305
7306         if (sc->tids.ftids_in_use > 0) {
7307                 rc = EBUSY;
7308                 goto done;
7309         }
7310
7311 #ifdef TCP_OFFLOAD
7312         if (uld_active(sc, ULD_TOM)) {
7313                 rc = EBUSY;
7314                 goto done;
7315         }
7316 #endif
7317
7318         rc = -t4_set_filter_mode(sc, fconf);
7319 done:
7320         end_synchronized_op(sc, LOCK_HELD);
7321         return (rc);
7322 }
7323
7324 static inline uint64_t
7325 get_filter_hits(struct adapter *sc, uint32_t fid)
7326 {
7327         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7328         uint64_t hits;
7329
7330         memwin_info(sc, 0, &mw_base, NULL);
7331         off = position_memwin(sc, 0,
7332             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7333         if (is_t4(sc)) {
7334                 hits = t4_read_reg64(sc, mw_base + off + 16);
7335                 hits = be64toh(hits);
7336         } else {
7337                 hits = t4_read_reg(sc, mw_base + off + 24);
7338                 hits = be32toh(hits);
7339         }
7340
7341         return (hits);
7342 }
7343
7344 static int
7345 get_filter(struct adapter *sc, struct t4_filter *t)
7346 {
7347         int i, rc, nfilters = sc->tids.nftids;
7348         struct filter_entry *f;
7349
7350         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7351             "t4getf");
7352         if (rc)
7353                 return (rc);
7354
7355         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7356             t->idx >= nfilters) {
7357                 t->idx = 0xffffffff;
7358                 goto done;
7359         }
7360
7361         f = &sc->tids.ftid_tab[t->idx];
7362         for (i = t->idx; i < nfilters; i++, f++) {
7363                 if (f->valid) {
7364                         t->idx = i;
7365                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7366                         t->smtidx = f->smtidx;
7367                         if (f->fs.hitcnts)
7368                                 t->hits = get_filter_hits(sc, t->idx);
7369                         else
7370                                 t->hits = UINT64_MAX;
7371                         t->fs = f->fs;
7372
7373                         goto done;
7374                 }
7375         }
7376
7377         t->idx = 0xffffffff;
7378 done:
7379         end_synchronized_op(sc, LOCK_HELD);
7380         return (0);
7381 }
7382
7383 static int
7384 set_filter(struct adapter *sc, struct t4_filter *t)
7385 {
7386         unsigned int nfilters, nports;
7387         struct filter_entry *f;
7388         int i, rc;
7389
7390         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7391         if (rc)
7392                 return (rc);
7393
7394         nfilters = sc->tids.nftids;
7395         nports = sc->params.nports;
7396
7397         if (nfilters == 0) {
7398                 rc = ENOTSUP;
7399                 goto done;
7400         }
7401
7402         if (!(sc->flags & FULL_INIT_DONE)) {
7403                 rc = EAGAIN;
7404                 goto done;
7405         }
7406
7407         if (t->idx >= nfilters) {
7408                 rc = EINVAL;
7409                 goto done;
7410         }
7411
7412         /* Validate against the global filter mode */
7413         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7414             sc->params.tp.vlan_pri_map) {
7415                 rc = E2BIG;
7416                 goto done;
7417         }
7418
7419         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7420                 rc = EINVAL;
7421                 goto done;
7422         }
7423
7424         if (t->fs.val.iport >= nports) {
7425                 rc = EINVAL;
7426                 goto done;
7427         }
7428
7429         /* Can't specify an iq if not steering to it */
7430         if (!t->fs.dirsteer && t->fs.iq) {
7431                 rc = EINVAL;
7432                 goto done;
7433         }
7434
7435         /* IPv6 filter idx must be 4 aligned */
7436         if (t->fs.type == 1 &&
7437             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7438                 rc = EINVAL;
7439                 goto done;
7440         }
7441
7442         if (sc->tids.ftid_tab == NULL) {
7443                 KASSERT(sc->tids.ftids_in_use == 0,
7444                     ("%s: no memory allocated but filters_in_use > 0",
7445                     __func__));
7446
7447                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7448                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7449                 if (sc->tids.ftid_tab == NULL) {
7450                         rc = ENOMEM;
7451                         goto done;
7452                 }
7453                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7454         }
7455
7456         for (i = 0; i < 4; i++) {
7457                 f = &sc->tids.ftid_tab[t->idx + i];
7458
7459                 if (f->pending || f->valid) {
7460                         rc = EBUSY;
7461                         goto done;
7462                 }
7463                 if (f->locked) {
7464                         rc = EPERM;
7465                         goto done;
7466                 }
7467
7468                 if (t->fs.type == 0)
7469                         break;
7470         }
7471
7472         f = &sc->tids.ftid_tab[t->idx];
7473         f->fs = t->fs;
7474
7475         rc = set_filter_wr(sc, t->idx);
7476 done:
7477         end_synchronized_op(sc, 0);
7478
7479         if (rc == 0) {
7480                 mtx_lock(&sc->tids.ftid_lock);
7481                 for (;;) {
7482                         if (f->pending == 0) {
7483                                 rc = f->valid ? 0 : EIO;
7484                                 break;
7485                         }
7486
7487                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7488                             PCATCH, "t4setfw", 0)) {
7489                                 rc = EINPROGRESS;
7490                                 break;
7491                         }
7492                 }
7493                 mtx_unlock(&sc->tids.ftid_lock);
7494         }
7495         return (rc);
7496 }
7497
7498 static int
7499 del_filter(struct adapter *sc, struct t4_filter *t)
7500 {
7501         unsigned int nfilters;
7502         struct filter_entry *f;
7503         int rc;
7504
7505         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7506         if (rc)
7507                 return (rc);
7508
7509         nfilters = sc->tids.nftids;
7510
7511         if (nfilters == 0) {
7512                 rc = ENOTSUP;
7513                 goto done;
7514         }
7515
7516         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7517             t->idx >= nfilters) {
7518                 rc = EINVAL;
7519                 goto done;
7520         }
7521
7522         if (!(sc->flags & FULL_INIT_DONE)) {
7523                 rc = EAGAIN;
7524                 goto done;
7525         }
7526
7527         f = &sc->tids.ftid_tab[t->idx];
7528
7529         if (f->pending) {
7530                 rc = EBUSY;
7531                 goto done;
7532         }
7533         if (f->locked) {
7534                 rc = EPERM;
7535                 goto done;
7536         }
7537
7538         if (f->valid) {
7539                 t->fs = f->fs;  /* extra info for the caller */
7540                 rc = del_filter_wr(sc, t->idx);
7541         }
7542
7543 done:
7544         end_synchronized_op(sc, 0);
7545
7546         if (rc == 0) {
7547                 mtx_lock(&sc->tids.ftid_lock);
7548                 for (;;) {
7549                         if (f->pending == 0) {
7550                                 rc = f->valid ? EIO : 0;
7551                                 break;
7552                         }
7553
7554                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7555                             PCATCH, "t4delfw", 0)) {
7556                                 rc = EINPROGRESS;
7557                                 break;
7558                         }
7559                 }
7560                 mtx_unlock(&sc->tids.ftid_lock);
7561         }
7562
7563         return (rc);
7564 }
7565
7566 static void
7567 clear_filter(struct filter_entry *f)
7568 {
7569         if (f->l2t)
7570                 t4_l2t_release(f->l2t);
7571
7572         bzero(f, sizeof (*f));
7573 }
7574
7575 static int
7576 set_filter_wr(struct adapter *sc, int fidx)
7577 {
7578         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7579         struct fw_filter_wr *fwr;
7580         unsigned int ftid;
7581         struct wrq_cookie cookie;
7582
7583         ASSERT_SYNCHRONIZED_OP(sc);
7584
7585         if (f->fs.newdmac || f->fs.newvlan) {
7586                 /* This filter needs an L2T entry; allocate one. */
7587                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7588                 if (f->l2t == NULL)
7589                         return (EAGAIN);
7590                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7591                     f->fs.dmac)) {
7592                         t4_l2t_release(f->l2t);
7593                         f->l2t = NULL;
7594                         return (ENOMEM);
7595                 }
7596         }
7597
7598         ftid = sc->tids.ftid_base + fidx;
7599
7600         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7601         if (fwr == NULL)
7602                 return (ENOMEM);
7603         bzero(fwr, sizeof(*fwr));
7604
7605         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7606         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7607         fwr->tid_to_iq =
7608             htobe32(V_FW_FILTER_WR_TID(ftid) |
7609                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7610                 V_FW_FILTER_WR_NOREPLY(0) |
7611                 V_FW_FILTER_WR_IQ(f->fs.iq));
7612         fwr->del_filter_to_l2tix =
7613             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7614                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7615                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7616                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7617                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7618                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7619                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7620                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7621                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7622                     f->fs.newvlan == VLAN_REWRITE) |
7623                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7624                     f->fs.newvlan == VLAN_REWRITE) |
7625                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7626                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7627                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7628                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7629         fwr->ethtype = htobe16(f->fs.val.ethtype);
7630         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7631         fwr->frag_to_ovlan_vldm =
7632             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7633                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7634                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7635                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7636                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7637                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7638         fwr->smac_sel = 0;
7639         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7640             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7641         fwr->maci_to_matchtypem =
7642             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7643                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7644                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7645                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7646                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7647                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7648                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7649                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7650         fwr->ptcl = f->fs.val.proto;
7651         fwr->ptclm = f->fs.mask.proto;
7652         fwr->ttyp = f->fs.val.tos;
7653         fwr->ttypm = f->fs.mask.tos;
7654         fwr->ivlan = htobe16(f->fs.val.vlan);
7655         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7656         fwr->ovlan = htobe16(f->fs.val.vnic);
7657         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7658         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7659         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7660         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7661         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7662         fwr->lp = htobe16(f->fs.val.dport);
7663         fwr->lpm = htobe16(f->fs.mask.dport);
7664         fwr->fp = htobe16(f->fs.val.sport);
7665         fwr->fpm = htobe16(f->fs.mask.sport);
7666         if (f->fs.newsmac)
7667                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7668
7669         f->pending = 1;
7670         sc->tids.ftids_in_use++;
7671
7672         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7673         return (0);
7674 }
7675
7676 static int
7677 del_filter_wr(struct adapter *sc, int fidx)
7678 {
7679         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7680         struct fw_filter_wr *fwr;
7681         unsigned int ftid;
7682         struct wrq_cookie cookie;
7683
7684         ftid = sc->tids.ftid_base + fidx;
7685
7686         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7687         if (fwr == NULL)
7688                 return (ENOMEM);
7689         bzero(fwr, sizeof (*fwr));
7690
7691         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7692
7693         f->pending = 1;
7694         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7695         return (0);
7696 }
7697
7698 int
7699 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7700 {
7701         struct adapter *sc = iq->adapter;
7702         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7703         unsigned int idx = GET_TID(rpl);
7704         unsigned int rc;
7705         struct filter_entry *f;
7706
7707         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7708             rss->opcode));
7709
7710         if (is_ftid(sc, idx)) {
7711
7712                 idx -= sc->tids.ftid_base;
7713                 f = &sc->tids.ftid_tab[idx];
7714                 rc = G_COOKIE(rpl->cookie);
7715
7716                 mtx_lock(&sc->tids.ftid_lock);
7717                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7718                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7719                             __func__, idx));
7720                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7721                         f->pending = 0;  /* asynchronous setup completed */
7722                         f->valid = 1;
7723                 } else {
7724                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7725                                 /* Add or delete failed, display an error */
7726                                 log(LOG_ERR,
7727                                     "filter %u setup failed with error %u\n",
7728                                     idx, rc);
7729                         }
7730
7731                         clear_filter(f);
7732                         sc->tids.ftids_in_use--;
7733                 }
7734                 wakeup(&sc->tids.ftid_tab);
7735                 mtx_unlock(&sc->tids.ftid_lock);
7736         }
7737
7738         return (0);
7739 }
7740
7741 static int
7742 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7743 {
7744         int rc;
7745
7746         if (cntxt->cid > M_CTXTQID)
7747                 return (EINVAL);
7748
7749         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7750             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7751                 return (EINVAL);
7752
7753         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7754         if (rc)
7755                 return (rc);
7756
7757         if (sc->flags & FW_OK) {
7758                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7759                     &cntxt->data[0]);
7760                 if (rc == 0)
7761                         goto done;
7762         }
7763
7764         /*
7765          * Read via firmware failed or wasn't even attempted.  Read directly via
7766          * the backdoor.
7767          */
7768         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7769 done:
7770         end_synchronized_op(sc, 0);
7771         return (rc);
7772 }
7773
7774 static int
7775 load_fw(struct adapter *sc, struct t4_data *fw)
7776 {
7777         int rc;
7778         uint8_t *fw_data;
7779
7780         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7781         if (rc)
7782                 return (rc);
7783
7784         if (sc->flags & FULL_INIT_DONE) {
7785                 rc = EBUSY;
7786                 goto done;
7787         }
7788
7789         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7790         if (fw_data == NULL) {
7791                 rc = ENOMEM;
7792                 goto done;
7793         }
7794
7795         rc = copyin(fw->data, fw_data, fw->len);
7796         if (rc == 0)
7797                 rc = -t4_load_fw(sc, fw_data, fw->len);
7798
7799         free(fw_data, M_CXGBE);
7800 done:
7801         end_synchronized_op(sc, 0);
7802         return (rc);
7803 }
7804
7805 static int
7806 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7807 {
7808         uint32_t addr, off, remaining, i, n;
7809         uint32_t *buf, *b;
7810         uint32_t mw_base, mw_aperture;
7811         int rc;
7812         uint8_t *dst;
7813
7814         rc = validate_mem_range(sc, mr->addr, mr->len);
7815         if (rc != 0)
7816                 return (rc);
7817
7818         memwin_info(sc, win, &mw_base, &mw_aperture);
7819         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7820         addr = mr->addr;
7821         remaining = mr->len;
7822         dst = (void *)mr->data;
7823
7824         while (remaining) {
7825                 off = position_memwin(sc, win, addr);
7826
7827                 /* number of bytes that we'll copy in the inner loop */
7828                 n = min(remaining, mw_aperture - off);
7829                 for (i = 0; i < n; i += 4)
7830                         *b++ = t4_read_reg(sc, mw_base + off + i);
7831
7832                 rc = copyout(buf, dst, n);
7833                 if (rc != 0)
7834                         break;
7835
7836                 b = buf;
7837                 dst += n;
7838                 remaining -= n;
7839                 addr += n;
7840         }
7841
7842         free(buf, M_CXGBE);
7843         return (rc);
7844 }
7845
7846 static int
7847 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7848 {
7849         int rc;
7850
7851         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7852                 return (EINVAL);
7853
7854         if (i2cd->len > sizeof(i2cd->data))
7855                 return (EFBIG);
7856
7857         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7858         if (rc)
7859                 return (rc);
7860         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7861             i2cd->offset, i2cd->len, &i2cd->data[0]);
7862         end_synchronized_op(sc, 0);
7863
7864         return (rc);
7865 }
7866
7867 static int
7868 in_range(int val, int lo, int hi)
7869 {
7870
7871         return (val < 0 || (val <= hi && val >= lo));
7872 }
7873
7874 static int
7875 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7876 {
7877         int fw_subcmd, fw_type, rc;
7878
7879         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7880         if (rc)
7881                 return (rc);
7882
7883         if (!(sc->flags & FULL_INIT_DONE)) {
7884                 rc = EAGAIN;
7885                 goto done;
7886         }
7887
7888         /*
7889          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7890          * sub-command and type are in common locations.)
7891          */
7892         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7893                 fw_subcmd = FW_SCHED_SC_CONFIG;
7894         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7895                 fw_subcmd = FW_SCHED_SC_PARAMS;
7896         else {
7897                 rc = EINVAL;
7898                 goto done;
7899         }
7900         if (p->type == SCHED_CLASS_TYPE_PACKET)
7901                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7902         else {
7903                 rc = EINVAL;
7904                 goto done;
7905         }
7906
7907         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7908                 /* Vet our parameters ..*/
7909                 if (p->u.config.minmax < 0) {
7910                         rc = EINVAL;
7911                         goto done;
7912                 }
7913
7914                 /* And pass the request to the firmware ...*/
7915                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7916                 goto done;
7917         }
7918
7919         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7920                 int fw_level;
7921                 int fw_mode;
7922                 int fw_rateunit;
7923                 int fw_ratemode;
7924
7925                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7926                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7927                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7928                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7929                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7930                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7931                 else {
7932                         rc = EINVAL;
7933                         goto done;
7934                 }
7935
7936                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7937                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7938                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7939                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7940                 else {
7941                         rc = EINVAL;
7942                         goto done;
7943                 }
7944
7945                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7946                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7947                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7948                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7949                 else {
7950                         rc = EINVAL;
7951                         goto done;
7952                 }
7953
7954                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7955                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7956                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7957                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7958                 else {
7959                         rc = EINVAL;
7960                         goto done;
7961                 }
7962
7963                 /* Vet our parameters ... */
7964                 if (!in_range(p->u.params.channel, 0, 3) ||
7965                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7966                     !in_range(p->u.params.minrate, 0, 10000000) ||
7967                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7968                     !in_range(p->u.params.weight, 0, 100)) {
7969                         rc = ERANGE;
7970                         goto done;
7971                 }
7972
7973                 /*
7974                  * Translate any unset parameters into the firmware's
7975                  * nomenclature and/or fail the call if the parameters
7976                  * are required ...
7977                  */
7978                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7979                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7980                         rc = EINVAL;
7981                         goto done;
7982                 }
7983                 if (p->u.params.minrate < 0)
7984                         p->u.params.minrate = 0;
7985                 if (p->u.params.maxrate < 0) {
7986                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7987                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7988                                 rc = EINVAL;
7989                                 goto done;
7990                         } else
7991                                 p->u.params.maxrate = 0;
7992                 }
7993                 if (p->u.params.weight < 0) {
7994                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7995                                 rc = EINVAL;
7996                                 goto done;
7997                         } else
7998                                 p->u.params.weight = 0;
7999                 }
8000                 if (p->u.params.pktsize < 0) {
8001                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
8002                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
8003                                 rc = EINVAL;
8004                                 goto done;
8005                         } else
8006                                 p->u.params.pktsize = 0;
8007                 }
8008
8009                 /* See what the firmware thinks of the request ... */
8010                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
8011                     fw_rateunit, fw_ratemode, p->u.params.channel,
8012                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
8013                     p->u.params.weight, p->u.params.pktsize, 1);
8014                 goto done;
8015         }
8016
8017         rc = EINVAL;
8018 done:
8019         end_synchronized_op(sc, 0);
8020         return (rc);
8021 }
8022
8023 static int
8024 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
8025 {
8026         struct port_info *pi = NULL;
8027         struct sge_txq *txq;
8028         uint32_t fw_mnem, fw_queue, fw_class;
8029         int i, rc;
8030
8031         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
8032         if (rc)
8033                 return (rc);
8034
8035         if (!(sc->flags & FULL_INIT_DONE)) {
8036                 rc = EAGAIN;
8037                 goto done;
8038         }
8039
8040         if (p->port >= sc->params.nports) {
8041                 rc = EINVAL;
8042                 goto done;
8043         }
8044
8045         pi = sc->port[p->port];
8046         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
8047                 rc = EINVAL;
8048                 goto done;
8049         }
8050
8051         /*
8052          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
8053          * Scheduling Class in this case).
8054          */
8055         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
8056             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
8057         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
8058
8059         /*
8060          * If op.queue is non-negative, then we're only changing the scheduling
8061          * on a single specified TX queue.
8062          */
8063         if (p->queue >= 0) {
8064                 txq = &sc->sge.txq[pi->first_txq + p->queue];
8065                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8066                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8067                     &fw_class);
8068                 goto done;
8069         }
8070
8071         /*
8072          * Change the scheduling on all the TX queues for the
8073          * interface.
8074          */
8075         for_each_txq(pi, i, txq) {
8076                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8077                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8078                     &fw_class);
8079                 if (rc)
8080                         goto done;
8081         }
8082
8083         rc = 0;
8084 done:
8085         end_synchronized_op(sc, 0);
8086         return (rc);
8087 }
8088
8089 int
8090 t4_os_find_pci_capability(struct adapter *sc, int cap)
8091 {
8092         int i;
8093
8094         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
8095 }
8096
8097 int
8098 t4_os_pci_save_state(struct adapter *sc)
8099 {
8100         device_t dev;
8101         struct pci_devinfo *dinfo;
8102
8103         dev = sc->dev;
8104         dinfo = device_get_ivars(dev);
8105
8106         pci_cfg_save(dev, dinfo, 0);
8107         return (0);
8108 }
8109
8110 int
8111 t4_os_pci_restore_state(struct adapter *sc)
8112 {
8113         device_t dev;
8114         struct pci_devinfo *dinfo;
8115
8116         dev = sc->dev;
8117         dinfo = device_get_ivars(dev);
8118
8119         pci_cfg_restore(dev, dinfo);
8120         return (0);
8121 }
8122
8123 void
8124 t4_os_portmod_changed(const struct adapter *sc, int idx)
8125 {
8126         struct port_info *pi = sc->port[idx];
8127         static const char *mod_str[] = {
8128                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
8129         };
8130
8131         build_medialist(pi, &pi->media);
8132 #ifdef DEV_NETMAP
8133         build_medialist(pi, &pi->nm_media);
8134 #endif
8135
8136         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
8137                 if_printf(pi->ifp, "transceiver unplugged.\n");
8138         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
8139                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
8140         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
8141                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
8142         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
8143                 if_printf(pi->ifp, "%s transceiver inserted.\n",
8144                     mod_str[pi->mod_type]);
8145         } else {
8146                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
8147                     pi->mod_type);
8148         }
8149 }
8150
8151 void
8152 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
8153 {
8154         struct port_info *pi = sc->port[idx];
8155         struct ifnet *ifp = pi->ifp;
8156
8157         if (link_stat) {
8158                 pi->linkdnrc = -1;
8159                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
8160                 if_link_state_change(ifp, LINK_STATE_UP);
8161         } else {
8162                 if (reason >= 0)
8163                         pi->linkdnrc = reason;
8164                 if_link_state_change(ifp, LINK_STATE_DOWN);
8165         }
8166 }
8167
8168 void
8169 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
8170 {
8171         struct adapter *sc;
8172
8173         sx_slock(&t4_list_lock);
8174         SLIST_FOREACH(sc, &t4_list, link) {
8175                 /*
8176                  * func should not make any assumptions about what state sc is
8177                  * in - the only guarantee is that sc->sc_lock is a valid lock.
8178                  */
8179                 func(sc, arg);
8180         }
8181         sx_sunlock(&t4_list_lock);
8182 }
8183
8184 static int
8185 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8186 {
8187        return (0);
8188 }
8189
8190 static int
8191 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8192 {
8193        return (0);
8194 }
8195
8196 static int
8197 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8198     struct thread *td)
8199 {
8200         int rc;
8201         struct adapter *sc = dev->si_drv1;
8202
8203         rc = priv_check(td, PRIV_DRIVER);
8204         if (rc != 0)
8205                 return (rc);
8206
8207         switch (cmd) {
8208         case CHELSIO_T4_GETREG: {
8209                 struct t4_reg *edata = (struct t4_reg *)data;
8210
8211                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8212                         return (EFAULT);
8213
8214                 if (edata->size == 4)
8215                         edata->val = t4_read_reg(sc, edata->addr);
8216                 else if (edata->size == 8)
8217                         edata->val = t4_read_reg64(sc, edata->addr);
8218                 else
8219                         return (EINVAL);
8220
8221                 break;
8222         }
8223         case CHELSIO_T4_SETREG: {
8224                 struct t4_reg *edata = (struct t4_reg *)data;
8225
8226                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8227                         return (EFAULT);
8228
8229                 if (edata->size == 4) {
8230                         if (edata->val & 0xffffffff00000000)
8231                                 return (EINVAL);
8232                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8233                 } else if (edata->size == 8)
8234                         t4_write_reg64(sc, edata->addr, edata->val);
8235                 else
8236                         return (EINVAL);
8237                 break;
8238         }
8239         case CHELSIO_T4_REGDUMP: {
8240                 struct t4_regdump *regs = (struct t4_regdump *)data;
8241                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8242                 uint8_t *buf;
8243
8244                 if (regs->len < reglen) {
8245                         regs->len = reglen; /* hint to the caller */
8246                         return (ENOBUFS);
8247                 }
8248
8249                 regs->len = reglen;
8250                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8251                 t4_get_regs(sc, regs, buf);
8252                 rc = copyout(buf, regs->data, reglen);
8253                 free(buf, M_CXGBE);
8254                 break;
8255         }
8256         case CHELSIO_T4_GET_FILTER_MODE:
8257                 rc = get_filter_mode(sc, (uint32_t *)data);
8258                 break;
8259         case CHELSIO_T4_SET_FILTER_MODE:
8260                 rc = set_filter_mode(sc, *(uint32_t *)data);
8261                 break;
8262         case CHELSIO_T4_GET_FILTER:
8263                 rc = get_filter(sc, (struct t4_filter *)data);
8264                 break;
8265         case CHELSIO_T4_SET_FILTER:
8266                 rc = set_filter(sc, (struct t4_filter *)data);
8267                 break;
8268         case CHELSIO_T4_DEL_FILTER:
8269                 rc = del_filter(sc, (struct t4_filter *)data);
8270                 break;
8271         case CHELSIO_T4_GET_SGE_CONTEXT:
8272                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8273                 break;
8274         case CHELSIO_T4_LOAD_FW:
8275                 rc = load_fw(sc, (struct t4_data *)data);
8276                 break;
8277         case CHELSIO_T4_GET_MEM:
8278                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8279                 break;
8280         case CHELSIO_T4_GET_I2C:
8281                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8282                 break;
8283         case CHELSIO_T4_CLEAR_STATS: {
8284                 int i;
8285                 u_int port_id = *(uint32_t *)data;
8286                 struct port_info *pi;
8287
8288                 if (port_id >= sc->params.nports)
8289                         return (EINVAL);
8290                 pi = sc->port[port_id];
8291
8292                 /* MAC stats */
8293                 t4_clr_port_stats(sc, pi->tx_chan);
8294                 pi->tx_parse_error = 0;
8295
8296                 if (pi->flags & PORT_INIT_DONE) {
8297                         struct sge_rxq *rxq;
8298                         struct sge_txq *txq;
8299                         struct sge_wrq *wrq;
8300
8301                         for_each_rxq(pi, i, rxq) {
8302 #if defined(INET) || defined(INET6)
8303                                 rxq->lro.lro_queued = 0;
8304                                 rxq->lro.lro_flushed = 0;
8305 #endif
8306                                 rxq->rxcsum = 0;
8307                                 rxq->vlan_extraction = 0;
8308                         }
8309
8310                         for_each_txq(pi, i, txq) {
8311                                 txq->txcsum = 0;
8312                                 txq->tso_wrs = 0;
8313                                 txq->vlan_insertion = 0;
8314                                 txq->imm_wrs = 0;
8315                                 txq->sgl_wrs = 0;
8316                                 txq->txpkt_wrs = 0;
8317                                 txq->txpkts0_wrs = 0;
8318                                 txq->txpkts1_wrs = 0;
8319                                 txq->txpkts0_pkts = 0;
8320                                 txq->txpkts1_pkts = 0;
8321                                 mp_ring_reset_stats(txq->r);
8322                         }
8323
8324 #ifdef TCP_OFFLOAD
8325                         /* nothing to clear for each ofld_rxq */
8326
8327                         for_each_ofld_txq(pi, i, wrq) {
8328                                 wrq->tx_wrs_direct = 0;
8329                                 wrq->tx_wrs_copied = 0;
8330                         }
8331 #endif
8332                         wrq = &sc->sge.ctrlq[pi->port_id];
8333                         wrq->tx_wrs_direct = 0;
8334                         wrq->tx_wrs_copied = 0;
8335                 }
8336                 break;
8337         }
8338         case CHELSIO_T4_SCHED_CLASS:
8339                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8340                 break;
8341         case CHELSIO_T4_SCHED_QUEUE:
8342                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8343                 break;
8344         case CHELSIO_T4_GET_TRACER:
8345                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8346                 break;
8347         case CHELSIO_T4_SET_TRACER:
8348                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8349                 break;
8350         default:
8351                 rc = EINVAL;
8352         }
8353
8354         return (rc);
8355 }
8356
8357 #ifdef TCP_OFFLOAD
8358 void
8359 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8360     const unsigned int *pgsz_order)
8361 {
8362         struct port_info *pi = ifp->if_softc;
8363         struct adapter *sc = pi->adapter;
8364
8365         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8366         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8367                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8368                 V_HPZ3(pgsz_order[3]));
8369 }
8370
8371 static int
8372 toe_capability(struct port_info *pi, int enable)
8373 {
8374         int rc;
8375         struct adapter *sc = pi->adapter;
8376
8377         ASSERT_SYNCHRONIZED_OP(sc);
8378
8379         if (!is_offload(sc))
8380                 return (ENODEV);
8381
8382         if (enable) {
8383                 /*
8384                  * We need the port's queues around so that we're able to send
8385                  * and receive CPLs to/from the TOE even if the ifnet for this
8386                  * port has never been UP'd administratively.
8387                  */
8388                 if (!(pi->flags & PORT_INIT_DONE)) {
8389                         rc = cxgbe_init_synchronized(pi);
8390                         if (rc)
8391                                 return (rc);
8392                 }
8393
8394                 if (isset(&sc->offload_map, pi->port_id))
8395                         return (0);
8396
8397                 if (!uld_active(sc, ULD_TOM)) {
8398                         rc = t4_activate_uld(sc, ULD_TOM);
8399                         if (rc == EAGAIN) {
8400                                 log(LOG_WARNING,
8401                                     "You must kldload t4_tom.ko before trying "
8402                                     "to enable TOE on a cxgbe interface.\n");
8403                         }
8404                         if (rc != 0)
8405                                 return (rc);
8406                         KASSERT(sc->tom_softc != NULL,
8407                             ("%s: TOM activated but softc NULL", __func__));
8408                         KASSERT(uld_active(sc, ULD_TOM),
8409                             ("%s: TOM activated but flag not set", __func__));
8410                 }
8411
8412                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8413                 if (!uld_active(sc, ULD_IWARP))
8414                         (void) t4_activate_uld(sc, ULD_IWARP);
8415                 if (!uld_active(sc, ULD_ISCSI))
8416                         (void) t4_activate_uld(sc, ULD_ISCSI);
8417
8418                 setbit(&sc->offload_map, pi->port_id);
8419         } else {
8420                 if (!isset(&sc->offload_map, pi->port_id))
8421                         return (0);
8422
8423                 KASSERT(uld_active(sc, ULD_TOM),
8424                     ("%s: TOM never initialized?", __func__));
8425                 clrbit(&sc->offload_map, pi->port_id);
8426         }
8427
8428         return (0);
8429 }
8430
8431 /*
8432  * Add an upper layer driver to the global list.
8433  */
8434 int
8435 t4_register_uld(struct uld_info *ui)
8436 {
8437         int rc = 0;
8438         struct uld_info *u;
8439
8440         sx_xlock(&t4_uld_list_lock);
8441         SLIST_FOREACH(u, &t4_uld_list, link) {
8442             if (u->uld_id == ui->uld_id) {
8443                     rc = EEXIST;
8444                     goto done;
8445             }
8446         }
8447
8448         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8449         ui->refcount = 0;
8450 done:
8451         sx_xunlock(&t4_uld_list_lock);
8452         return (rc);
8453 }
8454
8455 int
8456 t4_unregister_uld(struct uld_info *ui)
8457 {
8458         int rc = EINVAL;
8459         struct uld_info *u;
8460
8461         sx_xlock(&t4_uld_list_lock);
8462
8463         SLIST_FOREACH(u, &t4_uld_list, link) {
8464             if (u == ui) {
8465                     if (ui->refcount > 0) {
8466                             rc = EBUSY;
8467                             goto done;
8468                     }
8469
8470                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8471                     rc = 0;
8472                     goto done;
8473             }
8474         }
8475 done:
8476         sx_xunlock(&t4_uld_list_lock);
8477         return (rc);
8478 }
8479
8480 int
8481 t4_activate_uld(struct adapter *sc, int id)
8482 {
8483         int rc;
8484         struct uld_info *ui;
8485
8486         ASSERT_SYNCHRONIZED_OP(sc);
8487
8488         if (id < 0 || id > ULD_MAX)
8489                 return (EINVAL);
8490         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8491
8492         sx_slock(&t4_uld_list_lock);
8493
8494         SLIST_FOREACH(ui, &t4_uld_list, link) {
8495                 if (ui->uld_id == id) {
8496                         if (!(sc->flags & FULL_INIT_DONE)) {
8497                                 rc = adapter_full_init(sc);
8498                                 if (rc != 0)
8499                                         break;
8500                         }
8501
8502                         rc = ui->activate(sc);
8503                         if (rc == 0) {
8504                                 setbit(&sc->active_ulds, id);
8505                                 ui->refcount++;
8506                         }
8507                         break;
8508                 }
8509         }
8510
8511         sx_sunlock(&t4_uld_list_lock);
8512
8513         return (rc);
8514 }
8515
8516 int
8517 t4_deactivate_uld(struct adapter *sc, int id)
8518 {
8519         int rc;
8520         struct uld_info *ui;
8521
8522         ASSERT_SYNCHRONIZED_OP(sc);
8523
8524         if (id < 0 || id > ULD_MAX)
8525                 return (EINVAL);
8526         rc = ENXIO;
8527
8528         sx_slock(&t4_uld_list_lock);
8529
8530         SLIST_FOREACH(ui, &t4_uld_list, link) {
8531                 if (ui->uld_id == id) {
8532                         rc = ui->deactivate(sc);
8533                         if (rc == 0) {
8534                                 clrbit(&sc->active_ulds, id);
8535                                 ui->refcount--;
8536                         }
8537                         break;
8538                 }
8539         }
8540
8541         sx_sunlock(&t4_uld_list_lock);
8542
8543         return (rc);
8544 }
8545
8546 int
8547 uld_active(struct adapter *sc, int uld_id)
8548 {
8549
8550         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8551
8552         return (isset(&sc->active_ulds, uld_id));
8553 }
8554 #endif
8555
8556 /*
8557  * Come up with reasonable defaults for some of the tunables, provided they're
8558  * not set by the user (in which case we'll use the values as is).
8559  */
8560 static void
8561 tweak_tunables(void)
8562 {
8563         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8564
8565         if (t4_ntxq10g < 1) {
8566 #ifdef RSS
8567                 t4_ntxq10g = rss_getnumbuckets();
8568 #else
8569                 t4_ntxq10g = min(nc, NTXQ_10G);
8570 #endif
8571         }
8572
8573         if (t4_ntxq1g < 1) {
8574 #ifdef RSS
8575                 /* XXX: way too many for 1GbE? */
8576                 t4_ntxq1g = rss_getnumbuckets();
8577 #else
8578                 t4_ntxq1g = min(nc, NTXQ_1G);
8579 #endif
8580         }
8581
8582         if (t4_nrxq10g < 1) {
8583 #ifdef RSS
8584                 t4_nrxq10g = rss_getnumbuckets();
8585 #else
8586                 t4_nrxq10g = min(nc, NRXQ_10G);
8587 #endif
8588         }
8589
8590         if (t4_nrxq1g < 1) {
8591 #ifdef RSS
8592                 /* XXX: way too many for 1GbE? */
8593                 t4_nrxq1g = rss_getnumbuckets();
8594 #else
8595                 t4_nrxq1g = min(nc, NRXQ_1G);
8596 #endif
8597         }
8598
8599 #ifdef TCP_OFFLOAD
8600         if (t4_nofldtxq10g < 1)
8601                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8602
8603         if (t4_nofldtxq1g < 1)
8604                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8605
8606         if (t4_nofldrxq10g < 1)
8607                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8608
8609         if (t4_nofldrxq1g < 1)
8610                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8611
8612         if (t4_toecaps_allowed == -1)
8613                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8614 #else
8615         if (t4_toecaps_allowed == -1)
8616                 t4_toecaps_allowed = 0;
8617 #endif
8618
8619 #ifdef DEV_NETMAP
8620         if (t4_nnmtxq10g < 1)
8621                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8622
8623         if (t4_nnmtxq1g < 1)
8624                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8625
8626         if (t4_nnmrxq10g < 1)
8627                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8628
8629         if (t4_nnmrxq1g < 1)
8630                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8631 #endif
8632
8633         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8634                 t4_tmr_idx_10g = TMR_IDX_10G;
8635
8636         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8637                 t4_pktc_idx_10g = PKTC_IDX_10G;
8638
8639         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8640                 t4_tmr_idx_1g = TMR_IDX_1G;
8641
8642         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8643                 t4_pktc_idx_1g = PKTC_IDX_1G;
8644
8645         if (t4_qsize_txq < 128)
8646                 t4_qsize_txq = 128;
8647
8648         if (t4_qsize_rxq < 128)
8649                 t4_qsize_rxq = 128;
8650         while (t4_qsize_rxq & 7)
8651                 t4_qsize_rxq++;
8652
8653         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8654 }
8655
8656 static struct sx mlu;   /* mod load unload */
8657 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8658
8659 static int
8660 mod_event(module_t mod, int cmd, void *arg)
8661 {
8662         int rc = 0;
8663         static int loaded = 0;
8664
8665         switch (cmd) {
8666         case MOD_LOAD:
8667                 sx_xlock(&mlu);
8668                 if (loaded++ == 0) {
8669                         t4_sge_modload();
8670                         sx_init(&t4_list_lock, "T4/T5 adapters");
8671                         SLIST_INIT(&t4_list);
8672 #ifdef TCP_OFFLOAD
8673                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8674                         SLIST_INIT(&t4_uld_list);
8675 #endif
8676                         t4_tracer_modload();
8677                         tweak_tunables();
8678                 }
8679                 sx_xunlock(&mlu);
8680                 break;
8681
8682         case MOD_UNLOAD:
8683                 sx_xlock(&mlu);
8684                 if (--loaded == 0) {
8685                         int tries;
8686
8687                         sx_slock(&t4_list_lock);
8688                         if (!SLIST_EMPTY(&t4_list)) {
8689                                 rc = EBUSY;
8690                                 sx_sunlock(&t4_list_lock);
8691                                 goto done_unload;
8692                         }
8693 #ifdef TCP_OFFLOAD
8694                         sx_slock(&t4_uld_list_lock);
8695                         if (!SLIST_EMPTY(&t4_uld_list)) {
8696                                 rc = EBUSY;
8697                                 sx_sunlock(&t4_uld_list_lock);
8698                                 sx_sunlock(&t4_list_lock);
8699                                 goto done_unload;
8700                         }
8701 #endif
8702                         tries = 0;
8703                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8704                                 uprintf("%ju clusters with custom free routine "
8705                                     "still is use.\n", t4_sge_extfree_refs());
8706                                 pause("t4unload", 2 * hz);
8707                         }
8708 #ifdef TCP_OFFLOAD
8709                         sx_sunlock(&t4_uld_list_lock);
8710 #endif
8711                         sx_sunlock(&t4_list_lock);
8712
8713                         if (t4_sge_extfree_refs() == 0) {
8714                                 t4_tracer_modunload();
8715 #ifdef TCP_OFFLOAD
8716                                 sx_destroy(&t4_uld_list_lock);
8717 #endif
8718                                 sx_destroy(&t4_list_lock);
8719                                 t4_sge_modunload();
8720                                 loaded = 0;
8721                         } else {
8722                                 rc = EBUSY;
8723                                 loaded++;       /* undo earlier decrement */
8724                         }
8725                 }
8726 done_unload:
8727                 sx_xunlock(&mlu);
8728                 break;
8729         }
8730
8731         return (rc);
8732 }
8733
8734 static devclass_t t4_devclass, t5_devclass;
8735 static devclass_t cxgbe_devclass, cxl_devclass;
8736
8737 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8738 MODULE_VERSION(t4nex, 1);
8739 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8740 #ifdef DEV_NETMAP
8741 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
8742 #endif /* DEV_NETMAP */
8743
8744
8745 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8746 MODULE_VERSION(t5nex, 1);
8747 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8748 #ifdef DEV_NETMAP
8749 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
8750 #endif /* DEV_NETMAP */
8751
8752 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8753 MODULE_VERSION(cxgbe, 1);
8754
8755 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8756 MODULE_VERSION(cxl, 1);