sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69 #include "t4_mp_ring.h"
  70
  71 /* T4 bus driver interface */
  72 static int t4_probe(device_t);
  73 static int t4_attach(device_t);
  74 static int t4_detach(device_t);
  75 static device_method_t t4_methods[] = {
  76         DEVMETHOD(device_probe,         t4_probe),
  77         DEVMETHOD(device_attach,        t4_attach),
  78         DEVMETHOD(device_detach,        t4_detach),
  79
  80         DEVMETHOD_END
  81 };
  82 static driver_t t4_driver = {
  83         "t4nex",
  84         t4_methods,
  85         sizeof(struct adapter)
  86 };
  87
  88
  89 /* T4 port (cxgbe) interface */
  90 static int cxgbe_probe(device_t);
  91 static int cxgbe_attach(device_t);
  92 static int cxgbe_detach(device_t);
  93 static device_method_t cxgbe_methods[] = {
  94         DEVMETHOD(device_probe,         cxgbe_probe),
  95         DEVMETHOD(device_attach,        cxgbe_attach),
  96         DEVMETHOD(device_detach,        cxgbe_detach),
  97         { 0, 0 }
  98 };
  99 static driver_t cxgbe_driver = {
 100         "cxgbe",
 101         cxgbe_methods,
 102         sizeof(struct port_info)
 103 };
 104
 105 static d_ioctl_t t4_ioctl;
 106 static d_open_t t4_open;
 107 static d_close_t t4_close;
 108
 109 static struct cdevsw t4_cdevsw = {
 110        .d_version = D_VERSION,
 111        .d_flags = 0,
 112        .d_open = t4_open,
 113        .d_close = t4_close,
 114        .d_ioctl = t4_ioctl,
 115        .d_name = "t4nex",
 116 };
 117
 118 /* T5 bus driver interface */
 119 static int t5_probe(device_t);
 120 static device_method_t t5_methods[] = {
 121         DEVMETHOD(device_probe,         t5_probe),
 122         DEVMETHOD(device_attach,        t4_attach),
 123         DEVMETHOD(device_detach,        t4_detach),
 124
 125         DEVMETHOD_END
 126 };
 127 static driver_t t5_driver = {
 128         "t5nex",
 129         t5_methods,
 130         sizeof(struct adapter)
 131 };
 132
 133
 134 /* T5 port (cxl) interface */
 135 static driver_t cxl_driver = {
 136         "cxl",
 137         cxgbe_methods,
 138         sizeof(struct port_info)
 139 };
 140
 141 static struct cdevsw t5_cdevsw = {
 142        .d_version = D_VERSION,
 143        .d_flags = 0,
 144        .d_open = t4_open,
 145        .d_close = t4_close,
 146        .d_ioctl = t4_ioctl,
 147        .d_name = "t5nex",
 148 };
 149
 150 /* ifnet + media interface */
 151 static void cxgbe_init(void *);
 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 154 static void cxgbe_qflush(struct ifnet *);
 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 156 static int cxgbe_media_change(struct ifnet *);
 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 158
 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 160
 161 /*
 162  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 163  * then ADAPTER_LOCK, then t4_uld_list_lock.
 164  */
 165 static struct sx t4_list_lock;
 166 SLIST_HEAD(, adapter) t4_list;
 167 #ifdef TCP_OFFLOAD
 168 static struct sx t4_uld_list_lock;
 169 SLIST_HEAD(, uld_info) t4_uld_list;
 170 #endif
 171
 172 /*
 173  * Tunables.  See tweak_tunables() too.
 174  *
 175  * Each tunable is set to a default value here if it's known at compile-time.
 176  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 177  * provide a reasonable default when the driver is loaded.
 178  *
 179  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 180  * T5 are under hw.cxl.
 181  */
 182
 183 /*
 184  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 185  */
 186 #define NTXQ_10G 16
 187 static int t4_ntxq10g = -1;
 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 189
 190 #define NRXQ_10G 8
 191 static int t4_nrxq10g = -1;
 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 193
 194 #define NTXQ_1G 4
 195 static int t4_ntxq1g = -1;
 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 197
 198 #define NRXQ_1G 2
 199 static int t4_nrxq1g = -1;
 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 201
 202 static int t4_rsrv_noflowq = 0;
 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 204
 205 #ifdef TCP_OFFLOAD
 206 #define NOFLDTXQ_10G 8
 207 static int t4_nofldtxq10g = -1;
 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 209
 210 #define NOFLDRXQ_10G 2
 211 static int t4_nofldrxq10g = -1;
 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 213
 214 #define NOFLDTXQ_1G 2
 215 static int t4_nofldtxq1g = -1;
 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 217
 218 #define NOFLDRXQ_1G 1
 219 static int t4_nofldrxq1g = -1;
 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 221 #endif
 222
 223 #ifdef DEV_NETMAP
 224 #define NNMTXQ_10G 2
 225 static int t4_nnmtxq10g = -1;
 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 227
 228 #define NNMRXQ_10G 2
 229 static int t4_nnmrxq10g = -1;
 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 231
 232 #define NNMTXQ_1G 1
 233 static int t4_nnmtxq1g = -1;
 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 235
 236 #define NNMRXQ_1G 1
 237 static int t4_nnmrxq1g = -1;
 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 239 #endif
 240
 241 /*
 242  * Holdoff parameters for 10G and 1G ports.
 243  */
 244 #define TMR_IDX_10G 1
 245 static int t4_tmr_idx_10g = TMR_IDX_10G;
 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 247
 248 #define PKTC_IDX_10G (-1)
 249 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 251
 252 #define TMR_IDX_1G 1
 253 static int t4_tmr_idx_1g = TMR_IDX_1G;
 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 255
 256 #define PKTC_IDX_1G (-1)
 257 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 259
 260 /*
 261  * Size (# of entries) of each tx and rx queue.
 262  */
 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 265
 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 268
 269 /*
 270  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 271  */
 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 274
 275 /*
 276  * Configuration file.
 277  */
 278 #define DEFAULT_CF      "default"
 279 #define FLASH_CF        "flash"
 280 #define UWIRE_CF        "uwire"
 281 #define FPGA_CF         "fpga"
 282 static char t4_cfg_file[32] = DEFAULT_CF;
 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 284
 285 /*
 286  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 287  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 288  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 289  *            mark or when signalled to do so, 0 to never emit PAUSE.
 290  */
 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 293
 294 /*
 295  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 296  * encouraged respectively).
 297  */
 298 static unsigned int t4_fw_install = 1;
 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 300
 301 /*
 302  * ASIC features that will be used.  Disable the ones you don't want so that the
 303  * chip resources aren't wasted on features that will not be used.
 304  */
 305 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 307
 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 310
 311 static int t4_toecaps_allowed = -1;
 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 313
 314 static int t4_rdmacaps_allowed = 0;
 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 316
 317 static int t4_iscsicaps_allowed = 0;
 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 319
 320 static int t4_fcoecaps_allowed = 0;
 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 322
 323 static int t5_write_combine = 0;
 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 325
 326 struct intrs_and_queues {
 327         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 328         uint16_t nirq;          /* Total # of vectors */
 329         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 330         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 331         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 332         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 333         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 334         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 335         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 336 #ifdef TCP_OFFLOAD
 337         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 338         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 339         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 340         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 341 #endif
 342 #ifdef DEV_NETMAP
 343         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 344         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 345         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 346         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 347 #endif
 348 };
 349
 350 struct filter_entry {
 351         uint32_t valid:1;       /* filter allocated and valid */
 352         uint32_t locked:1;      /* filter is administratively locked */
 353         uint32_t pending:1;     /* filter action is pending firmware reply */
 354         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 355         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 356
 357         struct t4_filter_specification fs;
 358 };
 359
 360 static int map_bars_0_and_4(struct adapter *);
 361 static int map_bar_2(struct adapter *);
 362 static void setup_memwin(struct adapter *);
 363 static int validate_mem_range(struct adapter *, uint32_t, int);
 364 static int fwmtype_to_hwmtype(int);
 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 366     uint32_t *);
 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 368 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 369 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 370     struct intrs_and_queues *);
 371 static int prep_firmware(struct adapter *);
 372 static int partition_resources(struct adapter *, const struct firmware *,
 373     const char *);
 374 static int get_params__pre_init(struct adapter *);
 375 static int get_params__post_init(struct adapter *);
 376 static int set_params__post_init(struct adapter *);
 377 static void t4_set_desc(struct adapter *);
 378 static void build_medialist(struct port_info *, struct ifmedia *);
 379 static int cxgbe_init_synchronized(struct port_info *);
 380 static int cxgbe_uninit_synchronized(struct port_info *);
 381 static int setup_intr_handlers(struct adapter *);
 382 static void quiesce_txq(struct adapter *, struct sge_txq *);
 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 384 static void quiesce_iq(struct adapter *, struct sge_iq *);
 385 static void quiesce_fl(struct adapter *, struct sge_fl *);
 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 387     driver_intr_t *, void *, char *);
 388 static int t4_free_irq(struct adapter *, struct irq *);
 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 390     unsigned int);
 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 393 static void cxgbe_tick(void *);
 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 396     struct mbuf *);
 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 398 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 399 static int t4_sysctls(struct adapter *);
 400 static int cxgbe_sysctls(struct port_info *);
 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 412 #ifdef SBUF_DRAIN
 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 438 #endif
 439 static uint32_t fconf_to_mode(uint32_t);
 440 static uint32_t mode_to_fconf(uint32_t);
 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 442 static int get_filter_mode(struct adapter *, uint32_t *);
 443 static int set_filter_mode(struct adapter *, uint32_t);
 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 445 static int get_filter(struct adapter *, struct t4_filter *);
 446 static int set_filter(struct adapter *, struct t4_filter *);
 447 static int del_filter(struct adapter *, struct t4_filter *);
 448 static void clear_filter(struct filter_entry *);
 449 static int set_filter_wr(struct adapter *, int);
 450 static int del_filter_wr(struct adapter *, int);
 451 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 452 static int load_fw(struct adapter *, struct t4_data *);
 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 454 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 455 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 457 #ifdef TCP_OFFLOAD
 458 static int toe_capability(struct port_info *, int);
 459 #endif
 460 static int mod_event(module_t, int, void *);
 461
 462 struct {
 463         uint16_t device;
 464         char *desc;
 465 } t4_pciids[] = {
 466         {0xa000, "Chelsio Terminator 4 FPGA"},
 467         {0x4400, "Chelsio T440-dbg"},
 468         {0x4401, "Chelsio T420-CR"},
 469         {0x4402, "Chelsio T422-CR"},
 470         {0x4403, "Chelsio T440-CR"},
 471         {0x4404, "Chelsio T420-BCH"},
 472         {0x4405, "Chelsio T440-BCH"},
 473         {0x4406, "Chelsio T440-CH"},
 474         {0x4407, "Chelsio T420-SO"},
 475         {0x4408, "Chelsio T420-CX"},
 476         {0x4409, "Chelsio T420-BT"},
 477         {0x440a, "Chelsio T404-BT"},
 478         {0x440e, "Chelsio T440-LP-CR"},
 479 }, t5_pciids[] = {
 480         {0xb000, "Chelsio Terminator 5 FPGA"},
 481         {0x5400, "Chelsio T580-dbg"},
 482         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 483         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 484         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 485         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 486         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 487         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 488         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 489         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 490         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 491         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 492         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 493         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 494         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 495 #ifdef notyet
 496         {0x5404,  "Chelsio T520-BCH"},
 497         {0x5405,  "Chelsio T540-BCH"},
 498         {0x5406,  "Chelsio T540-CH"},
 499         {0x5408,  "Chelsio T520-CX"},
 500         {0x540b,  "Chelsio B520-SR"},
 501         {0x540c,  "Chelsio B504-BT"},
 502         {0x540f,  "Chelsio Amsterdam"},
 503         {0x5413,  "Chelsio T580-CHR"},
 504 #endif
 505 };
 506
 507 #ifdef TCP_OFFLOAD
 508 /*
 509  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 510  * exactly the same for both rxq and ofld_rxq.
 511  */
 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 514 #endif
 515
 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 519
 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 521
 522 static int
 523 t4_probe(device_t dev)
 524 {
 525         int i;
 526         uint16_t v = pci_get_vendor(dev);
 527         uint16_t d = pci_get_device(dev);
 528         uint8_t f = pci_get_function(dev);
 529
 530         if (v != PCI_VENDOR_ID_CHELSIO)
 531                 return (ENXIO);
 532
 533         /* Attach only to PF0 of the FPGA */
 534         if (d == 0xa000 && f != 0)
 535                 return (ENXIO);
 536
 537         for (i = 0; i < nitems(t4_pciids); i++) {
 538                 if (d == t4_pciids[i].device) {
 539                         device_set_desc(dev, t4_pciids[i].desc);
 540                         return (BUS_PROBE_DEFAULT);
 541                 }
 542         }
 543
 544         return (ENXIO);
 545 }
 546
 547 static int
 548 t5_probe(device_t dev)
 549 {
 550         int i;
 551         uint16_t v = pci_get_vendor(dev);
 552         uint16_t d = pci_get_device(dev);
 553         uint8_t f = pci_get_function(dev);
 554
 555         if (v != PCI_VENDOR_ID_CHELSIO)
 556                 return (ENXIO);
 557
 558         /* Attach only to PF0 of the FPGA */
 559         if (d == 0xb000 && f != 0)
 560                 return (ENXIO);
 561
 562         for (i = 0; i < nitems(t5_pciids); i++) {
 563                 if (d == t5_pciids[i].device) {
 564                         device_set_desc(dev, t5_pciids[i].desc);
 565                         return (BUS_PROBE_DEFAULT);
 566                 }
 567         }
 568
 569         return (ENXIO);
 570 }
 571
 572 static int
 573 t4_attach(device_t dev)
 574 {
 575         struct adapter *sc;
 576         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 577         struct intrs_and_queues iaq;
 578         struct sge *s;
 579 #ifdef TCP_OFFLOAD
 580         int ofld_rqidx, ofld_tqidx;
 581 #endif
 582 #ifdef DEV_NETMAP
 583         int nm_rqidx, nm_tqidx;
 584 #endif
 585
 586         sc = device_get_softc(dev);
 587         sc->dev = dev;
 588         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 589
 590         pci_enable_busmaster(dev);
 591         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 592                 uint32_t v;
 593
 594                 pci_set_max_read_req(dev, 4096);
 595                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 596                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 597                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 598
 599                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 600         }
 601
 602         sc->traceq = -1;
 603         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 604         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 605             device_get_nameunit(dev));
 606
 607         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 608             device_get_nameunit(dev));
 609         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 610         sx_xlock(&t4_list_lock);
 611         SLIST_INSERT_HEAD(&t4_list, sc, link);
 612         sx_xunlock(&t4_list_lock);
 613
 614         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 615         TAILQ_INIT(&sc->sfl);
 616         callout_init(&sc->sfl_callout, 1);
 617
 618         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 619
 620         rc = map_bars_0_and_4(sc);
 621         if (rc != 0)
 622                 goto done; /* error message displayed already */
 623
 624         /*
 625          * This is the real PF# to which we're attaching.  Works from within PCI
 626          * passthrough environments too, where pci_get_function() could return a
 627          * different PF# depending on the passthrough configuration.  We need to
 628          * use the real PF# in all our communication with the firmware.
 629          */
 630         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 631         sc->mbox = sc->pf;
 632
 633         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 634         sc->an_handler = an_not_handled;
 635         for (i = 0; i < nitems(sc->cpl_handler); i++)
 636                 sc->cpl_handler[i] = cpl_not_handled;
 637         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 638                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 639         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 642         t4_init_sge_cpl_handlers(sc);
 643
 644         /* Prepare the adapter for operation */
 645         rc = -t4_prep_adapter(sc);
 646         if (rc != 0) {
 647                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 648                 goto done;
 649         }
 650
 651         /*
 652          * Do this really early, with the memory windows set up even before the
 653          * character device.  The userland tool's register i/o and mem read
 654          * will work even in "recovery mode".
 655          */
 656         setup_memwin(sc);
 657         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 658             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 659             device_get_nameunit(dev));
 660         if (sc->cdev == NULL)
 661                 device_printf(dev, "failed to create nexus char device.\n");
 662         else
 663                 sc->cdev->si_drv1 = sc;
 664
 665         /* Go no further if recovery mode has been requested. */
 666         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 667                 device_printf(dev, "recovery mode.\n");
 668                 goto done;
 669         }
 670
 671 #if defined(__i386__)
 672         if ((cpu_feature & CPUID_CX8) == 0) {
 673                 device_printf(dev, "64 bit atomics not available.\n");
 674                 rc = ENOTSUP;
 675                 goto done;
 676         }
 677 #endif
 678
 679         /* Prepare the firmware for operation */
 680         rc = prep_firmware(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         rc = get_params__post_init(sc);
 685         if (rc != 0)
 686                 goto done; /* error message displayed already */
 687
 688         rc = set_params__post_init(sc);
 689         if (rc != 0)
 690                 goto done; /* error message displayed already */
 691
 692         rc = map_bar_2(sc);
 693         if (rc != 0)
 694                 goto done; /* error message displayed already */
 695
 696         rc = t4_create_dma_tag(sc);
 697         if (rc != 0)
 698                 goto done; /* error message displayed already */
 699
 700         /*
 701          * First pass over all the ports - allocate VIs and initialize some
 702          * basic parameters like mac address, port type, etc.  We also figure
 703          * out whether a port is 10G or 1G and use that information when
 704          * calculating how many interrupts to attempt to allocate.
 705          */
 706         n10g = n1g = 0;
 707         for_each_port(sc, i) {
 708                 struct port_info *pi;
 709
 710                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 711                 sc->port[i] = pi;
 712
 713                 /* These must be set before t4_port_init */
 714                 pi->adapter = sc;
 715                 pi->port_id = i;
 716
 717                 /* Allocate the vi and initialize parameters like mac addr */
 718                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 719                 if (rc != 0) {
 720                         device_printf(dev, "unable to initialize port %d: %d\n",
 721                             i, rc);
 722                         free(pi, M_CXGBE);
 723                         sc->port[i] = NULL;
 724                         goto done;
 725                 }
 726
 727                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 728                 pi->link_cfg.requested_fc |= t4_pause_settings;
 729                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 730                 pi->link_cfg.fc |= t4_pause_settings;
 731
 732                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 733                 if (rc != 0) {
 734                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 735                         free(pi, M_CXGBE);
 736                         sc->port[i] = NULL;
 737                         goto done;
 738                 }
 739
 740                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 741                     device_get_nameunit(dev), i);
 742                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 743                 sc->chan_map[pi->tx_chan] = i;
 744
 745                 if (is_10G_port(pi) || is_40G_port(pi)) {
 746                         n10g++;
 747                         pi->tmr_idx = t4_tmr_idx_10g;
 748                         pi->pktc_idx = t4_pktc_idx_10g;
 749                 } else {
 750                         n1g++;
 751                         pi->tmr_idx = t4_tmr_idx_1g;
 752                         pi->pktc_idx = t4_pktc_idx_1g;
 753                 }
 754
 755                 pi->xact_addr_filt = -1;
 756                 pi->linkdnrc = -1;
 757
 758                 pi->qsize_rxq = t4_qsize_rxq;
 759                 pi->qsize_txq = t4_qsize_txq;
 760
 761                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 762                 if (pi->dev == NULL) {
 763                         device_printf(dev,
 764                             "failed to add device for port %d.\n", i);
 765                         rc = ENXIO;
 766                         goto done;
 767                 }
 768                 device_set_softc(pi->dev, pi);
 769         }
 770
 771         /*
 772          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 773          */
 774         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 775         if (rc != 0)
 776                 goto done; /* error message displayed already */
 777
 778         sc->intr_type = iaq.intr_type;
 779         sc->intr_count = iaq.nirq;
 780
 781         s = &sc->sge;
 782         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 783         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 784         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 785         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 786         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 787 #ifdef TCP_OFFLOAD
 788         if (is_offload(sc)) {
 789                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 790                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 791                 s->neq += s->nofldtxq + s->nofldrxq;
 792                 s->niq += s->nofldrxq;
 793
 794                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 795                     M_CXGBE, M_ZERO | M_WAITOK);
 796                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 797                     M_CXGBE, M_ZERO | M_WAITOK);
 798         }
 799 #endif
 800 #ifdef DEV_NETMAP
 801         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 802         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 803         s->neq += s->nnmtxq + s->nnmrxq;
 804         s->niq += s->nnmrxq;
 805
 806         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 807             M_CXGBE, M_ZERO | M_WAITOK);
 808         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 809             M_CXGBE, M_ZERO | M_WAITOK);
 810 #endif
 811
 812         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 813             M_ZERO | M_WAITOK);
 814         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 815             M_ZERO | M_WAITOK);
 816         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 817             M_ZERO | M_WAITOK);
 818         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 819             M_ZERO | M_WAITOK);
 820         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 821             M_ZERO | M_WAITOK);
 822
 823         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 824             M_ZERO | M_WAITOK);
 825
 826         t4_init_l2t(sc, M_WAITOK);
 827
 828         /*
 829          * Second pass over the ports.  This time we know the number of rx and
 830          * tx queues that each port should get.
 831          */
 832         rqidx = tqidx = 0;
 833 #ifdef TCP_OFFLOAD
 834         ofld_rqidx = ofld_tqidx = 0;
 835 #endif
 836 #ifdef DEV_NETMAP
 837         nm_rqidx = nm_tqidx = 0;
 838 #endif
 839         for_each_port(sc, i) {
 840                 struct port_info *pi = sc->port[i];
 841
 842                 if (pi == NULL)
 843                         continue;
 844
 845                 pi->first_rxq = rqidx;
 846                 pi->first_txq = tqidx;
 847                 if (is_10G_port(pi) || is_40G_port(pi)) {
 848                         pi->flags |= iaq.intr_flags_10g;
 849                         pi->nrxq = iaq.nrxq10g;
 850                         pi->ntxq = iaq.ntxq10g;
 851                 } else {
 852                         pi->flags |= iaq.intr_flags_1g;
 853                         pi->nrxq = iaq.nrxq1g;
 854                         pi->ntxq = iaq.ntxq1g;
 855                 }
 856
 857                 if (pi->ntxq > 1)
 858                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 859                 else
 860                         pi->rsrv_noflowq = 0;
 861
 862                 rqidx += pi->nrxq;
 863                 tqidx += pi->ntxq;
 864 #ifdef TCP_OFFLOAD
 865                 if (is_offload(sc)) {
 866                         pi->first_ofld_rxq = ofld_rqidx;
 867                         pi->first_ofld_txq = ofld_tqidx;
 868                         if (is_10G_port(pi) || is_40G_port(pi)) {
 869                                 pi->nofldrxq = iaq.nofldrxq10g;
 870                                 pi->nofldtxq = iaq.nofldtxq10g;
 871                         } else {
 872                                 pi->nofldrxq = iaq.nofldrxq1g;
 873                                 pi->nofldtxq = iaq.nofldtxq1g;
 874                         }
 875                         ofld_rqidx += pi->nofldrxq;
 876                         ofld_tqidx += pi->nofldtxq;
 877                 }
 878 #endif
 879 #ifdef DEV_NETMAP
 880                 pi->first_nm_rxq = nm_rqidx;
 881                 pi->first_nm_txq = nm_tqidx;
 882                 if (is_10G_port(pi) || is_40G_port(pi)) {
 883                         pi->nnmrxq = iaq.nnmrxq10g;
 884                         pi->nnmtxq = iaq.nnmtxq10g;
 885                 } else {
 886                         pi->nnmrxq = iaq.nnmrxq1g;
 887                         pi->nnmtxq = iaq.nnmtxq1g;
 888                 }
 889                 nm_rqidx += pi->nnmrxq;
 890                 nm_tqidx += pi->nnmtxq;
 891 #endif
 892         }
 893
 894         rc = setup_intr_handlers(sc);
 895         if (rc != 0) {
 896                 device_printf(dev,
 897                     "failed to setup interrupt handlers: %d\n", rc);
 898                 goto done;
 899         }
 900
 901         rc = bus_generic_attach(dev);
 902         if (rc != 0) {
 903                 device_printf(dev,
 904                     "failed to attach all child ports: %d\n", rc);
 905                 goto done;
 906         }
 907
 908         device_printf(dev,
 909             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 910             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 911             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 912             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 913             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 914
 915         t4_set_desc(sc);
 916
 917 done:
 918         if (rc != 0 && sc->cdev) {
 919                 /* cdev was created and so cxgbetool works; recover that way. */
 920                 device_printf(dev,
 921                     "error during attach, adapter is now in recovery mode.\n");
 922                 rc = 0;
 923         }
 924
 925         if (rc != 0)
 926                 t4_detach(dev);
 927         else
 928                 t4_sysctls(sc);
 929
 930         return (rc);
 931 }
 932
 933 /*
 934  * Idempotent
 935  */
 936 static int
 937 t4_detach(device_t dev)
 938 {
 939         struct adapter *sc;
 940         struct port_info *pi;
 941         int i, rc;
 942
 943         sc = device_get_softc(dev);
 944
 945         if (sc->flags & FULL_INIT_DONE)
 946                 t4_intr_disable(sc);
 947
 948         if (sc->cdev) {
 949                 destroy_dev(sc->cdev);
 950                 sc->cdev = NULL;
 951         }
 952
 953         rc = bus_generic_detach(dev);
 954         if (rc) {
 955                 device_printf(dev,
 956                     "failed to detach child devices: %d\n", rc);
 957                 return (rc);
 958         }
 959
 960         for (i = 0; i < sc->intr_count; i++)
 961                 t4_free_irq(sc, &sc->irq[i]);
 962
 963         for (i = 0; i < MAX_NPORTS; i++) {
 964                 pi = sc->port[i];
 965                 if (pi) {
 966                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 967                         if (pi->dev)
 968                                 device_delete_child(dev, pi->dev);
 969
 970                         mtx_destroy(&pi->pi_lock);
 971                         free(pi, M_CXGBE);
 972                 }
 973         }
 974
 975         if (sc->flags & FULL_INIT_DONE)
 976                 adapter_full_uninit(sc);
 977
 978         if (sc->flags & FW_OK)
 979                 t4_fw_bye(sc, sc->mbox);
 980
 981         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 982                 pci_release_msi(dev);
 983
 984         if (sc->regs_res)
 985                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 986                     sc->regs_res);
 987
 988         if (sc->udbs_res)
 989                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 990                     sc->udbs_res);
 991
 992         if (sc->msix_res)
 993                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 994                     sc->msix_res);
 995
 996         if (sc->l2t)
 997                 t4_free_l2t(sc->l2t);
 998
 999 #ifdef TCP_OFFLOAD
1000         free(sc->sge.ofld_rxq, M_CXGBE);
1001         free(sc->sge.ofld_txq, M_CXGBE);
1002 #endif
1003 #ifdef DEV_NETMAP
1004         free(sc->sge.nm_rxq, M_CXGBE);
1005         free(sc->sge.nm_txq, M_CXGBE);
1006 #endif
1007         free(sc->irq, M_CXGBE);
1008         free(sc->sge.rxq, M_CXGBE);
1009         free(sc->sge.txq, M_CXGBE);
1010         free(sc->sge.ctrlq, M_CXGBE);
1011         free(sc->sge.iqmap, M_CXGBE);
1012         free(sc->sge.eqmap, M_CXGBE);
1013         free(sc->tids.ftid_tab, M_CXGBE);
1014         t4_destroy_dma_tag(sc);
1015         if (mtx_initialized(&sc->sc_lock)) {
1016                 sx_xlock(&t4_list_lock);
1017                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1018                 sx_xunlock(&t4_list_lock);
1019                 mtx_destroy(&sc->sc_lock);
1020         }
1021
1022         if (mtx_initialized(&sc->tids.ftid_lock))
1023                 mtx_destroy(&sc->tids.ftid_lock);
1024         if (mtx_initialized(&sc->sfl_lock))
1025                 mtx_destroy(&sc->sfl_lock);
1026         if (mtx_initialized(&sc->ifp_lock))
1027                 mtx_destroy(&sc->ifp_lock);
1028         if (mtx_initialized(&sc->regwin_lock))
1029                 mtx_destroy(&sc->regwin_lock);
1030
1031         bzero(sc, sizeof(*sc));
1032
1033         return (0);
1034 }
1035
1036 static int
1037 cxgbe_probe(device_t dev)
1038 {
1039         char buf[128];
1040         struct port_info *pi = device_get_softc(dev);
1041
1042         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1043         device_set_desc_copy(dev, buf);
1044
1045         return (BUS_PROBE_DEFAULT);
1046 }
1047
1048 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1049     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1050     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1051 #define T4_CAP_ENABLE (T4_CAP)
1052
1053 static int
1054 cxgbe_attach(device_t dev)
1055 {
1056         struct port_info *pi = device_get_softc(dev);
1057         struct ifnet *ifp;
1058         char *s;
1059         int n, o;
1060
1061         /* Allocate an ifnet and set it up */
1062         ifp = if_alloc(IFT_ETHER);
1063         if (ifp == NULL) {
1064                 device_printf(dev, "Cannot allocate ifnet\n");
1065                 return (ENOMEM);
1066         }
1067         pi->ifp = ifp;
1068         ifp->if_softc = pi;
1069
1070         callout_init(&pi->tick, 1);
1071
1072         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1073         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1074
1075         ifp->if_init = cxgbe_init;
1076         ifp->if_ioctl = cxgbe_ioctl;
1077         ifp->if_transmit = cxgbe_transmit;
1078         ifp->if_qflush = cxgbe_qflush;
1079         ifp->if_get_counter = cxgbe_get_counter;
1080
1081         ifp->if_capabilities = T4_CAP;
1082 #ifdef TCP_OFFLOAD
1083         if (is_offload(pi->adapter))
1084                 ifp->if_capabilities |= IFCAP_TOE;
1085 #endif
1086         ifp->if_capenable = T4_CAP_ENABLE;
1087         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1088             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1089
1090         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1091         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1092         ifp->if_hw_tsomaxsegsize = 65536;
1093
1094         /* Initialize ifmedia for this port */
1095         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1096             cxgbe_media_status);
1097         build_medialist(pi, &pi->media);
1098
1099         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1100             EVENTHANDLER_PRI_ANY);
1101
1102         ether_ifattach(ifp, pi->hw_addr);
1103
1104         n = 128;
1105         s = malloc(n, M_CXGBE, M_WAITOK);
1106         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1107         MPASS(n > o);
1108 #ifdef TCP_OFFLOAD
1109         if (is_offload(pi->adapter)) {
1110                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1111                     pi->nofldtxq, pi->nofldrxq);
1112                 MPASS(n > o);
1113         }
1114 #endif
1115 #ifdef DEV_NETMAP
1116         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1117             pi->nnmrxq);
1118         MPASS(n > o);
1119 #endif
1120         device_printf(dev, "%s\n", s);
1121         free(s, M_CXGBE);
1122
1123 #ifdef DEV_NETMAP
1124         /* nm_media handled here to keep implementation private to this file */
1125         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1126             cxgbe_media_status);
1127         build_medialist(pi, &pi->nm_media);
1128         create_netmap_ifnet(pi);        /* logs errors it something fails */
1129 #endif
1130         cxgbe_sysctls(pi);
1131
1132         return (0);
1133 }
1134
1135 static int
1136 cxgbe_detach(device_t dev)
1137 {
1138         struct port_info *pi = device_get_softc(dev);
1139         struct adapter *sc = pi->adapter;
1140         struct ifnet *ifp = pi->ifp;
1141
1142         /* Tell if_ioctl and if_init that the port is going away */
1143         ADAPTER_LOCK(sc);
1144         SET_DOOMED(pi);
1145         wakeup(&sc->flags);
1146         while (IS_BUSY(sc))
1147                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1148         SET_BUSY(sc);
1149 #ifdef INVARIANTS
1150         sc->last_op = "t4detach";
1151         sc->last_op_thr = curthread;
1152         sc->last_op_flags = 0;
1153 #endif
1154         ADAPTER_UNLOCK(sc);
1155
1156         if (pi->flags & HAS_TRACEQ) {
1157                 sc->traceq = -1;        /* cloner should not create ifnet */
1158                 t4_tracer_port_detach(sc);
1159         }
1160
1161         if (pi->vlan_c)
1162                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1163
1164         PORT_LOCK(pi);
1165         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1166         callout_stop(&pi->tick);
1167         PORT_UNLOCK(pi);
1168         callout_drain(&pi->tick);
1169
1170         /* Let detach proceed even if these fail. */
1171         cxgbe_uninit_synchronized(pi);
1172         port_full_uninit(pi);
1173
1174         ifmedia_removeall(&pi->media);
1175         ether_ifdetach(pi->ifp);
1176         if_free(pi->ifp);
1177
1178 #ifdef DEV_NETMAP
1179         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1180         destroy_netmap_ifnet(pi);
1181 #endif
1182
1183         ADAPTER_LOCK(sc);
1184         CLR_BUSY(sc);
1185         wakeup(&sc->flags);
1186         ADAPTER_UNLOCK(sc);
1187
1188         return (0);
1189 }
1190
1191 static void
1192 cxgbe_init(void *arg)
1193 {
1194         struct port_info *pi = arg;
1195         struct adapter *sc = pi->adapter;
1196
1197         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1198                 return;
1199         cxgbe_init_synchronized(pi);
1200         end_synchronized_op(sc, 0);
1201 }
1202
1203 static int
1204 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1205 {
1206         int rc = 0, mtu, flags, can_sleep;
1207         struct port_info *pi = ifp->if_softc;
1208         struct adapter *sc = pi->adapter;
1209         struct ifreq *ifr = (struct ifreq *)data;
1210         uint32_t mask;
1211
1212         switch (cmd) {
1213         case SIOCSIFMTU:
1214                 mtu = ifr->ifr_mtu;
1215                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1216                         return (EINVAL);
1217
1218                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1219                 if (rc)
1220                         return (rc);
1221                 ifp->if_mtu = mtu;
1222                 if (pi->flags & PORT_INIT_DONE) {
1223                         t4_update_fl_bufsize(ifp);
1224                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1225                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1226                 }
1227                 end_synchronized_op(sc, 0);
1228                 break;
1229
1230         case SIOCSIFFLAGS:
1231                 can_sleep = 0;
1232 redo_sifflags:
1233                 rc = begin_synchronized_op(sc, pi,
1234                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1235                 if (rc)
1236                         return (rc);
1237
1238                 if (ifp->if_flags & IFF_UP) {
1239                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1240                                 flags = pi->if_flags;
1241                                 if ((ifp->if_flags ^ flags) &
1242                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1243                                         if (can_sleep == 1) {
1244                                                 end_synchronized_op(sc, 0);
1245                                                 can_sleep = 0;
1246                                                 goto redo_sifflags;
1247                                         }
1248                                         rc = update_mac_settings(ifp,
1249                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1250                                 }
1251                         } else {
1252                                 if (can_sleep == 0) {
1253                                         end_synchronized_op(sc, LOCK_HELD);
1254                                         can_sleep = 1;
1255                                         goto redo_sifflags;
1256                                 }
1257                                 rc = cxgbe_init_synchronized(pi);
1258                         }
1259                         pi->if_flags = ifp->if_flags;
1260                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1261                         if (can_sleep == 0) {
1262                                 end_synchronized_op(sc, LOCK_HELD);
1263                                 can_sleep = 1;
1264                                 goto redo_sifflags;
1265                         }
1266                         rc = cxgbe_uninit_synchronized(pi);
1267                 }
1268                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1269                 break;
1270
1271         case SIOCADDMULTI:
1272         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1273                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1274                 if (rc)
1275                         return (rc);
1276                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1277                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1278                 end_synchronized_op(sc, LOCK_HELD);
1279                 break;
1280
1281         case SIOCSIFCAP:
1282                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1283                 if (rc)
1284                         return (rc);
1285
1286                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1287                 if (mask & IFCAP_TXCSUM) {
1288                         ifp->if_capenable ^= IFCAP_TXCSUM;
1289                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1290
1291                         if (IFCAP_TSO4 & ifp->if_capenable &&
1292                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1293                                 ifp->if_capenable &= ~IFCAP_TSO4;
1294                                 if_printf(ifp,
1295                                     "tso4 disabled due to -txcsum.\n");
1296                         }
1297                 }
1298                 if (mask & IFCAP_TXCSUM_IPV6) {
1299                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1300                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1301
1302                         if (IFCAP_TSO6 & ifp->if_capenable &&
1303                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1304                                 ifp->if_capenable &= ~IFCAP_TSO6;
1305                                 if_printf(ifp,
1306                                     "tso6 disabled due to -txcsum6.\n");
1307                         }
1308                 }
1309                 if (mask & IFCAP_RXCSUM)
1310                         ifp->if_capenable ^= IFCAP_RXCSUM;
1311                 if (mask & IFCAP_RXCSUM_IPV6)
1312                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1313
1314                 /*
1315                  * Note that we leave CSUM_TSO alone (it is always set).  The
1316                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1317                  * sending a TSO request our way, so it's sufficient to toggle
1318                  * IFCAP_TSOx only.
1319                  */
1320                 if (mask & IFCAP_TSO4) {
1321                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1322                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1323                                 if_printf(ifp, "enable txcsum first.\n");
1324                                 rc = EAGAIN;
1325                                 goto fail;
1326                         }
1327                         ifp->if_capenable ^= IFCAP_TSO4;
1328                 }
1329                 if (mask & IFCAP_TSO6) {
1330                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1331                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1332                                 if_printf(ifp, "enable txcsum6 first.\n");
1333                                 rc = EAGAIN;
1334                                 goto fail;
1335                         }
1336                         ifp->if_capenable ^= IFCAP_TSO6;
1337                 }
1338                 if (mask & IFCAP_LRO) {
1339 #if defined(INET) || defined(INET6)
1340                         int i;
1341                         struct sge_rxq *rxq;
1342
1343                         ifp->if_capenable ^= IFCAP_LRO;
1344                         for_each_rxq(pi, i, rxq) {
1345                                 if (ifp->if_capenable & IFCAP_LRO)
1346                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1347                                 else
1348                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1349                         }
1350 #endif
1351                 }
1352 #ifdef TCP_OFFLOAD
1353                 if (mask & IFCAP_TOE) {
1354                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1355
1356                         rc = toe_capability(pi, enable);
1357                         if (rc != 0)
1358                                 goto fail;
1359
1360                         ifp->if_capenable ^= mask;
1361                 }
1362 #endif
1363                 if (mask & IFCAP_VLAN_HWTAGGING) {
1364                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1365                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1366                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1367                 }
1368                 if (mask & IFCAP_VLAN_MTU) {
1369                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1370
1371                         /* Need to find out how to disable auto-mtu-inflation */
1372                 }
1373                 if (mask & IFCAP_VLAN_HWTSO)
1374                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1375                 if (mask & IFCAP_VLAN_HWCSUM)
1376                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1377
1378 #ifdef VLAN_CAPABILITIES
1379                 VLAN_CAPABILITIES(ifp);
1380 #endif
1381 fail:
1382                 end_synchronized_op(sc, 0);
1383                 break;
1384
1385         case SIOCSIFMEDIA:
1386         case SIOCGIFMEDIA:
1387                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1388                 break;
1389
1390         case SIOCGI2C: {
1391                 struct ifi2creq i2c;
1392
1393                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1394                 if (rc != 0)
1395                         break;
1396                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1397                         rc = EPERM;
1398                         break;
1399                 }
1400                 if (i2c.len > sizeof(i2c.data)) {
1401                         rc = EINVAL;
1402                         break;
1403                 }
1404                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1405                 if (rc)
1406                         return (rc);
1407                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1408                     i2c.offset, i2c.len, &i2c.data[0]);
1409                 end_synchronized_op(sc, 0);
1410                 if (rc == 0)
1411                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1412                 break;
1413         }
1414
1415         default:
1416                 rc = ether_ioctl(ifp, cmd, data);
1417         }
1418
1419         return (rc);
1420 }
1421
1422 static int
1423 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1424 {
1425         struct port_info *pi = ifp->if_softc;
1426         struct adapter *sc = pi->adapter;
1427         struct sge_txq *txq;
1428         void *items[1];
1429         int rc;
1430
1431         M_ASSERTPKTHDR(m);
1432         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1433
1434         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1435                 m_freem(m);
1436                 return (ENETDOWN);
1437         }
1438
1439         rc = parse_pkt(&m);
1440         if (__predict_false(rc != 0)) {
1441                 MPASS(m == NULL);                       /* was freed already */
1442                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1443                 return (rc);
1444         }
1445
1446         /* Select a txq. */
1447         txq = &sc->sge.txq[pi->first_txq];
1448         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1449                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1450                     pi->rsrv_noflowq);
1451
1452         items[0] = m;
1453         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1454         if (__predict_false(rc != 0))
1455                 m_freem(m);
1456
1457         return (rc);
1458 }
1459
1460 static void
1461 cxgbe_qflush(struct ifnet *ifp)
1462 {
1463         struct port_info *pi = ifp->if_softc;
1464         struct sge_txq *txq;
1465         int i;
1466
1467         /* queues do not exist if !PORT_INIT_DONE. */
1468         if (pi->flags & PORT_INIT_DONE) {
1469                 for_each_txq(pi, i, txq) {
1470                         TXQ_LOCK(txq);
1471                         txq->eq.flags &= ~EQ_ENABLED;
1472                         TXQ_UNLOCK(txq);
1473                         while (!mp_ring_is_idle(txq->r)) {
1474                                 mp_ring_check_drainage(txq->r, 0);
1475                                 pause("qflush", 1);
1476                         }
1477                 }
1478         }
1479         if_qflush(ifp);
1480 }
1481
1482 static uint64_t
1483 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1484 {
1485         struct port_info *pi = ifp->if_softc;
1486         struct adapter *sc = pi->adapter;
1487         struct port_stats *s = &pi->stats;
1488
1489         cxgbe_refresh_stats(sc, pi);
1490
1491         switch (c) {
1492         case IFCOUNTER_IPACKETS:
1493                 return (s->rx_frames - s->rx_pause);
1494
1495         case IFCOUNTER_IERRORS:
1496                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1497                     s->rx_fcs_err + s->rx_len_err);
1498
1499         case IFCOUNTER_OPACKETS:
1500                 return (s->tx_frames - s->tx_pause);
1501
1502         case IFCOUNTER_OERRORS:
1503                 return (s->tx_error_frames);
1504
1505         case IFCOUNTER_IBYTES:
1506                 return (s->rx_octets - s->rx_pause * 64);
1507
1508         case IFCOUNTER_OBYTES:
1509                 return (s->tx_octets - s->tx_pause * 64);
1510
1511         case IFCOUNTER_IMCASTS:
1512                 return (s->rx_mcast_frames - s->rx_pause);
1513
1514         case IFCOUNTER_OMCASTS:
1515                 return (s->tx_mcast_frames - s->tx_pause);
1516
1517         case IFCOUNTER_IQDROPS:
1518                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1519                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1520                     s->rx_trunc3 + pi->tnl_cong_drops);
1521
1522         case IFCOUNTER_OQDROPS: {
1523                 uint64_t drops;
1524
1525                 drops = s->tx_drop;
1526                 if (pi->flags & PORT_INIT_DONE) {
1527                         int i;
1528                         struct sge_txq *txq;
1529
1530                         for_each_txq(pi, i, txq)
1531                                 drops += counter_u64_fetch(txq->r->drops);
1532                 }
1533
1534                 return (drops);
1535
1536         }
1537
1538         default:
1539                 return (if_get_counter_default(ifp, c));
1540         }
1541 }
1542
1543 static int
1544 cxgbe_media_change(struct ifnet *ifp)
1545 {
1546         struct port_info *pi = ifp->if_softc;
1547
1548         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1549
1550         return (EOPNOTSUPP);
1551 }
1552
1553 static void
1554 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1555 {
1556         struct port_info *pi = ifp->if_softc;
1557         struct ifmedia *media = NULL;
1558         struct ifmedia_entry *cur;
1559         int speed = pi->link_cfg.speed;
1560
1561         if (ifp == pi->ifp)
1562                 media = &pi->media;
1563 #ifdef DEV_NETMAP
1564         else if (ifp == pi->nm_ifp)
1565                 media = &pi->nm_media;
1566 #endif
1567         MPASS(media != NULL);
1568
1569         cur = media->ifm_cur;
1570
1571         ifmr->ifm_status = IFM_AVALID;
1572         if (!pi->link_cfg.link_ok)
1573                 return;
1574
1575         ifmr->ifm_status |= IFM_ACTIVE;
1576
1577         /* active and current will differ iff current media is autoselect. */
1578         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1579                 return;
1580
1581         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1582         if (speed == SPEED_10000)
1583                 ifmr->ifm_active |= IFM_10G_T;
1584         else if (speed == SPEED_1000)
1585                 ifmr->ifm_active |= IFM_1000_T;
1586         else if (speed == SPEED_100)
1587                 ifmr->ifm_active |= IFM_100_TX;
1588         else if (speed == SPEED_10)
1589                 ifmr->ifm_active |= IFM_10_T;
1590         else
1591                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1592                             speed));
1593 }
1594
1595 void
1596 t4_fatal_err(struct adapter *sc)
1597 {
1598         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1599         t4_intr_disable(sc);
1600         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1601             device_get_nameunit(sc->dev));
1602 }
1603
1604 static int
1605 map_bars_0_and_4(struct adapter *sc)
1606 {
1607         sc->regs_rid = PCIR_BAR(0);
1608         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1609             &sc->regs_rid, RF_ACTIVE);
1610         if (sc->regs_res == NULL) {
1611                 device_printf(sc->dev, "cannot map registers.\n");
1612                 return (ENXIO);
1613         }
1614         sc->bt = rman_get_bustag(sc->regs_res);
1615         sc->bh = rman_get_bushandle(sc->regs_res);
1616         sc->mmio_len = rman_get_size(sc->regs_res);
1617         setbit(&sc->doorbells, DOORBELL_KDB);
1618
1619         sc->msix_rid = PCIR_BAR(4);
1620         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1621             &sc->msix_rid, RF_ACTIVE);
1622         if (sc->msix_res == NULL) {
1623                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1624                 return (ENXIO);
1625         }
1626
1627         return (0);
1628 }
1629
1630 static int
1631 map_bar_2(struct adapter *sc)
1632 {
1633
1634         /*
1635          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1636          * to map it if RDMA is disabled.
1637          */
1638         if (is_t4(sc) && sc->rdmacaps == 0)
1639                 return (0);
1640
1641         sc->udbs_rid = PCIR_BAR(2);
1642         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1643             &sc->udbs_rid, RF_ACTIVE);
1644         if (sc->udbs_res == NULL) {
1645                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1646                 return (ENXIO);
1647         }
1648         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1649
1650         if (is_t5(sc)) {
1651                 setbit(&sc->doorbells, DOORBELL_UDB);
1652 #if defined(__i386__) || defined(__amd64__)
1653                 if (t5_write_combine) {
1654                         int rc;
1655
1656                         /*
1657                          * Enable write combining on BAR2.  This is the
1658                          * userspace doorbell BAR and is split into 128B
1659                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1660                          * with an egress queue.  The first 64B has the doorbell
1661                          * and the second 64B can be used to submit a tx work
1662                          * request with an implicit doorbell.
1663                          */
1664
1665                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1666                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1667                         if (rc == 0) {
1668                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1669                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1670                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1671                         } else {
1672                                 device_printf(sc->dev,
1673                                     "couldn't enable write combining: %d\n",
1674                                     rc);
1675                         }
1676
1677                         t4_write_reg(sc, A_SGE_STAT_CFG,
1678                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1679                 }
1680 #endif
1681         }
1682
1683         return (0);
1684 }
1685
1686 static const struct memwin t4_memwin[] = {
1687         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1688         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1689         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1690 };
1691
1692 static const struct memwin t5_memwin[] = {
1693         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1694         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1695         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1696 };
1697
1698 static void
1699 setup_memwin(struct adapter *sc)
1700 {
1701         const struct memwin *mw;
1702         int i, n;
1703         uint32_t bar0;
1704
1705         if (is_t4(sc)) {
1706                 /*
1707                  * Read low 32b of bar0 indirectly via the hardware backdoor
1708                  * mechanism.  Works from within PCI passthrough environments
1709                  * too, where rman_get_start() can return a different value.  We
1710                  * need to program the T4 memory window decoders with the actual
1711                  * addresses that will be coming across the PCIe link.
1712                  */
1713                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1714                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1715
1716                 mw = &t4_memwin[0];
1717                 n = nitems(t4_memwin);
1718         } else {
1719                 /* T5 uses the relative offset inside the PCIe BAR */
1720                 bar0 = 0;
1721
1722                 mw = &t5_memwin[0];
1723                 n = nitems(t5_memwin);
1724         }
1725
1726         for (i = 0; i < n; i++, mw++) {
1727                 t4_write_reg(sc,
1728                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1729                     (mw->base + bar0) | V_BIR(0) |
1730                     V_WINDOW(ilog2(mw->aperture) - 10));
1731         }
1732
1733         /* flush */
1734         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1735 }
1736
1737 /*
1738  * Verify that the memory range specified by the addr/len pair is valid and lies
1739  * entirely within a single region (EDCx or MCx).
1740  */
1741 static int
1742 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1743 {
1744         uint32_t em, addr_len, maddr, mlen;
1745
1746         /* Memory can only be accessed in naturally aligned 4 byte units */
1747         if (addr & 3 || len & 3 || len == 0)
1748                 return (EINVAL);
1749
1750         /* Enabled memories */
1751         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1752         if (em & F_EDRAM0_ENABLE) {
1753                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1754                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1755                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1756                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1757                     addr + len <= maddr + mlen)
1758                         return (0);
1759         }
1760         if (em & F_EDRAM1_ENABLE) {
1761                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1762                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1763                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1764                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1765                     addr + len <= maddr + mlen)
1766                         return (0);
1767         }
1768         if (em & F_EXT_MEM_ENABLE) {
1769                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1770                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1771                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1772                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1773                     addr + len <= maddr + mlen)
1774                         return (0);
1775         }
1776         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1777                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1778                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1779                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1780                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1781                     addr + len <= maddr + mlen)
1782                         return (0);
1783         }
1784
1785         return (EFAULT);
1786 }
1787
1788 static int
1789 fwmtype_to_hwmtype(int mtype)
1790 {
1791
1792         switch (mtype) {
1793         case FW_MEMTYPE_EDC0:
1794                 return (MEM_EDC0);
1795         case FW_MEMTYPE_EDC1:
1796                 return (MEM_EDC1);
1797         case FW_MEMTYPE_EXTMEM:
1798                 return (MEM_MC0);
1799         case FW_MEMTYPE_EXTMEM1:
1800                 return (MEM_MC1);
1801         default:
1802                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1803         }
1804 }
1805
1806 /*
1807  * Verify that the memory range specified by the memtype/offset/len pair is
1808  * valid and lies entirely within the memtype specified.  The global address of
1809  * the start of the range is returned in addr.
1810  */
1811 static int
1812 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1813     uint32_t *addr)
1814 {
1815         uint32_t em, addr_len, maddr, mlen;
1816
1817         /* Memory can only be accessed in naturally aligned 4 byte units */
1818         if (off & 3 || len & 3 || len == 0)
1819                 return (EINVAL);
1820
1821         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1822         switch (fwmtype_to_hwmtype(mtype)) {
1823         case MEM_EDC0:
1824                 if (!(em & F_EDRAM0_ENABLE))
1825                         return (EINVAL);
1826                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1827                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1828                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1829                 break;
1830         case MEM_EDC1:
1831                 if (!(em & F_EDRAM1_ENABLE))
1832                         return (EINVAL);
1833                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1834                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1835                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1836                 break;
1837         case MEM_MC:
1838                 if (!(em & F_EXT_MEM_ENABLE))
1839                         return (EINVAL);
1840                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1841                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1842                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1843                 break;
1844         case MEM_MC1:
1845                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1846                         return (EINVAL);
1847                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1848                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1849                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1850                 break;
1851         default:
1852                 return (EINVAL);
1853         }
1854
1855         if (mlen > 0 && off < mlen && off + len <= mlen) {
1856                 *addr = maddr + off;    /* global address */
1857                 return (0);
1858         }
1859
1860         return (EFAULT);
1861 }
1862
1863 static void
1864 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1865 {
1866         const struct memwin *mw;
1867
1868         if (is_t4(sc)) {
1869                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1870                     ("%s: incorrect memwin# (%d)", __func__, win));
1871                 mw = &t4_memwin[win];
1872         } else {
1873                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1874                     ("%s: incorrect memwin# (%d)", __func__, win));
1875                 mw = &t5_memwin[win];
1876         }
1877
1878         if (base != NULL)
1879                 *base = mw->base;
1880         if (aperture != NULL)
1881                 *aperture = mw->aperture;
1882 }
1883
1884 /*
1885  * Positions the memory window such that it can be used to access the specified
1886  * address in the chip's address space.  The return value is the offset of addr
1887  * from the start of the window.
1888  */
1889 static uint32_t
1890 position_memwin(struct adapter *sc, int n, uint32_t addr)
1891 {
1892         uint32_t start, pf;
1893         uint32_t reg;
1894
1895         KASSERT(n >= 0 && n <= 3,
1896             ("%s: invalid window %d.", __func__, n));
1897         KASSERT((addr & 3) == 0,
1898             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1899
1900         if (is_t4(sc)) {
1901                 pf = 0;
1902                 start = addr & ~0xf;    /* start must be 16B aligned */
1903         } else {
1904                 pf = V_PFNUM(sc->pf);
1905                 start = addr & ~0x7f;   /* start must be 128B aligned */
1906         }
1907         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1908
1909         t4_write_reg(sc, reg, start | pf);
1910         t4_read_reg(sc, reg);
1911
1912         return (addr - start);
1913 }
1914
1915 static int
1916 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1917     struct intrs_and_queues *iaq)
1918 {
1919         int rc, itype, navail, nrxq10g, nrxq1g, n;
1920         int nofldrxq10g = 0, nofldrxq1g = 0;
1921         int nnmrxq10g = 0, nnmrxq1g = 0;
1922
1923         bzero(iaq, sizeof(*iaq));
1924
1925         iaq->ntxq10g = t4_ntxq10g;
1926         iaq->ntxq1g = t4_ntxq1g;
1927         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1928         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1929         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1930 #ifdef TCP_OFFLOAD
1931         if (is_offload(sc)) {
1932                 iaq->nofldtxq10g = t4_nofldtxq10g;
1933                 iaq->nofldtxq1g = t4_nofldtxq1g;
1934                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1935                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1936         }
1937 #endif
1938 #ifdef DEV_NETMAP
1939         iaq->nnmtxq10g = t4_nnmtxq10g;
1940         iaq->nnmtxq1g = t4_nnmtxq1g;
1941         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1942         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1943 #endif
1944
1945         for (itype = INTR_MSIX; itype; itype >>= 1) {
1946
1947                 if ((itype & t4_intr_types) == 0)
1948                         continue;       /* not allowed */
1949
1950                 if (itype == INTR_MSIX)
1951                         navail = pci_msix_count(sc->dev);
1952                 else if (itype == INTR_MSI)
1953                         navail = pci_msi_count(sc->dev);
1954                 else
1955                         navail = 1;
1956 restart:
1957                 if (navail == 0)
1958                         continue;
1959
1960                 iaq->intr_type = itype;
1961                 iaq->intr_flags_10g = 0;
1962                 iaq->intr_flags_1g = 0;
1963
1964                 /*
1965                  * Best option: an interrupt vector for errors, one for the
1966                  * firmware event queue, and one for every rxq (NIC, TOE, and
1967                  * netmap).
1968                  */
1969                 iaq->nirq = T4_EXTRA_INTR;
1970                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1971                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1972                 if (iaq->nirq <= navail &&
1973                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1974                         iaq->intr_flags_10g = INTR_ALL;
1975                         iaq->intr_flags_1g = INTR_ALL;
1976                         goto allocate;
1977                 }
1978
1979                 /*
1980                  * Second best option: a vector for errors, one for the firmware
1981                  * event queue, and vectors for either all the NIC rx queues or
1982                  * all the TOE rx queues.  The queues that don't get vectors
1983                  * will forward their interrupts to those that do.
1984                  *
1985                  * Note: netmap rx queues cannot be created early and so they
1986                  * can't be setup to receive forwarded interrupts for others.
1987                  */
1988                 iaq->nirq = T4_EXTRA_INTR;
1989                 if (nrxq10g >= nofldrxq10g) {
1990                         iaq->intr_flags_10g = INTR_RXQ;
1991                         iaq->nirq += n10g * nrxq10g;
1992 #ifdef DEV_NETMAP
1993                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1994 #endif
1995                 } else {
1996                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1997                         iaq->nirq += n10g * nofldrxq10g;
1998 #ifdef DEV_NETMAP
1999                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2000 #endif
2001                 }
2002                 if (nrxq1g >= nofldrxq1g) {
2003                         iaq->intr_flags_1g = INTR_RXQ;
2004                         iaq->nirq += n1g * nrxq1g;
2005 #ifdef DEV_NETMAP
2006                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2007 #endif
2008                 } else {
2009                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2010                         iaq->nirq += n1g * nofldrxq1g;
2011 #ifdef DEV_NETMAP
2012                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2013 #endif
2014                 }
2015                 if (iaq->nirq <= navail &&
2016                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2017                         goto allocate;
2018
2019                 /*
2020                  * Next best option: an interrupt vector for errors, one for the
2021                  * firmware event queue, and at least one per port.  At this
2022                  * point we know we'll have to downsize nrxq and/or nofldrxq
2023                  * and/or nnmrxq to fit what's available to us.
2024                  */
2025                 iaq->nirq = T4_EXTRA_INTR;
2026                 iaq->nirq += n10g + n1g;
2027                 if (iaq->nirq <= navail) {
2028                         int leftover = navail - iaq->nirq;
2029
2030                         if (n10g > 0) {
2031                                 int target = max(nrxq10g, nofldrxq10g);
2032
2033                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2034                                     INTR_RXQ : INTR_OFLD_RXQ;
2035
2036                                 n = 1;
2037                                 while (n < target && leftover >= n10g) {
2038                                         leftover -= n10g;
2039                                         iaq->nirq += n10g;
2040                                         n++;
2041                                 }
2042                                 iaq->nrxq10g = min(n, nrxq10g);
2043 #ifdef TCP_OFFLOAD
2044                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2045 #endif
2046 #ifdef DEV_NETMAP
2047                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2048 #endif
2049                         }
2050
2051                         if (n1g > 0) {
2052                                 int target = max(nrxq1g, nofldrxq1g);
2053
2054                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2055                                     INTR_RXQ : INTR_OFLD_RXQ;
2056
2057                                 n = 1;
2058                                 while (n < target && leftover >= n1g) {
2059                                         leftover -= n1g;
2060                                         iaq->nirq += n1g;
2061                                         n++;
2062                                 }
2063                                 iaq->nrxq1g = min(n, nrxq1g);
2064 #ifdef TCP_OFFLOAD
2065                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2066 #endif
2067 #ifdef DEV_NETMAP
2068                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2069 #endif
2070                         }
2071
2072                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2073                                 goto allocate;
2074                 }
2075
2076                 /*
2077                  * Least desirable option: one interrupt vector for everything.
2078                  */
2079                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2080                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2081 #ifdef TCP_OFFLOAD
2082                 if (is_offload(sc))
2083                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2084 #endif
2085 #ifdef DEV_NETMAP
2086                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2087 #endif
2088
2089 allocate:
2090                 navail = iaq->nirq;
2091                 rc = 0;
2092                 if (itype == INTR_MSIX)
2093                         rc = pci_alloc_msix(sc->dev, &navail);
2094                 else if (itype == INTR_MSI)
2095                         rc = pci_alloc_msi(sc->dev, &navail);
2096
2097                 if (rc == 0) {
2098                         if (navail == iaq->nirq)
2099                                 return (0);
2100
2101                         /*
2102                          * Didn't get the number requested.  Use whatever number
2103                          * the kernel is willing to allocate (it's in navail).
2104                          */
2105                         device_printf(sc->dev, "fewer vectors than requested, "
2106                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2107                             itype, iaq->nirq, navail);
2108                         pci_release_msi(sc->dev);
2109                         goto restart;
2110                 }
2111
2112                 device_printf(sc->dev,
2113                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2114                     itype, rc, iaq->nirq, navail);
2115         }
2116
2117         device_printf(sc->dev,
2118             "failed to find a usable interrupt type.  "
2119             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2120             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2121
2122         return (ENXIO);
2123 }
2124
2125 #define FW_VERSION(chip) ( \
2126     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2127     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2128     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2129     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2130 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2131
2132 struct fw_info {
2133         uint8_t chip;
2134         char *kld_name;
2135         char *fw_mod_name;
2136         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2137 } fw_info[] = {
2138         {
2139                 .chip = CHELSIO_T4,
2140                 .kld_name = "t4fw_cfg",
2141                 .fw_mod_name = "t4fw",
2142                 .fw_hdr = {
2143                         .chip = FW_HDR_CHIP_T4,
2144                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2145                         .intfver_nic = FW_INTFVER(T4, NIC),
2146                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2147                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2148                         .intfver_ri = FW_INTFVER(T4, RI),
2149                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2150                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2151                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2152                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2153                 },
2154         }, {
2155                 .chip = CHELSIO_T5,
2156                 .kld_name = "t5fw_cfg",
2157                 .fw_mod_name = "t5fw",
2158                 .fw_hdr = {
2159                         .chip = FW_HDR_CHIP_T5,
2160                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2161                         .intfver_nic = FW_INTFVER(T5, NIC),
2162                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2163                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2164                         .intfver_ri = FW_INTFVER(T5, RI),
2165                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2166                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2167                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2168                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2169                 },
2170         }
2171 };
2172
2173 static struct fw_info *
2174 find_fw_info(int chip)
2175 {
2176         int i;
2177
2178         for (i = 0; i < nitems(fw_info); i++) {
2179                 if (fw_info[i].chip == chip)
2180                         return (&fw_info[i]);
2181         }
2182         return (NULL);
2183 }
2184
2185 /*
2186  * Is the given firmware API compatible with the one the driver was compiled
2187  * with?
2188  */
2189 static int
2190 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2191 {
2192
2193         /* short circuit if it's the exact same firmware version */
2194         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2195                 return (1);
2196
2197         /*
2198          * XXX: Is this too conservative?  Perhaps I should limit this to the
2199          * features that are supported in the driver.
2200          */
2201 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2202         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2203             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2204             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2205                 return (1);
2206 #undef SAME_INTF
2207
2208         return (0);
2209 }
2210
2211 /*
2212  * The firmware in the KLD is usable, but should it be installed?  This routine
2213  * explains itself in detail if it indicates the KLD firmware should be
2214  * installed.
2215  */
2216 static int
2217 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2218 {
2219         const char *reason;
2220
2221         if (!card_fw_usable) {
2222                 reason = "incompatible or unusable";
2223                 goto install;
2224         }
2225
2226         if (k > c) {
2227                 reason = "older than the version bundled with this driver";
2228                 goto install;
2229         }
2230
2231         if (t4_fw_install == 2 && k != c) {
2232                 reason = "different than the version bundled with this driver";
2233                 goto install;
2234         }
2235
2236         return (0);
2237
2238 install:
2239         if (t4_fw_install == 0) {
2240                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2241                     "but the driver is prohibited from installing a different "
2242                     "firmware on the card.\n",
2243                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2244                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2245
2246                 return (0);
2247         }
2248
2249         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2250             "installing firmware %u.%u.%u.%u on card.\n",
2251             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2252             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2253             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2254             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2255
2256         return (1);
2257 }
2258 /*
2259  * Establish contact with the firmware and determine if we are the master driver
2260  * or not, and whether we are responsible for chip initialization.
2261  */
2262 static int
2263 prep_firmware(struct adapter *sc)
2264 {
2265         const struct firmware *fw = NULL, *default_cfg;
2266         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2267         enum dev_state state;
2268         struct fw_info *fw_info;
2269         struct fw_hdr *card_fw;         /* fw on the card */
2270         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2271         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2272                                            against */
2273
2274         /* Contact firmware. */
2275         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2276         if (rc < 0 || state == DEV_STATE_ERR) {
2277                 rc = -rc;
2278                 device_printf(sc->dev,
2279                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2280                 return (rc);
2281         }
2282         pf = rc;
2283         if (pf == sc->mbox)
2284                 sc->flags |= MASTER_PF;
2285         else if (state == DEV_STATE_UNINIT) {
2286                 /*
2287                  * We didn't get to be the master so we definitely won't be
2288                  * configuring the chip.  It's a bug if someone else hasn't
2289                  * configured it already.
2290                  */
2291                 device_printf(sc->dev, "couldn't be master(%d), "
2292                     "device not already initialized either(%d).\n", rc, state);
2293                 return (EDOOFUS);
2294         }
2295
2296         /* This is the firmware whose headers the driver was compiled against */
2297         fw_info = find_fw_info(chip_id(sc));
2298         if (fw_info == NULL) {
2299                 device_printf(sc->dev,
2300                     "unable to look up firmware information for chip %d.\n",
2301                     chip_id(sc));
2302                 return (EINVAL);
2303         }
2304         drv_fw = &fw_info->fw_hdr;
2305
2306         /*
2307          * The firmware KLD contains many modules.  The KLD name is also the
2308          * name of the module that contains the default config file.
2309          */
2310         default_cfg = firmware_get(fw_info->kld_name);
2311
2312         /* Read the header of the firmware on the card */
2313         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2314         rc = -t4_read_flash(sc, FLASH_FW_START,
2315             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2316         if (rc == 0)
2317                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2318         else {
2319                 device_printf(sc->dev,
2320                     "Unable to read card's firmware header: %d\n", rc);
2321                 card_fw_usable = 0;
2322         }
2323
2324         /* This is the firmware in the KLD */
2325         fw = firmware_get(fw_info->fw_mod_name);
2326         if (fw != NULL) {
2327                 kld_fw = (const void *)fw->data;
2328                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2329         } else {
2330                 kld_fw = NULL;
2331                 kld_fw_usable = 0;
2332         }
2333
2334         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2335             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2336                 /*
2337                  * Common case: the firmware on the card is an exact match and
2338                  * the KLD is an exact match too, or the KLD is
2339                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2340                  * here -- use cxgbetool loadfw if you want to reinstall the
2341                  * same firmware as the one on the card.
2342                  */
2343         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2344             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2345             be32toh(card_fw->fw_ver))) {
2346
2347                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2348                 if (rc != 0) {
2349                         device_printf(sc->dev,
2350                             "failed to install firmware: %d\n", rc);
2351                         goto done;
2352                 }
2353
2354                 /* Installed successfully, update the cached header too. */
2355                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2356                 card_fw_usable = 1;
2357                 need_fw_reset = 0;      /* already reset as part of load_fw */
2358         }
2359
2360         if (!card_fw_usable) {
2361                 uint32_t d, c, k;
2362
2363                 d = ntohl(drv_fw->fw_ver);
2364                 c = ntohl(card_fw->fw_ver);
2365                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2366
2367                 device_printf(sc->dev, "Cannot find a usable firmware: "
2368                     "fw_install %d, chip state %d, "
2369                     "driver compiled with %d.%d.%d.%d, "
2370                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2371                     t4_fw_install, state,
2372                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2373                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2374                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2375                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2376                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2377                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2378                 rc = EINVAL;
2379                 goto done;
2380         }
2381
2382         /* We're using whatever's on the card and it's known to be good. */
2383         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2384         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2385             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2386             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2387             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2388             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2389         t4_get_tp_version(sc, &sc->params.tp_vers);
2390
2391         /* Reset device */
2392         if (need_fw_reset &&
2393             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2394                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2395                 if (rc != ETIMEDOUT && rc != EIO)
2396                         t4_fw_bye(sc, sc->mbox);
2397                 goto done;
2398         }
2399         sc->flags |= FW_OK;
2400
2401         rc = get_params__pre_init(sc);
2402         if (rc != 0)
2403                 goto done; /* error message displayed already */
2404
2405         /* Partition adapter resources as specified in the config file. */
2406         if (state == DEV_STATE_UNINIT) {
2407
2408                 KASSERT(sc->flags & MASTER_PF,
2409                     ("%s: trying to change chip settings when not master.",
2410                     __func__));
2411
2412                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2413                 if (rc != 0)
2414                         goto done;      /* error message displayed already */
2415
2416                 t4_tweak_chip_settings(sc);
2417
2418                 /* get basic stuff going */
2419                 rc = -t4_fw_initialize(sc, sc->mbox);
2420                 if (rc != 0) {
2421                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2422                         goto done;
2423                 }
2424         } else {
2425                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2426                 sc->cfcsum = 0;
2427         }
2428
2429 done:
2430         free(card_fw, M_CXGBE);
2431         if (fw != NULL)
2432                 firmware_put(fw, FIRMWARE_UNLOAD);
2433         if (default_cfg != NULL)
2434                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2435
2436         return (rc);
2437 }
2438
2439 #define FW_PARAM_DEV(param) \
2440         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2441          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2442 #define FW_PARAM_PFVF(param) \
2443         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2444          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2445
2446 /*
2447  * Partition chip resources for use between various PFs, VFs, etc.
2448  */
2449 static int
2450 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2451     const char *name_prefix)
2452 {
2453         const struct firmware *cfg = NULL;
2454         int rc = 0;
2455         struct fw_caps_config_cmd caps;
2456         uint32_t mtype, moff, finicsum, cfcsum;
2457
2458         /*
2459          * Figure out what configuration file to use.  Pick the default config
2460          * file for the card if the user hasn't specified one explicitly.
2461          */
2462         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2463         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2464                 /* Card specific overrides go here. */
2465                 if (pci_get_device(sc->dev) == 0x440a)
2466                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2467                 if (is_fpga(sc))
2468                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2469         }
2470
2471         /*
2472          * We need to load another module if the profile is anything except
2473          * "default" or "flash".
2474          */
2475         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2476             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2477                 char s[32];
2478
2479                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2480                 cfg = firmware_get(s);
2481                 if (cfg == NULL) {
2482                         if (default_cfg != NULL) {
2483                                 device_printf(sc->dev,
2484                                     "unable to load module \"%s\" for "
2485                                     "configuration profile \"%s\", will use "
2486                                     "the default config file instead.\n",
2487                                     s, sc->cfg_file);
2488                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2489                                     "%s", DEFAULT_CF);
2490                         } else {
2491                                 device_printf(sc->dev,
2492                                     "unable to load module \"%s\" for "
2493                                     "configuration profile \"%s\", will use "
2494                                     "the config file on the card's flash "
2495                                     "instead.\n", s, sc->cfg_file);
2496                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2497                                     "%s", FLASH_CF);
2498                         }
2499                 }
2500         }
2501
2502         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2503             default_cfg == NULL) {
2504                 device_printf(sc->dev,
2505                     "default config file not available, will use the config "
2506                     "file on the card's flash instead.\n");
2507                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2508         }
2509
2510         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2511                 u_int cflen, i, n;
2512                 const uint32_t *cfdata;
2513                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2514
2515                 KASSERT(cfg != NULL || default_cfg != NULL,
2516                     ("%s: no config to upload", __func__));
2517
2518                 /*
2519                  * Ask the firmware where it wants us to upload the config file.
2520                  */
2521                 param = FW_PARAM_DEV(CF);
2522                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2523                 if (rc != 0) {
2524                         /* No support for config file?  Shouldn't happen. */
2525                         device_printf(sc->dev,
2526                             "failed to query config file location: %d.\n", rc);
2527                         goto done;
2528                 }
2529                 mtype = G_FW_PARAMS_PARAM_Y(val);
2530                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2531
2532                 /*
2533                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2534                  * useless stuffing/comments at the end of the config file so
2535                  * it's ok to simply throw away the last remaining bytes when
2536                  * the config file is not an exact multiple of 4.  This also
2537                  * helps with the validate_mt_off_len check.
2538                  */
2539                 if (cfg != NULL) {
2540                         cflen = cfg->datasize & ~3;
2541                         cfdata = cfg->data;
2542                 } else {
2543                         cflen = default_cfg->datasize & ~3;
2544                         cfdata = default_cfg->data;
2545                 }
2546
2547                 if (cflen > FLASH_CFG_MAX_SIZE) {
2548                         device_printf(sc->dev,
2549                             "config file too long (%d, max allowed is %d).  "
2550                             "Will try to use the config on the card, if any.\n",
2551                             cflen, FLASH_CFG_MAX_SIZE);
2552                         goto use_config_on_flash;
2553                 }
2554
2555                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2556                 if (rc != 0) {
2557                         device_printf(sc->dev,
2558                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2559                             "Will try to use the config on the card, if any.\n",
2560                             __func__, mtype, moff, cflen, rc);
2561                         goto use_config_on_flash;
2562                 }
2563
2564                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2565                 while (cflen) {
2566                         off = position_memwin(sc, 2, addr);
2567                         n = min(cflen, mw_aperture - off);
2568                         for (i = 0; i < n; i += 4)
2569                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2570                         cflen -= n;
2571                         addr += n;
2572                 }
2573         } else {
2574 use_config_on_flash:
2575                 mtype = FW_MEMTYPE_FLASH;
2576                 moff = t4_flash_cfg_addr(sc);
2577         }
2578
2579         bzero(&caps, sizeof(caps));
2580         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2581             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2582         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2583             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2584             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2585         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2586         if (rc != 0) {
2587                 device_printf(sc->dev,
2588                     "failed to pre-process config file: %d "
2589                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2590                 goto done;
2591         }
2592
2593         finicsum = be32toh(caps.finicsum);
2594         cfcsum = be32toh(caps.cfcsum);
2595         if (finicsum != cfcsum) {
2596                 device_printf(sc->dev,
2597                     "WARNING: config file checksum mismatch: %08x %08x\n",
2598                     finicsum, cfcsum);
2599         }
2600         sc->cfcsum = cfcsum;
2601
2602 #define LIMIT_CAPS(x) do { \
2603         caps.x &= htobe16(t4_##x##_allowed); \
2604 } while (0)
2605
2606         /*
2607          * Let the firmware know what features will (not) be used so it can tune
2608          * things accordingly.
2609          */
2610         LIMIT_CAPS(linkcaps);
2611         LIMIT_CAPS(niccaps);
2612         LIMIT_CAPS(toecaps);
2613         LIMIT_CAPS(rdmacaps);
2614         LIMIT_CAPS(iscsicaps);
2615         LIMIT_CAPS(fcoecaps);
2616 #undef LIMIT_CAPS
2617
2618         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2619             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2620         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2621         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2622         if (rc != 0) {
2623                 device_printf(sc->dev,
2624                     "failed to process config file: %d.\n", rc);
2625         }
2626 done:
2627         if (cfg != NULL)
2628                 firmware_put(cfg, FIRMWARE_UNLOAD);
2629         return (rc);
2630 }
2631
2632 /*
2633  * Retrieve parameters that are needed (or nice to have) very early.
2634  */
2635 static int
2636 get_params__pre_init(struct adapter *sc)
2637 {
2638         int rc;
2639         uint32_t param[2], val[2];
2640         struct fw_devlog_cmd cmd;
2641         struct devlog_params *dlog = &sc->params.devlog;
2642
2643         param[0] = FW_PARAM_DEV(PORTVEC);
2644         param[1] = FW_PARAM_DEV(CCLK);
2645         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2646         if (rc != 0) {
2647                 device_printf(sc->dev,
2648                     "failed to query parameters (pre_init): %d.\n", rc);
2649                 return (rc);
2650         }
2651
2652         sc->params.portvec = val[0];
2653         sc->params.nports = bitcount32(val[0]);
2654         sc->params.vpd.cclk = val[1];
2655
2656         /* Read device log parameters. */
2657         bzero(&cmd, sizeof(cmd));
2658         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2659             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2660         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2661         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2662         if (rc != 0) {
2663                 device_printf(sc->dev,
2664                     "failed to get devlog parameters: %d.\n", rc);
2665                 bzero(dlog, sizeof (*dlog));
2666                 rc = 0; /* devlog isn't critical for device operation */
2667         } else {
2668                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2669                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2670                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2671                 dlog->size = be32toh(cmd.memsize_devlog);
2672         }
2673
2674         return (rc);
2675 }
2676
2677 /*
2678  * Retrieve various parameters that are of interest to the driver.  The device
2679  * has been initialized by the firmware at this point.
2680  */
2681 static int
2682 get_params__post_init(struct adapter *sc)
2683 {
2684         int rc;
2685         uint32_t param[7], val[7];
2686         struct fw_caps_config_cmd caps;
2687
2688         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2689         param[1] = FW_PARAM_PFVF(EQ_START);
2690         param[2] = FW_PARAM_PFVF(FILTER_START);
2691         param[3] = FW_PARAM_PFVF(FILTER_END);
2692         param[4] = FW_PARAM_PFVF(L2T_START);
2693         param[5] = FW_PARAM_PFVF(L2T_END);
2694         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2695         if (rc != 0) {
2696                 device_printf(sc->dev,
2697                     "failed to query parameters (post_init): %d.\n", rc);
2698                 return (rc);
2699         }
2700
2701         sc->sge.iq_start = val[0];
2702         sc->sge.eq_start = val[1];
2703         sc->tids.ftid_base = val[2];
2704         sc->tids.nftids = val[3] - val[2] + 1;
2705         sc->params.ftid_min = val[2];
2706         sc->params.ftid_max = val[3];
2707         sc->vres.l2t.start = val[4];
2708         sc->vres.l2t.size = val[5] - val[4] + 1;
2709         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2710             ("%s: L2 table size (%u) larger than expected (%u)",
2711             __func__, sc->vres.l2t.size, L2T_SIZE));
2712
2713         /* get capabilites */
2714         bzero(&caps, sizeof(caps));
2715         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2716             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2717         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2718         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2719         if (rc != 0) {
2720                 device_printf(sc->dev,
2721                     "failed to get card capabilities: %d.\n", rc);
2722                 return (rc);
2723         }
2724
2725 #define READ_CAPS(x) do { \
2726         sc->x = htobe16(caps.x); \
2727 } while (0)
2728         READ_CAPS(linkcaps);
2729         READ_CAPS(niccaps);
2730         READ_CAPS(toecaps);
2731         READ_CAPS(rdmacaps);
2732         READ_CAPS(iscsicaps);
2733         READ_CAPS(fcoecaps);
2734
2735         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2736                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2737                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2738                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2739                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2740                 if (rc != 0) {
2741                         device_printf(sc->dev,
2742                             "failed to query NIC parameters: %d.\n", rc);
2743                         return (rc);
2744                 }
2745                 sc->tids.etid_base = val[0];
2746                 sc->params.etid_min = val[0];
2747                 sc->tids.netids = val[1] - val[0] + 1;
2748                 sc->params.netids = sc->tids.netids;
2749                 sc->params.eo_wr_cred = val[2];
2750                 sc->params.ethoffload = 1;
2751         }
2752
2753         if (sc->toecaps) {
2754                 /* query offload-related parameters */
2755                 param[0] = FW_PARAM_DEV(NTID);
2756                 param[1] = FW_PARAM_PFVF(SERVER_START);
2757                 param[2] = FW_PARAM_PFVF(SERVER_END);
2758                 param[3] = FW_PARAM_PFVF(TDDP_START);
2759                 param[4] = FW_PARAM_PFVF(TDDP_END);
2760                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2761                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2762                 if (rc != 0) {
2763                         device_printf(sc->dev,
2764                             "failed to query TOE parameters: %d.\n", rc);
2765                         return (rc);
2766                 }
2767                 sc->tids.ntids = val[0];
2768                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2769                 sc->tids.stid_base = val[1];
2770                 sc->tids.nstids = val[2] - val[1] + 1;
2771                 sc->vres.ddp.start = val[3];
2772                 sc->vres.ddp.size = val[4] - val[3] + 1;
2773                 sc->params.ofldq_wr_cred = val[5];
2774                 sc->params.offload = 1;
2775         }
2776         if (sc->rdmacaps) {
2777                 param[0] = FW_PARAM_PFVF(STAG_START);
2778                 param[1] = FW_PARAM_PFVF(STAG_END);
2779                 param[2] = FW_PARAM_PFVF(RQ_START);
2780                 param[3] = FW_PARAM_PFVF(RQ_END);
2781                 param[4] = FW_PARAM_PFVF(PBL_START);
2782                 param[5] = FW_PARAM_PFVF(PBL_END);
2783                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2784                 if (rc != 0) {
2785                         device_printf(sc->dev,
2786                             "failed to query RDMA parameters(1): %d.\n", rc);
2787                         return (rc);
2788                 }
2789                 sc->vres.stag.start = val[0];
2790                 sc->vres.stag.size = val[1] - val[0] + 1;
2791                 sc->vres.rq.start = val[2];
2792                 sc->vres.rq.size = val[3] - val[2] + 1;
2793                 sc->vres.pbl.start = val[4];
2794                 sc->vres.pbl.size = val[5] - val[4] + 1;
2795
2796                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2797                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2798                 param[2] = FW_PARAM_PFVF(CQ_START);
2799                 param[3] = FW_PARAM_PFVF(CQ_END);
2800                 param[4] = FW_PARAM_PFVF(OCQ_START);
2801                 param[5] = FW_PARAM_PFVF(OCQ_END);
2802                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2803                 if (rc != 0) {
2804                         device_printf(sc->dev,
2805                             "failed to query RDMA parameters(2): %d.\n", rc);
2806                         return (rc);
2807                 }
2808                 sc->vres.qp.start = val[0];
2809                 sc->vres.qp.size = val[1] - val[0] + 1;
2810                 sc->vres.cq.start = val[2];
2811                 sc->vres.cq.size = val[3] - val[2] + 1;
2812                 sc->vres.ocq.start = val[4];
2813                 sc->vres.ocq.size = val[5] - val[4] + 1;
2814         }
2815         if (sc->iscsicaps) {
2816                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2817                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2818                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2819                 if (rc != 0) {
2820                         device_printf(sc->dev,
2821                             "failed to query iSCSI parameters: %d.\n", rc);
2822                         return (rc);
2823                 }
2824                 sc->vres.iscsi.start = val[0];
2825                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2826         }
2827
2828         /*
2829          * We've got the params we wanted to query via the firmware.  Now grab
2830          * some others directly from the chip.
2831          */
2832         rc = t4_read_chip_settings(sc);
2833
2834         return (rc);
2835 }
2836
2837 static int
2838 set_params__post_init(struct adapter *sc)
2839 {
2840         uint32_t param, val;
2841
2842         /* ask for encapsulated CPLs */
2843         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2844         val = 1;
2845         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2846
2847         return (0);
2848 }
2849
2850 #undef FW_PARAM_PFVF
2851 #undef FW_PARAM_DEV
2852
2853 static void
2854 t4_set_desc(struct adapter *sc)
2855 {
2856         char buf[128];
2857         struct adapter_params *p = &sc->params;
2858
2859         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2860             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2861             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2862
2863         device_set_desc_copy(sc->dev, buf);
2864 }
2865
2866 static void
2867 build_medialist(struct port_info *pi, struct ifmedia *media)
2868 {
2869         int m;
2870
2871         PORT_LOCK(pi);
2872
2873         ifmedia_removeall(media);
2874
2875         m = IFM_ETHER | IFM_FDX;
2876
2877         switch(pi->port_type) {
2878         case FW_PORT_TYPE_BT_XFI:
2879         case FW_PORT_TYPE_BT_XAUI:
2880                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
2881                 /* fall through */
2882
2883         case FW_PORT_TYPE_BT_SGMII:
2884                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
2885                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
2886                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
2887                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2888                 break;
2889
2890         case FW_PORT_TYPE_CX4:
2891                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
2892                 ifmedia_set(media, m | IFM_10G_CX4);
2893                 break;
2894
2895         case FW_PORT_TYPE_QSFP_10G:
2896         case FW_PORT_TYPE_SFP:
2897         case FW_PORT_TYPE_FIBER_XFI:
2898         case FW_PORT_TYPE_FIBER_XAUI:
2899                 switch (pi->mod_type) {
2900
2901                 case FW_PORT_MOD_TYPE_LR:
2902                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
2903                         ifmedia_set(media, m | IFM_10G_LR);
2904                         break;
2905
2906                 case FW_PORT_MOD_TYPE_SR:
2907                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
2908                         ifmedia_set(media, m | IFM_10G_SR);
2909                         break;
2910
2911                 case FW_PORT_MOD_TYPE_LRM:
2912                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
2913                         ifmedia_set(media, m | IFM_10G_LRM);
2914                         break;
2915
2916                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2917                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2918                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
2919                         ifmedia_set(media, m | IFM_10G_TWINAX);
2920                         break;
2921
2922                 case FW_PORT_MOD_TYPE_NONE:
2923                         m &= ~IFM_FDX;
2924                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2925                         ifmedia_set(media, m | IFM_NONE);
2926                         break;
2927
2928                 case FW_PORT_MOD_TYPE_NA:
2929                 case FW_PORT_MOD_TYPE_ER:
2930                 default:
2931                         device_printf(pi->dev,
2932                             "unknown port_type (%d), mod_type (%d)\n",
2933                             pi->port_type, pi->mod_type);
2934                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2935                         ifmedia_set(media, m | IFM_UNKNOWN);
2936                         break;
2937                 }
2938                 break;
2939
2940         case FW_PORT_TYPE_QSFP:
2941                 switch (pi->mod_type) {
2942
2943                 case FW_PORT_MOD_TYPE_LR:
2944                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
2945                         ifmedia_set(media, m | IFM_40G_LR4);
2946                         break;
2947
2948                 case FW_PORT_MOD_TYPE_SR:
2949                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
2950                         ifmedia_set(media, m | IFM_40G_SR4);
2951                         break;
2952
2953                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2954                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2955                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
2956                         ifmedia_set(media, m | IFM_40G_CR4);
2957                         break;
2958
2959                 case FW_PORT_MOD_TYPE_NONE:
2960                         m &= ~IFM_FDX;
2961                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2962                         ifmedia_set(media, m | IFM_NONE);
2963                         break;
2964
2965                 default:
2966                         device_printf(pi->dev,
2967                             "unknown port_type (%d), mod_type (%d)\n",
2968                             pi->port_type, pi->mod_type);
2969                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2970                         ifmedia_set(media, m | IFM_UNKNOWN);
2971                         break;
2972                 }
2973                 break;
2974
2975         default:
2976                 device_printf(pi->dev,
2977                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2978                     pi->mod_type);
2979                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2980                 ifmedia_set(media, m | IFM_UNKNOWN);
2981                 break;
2982         }
2983
2984         PORT_UNLOCK(pi);
2985 }
2986
2987 #define FW_MAC_EXACT_CHUNK      7
2988
2989 /*
2990  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2991  * indicates which parameters should be programmed (the rest are left alone).
2992  */
2993 int
2994 update_mac_settings(struct ifnet *ifp, int flags)
2995 {
2996         int rc = 0;
2997         struct port_info *pi = ifp->if_softc;
2998         struct adapter *sc = pi->adapter;
2999         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3000         uint16_t viid = 0xffff;
3001         int16_t *xact_addr_filt = NULL;
3002
3003         ASSERT_SYNCHRONIZED_OP(sc);
3004         KASSERT(flags, ("%s: not told what to update.", __func__));
3005
3006         if (ifp == pi->ifp) {
3007                 viid = pi->viid;
3008                 xact_addr_filt = &pi->xact_addr_filt;
3009         }
3010 #ifdef DEV_NETMAP
3011         else if (ifp == pi->nm_ifp) {
3012                 viid = pi->nm_viid;
3013                 xact_addr_filt = &pi->nm_xact_addr_filt;
3014         }
3015 #endif
3016         if (flags & XGMAC_MTU)
3017                 mtu = ifp->if_mtu;
3018
3019         if (flags & XGMAC_PROMISC)
3020                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3021
3022         if (flags & XGMAC_ALLMULTI)
3023                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3024
3025         if (flags & XGMAC_VLANEX)
3026                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3027
3028         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3029                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3030                     1, vlanex, false);
3031                 if (rc) {
3032                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3033                             rc);
3034                         return (rc);
3035                 }
3036         }
3037
3038         if (flags & XGMAC_UCADDR) {
3039                 uint8_t ucaddr[ETHER_ADDR_LEN];
3040
3041                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3042                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3043                     true, true);
3044                 if (rc < 0) {
3045                         rc = -rc;
3046                         if_printf(ifp, "change_mac failed: %d\n", rc);
3047                         return (rc);
3048                 } else {
3049                         *xact_addr_filt = rc;
3050                         rc = 0;
3051                 }
3052         }
3053
3054         if (flags & XGMAC_MCADDRS) {
3055                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3056                 int del = 1;
3057                 uint64_t hash = 0;
3058                 struct ifmultiaddr *ifma;
3059                 int i = 0, j;
3060
3061                 if_maddr_rlock(ifp);
3062                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3063                         if (ifma->ifma_addr->sa_family != AF_LINK)
3064                                 continue;
3065                         mcaddr[i] =
3066                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3067                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3068                         i++;
3069
3070                         if (i == FW_MAC_EXACT_CHUNK) {
3071                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3072                                     i, mcaddr, NULL, &hash, 0);
3073                                 if (rc < 0) {
3074                                         rc = -rc;
3075                                         for (j = 0; j < i; j++) {
3076                                                 if_printf(ifp,
3077                                                     "failed to add mc address"
3078                                                     " %02x:%02x:%02x:"
3079                                                     "%02x:%02x:%02x rc=%d\n",
3080                                                     mcaddr[j][0], mcaddr[j][1],
3081                                                     mcaddr[j][2], mcaddr[j][3],
3082                                                     mcaddr[j][4], mcaddr[j][5],
3083                                                     rc);
3084                                         }
3085                                         goto mcfail;
3086                                 }
3087                                 del = 0;
3088                                 i = 0;
3089                         }
3090                 }
3091                 if (i > 0) {
3092                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3093                             mcaddr, NULL, &hash, 0);
3094                         if (rc < 0) {
3095                                 rc = -rc;
3096                                 for (j = 0; j < i; j++) {
3097                                         if_printf(ifp,
3098                                             "failed to add mc address"
3099                                             " %02x:%02x:%02x:"
3100                                             "%02x:%02x:%02x rc=%d\n",
3101                                             mcaddr[j][0], mcaddr[j][1],
3102                                             mcaddr[j][2], mcaddr[j][3],
3103                                             mcaddr[j][4], mcaddr[j][5],
3104                                             rc);
3105                                 }
3106                                 goto mcfail;
3107                         }
3108                 }
3109
3110                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3111                 if (rc != 0)
3112                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3113 mcfail:
3114                 if_maddr_runlock(ifp);
3115         }
3116
3117         return (rc);
3118 }
3119
3120 /*
3121  * {begin|end}_synchronized_op must be called from the same thread.
3122  */
3123 int
3124 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3125     char *wmesg)
3126 {
3127         int rc, pri;
3128
3129 #ifdef WITNESS
3130         /* the caller thinks it's ok to sleep, but is it really? */
3131         if (flags & SLEEP_OK)
3132                 pause("t4slptst", 1);
3133 #endif
3134
3135         if (INTR_OK)
3136                 pri = PCATCH;
3137         else
3138                 pri = 0;
3139
3140         ADAPTER_LOCK(sc);
3141         for (;;) {
3142
3143                 if (pi && IS_DOOMED(pi)) {
3144                         rc = ENXIO;
3145                         goto done;
3146                 }
3147
3148                 if (!IS_BUSY(sc)) {
3149                         rc = 0;
3150                         break;
3151                 }
3152
3153                 if (!(flags & SLEEP_OK)) {
3154                         rc = EBUSY;
3155                         goto done;
3156                 }
3157
3158                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3159                         rc = EINTR;
3160                         goto done;
3161                 }
3162         }
3163
3164         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3165         SET_BUSY(sc);
3166 #ifdef INVARIANTS
3167         sc->last_op = wmesg;
3168         sc->last_op_thr = curthread;
3169         sc->last_op_flags = flags;
3170 #endif
3171
3172 done:
3173         if (!(flags & HOLD_LOCK) || rc)
3174                 ADAPTER_UNLOCK(sc);
3175
3176         return (rc);
3177 }
3178
3179 /*
3180  * {begin|end}_synchronized_op must be called from the same thread.
3181  */
3182 void
3183 end_synchronized_op(struct adapter *sc, int flags)
3184 {
3185
3186         if (flags & LOCK_HELD)
3187                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3188         else
3189                 ADAPTER_LOCK(sc);
3190
3191         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3192         CLR_BUSY(sc);
3193         wakeup(&sc->flags);
3194         ADAPTER_UNLOCK(sc);
3195 }
3196
3197 static int
3198 cxgbe_init_synchronized(struct port_info *pi)
3199 {
3200         struct adapter *sc = pi->adapter;
3201         struct ifnet *ifp = pi->ifp;
3202         int rc = 0, i;
3203         struct sge_txq *txq;
3204
3205         ASSERT_SYNCHRONIZED_OP(sc);
3206
3207         if (isset(&sc->open_device_map, pi->port_id)) {
3208                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3209                     ("mismatch between open_device_map and if_drv_flags"));
3210                 return (0);     /* already running */
3211         }
3212
3213         if (!(sc->flags & FULL_INIT_DONE) &&
3214             ((rc = adapter_full_init(sc)) != 0))
3215                 return (rc);    /* error message displayed already */
3216
3217         if (!(pi->flags & PORT_INIT_DONE) &&
3218             ((rc = port_full_init(pi)) != 0))
3219                 return (rc); /* error message displayed already */
3220
3221         rc = update_mac_settings(ifp, XGMAC_ALL);
3222         if (rc)
3223                 goto done;      /* error message displayed already */
3224
3225         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3226         if (rc != 0) {
3227                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3228                 goto done;
3229         }
3230
3231         /*
3232          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3233          * if this changes.
3234          */
3235
3236         for_each_txq(pi, i, txq) {
3237                 TXQ_LOCK(txq);
3238                 txq->eq.flags |= EQ_ENABLED;
3239                 TXQ_UNLOCK(txq);
3240         }
3241
3242         /*
3243          * The first iq of the first port to come up is used for tracing.
3244          */
3245         if (sc->traceq < 0) {
3246                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3247                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3248                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3249                     V_QUEUENUMBER(sc->traceq));
3250                 pi->flags |= HAS_TRACEQ;
3251         }
3252
3253         /* all ok */
3254         setbit(&sc->open_device_map, pi->port_id);
3255         PORT_LOCK(pi);
3256         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3257         PORT_UNLOCK(pi);
3258
3259         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3260 done:
3261         if (rc != 0)
3262                 cxgbe_uninit_synchronized(pi);
3263
3264         return (rc);
3265 }
3266
3267 /*
3268  * Idempotent.
3269  */
3270 static int
3271 cxgbe_uninit_synchronized(struct port_info *pi)
3272 {
3273         struct adapter *sc = pi->adapter;
3274         struct ifnet *ifp = pi->ifp;
3275         int rc, i;
3276         struct sge_txq *txq;
3277
3278         ASSERT_SYNCHRONIZED_OP(sc);
3279
3280         if (!(pi->flags & PORT_INIT_DONE)) {
3281                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3282                     ("uninited port is running"));
3283                 return (0);
3284         }
3285
3286         /*
3287          * Disable the VI so that all its data in either direction is discarded
3288          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3289          * tick) intact as the TP can deliver negative advice or data that it's
3290          * holding in its RAM (for an offloaded connection) even after the VI is
3291          * disabled.
3292          */
3293         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3294         if (rc) {
3295                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3296                 return (rc);
3297         }
3298
3299         for_each_txq(pi, i, txq) {
3300                 TXQ_LOCK(txq);
3301                 txq->eq.flags &= ~EQ_ENABLED;
3302                 TXQ_UNLOCK(txq);
3303         }
3304
3305         clrbit(&sc->open_device_map, pi->port_id);
3306         PORT_LOCK(pi);
3307         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3308         PORT_UNLOCK(pi);
3309
3310         pi->link_cfg.link_ok = 0;
3311         pi->link_cfg.speed = 0;
3312         pi->linkdnrc = -1;
3313         t4_os_link_changed(sc, pi->port_id, 0, -1);
3314
3315         return (0);
3316 }
3317
3318 /*
3319  * It is ok for this function to fail midway and return right away.  t4_detach
3320  * will walk the entire sc->irq list and clean up whatever is valid.
3321  */
3322 static int
3323 setup_intr_handlers(struct adapter *sc)
3324 {
3325         int rc, rid, p, q;
3326         char s[8];
3327         struct irq *irq;
3328         struct port_info *pi;
3329         struct sge_rxq *rxq;
3330 #ifdef TCP_OFFLOAD
3331         struct sge_ofld_rxq *ofld_rxq;
3332 #endif
3333 #ifdef DEV_NETMAP
3334         struct sge_nm_rxq *nm_rxq;
3335 #endif
3336
3337         /*
3338          * Setup interrupts.
3339          */
3340         irq = &sc->irq[0];
3341         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3342         if (sc->intr_count == 1)
3343                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3344
3345         /* Multiple interrupts. */
3346         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3347             ("%s: too few intr.", __func__));
3348
3349         /* The first one is always error intr */
3350         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3351         if (rc != 0)
3352                 return (rc);
3353         irq++;
3354         rid++;
3355
3356         /* The second one is always the firmware event queue */
3357         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3358         if (rc != 0)
3359                 return (rc);
3360         irq++;
3361         rid++;
3362
3363         for_each_port(sc, p) {
3364                 pi = sc->port[p];
3365
3366                 if (pi->flags & INTR_RXQ) {
3367                         for_each_rxq(pi, q, rxq) {
3368                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3369                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3370                                     s);
3371                                 if (rc != 0)
3372                                         return (rc);
3373                                 irq++;
3374                                 rid++;
3375                         }
3376                 }
3377 #ifdef TCP_OFFLOAD
3378                 if (pi->flags & INTR_OFLD_RXQ) {
3379                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3380                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3381                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3382                                     ofld_rxq, s);
3383                                 if (rc != 0)
3384                                         return (rc);
3385                                 irq++;
3386                                 rid++;
3387                         }
3388                 }
3389 #endif
3390 #ifdef DEV_NETMAP
3391                 if (pi->flags & INTR_NM_RXQ) {
3392                         for_each_nm_rxq(pi, q, nm_rxq) {
3393                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3394                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3395                                     nm_rxq, s);
3396                                 if (rc != 0)
3397                                         return (rc);
3398                                 irq++;
3399                                 rid++;
3400                         }
3401                 }
3402 #endif
3403         }
3404         MPASS(irq == &sc->irq[sc->intr_count]);
3405
3406         return (0);
3407 }
3408
3409 int
3410 adapter_full_init(struct adapter *sc)
3411 {
3412         int rc, i;
3413
3414         ASSERT_SYNCHRONIZED_OP(sc);
3415         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3416         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3417             ("%s: FULL_INIT_DONE already", __func__));
3418
3419         /*
3420          * queues that belong to the adapter (not any particular port).
3421          */
3422         rc = t4_setup_adapter_queues(sc);
3423         if (rc != 0)
3424                 goto done;
3425
3426         for (i = 0; i < nitems(sc->tq); i++) {
3427                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3428                     taskqueue_thread_enqueue, &sc->tq[i]);
3429                 if (sc->tq[i] == NULL) {
3430                         device_printf(sc->dev,
3431                             "failed to allocate task queue %d\n", i);
3432                         rc = ENOMEM;
3433                         goto done;
3434                 }
3435                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3436                     device_get_nameunit(sc->dev), i);
3437         }
3438
3439         t4_intr_enable(sc);
3440         sc->flags |= FULL_INIT_DONE;
3441 done:
3442         if (rc != 0)
3443                 adapter_full_uninit(sc);
3444
3445         return (rc);
3446 }
3447
3448 int
3449 adapter_full_uninit(struct adapter *sc)
3450 {
3451         int i;
3452
3453         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3454
3455         t4_teardown_adapter_queues(sc);
3456
3457         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3458                 taskqueue_free(sc->tq[i]);
3459                 sc->tq[i] = NULL;
3460         }
3461
3462         sc->flags &= ~FULL_INIT_DONE;
3463
3464         return (0);
3465 }
3466
3467 int
3468 port_full_init(struct port_info *pi)
3469 {
3470         struct adapter *sc = pi->adapter;
3471         struct ifnet *ifp = pi->ifp;
3472         uint16_t *rss;
3473         struct sge_rxq *rxq;
3474         int rc, i, j;
3475
3476         ASSERT_SYNCHRONIZED_OP(sc);
3477         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3478             ("%s: PORT_INIT_DONE already", __func__));
3479
3480         sysctl_ctx_init(&pi->ctx);
3481         pi->flags |= PORT_SYSCTL_CTX;
3482
3483         /*
3484          * Allocate tx/rx/fl queues for this port.
3485          */
3486         rc = t4_setup_port_queues(pi);
3487         if (rc != 0)
3488                 goto done;      /* error message displayed already */
3489
3490         /*
3491          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3492          */
3493         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3494         for (i = 0; i < pi->rss_size;) {
3495                 for_each_rxq(pi, j, rxq) {
3496                         rss[i++] = rxq->iq.abs_id;
3497                         if (i == pi->rss_size)
3498                                 break;
3499                 }
3500         }
3501
3502         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3503             pi->rss_size);
3504         if (rc != 0) {
3505                 if_printf(ifp, "rss_config failed: %d\n", rc);
3506                 goto done;
3507         }
3508
3509         pi->rss = rss;
3510         pi->flags |= PORT_INIT_DONE;
3511 done:
3512         if (rc != 0)
3513                 port_full_uninit(pi);
3514
3515         return (rc);
3516 }
3517
3518 /*
3519  * Idempotent.
3520  */
3521 int
3522 port_full_uninit(struct port_info *pi)
3523 {
3524         struct adapter *sc = pi->adapter;
3525         int i;
3526         struct sge_rxq *rxq;
3527         struct sge_txq *txq;
3528 #ifdef TCP_OFFLOAD
3529         struct sge_ofld_rxq *ofld_rxq;
3530         struct sge_wrq *ofld_txq;
3531 #endif
3532
3533         if (pi->flags & PORT_INIT_DONE) {
3534
3535                 /* Need to quiesce queues.  */
3536
3537                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3538
3539                 for_each_txq(pi, i, txq) {
3540                         quiesce_txq(sc, txq);
3541                 }
3542
3543 #ifdef TCP_OFFLOAD
3544                 for_each_ofld_txq(pi, i, ofld_txq) {
3545                         quiesce_wrq(sc, ofld_txq);
3546                 }
3547 #endif
3548
3549                 for_each_rxq(pi, i, rxq) {
3550                         quiesce_iq(sc, &rxq->iq);
3551                         quiesce_fl(sc, &rxq->fl);
3552                 }
3553
3554 #ifdef TCP_OFFLOAD
3555                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3556                         quiesce_iq(sc, &ofld_rxq->iq);
3557                         quiesce_fl(sc, &ofld_rxq->fl);
3558                 }
3559 #endif
3560                 free(pi->rss, M_CXGBE);
3561         }
3562
3563         t4_teardown_port_queues(pi);
3564         pi->flags &= ~PORT_INIT_DONE;
3565
3566         return (0);
3567 }
3568
3569 static void
3570 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3571 {
3572         struct sge_eq *eq = &txq->eq;
3573         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3574
3575         (void) sc;      /* unused */
3576
3577 #ifdef INVARIANTS
3578         TXQ_LOCK(txq);
3579         MPASS((eq->flags & EQ_ENABLED) == 0);
3580         TXQ_UNLOCK(txq);
3581 #endif
3582
3583         /* Wait for the mp_ring to empty. */
3584         while (!mp_ring_is_idle(txq->r)) {
3585                 mp_ring_check_drainage(txq->r, 0);
3586                 pause("rquiesce", 1);
3587         }
3588
3589         /* Then wait for the hardware to finish. */
3590         while (spg->cidx != htobe16(eq->pidx))
3591                 pause("equiesce", 1);
3592
3593         /* Finally, wait for the driver to reclaim all descriptors. */
3594         while (eq->cidx != eq->pidx)
3595                 pause("dquiesce", 1);
3596 }
3597
3598 static void
3599 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3600 {
3601
3602         /* XXXTX */
3603 }
3604
3605 static void
3606 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3607 {
3608         (void) sc;      /* unused */
3609
3610         /* Synchronize with the interrupt handler */
3611         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3612                 pause("iqfree", 1);
3613 }
3614
3615 static void
3616 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3617 {
3618         mtx_lock(&sc->sfl_lock);
3619         FL_LOCK(fl);
3620         fl->flags |= FL_DOOMED;
3621         FL_UNLOCK(fl);
3622         mtx_unlock(&sc->sfl_lock);
3623
3624         callout_drain(&sc->sfl_callout);
3625         KASSERT((fl->flags & FL_STARVING) == 0,
3626             ("%s: still starving", __func__));
3627 }
3628
3629 static int
3630 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3631     driver_intr_t *handler, void *arg, char *name)
3632 {
3633         int rc;
3634
3635         irq->rid = rid;
3636         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3637             RF_SHAREABLE | RF_ACTIVE);
3638         if (irq->res == NULL) {
3639                 device_printf(sc->dev,
3640                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3641                 return (ENOMEM);
3642         }
3643
3644         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3645             NULL, handler, arg, &irq->tag);
3646         if (rc != 0) {
3647                 device_printf(sc->dev,
3648                     "failed to setup interrupt for rid %d, name %s: %d\n",
3649                     rid, name, rc);
3650         } else if (name)
3651                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3652
3653         return (rc);
3654 }
3655
3656 static int
3657 t4_free_irq(struct adapter *sc, struct irq *irq)
3658 {
3659         if (irq->tag)
3660                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3661         if (irq->res)
3662                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3663
3664         bzero(irq, sizeof(*irq));
3665
3666         return (0);
3667 }
3668
3669 static void
3670 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3671     unsigned int end)
3672 {
3673         uint32_t *p = (uint32_t *)(buf + start);
3674
3675         for ( ; start <= end; start += sizeof(uint32_t))
3676                 *p++ = t4_read_reg(sc, start);
3677 }
3678
3679 static void
3680 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3681 {
3682         int i, n;
3683         const unsigned int *reg_ranges;
3684         static const unsigned int t4_reg_ranges[] = {
3685                 0x1008, 0x1108,
3686                 0x1180, 0x11b4,
3687                 0x11fc, 0x123c,
3688                 0x1300, 0x173c,
3689                 0x1800, 0x18fc,
3690                 0x3000, 0x30d8,
3691                 0x30e0, 0x5924,
3692                 0x5960, 0x59d4,
3693                 0x5a00, 0x5af8,
3694                 0x6000, 0x6098,
3695                 0x6100, 0x6150,
3696                 0x6200, 0x6208,
3697                 0x6240, 0x6248,
3698                 0x6280, 0x6338,
3699                 0x6370, 0x638c,
3700                 0x6400, 0x643c,
3701                 0x6500, 0x6524,
3702                 0x6a00, 0x6a38,
3703                 0x6a60, 0x6a78,
3704                 0x6b00, 0x6b84,
3705                 0x6bf0, 0x6c84,
3706                 0x6cf0, 0x6d84,
3707                 0x6df0, 0x6e84,
3708                 0x6ef0, 0x6f84,
3709                 0x6ff0, 0x7084,
3710                 0x70f0, 0x7184,
3711                 0x71f0, 0x7284,
3712                 0x72f0, 0x7384,
3713                 0x73f0, 0x7450,
3714                 0x7500, 0x7530,
3715                 0x7600, 0x761c,
3716                 0x7680, 0x76cc,
3717                 0x7700, 0x7798,
3718                 0x77c0, 0x77fc,
3719                 0x7900, 0x79fc,
3720                 0x7b00, 0x7c38,
3721                 0x7d00, 0x7efc,
3722                 0x8dc0, 0x8e1c,
3723                 0x8e30, 0x8e78,
3724                 0x8ea0, 0x8f6c,
3725                 0x8fc0, 0x9074,
3726                 0x90fc, 0x90fc,
3727                 0x9400, 0x9458,
3728                 0x9600, 0x96bc,
3729                 0x9800, 0x9808,
3730                 0x9820, 0x983c,
3731                 0x9850, 0x9864,
3732                 0x9c00, 0x9c6c,
3733                 0x9c80, 0x9cec,
3734                 0x9d00, 0x9d6c,
3735                 0x9d80, 0x9dec,
3736                 0x9e00, 0x9e6c,
3737                 0x9e80, 0x9eec,
3738                 0x9f00, 0x9f6c,
3739                 0x9f80, 0x9fec,
3740                 0xd004, 0xd03c,
3741                 0xdfc0, 0xdfe0,
3742                 0xe000, 0xea7c,
3743                 0xf000, 0x11110,
3744                 0x11118, 0x11190,
3745                 0x19040, 0x1906c,
3746                 0x19078, 0x19080,
3747                 0x1908c, 0x19124,
3748                 0x19150, 0x191b0,
3749                 0x191d0, 0x191e8,
3750                 0x19238, 0x1924c,
3751                 0x193f8, 0x19474,
3752                 0x19490, 0x194f8,
3753                 0x19800, 0x19f30,
3754                 0x1a000, 0x1a06c,
3755                 0x1a0b0, 0x1a120,
3756                 0x1a128, 0x1a138,
3757                 0x1a190, 0x1a1c4,
3758                 0x1a1fc, 0x1a1fc,
3759                 0x1e040, 0x1e04c,
3760                 0x1e284, 0x1e28c,
3761                 0x1e2c0, 0x1e2c0,
3762                 0x1e2e0, 0x1e2e0,
3763                 0x1e300, 0x1e384,
3764                 0x1e3c0, 0x1e3c8,
3765                 0x1e440, 0x1e44c,
3766                 0x1e684, 0x1e68c,
3767                 0x1e6c0, 0x1e6c0,
3768                 0x1e6e0, 0x1e6e0,
3769                 0x1e700, 0x1e784,
3770                 0x1e7c0, 0x1e7c8,
3771                 0x1e840, 0x1e84c,
3772                 0x1ea84, 0x1ea8c,
3773                 0x1eac0, 0x1eac0,
3774                 0x1eae0, 0x1eae0,
3775                 0x1eb00, 0x1eb84,
3776                 0x1ebc0, 0x1ebc8,
3777                 0x1ec40, 0x1ec4c,
3778                 0x1ee84, 0x1ee8c,
3779                 0x1eec0, 0x1eec0,
3780                 0x1eee0, 0x1eee0,
3781                 0x1ef00, 0x1ef84,
3782                 0x1efc0, 0x1efc8,
3783                 0x1f040, 0x1f04c,
3784                 0x1f284, 0x1f28c,
3785                 0x1f2c0, 0x1f2c0,
3786                 0x1f2e0, 0x1f2e0,
3787                 0x1f300, 0x1f384,
3788                 0x1f3c0, 0x1f3c8,
3789                 0x1f440, 0x1f44c,
3790                 0x1f684, 0x1f68c,
3791                 0x1f6c0, 0x1f6c0,
3792                 0x1f6e0, 0x1f6e0,
3793                 0x1f700, 0x1f784,
3794                 0x1f7c0, 0x1f7c8,
3795                 0x1f840, 0x1f84c,
3796                 0x1fa84, 0x1fa8c,
3797                 0x1fac0, 0x1fac0,
3798                 0x1fae0, 0x1fae0,
3799                 0x1fb00, 0x1fb84,
3800                 0x1fbc0, 0x1fbc8,
3801                 0x1fc40, 0x1fc4c,
3802                 0x1fe84, 0x1fe8c,
3803                 0x1fec0, 0x1fec0,
3804                 0x1fee0, 0x1fee0,
3805                 0x1ff00, 0x1ff84,
3806                 0x1ffc0, 0x1ffc8,
3807                 0x20000, 0x2002c,
3808                 0x20100, 0x2013c,
3809                 0x20190, 0x201c8,
3810                 0x20200, 0x20318,
3811                 0x20400, 0x20528,
3812                 0x20540, 0x20614,
3813                 0x21000, 0x21040,
3814                 0x2104c, 0x21060,
3815                 0x210c0, 0x210ec,
3816                 0x21200, 0x21268,
3817                 0x21270, 0x21284,
3818                 0x212fc, 0x21388,
3819                 0x21400, 0x21404,
3820                 0x21500, 0x21518,
3821                 0x2152c, 0x2153c,
3822                 0x21550, 0x21554,
3823                 0x21600, 0x21600,
3824                 0x21608, 0x21628,
3825                 0x21630, 0x2163c,
3826                 0x21700, 0x2171c,
3827                 0x21780, 0x2178c,
3828                 0x21800, 0x21c38,
3829                 0x21c80, 0x21d7c,
3830                 0x21e00, 0x21e04,
3831                 0x22000, 0x2202c,
3832                 0x22100, 0x2213c,
3833                 0x22190, 0x221c8,
3834                 0x22200, 0x22318,
3835                 0x22400, 0x22528,
3836                 0x22540, 0x22614,
3837                 0x23000, 0x23040,
3838                 0x2304c, 0x23060,
3839                 0x230c0, 0x230ec,
3840                 0x23200, 0x23268,
3841                 0x23270, 0x23284,
3842                 0x232fc, 0x23388,
3843                 0x23400, 0x23404,
3844                 0x23500, 0x23518,
3845                 0x2352c, 0x2353c,
3846                 0x23550, 0x23554,
3847                 0x23600, 0x23600,
3848                 0x23608, 0x23628,
3849                 0x23630, 0x2363c,
3850                 0x23700, 0x2371c,
3851                 0x23780, 0x2378c,
3852                 0x23800, 0x23c38,
3853                 0x23c80, 0x23d7c,
3854                 0x23e00, 0x23e04,
3855                 0x24000, 0x2402c,
3856                 0x24100, 0x2413c,
3857                 0x24190, 0x241c8,
3858                 0x24200, 0x24318,
3859                 0x24400, 0x24528,
3860                 0x24540, 0x24614,
3861                 0x25000, 0x25040,
3862                 0x2504c, 0x25060,
3863                 0x250c0, 0x250ec,
3864                 0x25200, 0x25268,
3865                 0x25270, 0x25284,
3866                 0x252fc, 0x25388,
3867                 0x25400, 0x25404,
3868                 0x25500, 0x25518,
3869                 0x2552c, 0x2553c,
3870                 0x25550, 0x25554,
3871                 0x25600, 0x25600,
3872                 0x25608, 0x25628,
3873                 0x25630, 0x2563c,
3874                 0x25700, 0x2571c,
3875                 0x25780, 0x2578c,
3876                 0x25800, 0x25c38,
3877                 0x25c80, 0x25d7c,
3878                 0x25e00, 0x25e04,
3879                 0x26000, 0x2602c,
3880                 0x26100, 0x2613c,
3881                 0x26190, 0x261c8,
3882                 0x26200, 0x26318,
3883                 0x26400, 0x26528,
3884                 0x26540, 0x26614,
3885                 0x27000, 0x27040,
3886                 0x2704c, 0x27060,
3887                 0x270c0, 0x270ec,
3888                 0x27200, 0x27268,
3889                 0x27270, 0x27284,
3890                 0x272fc, 0x27388,
3891                 0x27400, 0x27404,
3892                 0x27500, 0x27518,
3893                 0x2752c, 0x2753c,
3894                 0x27550, 0x27554,
3895                 0x27600, 0x27600,
3896                 0x27608, 0x27628,
3897                 0x27630, 0x2763c,
3898                 0x27700, 0x2771c,
3899                 0x27780, 0x2778c,
3900                 0x27800, 0x27c38,
3901                 0x27c80, 0x27d7c,
3902                 0x27e00, 0x27e04
3903         };
3904         static const unsigned int t5_reg_ranges[] = {
3905                 0x1008, 0x1148,
3906                 0x1180, 0x11b4,
3907                 0x11fc, 0x123c,
3908                 0x1280, 0x173c,
3909                 0x1800, 0x18fc,
3910                 0x3000, 0x3028,
3911                 0x3060, 0x30d8,
3912                 0x30e0, 0x30fc,
3913                 0x3140, 0x357c,
3914                 0x35a8, 0x35cc,
3915                 0x35ec, 0x35ec,
3916                 0x3600, 0x5624,
3917                 0x56cc, 0x575c,
3918                 0x580c, 0x5814,
3919                 0x5890, 0x58bc,
3920                 0x5940, 0x59dc,
3921                 0x59fc, 0x5a18,
3922                 0x5a60, 0x5a9c,
3923                 0x5b94, 0x5bfc,
3924                 0x6000, 0x6040,
3925                 0x6058, 0x614c,
3926                 0x7700, 0x7798,
3927                 0x77c0, 0x78fc,
3928                 0x7b00, 0x7c54,
3929                 0x7d00, 0x7efc,
3930                 0x8dc0, 0x8de0,
3931                 0x8df8, 0x8e84,
3932                 0x8ea0, 0x8f84,
3933                 0x8fc0, 0x90f8,
3934                 0x9400, 0x9470,
3935                 0x9600, 0x96f4,
3936                 0x9800, 0x9808,
3937                 0x9820, 0x983c,
3938                 0x9850, 0x9864,
3939                 0x9c00, 0x9c6c,
3940                 0x9c80, 0x9cec,
3941                 0x9d00, 0x9d6c,
3942                 0x9d80, 0x9dec,
3943                 0x9e00, 0x9e6c,
3944                 0x9e80, 0x9eec,
3945                 0x9f00, 0x9f6c,
3946                 0x9f80, 0xa020,
3947                 0xd004, 0xd03c,
3948                 0xdfc0, 0xdfe0,
3949                 0xe000, 0x11088,
3950                 0x1109c, 0x11110,
3951                 0x11118, 0x1117c,
3952                 0x11190, 0x11204,
3953                 0x19040, 0x1906c,
3954                 0x19078, 0x19080,
3955                 0x1908c, 0x19124,
3956                 0x19150, 0x191b0,
3957                 0x191d0, 0x191e8,
3958                 0x19238, 0x19290,
3959                 0x193f8, 0x19474,
3960                 0x19490, 0x194cc,
3961                 0x194f0, 0x194f8,
3962                 0x19c00, 0x19c60,
3963                 0x19c94, 0x19e10,
3964                 0x19e50, 0x19f34,
3965                 0x19f40, 0x19f50,
3966                 0x19f90, 0x19fe4,
3967                 0x1a000, 0x1a06c,
3968                 0x1a0b0, 0x1a120,
3969                 0x1a128, 0x1a138,
3970                 0x1a190, 0x1a1c4,
3971                 0x1a1fc, 0x1a1fc,
3972                 0x1e008, 0x1e00c,
3973                 0x1e040, 0x1e04c,
3974                 0x1e284, 0x1e290,
3975                 0x1e2c0, 0x1e2c0,
3976                 0x1e2e0, 0x1e2e0,
3977                 0x1e300, 0x1e384,
3978                 0x1e3c0, 0x1e3c8,
3979                 0x1e408, 0x1e40c,
3980                 0x1e440, 0x1e44c,
3981                 0x1e684, 0x1e690,
3982                 0x1e6c0, 0x1e6c0,
3983                 0x1e6e0, 0x1e6e0,
3984                 0x1e700, 0x1e784,
3985                 0x1e7c0, 0x1e7c8,
3986                 0x1e808, 0x1e80c,
3987                 0x1e840, 0x1e84c,
3988                 0x1ea84, 0x1ea90,
3989                 0x1eac0, 0x1eac0,
3990                 0x1eae0, 0x1eae0,
3991                 0x1eb00, 0x1eb84,
3992                 0x1ebc0, 0x1ebc8,
3993                 0x1ec08, 0x1ec0c,
3994                 0x1ec40, 0x1ec4c,
3995                 0x1ee84, 0x1ee90,
3996                 0x1eec0, 0x1eec0,
3997                 0x1eee0, 0x1eee0,
3998                 0x1ef00, 0x1ef84,
3999                 0x1efc0, 0x1efc8,
4000                 0x1f008, 0x1f00c,
4001                 0x1f040, 0x1f04c,
4002                 0x1f284, 0x1f290,
4003                 0x1f2c0, 0x1f2c0,
4004                 0x1f2e0, 0x1f2e0,
4005                 0x1f300, 0x1f384,
4006                 0x1f3c0, 0x1f3c8,
4007                 0x1f408, 0x1f40c,
4008                 0x1f440, 0x1f44c,
4009                 0x1f684, 0x1f690,
4010                 0x1f6c0, 0x1f6c0,
4011                 0x1f6e0, 0x1f6e0,
4012                 0x1f700, 0x1f784,
4013                 0x1f7c0, 0x1f7c8,
4014                 0x1f808, 0x1f80c,
4015                 0x1f840, 0x1f84c,
4016                 0x1fa84, 0x1fa90,
4017                 0x1fac0, 0x1fac0,
4018                 0x1fae0, 0x1fae0,
4019                 0x1fb00, 0x1fb84,
4020                 0x1fbc0, 0x1fbc8,
4021                 0x1fc08, 0x1fc0c,
4022                 0x1fc40, 0x1fc4c,
4023                 0x1fe84, 0x1fe90,
4024                 0x1fec0, 0x1fec0,
4025                 0x1fee0, 0x1fee0,
4026                 0x1ff00, 0x1ff84,
4027                 0x1ffc0, 0x1ffc8,
4028                 0x30000, 0x30030,
4029                 0x30100, 0x30144,
4030                 0x30190, 0x301d0,
4031                 0x30200, 0x30318,
4032                 0x30400, 0x3052c,
4033                 0x30540, 0x3061c,
4034                 0x30800, 0x30834,
4035                 0x308c0, 0x30908,
4036                 0x30910, 0x309ac,
4037                 0x30a00, 0x30a2c,
4038                 0x30a44, 0x30a50,
4039                 0x30a74, 0x30c24,
4040                 0x30d00, 0x30d00,
4041                 0x30d08, 0x30d14,
4042                 0x30d1c, 0x30d20,
4043                 0x30d3c, 0x30d50,
4044                 0x31200, 0x3120c,
4045                 0x31220, 0x31220,
4046                 0x31240, 0x31240,
4047                 0x31600, 0x3160c,
4048                 0x31a00, 0x31a1c,
4049                 0x31e00, 0x31e20,
4050                 0x31e38, 0x31e3c,
4051                 0x31e80, 0x31e80,
4052                 0x31e88, 0x31ea8,
4053                 0x31eb0, 0x31eb4,
4054                 0x31ec8, 0x31ed4,
4055                 0x31fb8, 0x32004,
4056                 0x32200, 0x32200,
4057                 0x32208, 0x32240,
4058                 0x32248, 0x32280,
4059                 0x32288, 0x322c0,
4060                 0x322c8, 0x322fc,
4061                 0x32600, 0x32630,
4062                 0x32a00, 0x32abc,
4063                 0x32b00, 0x32b70,
4064                 0x33000, 0x33048,
4065                 0x33060, 0x3309c,
4066                 0x330f0, 0x33148,
4067                 0x33160, 0x3319c,
4068                 0x331f0, 0x332e4,
4069                 0x332f8, 0x333e4,
4070                 0x333f8, 0x33448,
4071                 0x33460, 0x3349c,
4072                 0x334f0, 0x33548,
4073                 0x33560, 0x3359c,
4074                 0x335f0, 0x336e4,
4075                 0x336f8, 0x337e4,
4076                 0x337f8, 0x337fc,
4077                 0x33814, 0x33814,
4078                 0x3382c, 0x3382c,
4079                 0x33880, 0x3388c,
4080                 0x338e8, 0x338ec,
4081                 0x33900, 0x33948,
4082                 0x33960, 0x3399c,
4083                 0x339f0, 0x33ae4,
4084                 0x33af8, 0x33b10,
4085                 0x33b28, 0x33b28,
4086                 0x33b3c, 0x33b50,
4087                 0x33bf0, 0x33c10,
4088                 0x33c28, 0x33c28,
4089                 0x33c3c, 0x33c50,
4090                 0x33cf0, 0x33cfc,
4091                 0x34000, 0x34030,
4092                 0x34100, 0x34144,
4093                 0x34190, 0x341d0,
4094                 0x34200, 0x34318,
4095                 0x34400, 0x3452c,
4096                 0x34540, 0x3461c,
4097                 0x34800, 0x34834,
4098                 0x348c0, 0x34908,
4099                 0x34910, 0x349ac,
4100                 0x34a00, 0x34a2c,
4101                 0x34a44, 0x34a50,
4102                 0x34a74, 0x34c24,
4103                 0x34d00, 0x34d00,
4104                 0x34d08, 0x34d14,
4105                 0x34d1c, 0x34d20,
4106                 0x34d3c, 0x34d50,
4107                 0x35200, 0x3520c,
4108                 0x35220, 0x35220,
4109                 0x35240, 0x35240,
4110                 0x35600, 0x3560c,
4111                 0x35a00, 0x35a1c,
4112                 0x35e00, 0x35e20,
4113                 0x35e38, 0x35e3c,
4114                 0x35e80, 0x35e80,
4115                 0x35e88, 0x35ea8,
4116                 0x35eb0, 0x35eb4,
4117                 0x35ec8, 0x35ed4,
4118                 0x35fb8, 0x36004,
4119                 0x36200, 0x36200,
4120                 0x36208, 0x36240,
4121                 0x36248, 0x36280,
4122                 0x36288, 0x362c0,
4123                 0x362c8, 0x362fc,
4124                 0x36600, 0x36630,
4125                 0x36a00, 0x36abc,
4126                 0x36b00, 0x36b70,
4127                 0x37000, 0x37048,
4128                 0x37060, 0x3709c,
4129                 0x370f0, 0x37148,
4130                 0x37160, 0x3719c,
4131                 0x371f0, 0x372e4,
4132                 0x372f8, 0x373e4,
4133                 0x373f8, 0x37448,
4134                 0x37460, 0x3749c,
4135                 0x374f0, 0x37548,
4136                 0x37560, 0x3759c,
4137                 0x375f0, 0x376e4,
4138                 0x376f8, 0x377e4,
4139                 0x377f8, 0x377fc,
4140                 0x37814, 0x37814,
4141                 0x3782c, 0x3782c,
4142                 0x37880, 0x3788c,
4143                 0x378e8, 0x378ec,
4144                 0x37900, 0x37948,
4145                 0x37960, 0x3799c,
4146                 0x379f0, 0x37ae4,
4147                 0x37af8, 0x37b10,
4148                 0x37b28, 0x37b28,
4149                 0x37b3c, 0x37b50,
4150                 0x37bf0, 0x37c10,
4151                 0x37c28, 0x37c28,
4152                 0x37c3c, 0x37c50,
4153                 0x37cf0, 0x37cfc,
4154                 0x38000, 0x38030,
4155                 0x38100, 0x38144,
4156                 0x38190, 0x381d0,
4157                 0x38200, 0x38318,
4158                 0x38400, 0x3852c,
4159                 0x38540, 0x3861c,
4160                 0x38800, 0x38834,
4161                 0x388c0, 0x38908,
4162                 0x38910, 0x389ac,
4163                 0x38a00, 0x38a2c,
4164                 0x38a44, 0x38a50,
4165                 0x38a74, 0x38c24,
4166                 0x38d00, 0x38d00,
4167                 0x38d08, 0x38d14,
4168                 0x38d1c, 0x38d20,
4169                 0x38d3c, 0x38d50,
4170                 0x39200, 0x3920c,
4171                 0x39220, 0x39220,
4172                 0x39240, 0x39240,
4173                 0x39600, 0x3960c,
4174                 0x39a00, 0x39a1c,
4175                 0x39e00, 0x39e20,
4176                 0x39e38, 0x39e3c,
4177                 0x39e80, 0x39e80,
4178                 0x39e88, 0x39ea8,
4179                 0x39eb0, 0x39eb4,
4180                 0x39ec8, 0x39ed4,
4181                 0x39fb8, 0x3a004,
4182                 0x3a200, 0x3a200,
4183                 0x3a208, 0x3a240,
4184                 0x3a248, 0x3a280,
4185                 0x3a288, 0x3a2c0,
4186                 0x3a2c8, 0x3a2fc,
4187                 0x3a600, 0x3a630,
4188                 0x3aa00, 0x3aabc,
4189                 0x3ab00, 0x3ab70,
4190                 0x3b000, 0x3b048,
4191                 0x3b060, 0x3b09c,
4192                 0x3b0f0, 0x3b148,
4193                 0x3b160, 0x3b19c,
4194                 0x3b1f0, 0x3b2e4,
4195                 0x3b2f8, 0x3b3e4,
4196                 0x3b3f8, 0x3b448,
4197                 0x3b460, 0x3b49c,
4198                 0x3b4f0, 0x3b548,
4199                 0x3b560, 0x3b59c,
4200                 0x3b5f0, 0x3b6e4,
4201                 0x3b6f8, 0x3b7e4,
4202                 0x3b7f8, 0x3b7fc,
4203                 0x3b814, 0x3b814,
4204                 0x3b82c, 0x3b82c,
4205                 0x3b880, 0x3b88c,
4206                 0x3b8e8, 0x3b8ec,
4207                 0x3b900, 0x3b948,
4208                 0x3b960, 0x3b99c,
4209                 0x3b9f0, 0x3bae4,
4210                 0x3baf8, 0x3bb10,
4211                 0x3bb28, 0x3bb28,
4212                 0x3bb3c, 0x3bb50,
4213                 0x3bbf0, 0x3bc10,
4214                 0x3bc28, 0x3bc28,
4215                 0x3bc3c, 0x3bc50,
4216                 0x3bcf0, 0x3bcfc,
4217                 0x3c000, 0x3c030,
4218                 0x3c100, 0x3c144,
4219                 0x3c190, 0x3c1d0,
4220                 0x3c200, 0x3c318,
4221                 0x3c400, 0x3c52c,
4222                 0x3c540, 0x3c61c,
4223                 0x3c800, 0x3c834,
4224                 0x3c8c0, 0x3c908,
4225                 0x3c910, 0x3c9ac,
4226                 0x3ca00, 0x3ca2c,
4227                 0x3ca44, 0x3ca50,
4228                 0x3ca74, 0x3cc24,
4229                 0x3cd00, 0x3cd00,
4230                 0x3cd08, 0x3cd14,
4231                 0x3cd1c, 0x3cd20,
4232                 0x3cd3c, 0x3cd50,
4233                 0x3d200, 0x3d20c,
4234                 0x3d220, 0x3d220,
4235                 0x3d240, 0x3d240,
4236                 0x3d600, 0x3d60c,
4237                 0x3da00, 0x3da1c,
4238                 0x3de00, 0x3de20,
4239                 0x3de38, 0x3de3c,
4240                 0x3de80, 0x3de80,
4241                 0x3de88, 0x3dea8,
4242                 0x3deb0, 0x3deb4,
4243                 0x3dec8, 0x3ded4,
4244                 0x3dfb8, 0x3e004,
4245                 0x3e200, 0x3e200,
4246                 0x3e208, 0x3e240,
4247                 0x3e248, 0x3e280,
4248                 0x3e288, 0x3e2c0,
4249                 0x3e2c8, 0x3e2fc,
4250                 0x3e600, 0x3e630,
4251                 0x3ea00, 0x3eabc,
4252                 0x3eb00, 0x3eb70,
4253                 0x3f000, 0x3f048,
4254                 0x3f060, 0x3f09c,
4255                 0x3f0f0, 0x3f148,
4256                 0x3f160, 0x3f19c,
4257                 0x3f1f0, 0x3f2e4,
4258                 0x3f2f8, 0x3f3e4,
4259                 0x3f3f8, 0x3f448,
4260                 0x3f460, 0x3f49c,
4261                 0x3f4f0, 0x3f548,
4262                 0x3f560, 0x3f59c,
4263                 0x3f5f0, 0x3f6e4,
4264                 0x3f6f8, 0x3f7e4,
4265                 0x3f7f8, 0x3f7fc,
4266                 0x3f814, 0x3f814,
4267                 0x3f82c, 0x3f82c,
4268                 0x3f880, 0x3f88c,
4269                 0x3f8e8, 0x3f8ec,
4270                 0x3f900, 0x3f948,
4271                 0x3f960, 0x3f99c,
4272                 0x3f9f0, 0x3fae4,
4273                 0x3faf8, 0x3fb10,
4274                 0x3fb28, 0x3fb28,
4275                 0x3fb3c, 0x3fb50,
4276                 0x3fbf0, 0x3fc10,
4277                 0x3fc28, 0x3fc28,
4278                 0x3fc3c, 0x3fc50,
4279                 0x3fcf0, 0x3fcfc,
4280                 0x40000, 0x4000c,
4281                 0x40040, 0x40068,
4282                 0x4007c, 0x40144,
4283                 0x40180, 0x4018c,
4284                 0x40200, 0x40298,
4285                 0x402ac, 0x4033c,
4286                 0x403f8, 0x403fc,
4287                 0x41304, 0x413c4,
4288                 0x41400, 0x4141c,
4289                 0x41480, 0x414d0,
4290                 0x44000, 0x44078,
4291                 0x440c0, 0x44278,
4292                 0x442c0, 0x44478,
4293                 0x444c0, 0x44678,
4294                 0x446c0, 0x44878,
4295                 0x448c0, 0x449fc,
4296                 0x45000, 0x45068,
4297                 0x45080, 0x45084,
4298                 0x450a0, 0x450b0,
4299                 0x45200, 0x45268,
4300                 0x45280, 0x45284,
4301                 0x452a0, 0x452b0,
4302                 0x460c0, 0x460e4,
4303                 0x47000, 0x4708c,
4304                 0x47200, 0x47250,
4305                 0x47400, 0x47420,
4306                 0x47600, 0x47618,
4307                 0x47800, 0x47814,
4308                 0x48000, 0x4800c,
4309                 0x48040, 0x48068,
4310                 0x4807c, 0x48144,
4311                 0x48180, 0x4818c,
4312                 0x48200, 0x48298,
4313                 0x482ac, 0x4833c,
4314                 0x483f8, 0x483fc,
4315                 0x49304, 0x493c4,
4316                 0x49400, 0x4941c,
4317                 0x49480, 0x494d0,
4318                 0x4c000, 0x4c078,
4319                 0x4c0c0, 0x4c278,
4320                 0x4c2c0, 0x4c478,
4321                 0x4c4c0, 0x4c678,
4322                 0x4c6c0, 0x4c878,
4323                 0x4c8c0, 0x4c9fc,
4324                 0x4d000, 0x4d068,
4325                 0x4d080, 0x4d084,
4326                 0x4d0a0, 0x4d0b0,
4327                 0x4d200, 0x4d268,
4328                 0x4d280, 0x4d284,
4329                 0x4d2a0, 0x4d2b0,
4330                 0x4e0c0, 0x4e0e4,
4331                 0x4f000, 0x4f08c,
4332                 0x4f200, 0x4f250,
4333                 0x4f400, 0x4f420,
4334                 0x4f600, 0x4f618,
4335                 0x4f800, 0x4f814,
4336                 0x50000, 0x500cc,
4337                 0x50400, 0x50400,
4338                 0x50800, 0x508cc,
4339                 0x50c00, 0x50c00,
4340                 0x51000, 0x5101c,
4341                 0x51300, 0x51308,
4342         };
4343
4344         if (is_t4(sc)) {
4345                 reg_ranges = &t4_reg_ranges[0];
4346                 n = nitems(t4_reg_ranges);
4347         } else {
4348                 reg_ranges = &t5_reg_ranges[0];
4349                 n = nitems(t5_reg_ranges);
4350         }
4351
4352         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4353         for (i = 0; i < n; i += 2)
4354                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4355 }
4356
4357 static void
4358 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4359 {
4360         int i;
4361         u_int v, tnl_cong_drops;
4362         struct timeval tv;
4363         const struct timeval interval = {0, 250000};    /* 250ms */
4364
4365         getmicrotime(&tv);
4366         timevalsub(&tv, &interval);
4367         if (timevalcmp(&tv, &pi->last_refreshed, <))
4368                 return;
4369
4370         tnl_cong_drops = 0;
4371         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4372         for (i = 0; i < NCHAN; i++) {
4373                 if (pi->rx_chan_map & (1 << i)) {
4374                         mtx_lock(&sc->regwin_lock);
4375                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4376                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4377                         mtx_unlock(&sc->regwin_lock);
4378                         tnl_cong_drops += v;
4379                 }
4380         }
4381         pi->tnl_cong_drops = tnl_cong_drops;
4382         getmicrotime(&pi->last_refreshed);
4383 }
4384
4385 static void
4386 cxgbe_tick(void *arg)
4387 {
4388         struct port_info *pi = arg;
4389         struct adapter *sc = pi->adapter;
4390         struct ifnet *ifp = pi->ifp;
4391
4392         PORT_LOCK(pi);
4393         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4394                 PORT_UNLOCK(pi);
4395                 return; /* without scheduling another callout */
4396         }
4397
4398         cxgbe_refresh_stats(sc, pi);
4399
4400         callout_schedule(&pi->tick, hz);
4401         PORT_UNLOCK(pi);
4402 }
4403
4404 static void
4405 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4406 {
4407         struct ifnet *vlan;
4408
4409         if (arg != ifp || ifp->if_type != IFT_ETHER)
4410                 return;
4411
4412         vlan = VLAN_DEVAT(ifp, vid);
4413         VLAN_SETCOOKIE(vlan, ifp);
4414 }
4415
4416 static int
4417 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4418 {
4419
4420 #ifdef INVARIANTS
4421         panic("%s: opcode 0x%02x on iq %p with payload %p",
4422             __func__, rss->opcode, iq, m);
4423 #else
4424         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4425             __func__, rss->opcode, iq, m);
4426         m_freem(m);
4427 #endif
4428         return (EDOOFUS);
4429 }
4430
4431 int
4432 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4433 {
4434         uintptr_t *loc, new;
4435
4436         if (opcode >= nitems(sc->cpl_handler))
4437                 return (EINVAL);
4438
4439         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4440         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4441         atomic_store_rel_ptr(loc, new);
4442
4443         return (0);
4444 }
4445
4446 static int
4447 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4448 {
4449
4450 #ifdef INVARIANTS
4451         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4452 #else
4453         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4454             __func__, iq, ctrl);
4455 #endif
4456         return (EDOOFUS);
4457 }
4458
4459 int
4460 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4461 {
4462         uintptr_t *loc, new;
4463
4464         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4465         loc = (uintptr_t *) &sc->an_handler;
4466         atomic_store_rel_ptr(loc, new);
4467
4468         return (0);
4469 }
4470
4471 static int
4472 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4473 {
4474         const struct cpl_fw6_msg *cpl =
4475             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4476
4477 #ifdef INVARIANTS
4478         panic("%s: fw_msg type %d", __func__, cpl->type);
4479 #else
4480         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4481 #endif
4482         return (EDOOFUS);
4483 }
4484
4485 int
4486 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4487 {
4488         uintptr_t *loc, new;
4489
4490         if (type >= nitems(sc->fw_msg_handler))
4491                 return (EINVAL);
4492
4493         /*
4494          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4495          * handler dispatch table.  Reject any attempt to install a handler for
4496          * this subtype.
4497          */
4498         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4499                 return (EINVAL);
4500
4501         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4502         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4503         atomic_store_rel_ptr(loc, new);
4504
4505         return (0);
4506 }
4507
4508 static int
4509 t4_sysctls(struct adapter *sc)
4510 {
4511         struct sysctl_ctx_list *ctx;
4512         struct sysctl_oid *oid;
4513         struct sysctl_oid_list *children, *c0;
4514         static char *caps[] = {
4515                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4516                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4517                     "\6HASHFILTER\7ETHOFLD",
4518                 "\20\1TOE",                             /* caps[2] toecaps */
4519                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4520                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4521                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4522                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4523                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4524                     "\4PO_INITIAOR\5PO_TARGET"
4525         };
4526         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4527
4528         ctx = device_get_sysctl_ctx(sc->dev);
4529
4530         /*
4531          * dev.t4nex.X.
4532          */
4533         oid = device_get_sysctl_tree(sc->dev);
4534         c0 = children = SYSCTL_CHILDREN(oid);
4535
4536         sc->sc_do_rxcopy = 1;
4537         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4538             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4539
4540         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4541             sc->params.nports, "# of ports");
4542
4543         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4544             NULL, chip_rev(sc), "chip hardware revision");
4545
4546         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4547             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4548
4549         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4550             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4551
4552         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4553             sc->cfcsum, "config file checksum");
4554
4555         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4556             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4557             sysctl_bitfield, "A", "available doorbells");
4558
4559         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4560             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4561             sysctl_bitfield, "A", "available link capabilities");
4562
4563         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4564             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4565             sysctl_bitfield, "A", "available NIC capabilities");
4566
4567         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4568             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4569             sysctl_bitfield, "A", "available TCP offload capabilities");
4570
4571         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4572             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4573             sysctl_bitfield, "A", "available RDMA capabilities");
4574
4575         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4576             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4577             sysctl_bitfield, "A", "available iSCSI capabilities");
4578
4579         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4580             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4581             sysctl_bitfield, "A", "available FCoE capabilities");
4582
4583         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4584             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4588             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4589             "interrupt holdoff timer values (us)");
4590
4591         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4592             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4593             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4594             "interrupt holdoff packet counter values");
4595
4596         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4597             NULL, sc->tids.nftids, "number of filters");
4598
4599         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4600             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4601             "chip temperature (in Celsius)");
4602
4603         t4_sge_sysctls(sc, ctx, children);
4604
4605         sc->lro_timeout = 100;
4606         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4607             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4608
4609         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4610             &sc->debug_flags, 0, "flags to enable runtime debugging");
4611
4612 #ifdef SBUF_DRAIN
4613         /*
4614          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4615          */
4616         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4617             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4618             "logs and miscellaneous information");
4619         children = SYSCTL_CHILDREN(oid);
4620
4621         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4622             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4623             sysctl_cctrl, "A", "congestion control");
4624
4625         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4626             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4627             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4628
4629         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4630             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4631             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4632
4633         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4634             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4635             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4636
4637         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4638             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4639             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4640
4641         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4642             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4643             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4644
4645         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4646             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4647             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4648
4649         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4650             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4651             sysctl_cim_la, "A", "CIM logic analyzer");
4652
4653         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4654             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4655             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4656
4657         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4658             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4659             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4660
4661         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4662             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4663             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4664
4665         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4666             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4667             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4668
4669         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4670             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4671             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4672
4673         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4674             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4675             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4676
4677         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4678             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4679             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4680
4681         if (is_t5(sc)) {
4682                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4683                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4684                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4685
4686                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4687                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4688                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4689         }
4690
4691         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4692             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4693             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4694
4695         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4696             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4697             sysctl_cim_qcfg, "A", "CIM queue configuration");
4698
4699         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4700             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4701             sysctl_cpl_stats, "A", "CPL statistics");
4702
4703         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4704             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4705             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4706
4707         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4708             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4709             sysctl_devlog, "A", "firmware's device log");
4710
4711         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4712             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4713             sysctl_fcoe_stats, "A", "FCoE statistics");
4714
4715         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4716             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4717             sysctl_hw_sched, "A", "hardware scheduler ");
4718
4719         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4720             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4721             sysctl_l2t, "A", "hardware L2 table");
4722
4723         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4724             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4725             sysctl_lb_stats, "A", "loopback statistics");
4726
4727         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4728             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4729             sysctl_meminfo, "A", "memory regions");
4730
4731         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4732             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4733             sysctl_mps_tcam, "A", "MPS TCAM entries");
4734
4735         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4736             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4737             sysctl_path_mtus, "A", "path MTUs");
4738
4739         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4740             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4741             sysctl_pm_stats, "A", "PM statistics");
4742
4743         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4744             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4745             sysctl_rdma_stats, "A", "RDMA statistics");
4746
4747         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4748             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4749             sysctl_tcp_stats, "A", "TCP statistics");
4750
4751         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4752             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4753             sysctl_tids, "A", "TID information");
4754
4755         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4756             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4757             sysctl_tp_err_stats, "A", "TP error statistics");
4758
4759         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4760             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4761             sysctl_tp_la, "A", "TP logic analyzer");
4762
4763         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4764             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4765             sysctl_tx_rate, "A", "Tx rate");
4766
4767         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4768             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4769             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4770
4771         if (is_t5(sc)) {
4772                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4773                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4774                     sysctl_wcwr_stats, "A", "write combined work requests");
4775         }
4776 #endif
4777
4778 #ifdef TCP_OFFLOAD
4779         if (is_offload(sc)) {
4780                 /*
4781                  * dev.t4nex.X.toe.
4782                  */
4783                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4784                     NULL, "TOE parameters");
4785                 children = SYSCTL_CHILDREN(oid);
4786
4787                 sc->tt.sndbuf = 256 * 1024;
4788                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4789                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4790
4791                 sc->tt.ddp = 0;
4792                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4793                     &sc->tt.ddp, 0, "DDP allowed");
4794
4795                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4796                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4797                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4798
4799                 sc->tt.ddp_thres =
4800                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4801                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4802                     &sc->tt.ddp_thres, 0, "DDP threshold");
4803
4804                 sc->tt.rx_coalesce = 1;
4805                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4806                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4807
4808                 sc->tt.tx_align = 1;
4809                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4810                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4811         }
4812 #endif
4813
4814
4815         return (0);
4816 }
4817
4818 static int
4819 cxgbe_sysctls(struct port_info *pi)
4820 {
4821         struct sysctl_ctx_list *ctx;
4822         struct sysctl_oid *oid;
4823         struct sysctl_oid_list *children;
4824         struct adapter *sc = pi->adapter;
4825
4826         ctx = device_get_sysctl_ctx(pi->dev);
4827
4828         /*
4829          * dev.cxgbe.X.
4830          */
4831         oid = device_get_sysctl_tree(pi->dev);
4832         children = SYSCTL_CHILDREN(oid);
4833
4834         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4835            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4836         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4837                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4838                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4839                     "PHY temperature (in Celsius)");
4840                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4841                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4842                     "PHY firmware version");
4843         }
4844         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4845             &pi->nrxq, 0, "# of rx queues");
4846         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4847             &pi->ntxq, 0, "# of tx queues");
4848         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4849             &pi->first_rxq, 0, "index of first rx queue");
4850         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4851             &pi->first_txq, 0, "index of first tx queue");
4852         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4853             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4854             "Reserve queue 0 for non-flowid packets");
4855
4856 #ifdef TCP_OFFLOAD
4857         if (is_offload(sc)) {
4858                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4859                     &pi->nofldrxq, 0,
4860                     "# of rx queues for offloaded TCP connections");
4861                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4862                     &pi->nofldtxq, 0,
4863                     "# of tx queues for offloaded TCP connections");
4864                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4865                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4866                     "index of first TOE rx queue");
4867                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4868                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4869                     "index of first TOE tx queue");
4870         }
4871 #endif
4872 #ifdef DEV_NETMAP
4873         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4874             &pi->nnmrxq, 0, "# of rx queues for netmap");
4875         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4876             &pi->nnmtxq, 0, "# of tx queues for netmap");
4877         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4878             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4879             "index of first netmap rx queue");
4880         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4881             CTLFLAG_RD, &pi->first_nm_txq, 0,
4882             "index of first netmap tx queue");
4883 #endif
4884
4885         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4886             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4887             "holdoff timer index");
4888         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4889             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4890             "holdoff packet counter index");
4891
4892         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4893             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4894             "rx queue size");
4895         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4896             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4897             "tx queue size");
4898
4899         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4900             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4901             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4902
4903         /*
4904          * dev.cxgbe.X.stats.
4905          */
4906         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4907             NULL, "port statistics");
4908         children = SYSCTL_CHILDREN(oid);
4909         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4910             &pi->tx_parse_error, 0,
4911             "# of tx packets with invalid length or # of segments");
4912
4913 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4914         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4915             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4916             sysctl_handle_t4_reg64, "QU", desc)
4917
4918         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4919             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4920         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4921             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4922         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4923             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4924         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4925             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4926         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4927             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4928         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4929             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4930         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4931             "# of tx frames in this range",
4932             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4933         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4934             "# of tx frames in this range",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4936         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4937             "# of tx frames in this range",
4938             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4939         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4940             "# of tx frames in this range",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4942         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4943             "# of tx frames in this range",
4944             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4945         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4946             "# of tx frames in this range",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4948         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4949             "# of tx frames in this range",
4950             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4951         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4952             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4953         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4954             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4955         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4956             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4957         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4958             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4959         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4960             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4961         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4962             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4963         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4964             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4965         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4966             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4967         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4968             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4969         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4971
4972         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4973             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4974         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4975             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4976         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4977             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4978         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4979             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4980         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4981             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4982         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4983             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4984         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4985             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4986         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4987             "# of frames received with bad FCS",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4990             "# of frames received with length error",
4991             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4992         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4993             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4994         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4995             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4996         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4997             "# of rx frames in this range",
4998             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4999         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5000             "# of rx frames in this range",
5001             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5002         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5003             "# of rx frames in this range",
5004             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5005         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5006             "# of rx frames in this range",
5007             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5008         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5009             "# of rx frames in this range",
5010             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5011         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5012             "# of rx frames in this range",
5013             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5014         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5015             "# of rx frames in this range",
5016             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5017         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5018             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5019         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5020             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5021         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5022             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5023         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5024             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5025         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5026             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5027         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5028             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5029         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5030             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5031         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5032             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5033         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5034             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5035
5036 #undef SYSCTL_ADD_T4_REG64
5037
5038 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5039         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5040             &pi->stats.name, desc)
5041
5042         /* We get these from port_stats and they may be stale by upto 1s */
5043         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5044             "# drops due to buffer-group 0 overflows");
5045         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5046             "# drops due to buffer-group 1 overflows");
5047         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5048             "# drops due to buffer-group 2 overflows");
5049         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5050             "# drops due to buffer-group 3 overflows");
5051         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5052             "# of buffer-group 0 truncated packets");
5053         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5054             "# of buffer-group 1 truncated packets");
5055         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5056             "# of buffer-group 2 truncated packets");
5057         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5058             "# of buffer-group 3 truncated packets");
5059
5060 #undef SYSCTL_ADD_T4_PORTSTAT
5061
5062         return (0);
5063 }
5064
5065 static int
5066 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5067 {
5068         int rc, *i, space = 0;
5069         struct sbuf sb;
5070
5071         sbuf_new_for_sysctl(&sb, NULL, 64, req);
5072         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5073                 if (space)
5074                         sbuf_printf(&sb, " ");
5075                 sbuf_printf(&sb, "%d", *i);
5076                 space = 1;
5077         }
5078         rc = sbuf_finish(&sb);
5079         sbuf_delete(&sb);
5080         return (rc);
5081 }
5082
5083 static int
5084 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5085 {
5086         int rc;
5087         struct sbuf *sb;
5088
5089         rc = sysctl_wire_old_buffer(req, 0);
5090         if (rc != 0)
5091                 return(rc);
5092
5093         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5094         if (sb == NULL)
5095                 return (ENOMEM);
5096
5097         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5098         rc = sbuf_finish(sb);
5099         sbuf_delete(sb);
5100
5101         return (rc);
5102 }
5103
5104 static int
5105 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5106 {
5107         struct port_info *pi = arg1;
5108         int op = arg2;
5109         struct adapter *sc = pi->adapter;
5110         u_int v;
5111         int rc;
5112
5113         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5114         if (rc)
5115                 return (rc);
5116         /* XXX: magic numbers */
5117         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5118             &v);
5119         end_synchronized_op(sc, 0);
5120         if (rc)
5121                 return (rc);
5122         if (op == 0)
5123                 v /= 256;
5124
5125         rc = sysctl_handle_int(oidp, &v, 0, req);
5126         return (rc);
5127 }
5128
5129 static int
5130 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5131 {
5132         struct port_info *pi = arg1;
5133         int rc, val;
5134
5135         val = pi->rsrv_noflowq;
5136         rc = sysctl_handle_int(oidp, &val, 0, req);
5137         if (rc != 0 || req->newptr == NULL)
5138                 return (rc);
5139
5140         if ((val >= 1) && (pi->ntxq > 1))
5141                 pi->rsrv_noflowq = 1;
5142         else
5143                 pi->rsrv_noflowq = 0;
5144
5145         return (rc);
5146 }
5147
5148 static int
5149 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5150 {
5151         struct port_info *pi = arg1;
5152         struct adapter *sc = pi->adapter;
5153         int idx, rc, i;
5154         struct sge_rxq *rxq;
5155 #ifdef TCP_OFFLOAD
5156         struct sge_ofld_rxq *ofld_rxq;
5157 #endif
5158         uint8_t v;
5159
5160         idx = pi->tmr_idx;
5161
5162         rc = sysctl_handle_int(oidp, &idx, 0, req);
5163         if (rc != 0 || req->newptr == NULL)
5164                 return (rc);
5165
5166         if (idx < 0 || idx >= SGE_NTIMERS)
5167                 return (EINVAL);
5168
5169         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5170             "t4tmr");
5171         if (rc)
5172                 return (rc);
5173
5174         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5175         for_each_rxq(pi, i, rxq) {
5176 #ifdef atomic_store_rel_8
5177                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5178 #else
5179                 rxq->iq.intr_params = v;
5180 #endif
5181         }
5182 #ifdef TCP_OFFLOAD
5183         for_each_ofld_rxq(pi, i, ofld_rxq) {
5184 #ifdef atomic_store_rel_8
5185                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5186 #else
5187                 ofld_rxq->iq.intr_params = v;
5188 #endif
5189         }
5190 #endif
5191         pi->tmr_idx = idx;
5192
5193         end_synchronized_op(sc, LOCK_HELD);
5194         return (0);
5195 }
5196
5197 static int
5198 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5199 {
5200         struct port_info *pi = arg1;
5201         struct adapter *sc = pi->adapter;
5202         int idx, rc;
5203
5204         idx = pi->pktc_idx;
5205
5206         rc = sysctl_handle_int(oidp, &idx, 0, req);
5207         if (rc != 0 || req->newptr == NULL)
5208                 return (rc);
5209
5210         if (idx < -1 || idx >= SGE_NCOUNTERS)
5211                 return (EINVAL);
5212
5213         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5214             "t4pktc");
5215         if (rc)
5216                 return (rc);
5217
5218         if (pi->flags & PORT_INIT_DONE)
5219                 rc = EBUSY; /* cannot be changed once the queues are created */
5220         else
5221                 pi->pktc_idx = idx;
5222
5223         end_synchronized_op(sc, LOCK_HELD);
5224         return (rc);
5225 }
5226
5227 static int
5228 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5229 {
5230         struct port_info *pi = arg1;
5231         struct adapter *sc = pi->adapter;
5232         int qsize, rc;
5233
5234         qsize = pi->qsize_rxq;
5235
5236         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5237         if (rc != 0 || req->newptr == NULL)
5238                 return (rc);
5239
5240         if (qsize < 128 || (qsize & 7))
5241                 return (EINVAL);
5242
5243         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5244             "t4rxqs");
5245         if (rc)
5246                 return (rc);
5247
5248         if (pi->flags & PORT_INIT_DONE)
5249                 rc = EBUSY; /* cannot be changed once the queues are created */
5250         else
5251                 pi->qsize_rxq = qsize;
5252
5253         end_synchronized_op(sc, LOCK_HELD);
5254         return (rc);
5255 }
5256
5257 static int
5258 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5259 {
5260         struct port_info *pi = arg1;
5261         struct adapter *sc = pi->adapter;
5262         int qsize, rc;
5263
5264         qsize = pi->qsize_txq;
5265
5266         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5267         if (rc != 0 || req->newptr == NULL)
5268                 return (rc);
5269
5270         if (qsize < 128 || qsize > 65536)
5271                 return (EINVAL);
5272
5273         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5274             "t4txqs");
5275         if (rc)
5276                 return (rc);
5277
5278         if (pi->flags & PORT_INIT_DONE)
5279                 rc = EBUSY; /* cannot be changed once the queues are created */
5280         else
5281                 pi->qsize_txq = qsize;
5282
5283         end_synchronized_op(sc, LOCK_HELD);
5284         return (rc);
5285 }
5286
5287 static int
5288 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5289 {
5290         struct port_info *pi = arg1;
5291         struct adapter *sc = pi->adapter;
5292         struct link_config *lc = &pi->link_cfg;
5293         int rc;
5294
5295         if (req->newptr == NULL) {
5296                 struct sbuf *sb;
5297                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5298
5299                 rc = sysctl_wire_old_buffer(req, 0);
5300                 if (rc != 0)
5301                         return(rc);
5302
5303                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5304                 if (sb == NULL)
5305                         return (ENOMEM);
5306
5307                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5308                 rc = sbuf_finish(sb);
5309                 sbuf_delete(sb);
5310         } else {
5311                 char s[2];
5312                 int n;
5313
5314                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5315                 s[1] = 0;
5316
5317                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5318                 if (rc != 0)
5319                         return(rc);
5320
5321                 if (s[1] != 0)
5322                         return (EINVAL);
5323                 if (s[0] < '0' || s[0] > '9')
5324                         return (EINVAL);        /* not a number */
5325                 n = s[0] - '0';
5326                 if (n & ~(PAUSE_TX | PAUSE_RX))
5327                         return (EINVAL);        /* some other bit is set too */
5328
5329                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5330                 if (rc)
5331                         return (rc);
5332                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5333                         int link_ok = lc->link_ok;
5334
5335                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5336                         lc->requested_fc |= n;
5337                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5338                         lc->link_ok = link_ok;  /* restore */
5339                 }
5340                 end_synchronized_op(sc, 0);
5341         }
5342
5343         return (rc);
5344 }
5345
5346 static int
5347 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5348 {
5349         struct adapter *sc = arg1;
5350         int reg = arg2;
5351         uint64_t val;
5352
5353         val = t4_read_reg64(sc, reg);
5354
5355         return (sysctl_handle_64(oidp, &val, 0, req));
5356 }
5357
5358 static int
5359 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5360 {
5361         struct adapter *sc = arg1;
5362         int rc, t;
5363         uint32_t param, val;
5364
5365         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5366         if (rc)
5367                 return (rc);
5368         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5369             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5370             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5371         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5372         end_synchronized_op(sc, 0);
5373         if (rc)
5374                 return (rc);
5375
5376         /* unknown is returned as 0 but we display -1 in that case */
5377         t = val == 0 ? -1 : val;
5378
5379         rc = sysctl_handle_int(oidp, &t, 0, req);
5380         return (rc);
5381 }
5382
5383 #ifdef SBUF_DRAIN
5384 static int
5385 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5386 {
5387         struct adapter *sc = arg1;
5388         struct sbuf *sb;
5389         int rc, i;
5390         uint16_t incr[NMTUS][NCCTRL_WIN];
5391         static const char *dec_fac[] = {
5392                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5393                 "0.9375"
5394         };
5395
5396         rc = sysctl_wire_old_buffer(req, 0);
5397         if (rc != 0)
5398                 return (rc);
5399
5400         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5401         if (sb == NULL)
5402                 return (ENOMEM);
5403
5404         t4_read_cong_tbl(sc, incr);
5405
5406         for (i = 0; i < NCCTRL_WIN; ++i) {
5407                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5408                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5409                     incr[5][i], incr[6][i], incr[7][i]);
5410                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5411                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5412                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5413                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5414         }
5415
5416         rc = sbuf_finish(sb);
5417         sbuf_delete(sb);
5418
5419         return (rc);
5420 }
5421
5422 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5423         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5424         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5425         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5426 };
5427
5428 static int
5429 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5430 {
5431         struct adapter *sc = arg1;
5432         struct sbuf *sb;
5433         int rc, i, n, qid = arg2;
5434         uint32_t *buf, *p;
5435         char *qtype;
5436         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5437
5438         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5439             ("%s: bad qid %d\n", __func__, qid));
5440
5441         if (qid < CIM_NUM_IBQ) {
5442                 /* inbound queue */
5443                 qtype = "IBQ";
5444                 n = 4 * CIM_IBQ_SIZE;
5445                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5446                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5447         } else {
5448                 /* outbound queue */
5449                 qtype = "OBQ";
5450                 qid -= CIM_NUM_IBQ;
5451                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5452                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5453                 rc = t4_read_cim_obq(sc, qid, buf, n);
5454         }
5455
5456         if (rc < 0) {
5457                 rc = -rc;
5458                 goto done;
5459         }
5460         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5461
5462         rc = sysctl_wire_old_buffer(req, 0);
5463         if (rc != 0)
5464                 goto done;
5465
5466         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5467         if (sb == NULL) {
5468                 rc = ENOMEM;
5469                 goto done;
5470         }
5471
5472         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5473         for (i = 0, p = buf; i < n; i += 16, p += 4)
5474                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5475                     p[2], p[3]);
5476
5477         rc = sbuf_finish(sb);
5478         sbuf_delete(sb);
5479 done:
5480         free(buf, M_CXGBE);
5481         return (rc);
5482 }
5483
5484 static int
5485 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5486 {
5487         struct adapter *sc = arg1;
5488         u_int cfg;
5489         struct sbuf *sb;
5490         uint32_t *buf, *p;
5491         int rc;
5492
5493         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5494         if (rc != 0)
5495                 return (rc);
5496
5497         rc = sysctl_wire_old_buffer(req, 0);
5498         if (rc != 0)
5499                 return (rc);
5500
5501         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5502         if (sb == NULL)
5503                 return (ENOMEM);
5504
5505         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5506             M_ZERO | M_WAITOK);
5507
5508         rc = -t4_cim_read_la(sc, buf, NULL);
5509         if (rc != 0)
5510                 goto done;
5511
5512         sbuf_printf(sb, "Status   Data      PC%s",
5513             cfg & F_UPDBGLACAPTPCONLY ? "" :
5514             "     LS0Stat  LS0Addr             LS0Data");
5515
5516         KASSERT((sc->params.cim_la_size & 7) == 0,
5517             ("%s: p will walk off the end of buf", __func__));
5518
5519         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5520                 if (cfg & F_UPDBGLACAPTPCONLY) {
5521                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5522                             p[6], p[7]);
5523                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5524                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5525                             p[4] & 0xff, p[5] >> 8);
5526                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5527                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5528                             p[1] & 0xf, p[2] >> 4);
5529                 } else {
5530                         sbuf_printf(sb,
5531                             "\n  %02x   %x%07x %x%07x %08x %08x "
5532                             "%08x%08x%08x%08x",
5533                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5534                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5535                             p[6], p[7]);
5536                 }
5537         }
5538
5539         rc = sbuf_finish(sb);
5540         sbuf_delete(sb);
5541 done:
5542         free(buf, M_CXGBE);
5543         return (rc);
5544 }
5545
5546 static int
5547 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5548 {
5549         struct adapter *sc = arg1;
5550         u_int i;
5551         struct sbuf *sb;
5552         uint32_t *buf, *p;
5553         int rc;
5554
5555         rc = sysctl_wire_old_buffer(req, 0);
5556         if (rc != 0)
5557                 return (rc);
5558
5559         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5560         if (sb == NULL)
5561                 return (ENOMEM);
5562
5563         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5564             M_ZERO | M_WAITOK);
5565
5566         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5567         p = buf;
5568
5569         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5570                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5571                     p[1], p[0]);
5572         }
5573
5574         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5575         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5576                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5577                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5578                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5579                     (p[1] >> 2) | ((p[2] & 3) << 30),
5580                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5581                     p[0] & 1);
5582         }
5583
5584         rc = sbuf_finish(sb);
5585         sbuf_delete(sb);
5586         free(buf, M_CXGBE);
5587         return (rc);
5588 }
5589
5590 static int
5591 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5592 {
5593         struct adapter *sc = arg1;
5594         u_int i;
5595         struct sbuf *sb;
5596         uint32_t *buf, *p;
5597         int rc;
5598
5599         rc = sysctl_wire_old_buffer(req, 0);
5600         if (rc != 0)
5601                 return (rc);
5602
5603         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5604         if (sb == NULL)
5605                 return (ENOMEM);
5606
5607         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5608             M_ZERO | M_WAITOK);
5609
5610         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5611         p = buf;
5612
5613         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5614         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5615                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5616                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5617                     p[4], p[3], p[2], p[1], p[0]);
5618         }
5619
5620         sbuf_printf(sb, "\n\nCntl ID               Data");
5621         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5622                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5623                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5624         }
5625
5626         rc = sbuf_finish(sb);
5627         sbuf_delete(sb);
5628         free(buf, M_CXGBE);
5629         return (rc);
5630 }
5631
5632 static int
5633 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5634 {
5635         struct adapter *sc = arg1;
5636         struct sbuf *sb;
5637         int rc, i;
5638         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5639         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5640         uint16_t thres[CIM_NUM_IBQ];
5641         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5642         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5643         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5644
5645         if (is_t4(sc)) {
5646                 cim_num_obq = CIM_NUM_OBQ;
5647                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5648                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5649         } else {
5650                 cim_num_obq = CIM_NUM_OBQ_T5;
5651                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5652                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5653         }
5654         nq = CIM_NUM_IBQ + cim_num_obq;
5655
5656         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5657         if (rc == 0)
5658                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5659         if (rc != 0)
5660                 return (rc);
5661
5662         t4_read_cimq_cfg(sc, base, size, thres);
5663
5664         rc = sysctl_wire_old_buffer(req, 0);
5665         if (rc != 0)
5666                 return (rc);
5667
5668         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5669         if (sb == NULL)
5670                 return (ENOMEM);
5671
5672         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5673
5674         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5675                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5676                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5677                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5678                     G_QUEREMFLITS(p[2]) * 16);
5679         for ( ; i < nq; i++, p += 4, wr += 2)
5680                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5681                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5682                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5683                     G_QUEREMFLITS(p[2]) * 16);
5684
5685         rc = sbuf_finish(sb);
5686         sbuf_delete(sb);
5687
5688         return (rc);
5689 }
5690
5691 static int
5692 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5693 {
5694         struct adapter *sc = arg1;
5695         struct sbuf *sb;
5696         int rc;
5697         struct tp_cpl_stats stats;
5698
5699         rc = sysctl_wire_old_buffer(req, 0);
5700         if (rc != 0)
5701                 return (rc);
5702
5703         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5704         if (sb == NULL)
5705                 return (ENOMEM);
5706
5707         t4_tp_get_cpl_stats(sc, &stats);
5708
5709         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5710             "channel 3\n");
5711         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5712                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5713         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5714                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5715
5716         rc = sbuf_finish(sb);
5717         sbuf_delete(sb);
5718
5719         return (rc);
5720 }
5721
5722 static int
5723 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5724 {
5725         struct adapter *sc = arg1;
5726         struct sbuf *sb;
5727         int rc;
5728         struct tp_usm_stats stats;
5729
5730         rc = sysctl_wire_old_buffer(req, 0);
5731         if (rc != 0)
5732                 return(rc);
5733
5734         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5735         if (sb == NULL)
5736                 return (ENOMEM);
5737
5738         t4_get_usm_stats(sc, &stats);
5739
5740         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5741         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5742         sbuf_printf(sb, "Drops:  %u", stats.drops);
5743
5744         rc = sbuf_finish(sb);
5745         sbuf_delete(sb);
5746
5747         return (rc);
5748 }
5749
5750 const char *devlog_level_strings[] = {
5751         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5752         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5753         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5754         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5755         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5756         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5757 };
5758
5759 const char *devlog_facility_strings[] = {
5760         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5761         [FW_DEVLOG_FACILITY_CF]         = "CF",
5762         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5763         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5764         [FW_DEVLOG_FACILITY_RES]        = "RES",
5765         [FW_DEVLOG_FACILITY_HW]         = "HW",
5766         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5767         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5768         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5769         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5770         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5771         [FW_DEVLOG_FACILITY_VI]         = "VI",
5772         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5773         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5774         [FW_DEVLOG_FACILITY_TM]         = "TM",
5775         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5776         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5777         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5778         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5779         [FW_DEVLOG_FACILITY_RI]         = "RI",
5780         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5781         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5782         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5783         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5784 };
5785
5786 static int
5787 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5788 {
5789         struct adapter *sc = arg1;
5790         struct devlog_params *dparams = &sc->params.devlog;
5791         struct fw_devlog_e *buf, *e;
5792         int i, j, rc, nentries, first = 0, m;
5793         struct sbuf *sb;
5794         uint64_t ftstamp = UINT64_MAX;
5795
5796         if (dparams->start == 0) {
5797                 dparams->memtype = FW_MEMTYPE_EDC0;
5798                 dparams->start = 0x84000;
5799                 dparams->size = 32768;
5800         }
5801
5802         nentries = dparams->size / sizeof(struct fw_devlog_e);
5803
5804         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5805         if (buf == NULL)
5806                 return (ENOMEM);
5807
5808         m = fwmtype_to_hwmtype(dparams->memtype);
5809         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5810         if (rc != 0)
5811                 goto done;
5812
5813         for (i = 0; i < nentries; i++) {
5814                 e = &buf[i];
5815
5816                 if (e->timestamp == 0)
5817                         break;  /* end */
5818
5819                 e->timestamp = be64toh(e->timestamp);
5820                 e->seqno = be32toh(e->seqno);
5821                 for (j = 0; j < 8; j++)
5822                         e->params[j] = be32toh(e->params[j]);
5823
5824                 if (e->timestamp < ftstamp) {
5825                         ftstamp = e->timestamp;
5826                         first = i;
5827                 }
5828         }
5829
5830         if (buf[first].timestamp == 0)
5831                 goto done;      /* nothing in the log */
5832
5833         rc = sysctl_wire_old_buffer(req, 0);
5834         if (rc != 0)
5835                 goto done;
5836
5837         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5838         if (sb == NULL) {
5839                 rc = ENOMEM;
5840                 goto done;
5841         }
5842         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5843             "Seq#", "Tstamp", "Level", "Facility", "Message");
5844
5845         i = first;
5846         do {
5847                 e = &buf[i];
5848                 if (e->timestamp == 0)
5849                         break;  /* end */
5850
5851                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5852                     e->seqno, e->timestamp,
5853                     (e->level < nitems(devlog_level_strings) ?
5854                         devlog_level_strings[e->level] : "UNKNOWN"),
5855                     (e->facility < nitems(devlog_facility_strings) ?
5856                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5857                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5858                     e->params[2], e->params[3], e->params[4],
5859                     e->params[5], e->params[6], e->params[7]);
5860
5861                 if (++i == nentries)
5862                         i = 0;
5863         } while (i != first);
5864
5865         rc = sbuf_finish(sb);
5866         sbuf_delete(sb);
5867 done:
5868         free(buf, M_CXGBE);
5869         return (rc);
5870 }
5871
5872 static int
5873 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5874 {
5875         struct adapter *sc = arg1;
5876         struct sbuf *sb;
5877         int rc;
5878         struct tp_fcoe_stats stats[4];
5879
5880         rc = sysctl_wire_old_buffer(req, 0);
5881         if (rc != 0)
5882                 return (rc);
5883
5884         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5885         if (sb == NULL)
5886                 return (ENOMEM);
5887
5888         t4_get_fcoe_stats(sc, 0, &stats[0]);
5889         t4_get_fcoe_stats(sc, 1, &stats[1]);
5890         t4_get_fcoe_stats(sc, 2, &stats[2]);
5891         t4_get_fcoe_stats(sc, 3, &stats[3]);
5892
5893         sbuf_printf(sb, "                   channel 0        channel 1        "
5894             "channel 2        channel 3\n");
5895         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5896             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5897             stats[3].octetsDDP);
5898         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5899             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5900         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5901             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5902             stats[3].framesDrop);
5903
5904         rc = sbuf_finish(sb);
5905         sbuf_delete(sb);
5906
5907         return (rc);
5908 }
5909
5910 static int
5911 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5912 {
5913         struct adapter *sc = arg1;
5914         struct sbuf *sb;
5915         int rc, i;
5916         unsigned int map, kbps, ipg, mode;
5917         unsigned int pace_tab[NTX_SCHED];
5918
5919         rc = sysctl_wire_old_buffer(req, 0);
5920         if (rc != 0)
5921                 return (rc);
5922
5923         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5924         if (sb == NULL)
5925                 return (ENOMEM);
5926
5927         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5928         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5929         t4_read_pace_tbl(sc, pace_tab);
5930
5931         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5932             "Class IPG (0.1 ns)   Flow IPG (us)");
5933
5934         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5935                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5936                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5937                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5938                 if (kbps)
5939                         sbuf_printf(sb, "%9u     ", kbps);
5940                 else
5941                         sbuf_printf(sb, " disabled     ");
5942
5943                 if (ipg)
5944                         sbuf_printf(sb, "%13u        ", ipg);
5945                 else
5946                         sbuf_printf(sb, "     disabled        ");
5947
5948                 if (pace_tab[i])
5949                         sbuf_printf(sb, "%10u", pace_tab[i]);
5950                 else
5951                         sbuf_printf(sb, "  disabled");
5952         }
5953
5954         rc = sbuf_finish(sb);
5955         sbuf_delete(sb);
5956
5957         return (rc);
5958 }
5959
5960 static int
5961 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5962 {
5963         struct adapter *sc = arg1;
5964         struct sbuf *sb;
5965         int rc, i, j;
5966         uint64_t *p0, *p1;
5967         struct lb_port_stats s[2];
5968         static const char *stat_name[] = {
5969                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5970                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5971                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5972                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5973                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5974                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5975                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5976         };
5977
5978         rc = sysctl_wire_old_buffer(req, 0);
5979         if (rc != 0)
5980                 return (rc);
5981
5982         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5983         if (sb == NULL)
5984                 return (ENOMEM);
5985
5986         memset(s, 0, sizeof(s));
5987
5988         for (i = 0; i < 4; i += 2) {
5989                 t4_get_lb_stats(sc, i, &s[0]);
5990                 t4_get_lb_stats(sc, i + 1, &s[1]);
5991
5992                 p0 = &s[0].octets;
5993                 p1 = &s[1].octets;
5994                 sbuf_printf(sb, "%s                       Loopback %u"
5995                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5996
5997                 for (j = 0; j < nitems(stat_name); j++)
5998                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5999                                    *p0++, *p1++);
6000         }
6001
6002         rc = sbuf_finish(sb);
6003         sbuf_delete(sb);
6004
6005         return (rc);
6006 }
6007
6008 static int
6009 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6010 {
6011         int rc = 0;
6012         struct port_info *pi = arg1;
6013         struct sbuf *sb;
6014         static const char *linkdnreasons[] = {
6015                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6016                 "PHY overheated", "unknown", "rx los", "reserved7"
6017         };
6018
6019         rc = sysctl_wire_old_buffer(req, 0);
6020         if (rc != 0)
6021                 return(rc);
6022         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6023         if (sb == NULL)
6024                 return (ENOMEM);
6025
6026         if (pi->linkdnrc < 0)
6027                 sbuf_printf(sb, "n/a");
6028         else if (pi->linkdnrc < nitems(linkdnreasons))
6029                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6030         else
6031                 sbuf_printf(sb, "%d", pi->linkdnrc);
6032
6033         rc = sbuf_finish(sb);
6034         sbuf_delete(sb);
6035
6036         return (rc);
6037 }
6038
6039 struct mem_desc {
6040         unsigned int base;
6041         unsigned int limit;
6042         unsigned int idx;
6043 };
6044
6045 static int
6046 mem_desc_cmp(const void *a, const void *b)
6047 {
6048         return ((const struct mem_desc *)a)->base -
6049                ((const struct mem_desc *)b)->base;
6050 }
6051
6052 static void
6053 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6054     unsigned int to)
6055 {
6056         unsigned int size;
6057
6058         size = to - from + 1;
6059         if (size == 0)
6060                 return;
6061
6062         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6063         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6064 }
6065
6066 static int
6067 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6068 {
6069         struct adapter *sc = arg1;
6070         struct sbuf *sb;
6071         int rc, i, n;
6072         uint32_t lo, hi, used, alloc;
6073         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6074         static const char *region[] = {
6075                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6076                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6077                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6078                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6079                 "RQUDP region:", "PBL region:", "TXPBL region:",
6080                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6081                 "On-chip queues:"
6082         };
6083         struct mem_desc avail[4];
6084         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6085         struct mem_desc *md = mem;
6086
6087         rc = sysctl_wire_old_buffer(req, 0);
6088         if (rc != 0)
6089                 return (rc);
6090
6091         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6092         if (sb == NULL)
6093                 return (ENOMEM);
6094
6095         for (i = 0; i < nitems(mem); i++) {
6096                 mem[i].limit = 0;
6097                 mem[i].idx = i;
6098         }
6099
6100         /* Find and sort the populated memory ranges */
6101         i = 0;
6102         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6103         if (lo & F_EDRAM0_ENABLE) {
6104                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6105                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6106                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6107                 avail[i].idx = 0;
6108                 i++;
6109         }
6110         if (lo & F_EDRAM1_ENABLE) {
6111                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6112                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6113                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6114                 avail[i].idx = 1;
6115                 i++;
6116         }
6117         if (lo & F_EXT_MEM_ENABLE) {
6118                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6119                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6120                 avail[i].limit = avail[i].base +
6121                     (G_EXT_MEM_SIZE(hi) << 20);
6122                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6123                 i++;
6124         }
6125         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6126                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6127                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6128                 avail[i].limit = avail[i].base +
6129                     (G_EXT_MEM1_SIZE(hi) << 20);
6130                 avail[i].idx = 4;
6131                 i++;
6132         }
6133         if (!i)                                    /* no memory available */
6134                 return 0;
6135         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6136
6137         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6138         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6139         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6140         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6141         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6142         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6143         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6144         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6145         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6146
6147         /* the next few have explicit upper bounds */
6148         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6149         md->limit = md->base - 1 +
6150                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6151                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6152         md++;
6153
6154         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6155         md->limit = md->base - 1 +
6156                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6157                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6158         md++;
6159
6160         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6161                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6162                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6163                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6164         } else {
6165                 md->base = 0;
6166                 md->idx = nitems(region);  /* hide it */
6167         }
6168         md++;
6169
6170 #define ulp_region(reg) \
6171         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6172         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6173
6174         ulp_region(RX_ISCSI);
6175         ulp_region(RX_TDDP);
6176         ulp_region(TX_TPT);
6177         ulp_region(RX_STAG);
6178         ulp_region(RX_RQ);
6179         ulp_region(RX_RQUDP);
6180         ulp_region(RX_PBL);
6181         ulp_region(TX_PBL);
6182 #undef ulp_region
6183
6184         md->base = 0;
6185         md->idx = nitems(region);
6186         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6187                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6188                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6189                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6190         }
6191         md++;
6192
6193         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6194         md->limit = md->base + sc->tids.ntids - 1;
6195         md++;
6196         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6197         md->limit = md->base + sc->tids.ntids - 1;
6198         md++;
6199
6200         md->base = sc->vres.ocq.start;
6201         if (sc->vres.ocq.size)
6202                 md->limit = md->base + sc->vres.ocq.size - 1;
6203         else
6204                 md->idx = nitems(region);  /* hide it */
6205         md++;
6206
6207         /* add any address-space holes, there can be up to 3 */
6208         for (n = 0; n < i - 1; n++)
6209                 if (avail[n].limit < avail[n + 1].base)
6210                         (md++)->base = avail[n].limit;
6211         if (avail[n].limit)
6212                 (md++)->base = avail[n].limit;
6213
6214         n = md - mem;
6215         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6216
6217         for (lo = 0; lo < i; lo++)
6218                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6219                                 avail[lo].limit - 1);
6220
6221         sbuf_printf(sb, "\n");
6222         for (i = 0; i < n; i++) {
6223                 if (mem[i].idx >= nitems(region))
6224                         continue;                        /* skip holes */
6225                 if (!mem[i].limit)
6226                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6227                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6228                                 mem[i].limit);
6229         }
6230
6231         sbuf_printf(sb, "\n");
6232         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6233         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6234         mem_region_show(sb, "uP RAM:", lo, hi);
6235
6236         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6237         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6238         mem_region_show(sb, "uP Extmem2:", lo, hi);
6239
6240         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6241         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6242                    G_PMRXMAXPAGE(lo),
6243                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6244                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6245
6246         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6247         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6248         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6249                    G_PMTXMAXPAGE(lo),
6250                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6251                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6252         sbuf_printf(sb, "%u p-structs\n",
6253                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6254
6255         for (i = 0; i < 4; i++) {
6256                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6257                 if (is_t4(sc)) {
6258                         used = G_USED(lo);
6259                         alloc = G_ALLOC(lo);
6260                 } else {
6261                         used = G_T5_USED(lo);
6262                         alloc = G_T5_ALLOC(lo);
6263                 }
6264                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6265                            i, used, alloc);
6266         }
6267         for (i = 0; i < 4; i++) {
6268                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6269                 if (is_t4(sc)) {
6270                         used = G_USED(lo);
6271                         alloc = G_ALLOC(lo);
6272                 } else {
6273                         used = G_T5_USED(lo);
6274                         alloc = G_T5_ALLOC(lo);
6275                 }
6276                 sbuf_printf(sb,
6277                            "\nLoopback %d using %u pages out of %u allocated",
6278                            i, used, alloc);
6279         }
6280
6281         rc = sbuf_finish(sb);
6282         sbuf_delete(sb);
6283
6284         return (rc);
6285 }
6286
6287 static inline void
6288 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6289 {
6290         *mask = x | y;
6291         y = htobe64(y);
6292         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6293 }
6294
6295 static int
6296 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6297 {
6298         struct adapter *sc = arg1;
6299         struct sbuf *sb;
6300         int rc, i, n;
6301
6302         rc = sysctl_wire_old_buffer(req, 0);
6303         if (rc != 0)
6304                 return (rc);
6305
6306         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6307         if (sb == NULL)
6308                 return (ENOMEM);
6309
6310         sbuf_printf(sb,
6311             "Idx  Ethernet address     Mask     Vld Ports PF"
6312             "  VF              Replication             P0 P1 P2 P3  ML");
6313         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6314             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6315         for (i = 0; i < n; i++) {
6316                 uint64_t tcamx, tcamy, mask;
6317                 uint32_t cls_lo, cls_hi;
6318                 uint8_t addr[ETHER_ADDR_LEN];
6319
6320                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6321                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6322                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6323                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6324
6325                 if (tcamx & tcamy)
6326                         continue;
6327
6328                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6329                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6330                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6331                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6332                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6333                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6334                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6335
6336                 if (cls_lo & F_REPLICATE) {
6337                         struct fw_ldst_cmd ldst_cmd;
6338
6339                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6340                         ldst_cmd.op_to_addrspace =
6341                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6342                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6343                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6344                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6345                         ldst_cmd.u.mps.rplc.fid_idx =
6346                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6347                                 V_FW_LDST_CMD_IDX(i));
6348
6349                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6350                             "t4mps");
6351                         if (rc)
6352                                 break;
6353                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6354                             sizeof(ldst_cmd), &ldst_cmd);
6355                         end_synchronized_op(sc, 0);
6356
6357                         if (rc != 0) {
6358                                 sbuf_printf(sb,
6359                                     " ------------ error %3u ------------", rc);
6360                                 rc = 0;
6361                         } else {
6362                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6363                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6364                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6365                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6366                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6367                         }
6368                 } else
6369                         sbuf_printf(sb, "%36s", "");
6370
6371                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6372                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6373                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6374         }
6375
6376         if (rc)
6377                 (void) sbuf_finish(sb);
6378         else
6379                 rc = sbuf_finish(sb);
6380         sbuf_delete(sb);
6381
6382         return (rc);
6383 }
6384
6385 static int
6386 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6387 {
6388         struct adapter *sc = arg1;
6389         struct sbuf *sb;
6390         int rc;
6391         uint16_t mtus[NMTUS];
6392
6393         rc = sysctl_wire_old_buffer(req, 0);
6394         if (rc != 0)
6395                 return (rc);
6396
6397         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6398         if (sb == NULL)
6399                 return (ENOMEM);
6400
6401         t4_read_mtu_tbl(sc, mtus, NULL);
6402
6403         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6404             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6405             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6406             mtus[14], mtus[15]);
6407
6408         rc = sbuf_finish(sb);
6409         sbuf_delete(sb);
6410
6411         return (rc);
6412 }
6413
6414 static int
6415 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6416 {
6417         struct adapter *sc = arg1;
6418         struct sbuf *sb;
6419         int rc, i;
6420         uint32_t cnt[PM_NSTATS];
6421         uint64_t cyc[PM_NSTATS];
6422         static const char *rx_stats[] = {
6423                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6424         };
6425         static const char *tx_stats[] = {
6426                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6427         };
6428
6429         rc = sysctl_wire_old_buffer(req, 0);
6430         if (rc != 0)
6431                 return (rc);
6432
6433         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6434         if (sb == NULL)
6435                 return (ENOMEM);
6436
6437         t4_pmtx_get_stats(sc, cnt, cyc);
6438         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6439         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6440                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6441                     cyc[i]);
6442
6443         t4_pmrx_get_stats(sc, cnt, cyc);
6444         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6445         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6446                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6447                     cyc[i]);
6448
6449         rc = sbuf_finish(sb);
6450         sbuf_delete(sb);
6451
6452         return (rc);
6453 }
6454
6455 static int
6456 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6457 {
6458         struct adapter *sc = arg1;
6459         struct sbuf *sb;
6460         int rc;
6461         struct tp_rdma_stats stats;
6462
6463         rc = sysctl_wire_old_buffer(req, 0);
6464         if (rc != 0)
6465                 return (rc);
6466
6467         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6468         if (sb == NULL)
6469                 return (ENOMEM);
6470
6471         t4_tp_get_rdma_stats(sc, &stats);
6472         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6473         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6474
6475         rc = sbuf_finish(sb);
6476         sbuf_delete(sb);
6477
6478         return (rc);
6479 }
6480
6481 static int
6482 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6483 {
6484         struct adapter *sc = arg1;
6485         struct sbuf *sb;
6486         int rc;
6487         struct tp_tcp_stats v4, v6;
6488
6489         rc = sysctl_wire_old_buffer(req, 0);
6490         if (rc != 0)
6491                 return (rc);
6492
6493         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6494         if (sb == NULL)
6495                 return (ENOMEM);
6496
6497         t4_tp_get_tcp_stats(sc, &v4, &v6);
6498         sbuf_printf(sb,
6499             "                                IP                 IPv6\n");
6500         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6501             v4.tcpOutRsts, v6.tcpOutRsts);
6502         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6503             v4.tcpInSegs, v6.tcpInSegs);
6504         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6505             v4.tcpOutSegs, v6.tcpOutSegs);
6506         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6507             v4.tcpRetransSegs, v6.tcpRetransSegs);
6508
6509         rc = sbuf_finish(sb);
6510         sbuf_delete(sb);
6511
6512         return (rc);
6513 }
6514
6515 static int
6516 sysctl_tids(SYSCTL_HANDLER_ARGS)
6517 {
6518         struct adapter *sc = arg1;
6519         struct sbuf *sb;
6520         int rc;
6521         struct tid_info *t = &sc->tids;
6522
6523         rc = sysctl_wire_old_buffer(req, 0);
6524         if (rc != 0)
6525                 return (rc);
6526
6527         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6528         if (sb == NULL)
6529                 return (ENOMEM);
6530
6531         if (t->natids) {
6532                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6533                     t->atids_in_use);
6534         }
6535
6536         if (t->ntids) {
6537                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6538                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6539
6540                         if (b) {
6541                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6542                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6543                                     t->ntids - 1);
6544                         } else {
6545                                 sbuf_printf(sb, "TID range: %u-%u",
6546                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6547                                     t->ntids - 1);
6548                         }
6549                 } else
6550                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6551                 sbuf_printf(sb, ", in use: %u\n",
6552                     atomic_load_acq_int(&t->tids_in_use));
6553         }
6554
6555         if (t->nstids) {
6556                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6557                     t->stid_base + t->nstids - 1, t->stids_in_use);
6558         }
6559
6560         if (t->nftids) {
6561                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6562                     t->ftid_base + t->nftids - 1);
6563         }
6564
6565         if (t->netids) {
6566                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6567                     t->etid_base + t->netids - 1);
6568         }
6569
6570         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6571             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6572             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6573
6574         rc = sbuf_finish(sb);
6575         sbuf_delete(sb);
6576
6577         return (rc);
6578 }
6579
6580 static int
6581 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6582 {
6583         struct adapter *sc = arg1;
6584         struct sbuf *sb;
6585         int rc;
6586         struct tp_err_stats stats;
6587
6588         rc = sysctl_wire_old_buffer(req, 0);
6589         if (rc != 0)
6590                 return (rc);
6591
6592         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6593         if (sb == NULL)
6594                 return (ENOMEM);
6595
6596         t4_tp_get_err_stats(sc, &stats);
6597
6598         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6599                       "channel 3\n");
6600         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6601             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6602             stats.macInErrs[3]);
6603         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6604             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6605             stats.hdrInErrs[3]);
6606         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6607             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6608             stats.tcpInErrs[3]);
6609         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6610             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6611             stats.tcp6InErrs[3]);
6612         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6613             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6614             stats.tnlCongDrops[3]);
6615         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6616             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6617             stats.tnlTxDrops[3]);
6618         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6619             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6620             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6621         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6622             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6623             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6624         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6625             stats.ofldNoNeigh, stats.ofldCongDefer);
6626
6627         rc = sbuf_finish(sb);
6628         sbuf_delete(sb);
6629
6630         return (rc);
6631 }
6632
6633 struct field_desc {
6634         const char *name;
6635         u_int start;
6636         u_int width;
6637 };
6638
6639 static void
6640 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6641 {
6642         char buf[32];
6643         int line_size = 0;
6644
6645         while (f->name) {
6646                 uint64_t mask = (1ULL << f->width) - 1;
6647                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6648                     ((uintmax_t)v >> f->start) & mask);
6649
6650                 if (line_size + len >= 79) {
6651                         line_size = 8;
6652                         sbuf_printf(sb, "\n        ");
6653                 }
6654                 sbuf_printf(sb, "%s ", buf);
6655                 line_size += len + 1;
6656                 f++;
6657         }
6658         sbuf_printf(sb, "\n");
6659 }
6660
6661 static struct field_desc tp_la0[] = {
6662         { "RcfOpCodeOut", 60, 4 },
6663         { "State", 56, 4 },
6664         { "WcfState", 52, 4 },
6665         { "RcfOpcSrcOut", 50, 2 },
6666         { "CRxError", 49, 1 },
6667         { "ERxError", 48, 1 },
6668         { "SanityFailed", 47, 1 },
6669         { "SpuriousMsg", 46, 1 },
6670         { "FlushInputMsg", 45, 1 },
6671         { "FlushInputCpl", 44, 1 },
6672         { "RssUpBit", 43, 1 },
6673         { "RssFilterHit", 42, 1 },
6674         { "Tid", 32, 10 },
6675         { "InitTcb", 31, 1 },
6676         { "LineNumber", 24, 7 },
6677         { "Emsg", 23, 1 },
6678         { "EdataOut", 22, 1 },
6679         { "Cmsg", 21, 1 },
6680         { "CdataOut", 20, 1 },
6681         { "EreadPdu", 19, 1 },
6682         { "CreadPdu", 18, 1 },
6683         { "TunnelPkt", 17, 1 },
6684         { "RcfPeerFin", 16, 1 },
6685         { "RcfReasonOut", 12, 4 },
6686         { "TxCchannel", 10, 2 },
6687         { "RcfTxChannel", 8, 2 },
6688         { "RxEchannel", 6, 2 },
6689         { "RcfRxChannel", 5, 1 },
6690         { "RcfDataOutSrdy", 4, 1 },
6691         { "RxDvld", 3, 1 },
6692         { "RxOoDvld", 2, 1 },
6693         { "RxCongestion", 1, 1 },
6694         { "TxCongestion", 0, 1 },
6695         { NULL }
6696 };
6697
6698 static struct field_desc tp_la1[] = {
6699         { "CplCmdIn", 56, 8 },
6700         { "CplCmdOut", 48, 8 },
6701         { "ESynOut", 47, 1 },
6702         { "EAckOut", 46, 1 },
6703         { "EFinOut", 45, 1 },
6704         { "ERstOut", 44, 1 },
6705         { "SynIn", 43, 1 },
6706         { "AckIn", 42, 1 },
6707         { "FinIn", 41, 1 },
6708         { "RstIn", 40, 1 },
6709         { "DataIn", 39, 1 },
6710         { "DataInVld", 38, 1 },
6711         { "PadIn", 37, 1 },
6712         { "RxBufEmpty", 36, 1 },
6713         { "RxDdp", 35, 1 },
6714         { "RxFbCongestion", 34, 1 },
6715         { "TxFbCongestion", 33, 1 },
6716         { "TxPktSumSrdy", 32, 1 },
6717         { "RcfUlpType", 28, 4 },
6718         { "Eread", 27, 1 },
6719         { "Ebypass", 26, 1 },
6720         { "Esave", 25, 1 },
6721         { "Static0", 24, 1 },
6722         { "Cread", 23, 1 },
6723         { "Cbypass", 22, 1 },
6724         { "Csave", 21, 1 },
6725         { "CPktOut", 20, 1 },
6726         { "RxPagePoolFull", 18, 2 },
6727         { "RxLpbkPkt", 17, 1 },
6728         { "TxLpbkPkt", 16, 1 },
6729         { "RxVfValid", 15, 1 },
6730         { "SynLearned", 14, 1 },
6731         { "SetDelEntry", 13, 1 },
6732         { "SetInvEntry", 12, 1 },
6733         { "CpcmdDvld", 11, 1 },
6734         { "CpcmdSave", 10, 1 },
6735         { "RxPstructsFull", 8, 2 },
6736         { "EpcmdDvld", 7, 1 },
6737         { "EpcmdFlush", 6, 1 },
6738         { "EpcmdTrimPrefix", 5, 1 },
6739         { "EpcmdTrimPostfix", 4, 1 },
6740         { "ERssIp4Pkt", 3, 1 },
6741         { "ERssIp6Pkt", 2, 1 },
6742         { "ERssTcpUdpPkt", 1, 1 },
6743         { "ERssFceFipPkt", 0, 1 },
6744         { NULL }
6745 };
6746
6747 static struct field_desc tp_la2[] = {
6748         { "CplCmdIn", 56, 8 },
6749         { "MpsVfVld", 55, 1 },
6750         { "MpsPf", 52, 3 },
6751         { "MpsVf", 44, 8 },
6752         { "SynIn", 43, 1 },
6753         { "AckIn", 42, 1 },
6754         { "FinIn", 41, 1 },
6755         { "RstIn", 40, 1 },
6756         { "DataIn", 39, 1 },
6757         { "DataInVld", 38, 1 },
6758         { "PadIn", 37, 1 },
6759         { "RxBufEmpty", 36, 1 },
6760         { "RxDdp", 35, 1 },
6761         { "RxFbCongestion", 34, 1 },
6762         { "TxFbCongestion", 33, 1 },
6763         { "TxPktSumSrdy", 32, 1 },
6764         { "RcfUlpType", 28, 4 },
6765         { "Eread", 27, 1 },
6766         { "Ebypass", 26, 1 },
6767         { "Esave", 25, 1 },
6768         { "Static0", 24, 1 },
6769         { "Cread", 23, 1 },
6770         { "Cbypass", 22, 1 },
6771         { "Csave", 21, 1 },
6772         { "CPktOut", 20, 1 },
6773         { "RxPagePoolFull", 18, 2 },
6774         { "RxLpbkPkt", 17, 1 },
6775         { "TxLpbkPkt", 16, 1 },
6776         { "RxVfValid", 15, 1 },
6777         { "SynLearned", 14, 1 },
6778         { "SetDelEntry", 13, 1 },
6779         { "SetInvEntry", 12, 1 },
6780         { "CpcmdDvld", 11, 1 },
6781         { "CpcmdSave", 10, 1 },
6782         { "RxPstructsFull", 8, 2 },
6783         { "EpcmdDvld", 7, 1 },
6784         { "EpcmdFlush", 6, 1 },
6785         { "EpcmdTrimPrefix", 5, 1 },
6786         { "EpcmdTrimPostfix", 4, 1 },
6787         { "ERssIp4Pkt", 3, 1 },
6788         { "ERssIp6Pkt", 2, 1 },
6789         { "ERssTcpUdpPkt", 1, 1 },
6790         { "ERssFceFipPkt", 0, 1 },
6791         { NULL }
6792 };
6793
6794 static void
6795 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6796 {
6797
6798         field_desc_show(sb, *p, tp_la0);
6799 }
6800
6801 static void
6802 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6803 {
6804
6805         if (idx)
6806                 sbuf_printf(sb, "\n");
6807         field_desc_show(sb, p[0], tp_la0);
6808         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6809                 field_desc_show(sb, p[1], tp_la0);
6810 }
6811
6812 static void
6813 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6814 {
6815
6816         if (idx)
6817                 sbuf_printf(sb, "\n");
6818         field_desc_show(sb, p[0], tp_la0);
6819         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6820                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6821 }
6822
6823 static int
6824 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6825 {
6826         struct adapter *sc = arg1;
6827         struct sbuf *sb;
6828         uint64_t *buf, *p;
6829         int rc;
6830         u_int i, inc;
6831         void (*show_func)(struct sbuf *, uint64_t *, int);
6832
6833         rc = sysctl_wire_old_buffer(req, 0);
6834         if (rc != 0)
6835                 return (rc);
6836
6837         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6838         if (sb == NULL)
6839                 return (ENOMEM);
6840
6841         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6842
6843         t4_tp_read_la(sc, buf, NULL);
6844         p = buf;
6845
6846         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6847         case 2:
6848                 inc = 2;
6849                 show_func = tp_la_show2;
6850                 break;
6851         case 3:
6852                 inc = 2;
6853                 show_func = tp_la_show3;
6854                 break;
6855         default:
6856                 inc = 1;
6857                 show_func = tp_la_show;
6858         }
6859
6860         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6861                 (*show_func)(sb, p, i);
6862
6863         rc = sbuf_finish(sb);
6864         sbuf_delete(sb);
6865         free(buf, M_CXGBE);
6866         return (rc);
6867 }
6868
6869 static int
6870 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6871 {
6872         struct adapter *sc = arg1;
6873         struct sbuf *sb;
6874         int rc;
6875         u64 nrate[NCHAN], orate[NCHAN];
6876
6877         rc = sysctl_wire_old_buffer(req, 0);
6878         if (rc != 0)
6879                 return (rc);
6880
6881         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6882         if (sb == NULL)
6883                 return (ENOMEM);
6884
6885         t4_get_chan_txrate(sc, nrate, orate);
6886         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6887                  "channel 3\n");
6888         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6889             nrate[0], nrate[1], nrate[2], nrate[3]);
6890         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6891             orate[0], orate[1], orate[2], orate[3]);
6892
6893         rc = sbuf_finish(sb);
6894         sbuf_delete(sb);
6895
6896         return (rc);
6897 }
6898
6899 static int
6900 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6901 {
6902         struct adapter *sc = arg1;
6903         struct sbuf *sb;
6904         uint32_t *buf, *p;
6905         int rc, i;
6906
6907         rc = sysctl_wire_old_buffer(req, 0);
6908         if (rc != 0)
6909                 return (rc);
6910
6911         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6912         if (sb == NULL)
6913                 return (ENOMEM);
6914
6915         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6916             M_ZERO | M_WAITOK);
6917
6918         t4_ulprx_read_la(sc, buf);
6919         p = buf;
6920
6921         sbuf_printf(sb, "      Pcmd        Type   Message"
6922             "                Data");
6923         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6924                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6925                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6926         }
6927
6928         rc = sbuf_finish(sb);
6929         sbuf_delete(sb);
6930         free(buf, M_CXGBE);
6931         return (rc);
6932 }
6933
6934 static int
6935 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6936 {
6937         struct adapter *sc = arg1;
6938         struct sbuf *sb;
6939         int rc, v;
6940
6941         rc = sysctl_wire_old_buffer(req, 0);
6942         if (rc != 0)
6943                 return (rc);
6944
6945         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6946         if (sb == NULL)
6947                 return (ENOMEM);
6948
6949         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6950         if (G_STATSOURCE_T5(v) == 7) {
6951                 if (G_STATMODE(v) == 0) {
6952                         sbuf_printf(sb, "total %d, incomplete %d",
6953                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6954                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6955                 } else if (G_STATMODE(v) == 1) {
6956                         sbuf_printf(sb, "total %d, data overflow %d",
6957                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6958                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6959                 }
6960         }
6961         rc = sbuf_finish(sb);
6962         sbuf_delete(sb);
6963
6964         return (rc);
6965 }
6966 #endif
6967
6968 static uint32_t
6969 fconf_to_mode(uint32_t fconf)
6970 {
6971         uint32_t mode;
6972
6973         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6974             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6975
6976         if (fconf & F_FRAGMENTATION)
6977                 mode |= T4_FILTER_IP_FRAGMENT;
6978
6979         if (fconf & F_MPSHITTYPE)
6980                 mode |= T4_FILTER_MPS_HIT_TYPE;
6981
6982         if (fconf & F_MACMATCH)
6983                 mode |= T4_FILTER_MAC_IDX;
6984
6985         if (fconf & F_ETHERTYPE)
6986                 mode |= T4_FILTER_ETH_TYPE;
6987
6988         if (fconf & F_PROTOCOL)
6989                 mode |= T4_FILTER_IP_PROTO;
6990
6991         if (fconf & F_TOS)
6992                 mode |= T4_FILTER_IP_TOS;
6993
6994         if (fconf & F_VLAN)
6995                 mode |= T4_FILTER_VLAN;
6996
6997         if (fconf & F_VNIC_ID)
6998                 mode |= T4_FILTER_VNIC;
6999
7000         if (fconf & F_PORT)
7001                 mode |= T4_FILTER_PORT;
7002
7003         if (fconf & F_FCOE)
7004                 mode |= T4_FILTER_FCoE;
7005
7006         return (mode);
7007 }
7008
7009 static uint32_t
7010 mode_to_fconf(uint32_t mode)
7011 {
7012         uint32_t fconf = 0;
7013
7014         if (mode & T4_FILTER_IP_FRAGMENT)
7015                 fconf |= F_FRAGMENTATION;
7016
7017         if (mode & T4_FILTER_MPS_HIT_TYPE)
7018                 fconf |= F_MPSHITTYPE;
7019
7020         if (mode & T4_FILTER_MAC_IDX)
7021                 fconf |= F_MACMATCH;
7022
7023         if (mode & T4_FILTER_ETH_TYPE)
7024                 fconf |= F_ETHERTYPE;
7025
7026         if (mode & T4_FILTER_IP_PROTO)
7027                 fconf |= F_PROTOCOL;
7028
7029         if (mode & T4_FILTER_IP_TOS)
7030                 fconf |= F_TOS;
7031
7032         if (mode & T4_FILTER_VLAN)
7033                 fconf |= F_VLAN;
7034
7035         if (mode & T4_FILTER_VNIC)
7036                 fconf |= F_VNIC_ID;
7037
7038         if (mode & T4_FILTER_PORT)
7039                 fconf |= F_PORT;
7040
7041         if (mode & T4_FILTER_FCoE)
7042                 fconf |= F_FCOE;
7043
7044         return (fconf);
7045 }
7046
7047 static uint32_t
7048 fspec_to_fconf(struct t4_filter_specification *fs)
7049 {
7050         uint32_t fconf = 0;
7051
7052         if (fs->val.frag || fs->mask.frag)
7053                 fconf |= F_FRAGMENTATION;
7054
7055         if (fs->val.matchtype || fs->mask.matchtype)
7056                 fconf |= F_MPSHITTYPE;
7057
7058         if (fs->val.macidx || fs->mask.macidx)
7059                 fconf |= F_MACMATCH;
7060
7061         if (fs->val.ethtype || fs->mask.ethtype)
7062                 fconf |= F_ETHERTYPE;
7063
7064         if (fs->val.proto || fs->mask.proto)
7065                 fconf |= F_PROTOCOL;
7066
7067         if (fs->val.tos || fs->mask.tos)
7068                 fconf |= F_TOS;
7069
7070         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7071                 fconf |= F_VLAN;
7072
7073         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7074                 fconf |= F_VNIC_ID;
7075
7076         if (fs->val.iport || fs->mask.iport)
7077                 fconf |= F_PORT;
7078
7079         if (fs->val.fcoe || fs->mask.fcoe)
7080                 fconf |= F_FCOE;
7081
7082         return (fconf);
7083 }
7084
7085 static int
7086 get_filter_mode(struct adapter *sc, uint32_t *mode)
7087 {
7088         int rc;
7089         uint32_t fconf;
7090
7091         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7092             "t4getfm");
7093         if (rc)
7094                 return (rc);
7095
7096         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7097             A_TP_VLAN_PRI_MAP);
7098
7099         if (sc->params.tp.vlan_pri_map != fconf) {
7100                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7101                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7102                     fconf);
7103         }
7104
7105         *mode = fconf_to_mode(fconf);
7106
7107         end_synchronized_op(sc, LOCK_HELD);
7108         return (0);
7109 }
7110
7111 static int
7112 set_filter_mode(struct adapter *sc, uint32_t mode)
7113 {
7114         uint32_t fconf;
7115         int rc;
7116
7117         fconf = mode_to_fconf(mode);
7118
7119         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7120             "t4setfm");
7121         if (rc)
7122                 return (rc);
7123
7124         if (sc->tids.ftids_in_use > 0) {
7125                 rc = EBUSY;
7126                 goto done;
7127         }
7128
7129 #ifdef TCP_OFFLOAD
7130         if (uld_active(sc, ULD_TOM)) {
7131                 rc = EBUSY;
7132                 goto done;
7133         }
7134 #endif
7135
7136         rc = -t4_set_filter_mode(sc, fconf);
7137 done:
7138         end_synchronized_op(sc, LOCK_HELD);
7139         return (rc);
7140 }
7141
7142 static inline uint64_t
7143 get_filter_hits(struct adapter *sc, uint32_t fid)
7144 {
7145         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7146         uint64_t hits;
7147
7148         memwin_info(sc, 0, &mw_base, NULL);
7149         off = position_memwin(sc, 0,
7150             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7151         if (is_t4(sc)) {
7152                 hits = t4_read_reg64(sc, mw_base + off + 16);
7153                 hits = be64toh(hits);
7154         } else {
7155                 hits = t4_read_reg(sc, mw_base + off + 24);
7156                 hits = be32toh(hits);
7157         }
7158
7159         return (hits);
7160 }
7161
7162 static int
7163 get_filter(struct adapter *sc, struct t4_filter *t)
7164 {
7165         int i, rc, nfilters = sc->tids.nftids;
7166         struct filter_entry *f;
7167
7168         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7169             "t4getf");
7170         if (rc)
7171                 return (rc);
7172
7173         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7174             t->idx >= nfilters) {
7175                 t->idx = 0xffffffff;
7176                 goto done;
7177         }
7178
7179         f = &sc->tids.ftid_tab[t->idx];
7180         for (i = t->idx; i < nfilters; i++, f++) {
7181                 if (f->valid) {
7182                         t->idx = i;
7183                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7184                         t->smtidx = f->smtidx;
7185                         if (f->fs.hitcnts)
7186                                 t->hits = get_filter_hits(sc, t->idx);
7187                         else
7188                                 t->hits = UINT64_MAX;
7189                         t->fs = f->fs;
7190
7191                         goto done;
7192                 }
7193         }
7194
7195         t->idx = 0xffffffff;
7196 done:
7197         end_synchronized_op(sc, LOCK_HELD);
7198         return (0);
7199 }
7200
7201 static int
7202 set_filter(struct adapter *sc, struct t4_filter *t)
7203 {
7204         unsigned int nfilters, nports;
7205         struct filter_entry *f;
7206         int i, rc;
7207
7208         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7209         if (rc)
7210                 return (rc);
7211
7212         nfilters = sc->tids.nftids;
7213         nports = sc->params.nports;
7214
7215         if (nfilters == 0) {
7216                 rc = ENOTSUP;
7217                 goto done;
7218         }
7219
7220         if (!(sc->flags & FULL_INIT_DONE)) {
7221                 rc = EAGAIN;
7222                 goto done;
7223         }
7224
7225         if (t->idx >= nfilters) {
7226                 rc = EINVAL;
7227                 goto done;
7228         }
7229
7230         /* Validate against the global filter mode */
7231         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7232             sc->params.tp.vlan_pri_map) {
7233                 rc = E2BIG;
7234                 goto done;
7235         }
7236
7237         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7238                 rc = EINVAL;
7239                 goto done;
7240         }
7241
7242         if (t->fs.val.iport >= nports) {
7243                 rc = EINVAL;
7244                 goto done;
7245         }
7246
7247         /* Can't specify an iq if not steering to it */
7248         if (!t->fs.dirsteer && t->fs.iq) {
7249                 rc = EINVAL;
7250                 goto done;
7251         }
7252
7253         /* IPv6 filter idx must be 4 aligned */
7254         if (t->fs.type == 1 &&
7255             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7256                 rc = EINVAL;
7257                 goto done;
7258         }
7259
7260         if (sc->tids.ftid_tab == NULL) {
7261                 KASSERT(sc->tids.ftids_in_use == 0,
7262                     ("%s: no memory allocated but filters_in_use > 0",
7263                     __func__));
7264
7265                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7266                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7267                 if (sc->tids.ftid_tab == NULL) {
7268                         rc = ENOMEM;
7269                         goto done;
7270                 }
7271                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7272         }
7273
7274         for (i = 0; i < 4; i++) {
7275                 f = &sc->tids.ftid_tab[t->idx + i];
7276
7277                 if (f->pending || f->valid) {
7278                         rc = EBUSY;
7279                         goto done;
7280                 }
7281                 if (f->locked) {
7282                         rc = EPERM;
7283                         goto done;
7284                 }
7285
7286                 if (t->fs.type == 0)
7287                         break;
7288         }
7289
7290         f = &sc->tids.ftid_tab[t->idx];
7291         f->fs = t->fs;
7292
7293         rc = set_filter_wr(sc, t->idx);
7294 done:
7295         end_synchronized_op(sc, 0);
7296
7297         if (rc == 0) {
7298                 mtx_lock(&sc->tids.ftid_lock);
7299                 for (;;) {
7300                         if (f->pending == 0) {
7301                                 rc = f->valid ? 0 : EIO;
7302                                 break;
7303                         }
7304
7305                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7306                             PCATCH, "t4setfw", 0)) {
7307                                 rc = EINPROGRESS;
7308                                 break;
7309                         }
7310                 }
7311                 mtx_unlock(&sc->tids.ftid_lock);
7312         }
7313         return (rc);
7314 }
7315
7316 static int
7317 del_filter(struct adapter *sc, struct t4_filter *t)
7318 {
7319         unsigned int nfilters;
7320         struct filter_entry *f;
7321         int rc;
7322
7323         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7324         if (rc)
7325                 return (rc);
7326
7327         nfilters = sc->tids.nftids;
7328
7329         if (nfilters == 0) {
7330                 rc = ENOTSUP;
7331                 goto done;
7332         }
7333
7334         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7335             t->idx >= nfilters) {
7336                 rc = EINVAL;
7337                 goto done;
7338         }
7339
7340         if (!(sc->flags & FULL_INIT_DONE)) {
7341                 rc = EAGAIN;
7342                 goto done;
7343         }
7344
7345         f = &sc->tids.ftid_tab[t->idx];
7346
7347         if (f->pending) {
7348                 rc = EBUSY;
7349                 goto done;
7350         }
7351         if (f->locked) {
7352                 rc = EPERM;
7353                 goto done;
7354         }
7355
7356         if (f->valid) {
7357                 t->fs = f->fs;  /* extra info for the caller */
7358                 rc = del_filter_wr(sc, t->idx);
7359         }
7360
7361 done:
7362         end_synchronized_op(sc, 0);
7363
7364         if (rc == 0) {
7365                 mtx_lock(&sc->tids.ftid_lock);
7366                 for (;;) {
7367                         if (f->pending == 0) {
7368                                 rc = f->valid ? EIO : 0;
7369                                 break;
7370                         }
7371
7372                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7373                             PCATCH, "t4delfw", 0)) {
7374                                 rc = EINPROGRESS;
7375                                 break;
7376                         }
7377                 }
7378                 mtx_unlock(&sc->tids.ftid_lock);
7379         }
7380
7381         return (rc);
7382 }
7383
7384 static void
7385 clear_filter(struct filter_entry *f)
7386 {
7387         if (f->l2t)
7388                 t4_l2t_release(f->l2t);
7389
7390         bzero(f, sizeof (*f));
7391 }
7392
7393 static int
7394 set_filter_wr(struct adapter *sc, int fidx)
7395 {
7396         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7397         struct fw_filter_wr *fwr;
7398         unsigned int ftid;
7399         struct wrq_cookie cookie;
7400
7401         ASSERT_SYNCHRONIZED_OP(sc);
7402
7403         if (f->fs.newdmac || f->fs.newvlan) {
7404                 /* This filter needs an L2T entry; allocate one. */
7405                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7406                 if (f->l2t == NULL)
7407                         return (EAGAIN);
7408                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7409                     f->fs.dmac)) {
7410                         t4_l2t_release(f->l2t);
7411                         f->l2t = NULL;
7412                         return (ENOMEM);
7413                 }
7414         }
7415
7416         ftid = sc->tids.ftid_base + fidx;
7417
7418         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7419         if (fwr == NULL)
7420                 return (ENOMEM);
7421         bzero(fwr, sizeof(*fwr));
7422
7423         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7424         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7425         fwr->tid_to_iq =
7426             htobe32(V_FW_FILTER_WR_TID(ftid) |
7427                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7428                 V_FW_FILTER_WR_NOREPLY(0) |
7429                 V_FW_FILTER_WR_IQ(f->fs.iq));
7430         fwr->del_filter_to_l2tix =
7431             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7432                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7433                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7434                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7435                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7436                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7437                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7438                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7439                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7440                     f->fs.newvlan == VLAN_REWRITE) |
7441                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7442                     f->fs.newvlan == VLAN_REWRITE) |
7443                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7444                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7445                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7446                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7447         fwr->ethtype = htobe16(f->fs.val.ethtype);
7448         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7449         fwr->frag_to_ovlan_vldm =
7450             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7451                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7452                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7453                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7454                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7455                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7456         fwr->smac_sel = 0;
7457         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7458             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7459         fwr->maci_to_matchtypem =
7460             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7461                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7462                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7463                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7464                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7465                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7466                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7467                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7468         fwr->ptcl = f->fs.val.proto;
7469         fwr->ptclm = f->fs.mask.proto;
7470         fwr->ttyp = f->fs.val.tos;
7471         fwr->ttypm = f->fs.mask.tos;
7472         fwr->ivlan = htobe16(f->fs.val.vlan);
7473         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7474         fwr->ovlan = htobe16(f->fs.val.vnic);
7475         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7476         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7477         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7478         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7479         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7480         fwr->lp = htobe16(f->fs.val.dport);
7481         fwr->lpm = htobe16(f->fs.mask.dport);
7482         fwr->fp = htobe16(f->fs.val.sport);
7483         fwr->fpm = htobe16(f->fs.mask.sport);
7484         if (f->fs.newsmac)
7485                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7486
7487         f->pending = 1;
7488         sc->tids.ftids_in_use++;
7489
7490         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7491         return (0);
7492 }
7493
7494 static int
7495 del_filter_wr(struct adapter *sc, int fidx)
7496 {
7497         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7498         struct fw_filter_wr *fwr;
7499         unsigned int ftid;
7500         struct wrq_cookie cookie;
7501
7502         ftid = sc->tids.ftid_base + fidx;
7503
7504         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7505         if (fwr == NULL)
7506                 return (ENOMEM);
7507         bzero(fwr, sizeof (*fwr));
7508
7509         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7510
7511         f->pending = 1;
7512         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7513         return (0);
7514 }
7515
7516 int
7517 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7518 {
7519         struct adapter *sc = iq->adapter;
7520         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7521         unsigned int idx = GET_TID(rpl);
7522         unsigned int rc;
7523         struct filter_entry *f;
7524
7525         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7526             rss->opcode));
7527
7528         if (is_ftid(sc, idx)) {
7529
7530                 idx -= sc->tids.ftid_base;
7531                 f = &sc->tids.ftid_tab[idx];
7532                 rc = G_COOKIE(rpl->cookie);
7533
7534                 mtx_lock(&sc->tids.ftid_lock);
7535                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7536                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7537                             __func__, idx));
7538                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7539                         f->pending = 0;  /* asynchronous setup completed */
7540                         f->valid = 1;
7541                 } else {
7542                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7543                                 /* Add or delete failed, display an error */
7544                                 log(LOG_ERR,
7545                                     "filter %u setup failed with error %u\n",
7546                                     idx, rc);
7547                         }
7548
7549                         clear_filter(f);
7550                         sc->tids.ftids_in_use--;
7551                 }
7552                 wakeup(&sc->tids.ftid_tab);
7553                 mtx_unlock(&sc->tids.ftid_lock);
7554         }
7555
7556         return (0);
7557 }
7558
7559 static int
7560 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7561 {
7562         int rc;
7563
7564         if (cntxt->cid > M_CTXTQID)
7565                 return (EINVAL);
7566
7567         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7568             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7569                 return (EINVAL);
7570
7571         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7572         if (rc)
7573                 return (rc);
7574
7575         if (sc->flags & FW_OK) {
7576                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7577                     &cntxt->data[0]);
7578                 if (rc == 0)
7579                         goto done;
7580         }
7581
7582         /*
7583          * Read via firmware failed or wasn't even attempted.  Read directly via
7584          * the backdoor.
7585          */
7586         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7587 done:
7588         end_synchronized_op(sc, 0);
7589         return (rc);
7590 }
7591
7592 static int
7593 load_fw(struct adapter *sc, struct t4_data *fw)
7594 {
7595         int rc;
7596         uint8_t *fw_data;
7597
7598         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7599         if (rc)
7600                 return (rc);
7601
7602         if (sc->flags & FULL_INIT_DONE) {
7603                 rc = EBUSY;
7604                 goto done;
7605         }
7606
7607         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7608         if (fw_data == NULL) {
7609                 rc = ENOMEM;
7610                 goto done;
7611         }
7612
7613         rc = copyin(fw->data, fw_data, fw->len);
7614         if (rc == 0)
7615                 rc = -t4_load_fw(sc, fw_data, fw->len);
7616
7617         free(fw_data, M_CXGBE);
7618 done:
7619         end_synchronized_op(sc, 0);
7620         return (rc);
7621 }
7622
7623 static int
7624 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7625 {
7626         uint32_t addr, off, remaining, i, n;
7627         uint32_t *buf, *b;
7628         uint32_t mw_base, mw_aperture;
7629         int rc;
7630         uint8_t *dst;
7631
7632         rc = validate_mem_range(sc, mr->addr, mr->len);
7633         if (rc != 0)
7634                 return (rc);
7635
7636         memwin_info(sc, win, &mw_base, &mw_aperture);
7637         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7638         addr = mr->addr;
7639         remaining = mr->len;
7640         dst = (void *)mr->data;
7641
7642         while (remaining) {
7643                 off = position_memwin(sc, win, addr);
7644
7645                 /* number of bytes that we'll copy in the inner loop */
7646                 n = min(remaining, mw_aperture - off);
7647                 for (i = 0; i < n; i += 4)
7648                         *b++ = t4_read_reg(sc, mw_base + off + i);
7649
7650                 rc = copyout(buf, dst, n);
7651                 if (rc != 0)
7652                         break;
7653
7654                 b = buf;
7655                 dst += n;
7656                 remaining -= n;
7657                 addr += n;
7658         }
7659
7660         free(buf, M_CXGBE);
7661         return (rc);
7662 }
7663
7664 static int
7665 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7666 {
7667         int rc;
7668
7669         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7670                 return (EINVAL);
7671
7672         if (i2cd->len > sizeof(i2cd->data))
7673                 return (EFBIG);
7674
7675         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7676         if (rc)
7677                 return (rc);
7678         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7679             i2cd->offset, i2cd->len, &i2cd->data[0]);
7680         end_synchronized_op(sc, 0);
7681
7682         return (rc);
7683 }
7684
7685 static int
7686 in_range(int val, int lo, int hi)
7687 {
7688
7689         return (val < 0 || (val <= hi && val >= lo));
7690 }
7691
7692 static int
7693 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7694 {
7695         int fw_subcmd, fw_type, rc;
7696
7697         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7698         if (rc)
7699                 return (rc);
7700
7701         if (!(sc->flags & FULL_INIT_DONE)) {
7702                 rc = EAGAIN;
7703                 goto done;
7704         }
7705
7706         /*
7707          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7708          * sub-command and type are in common locations.)
7709          */
7710         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7711                 fw_subcmd = FW_SCHED_SC_CONFIG;
7712         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7713                 fw_subcmd = FW_SCHED_SC_PARAMS;
7714         else {
7715                 rc = EINVAL;
7716                 goto done;
7717         }
7718         if (p->type == SCHED_CLASS_TYPE_PACKET)
7719                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7720         else {
7721                 rc = EINVAL;
7722                 goto done;
7723         }
7724
7725         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7726                 /* Vet our parameters ..*/
7727                 if (p->u.config.minmax < 0) {
7728                         rc = EINVAL;
7729                         goto done;
7730                 }
7731
7732                 /* And pass the request to the firmware ...*/
7733                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7734                 goto done;
7735         }
7736
7737         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7738                 int fw_level;
7739                 int fw_mode;
7740                 int fw_rateunit;
7741                 int fw_ratemode;
7742
7743                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7744                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7745                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7746                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7747                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7748                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7749                 else {
7750                         rc = EINVAL;
7751                         goto done;
7752                 }
7753
7754                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7755                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7756                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7757                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7758                 else {
7759                         rc = EINVAL;
7760                         goto done;
7761                 }
7762
7763                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7764                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7765                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7766                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7767                 else {
7768                         rc = EINVAL;
7769                         goto done;
7770                 }
7771
7772                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7773                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7774                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7775                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7776                 else {
7777                         rc = EINVAL;
7778                         goto done;
7779                 }
7780
7781                 /* Vet our parameters ... */
7782                 if (!in_range(p->u.params.channel, 0, 3) ||
7783                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7784                     !in_range(p->u.params.minrate, 0, 10000000) ||
7785                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7786                     !in_range(p->u.params.weight, 0, 100)) {
7787                         rc = ERANGE;
7788                         goto done;
7789                 }
7790
7791                 /*
7792                  * Translate any unset parameters into the firmware's
7793                  * nomenclature and/or fail the call if the parameters
7794                  * are required ...
7795                  */
7796                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7797                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7798                         rc = EINVAL;
7799                         goto done;
7800                 }
7801                 if (p->u.params.minrate < 0)
7802                         p->u.params.minrate = 0;
7803                 if (p->u.params.maxrate < 0) {
7804                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7805                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7806                                 rc = EINVAL;
7807                                 goto done;
7808                         } else
7809                                 p->u.params.maxrate = 0;
7810                 }
7811                 if (p->u.params.weight < 0) {
7812                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7813                                 rc = EINVAL;
7814                                 goto done;
7815                         } else
7816                                 p->u.params.weight = 0;
7817                 }
7818                 if (p->u.params.pktsize < 0) {
7819                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7820                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7821                                 rc = EINVAL;
7822                                 goto done;
7823                         } else
7824                                 p->u.params.pktsize = 0;
7825                 }
7826
7827                 /* See what the firmware thinks of the request ... */
7828                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7829                     fw_rateunit, fw_ratemode, p->u.params.channel,
7830                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7831                     p->u.params.weight, p->u.params.pktsize, 1);
7832                 goto done;
7833         }
7834
7835         rc = EINVAL;
7836 done:
7837         end_synchronized_op(sc, 0);
7838         return (rc);
7839 }
7840
7841 static int
7842 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7843 {
7844         struct port_info *pi = NULL;
7845         struct sge_txq *txq;
7846         uint32_t fw_mnem, fw_queue, fw_class;
7847         int i, rc;
7848
7849         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7850         if (rc)
7851                 return (rc);
7852
7853         if (!(sc->flags & FULL_INIT_DONE)) {
7854                 rc = EAGAIN;
7855                 goto done;
7856         }
7857
7858         if (p->port >= sc->params.nports) {
7859                 rc = EINVAL;
7860                 goto done;
7861         }
7862
7863         pi = sc->port[p->port];
7864         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7865                 rc = EINVAL;
7866                 goto done;
7867         }
7868
7869         /*
7870          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7871          * Scheduling Class in this case).
7872          */
7873         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7874             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7875         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7876
7877         /*
7878          * If op.queue is non-negative, then we're only changing the scheduling
7879          * on a single specified TX queue.
7880          */
7881         if (p->queue >= 0) {
7882                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7883                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7884                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7885                     &fw_class);
7886                 goto done;
7887         }
7888
7889         /*
7890          * Change the scheduling on all the TX queues for the
7891          * interface.
7892          */
7893         for_each_txq(pi, i, txq) {
7894                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7895                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7896                     &fw_class);
7897                 if (rc)
7898                         goto done;
7899         }
7900
7901         rc = 0;
7902 done:
7903         end_synchronized_op(sc, 0);
7904         return (rc);
7905 }
7906
7907 int
7908 t4_os_find_pci_capability(struct adapter *sc, int cap)
7909 {
7910         int i;
7911
7912         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7913 }
7914
7915 int
7916 t4_os_pci_save_state(struct adapter *sc)
7917 {
7918         device_t dev;
7919         struct pci_devinfo *dinfo;
7920
7921         dev = sc->dev;
7922         dinfo = device_get_ivars(dev);
7923
7924         pci_cfg_save(dev, dinfo, 0);
7925         return (0);
7926 }
7927
7928 int
7929 t4_os_pci_restore_state(struct adapter *sc)
7930 {
7931         device_t dev;
7932         struct pci_devinfo *dinfo;
7933
7934         dev = sc->dev;
7935         dinfo = device_get_ivars(dev);
7936
7937         pci_cfg_restore(dev, dinfo);
7938         return (0);
7939 }
7940
7941 void
7942 t4_os_portmod_changed(const struct adapter *sc, int idx)
7943 {
7944         struct port_info *pi = sc->port[idx];
7945         static const char *mod_str[] = {
7946                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7947         };
7948
7949         build_medialist(pi, &pi->media);
7950 #ifdef DEV_NETMAP
7951         build_medialist(pi, &pi->nm_media);
7952 #endif
7953
7954         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7955                 if_printf(pi->ifp, "transceiver unplugged.\n");
7956         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7957                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7958         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7959                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7960         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7961                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7962                     mod_str[pi->mod_type]);
7963         } else {
7964                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7965                     pi->mod_type);
7966         }
7967 }
7968
7969 void
7970 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7971 {
7972         struct port_info *pi = sc->port[idx];
7973         struct ifnet *ifp = pi->ifp;
7974
7975         if (link_stat) {
7976                 pi->linkdnrc = -1;
7977                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7978                 if_link_state_change(ifp, LINK_STATE_UP);
7979         } else {
7980                 if (reason >= 0)
7981                         pi->linkdnrc = reason;
7982                 if_link_state_change(ifp, LINK_STATE_DOWN);
7983         }
7984 }
7985
7986 void
7987 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7988 {
7989         struct adapter *sc;
7990
7991         sx_slock(&t4_list_lock);
7992         SLIST_FOREACH(sc, &t4_list, link) {
7993                 /*
7994                  * func should not make any assumptions about what state sc is
7995                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7996                  */
7997                 func(sc, arg);
7998         }
7999         sx_sunlock(&t4_list_lock);
8000 }
8001
8002 static int
8003 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8004 {
8005        return (0);
8006 }
8007
8008 static int
8009 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8010 {
8011        return (0);
8012 }
8013
8014 static int
8015 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8016     struct thread *td)
8017 {
8018         int rc;
8019         struct adapter *sc = dev->si_drv1;
8020
8021         rc = priv_check(td, PRIV_DRIVER);
8022         if (rc != 0)
8023                 return (rc);
8024
8025         switch (cmd) {
8026         case CHELSIO_T4_GETREG: {
8027                 struct t4_reg *edata = (struct t4_reg *)data;
8028
8029                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8030                         return (EFAULT);
8031
8032                 if (edata->size == 4)
8033                         edata->val = t4_read_reg(sc, edata->addr);
8034                 else if (edata->size == 8)
8035                         edata->val = t4_read_reg64(sc, edata->addr);
8036                 else
8037                         return (EINVAL);
8038
8039                 break;
8040         }
8041         case CHELSIO_T4_SETREG: {
8042                 struct t4_reg *edata = (struct t4_reg *)data;
8043
8044                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8045                         return (EFAULT);
8046
8047                 if (edata->size == 4) {
8048                         if (edata->val & 0xffffffff00000000)
8049                                 return (EINVAL);
8050                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8051                 } else if (edata->size == 8)
8052                         t4_write_reg64(sc, edata->addr, edata->val);
8053                 else
8054                         return (EINVAL);
8055                 break;
8056         }
8057         case CHELSIO_T4_REGDUMP: {
8058                 struct t4_regdump *regs = (struct t4_regdump *)data;
8059                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8060                 uint8_t *buf;
8061
8062                 if (regs->len < reglen) {
8063                         regs->len = reglen; /* hint to the caller */
8064                         return (ENOBUFS);
8065                 }
8066
8067                 regs->len = reglen;
8068                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8069                 t4_get_regs(sc, regs, buf);
8070                 rc = copyout(buf, regs->data, reglen);
8071                 free(buf, M_CXGBE);
8072                 break;
8073         }
8074         case CHELSIO_T4_GET_FILTER_MODE:
8075                 rc = get_filter_mode(sc, (uint32_t *)data);
8076                 break;
8077         case CHELSIO_T4_SET_FILTER_MODE:
8078                 rc = set_filter_mode(sc, *(uint32_t *)data);
8079                 break;
8080         case CHELSIO_T4_GET_FILTER:
8081                 rc = get_filter(sc, (struct t4_filter *)data);
8082                 break;
8083         case CHELSIO_T4_SET_FILTER:
8084                 rc = set_filter(sc, (struct t4_filter *)data);
8085                 break;
8086         case CHELSIO_T4_DEL_FILTER:
8087                 rc = del_filter(sc, (struct t4_filter *)data);
8088                 break;
8089         case CHELSIO_T4_GET_SGE_CONTEXT:
8090                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8091                 break;
8092         case CHELSIO_T4_LOAD_FW:
8093                 rc = load_fw(sc, (struct t4_data *)data);
8094                 break;
8095         case CHELSIO_T4_GET_MEM:
8096                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8097                 break;
8098         case CHELSIO_T4_GET_I2C:
8099                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8100                 break;
8101         case CHELSIO_T4_CLEAR_STATS: {
8102                 int i;
8103                 u_int port_id = *(uint32_t *)data;
8104                 struct port_info *pi;
8105
8106                 if (port_id >= sc->params.nports)
8107                         return (EINVAL);
8108                 pi = sc->port[port_id];
8109
8110                 /* MAC stats */
8111                 t4_clr_port_stats(sc, pi->tx_chan);
8112                 pi->tx_parse_error = 0;
8113
8114                 if (pi->flags & PORT_INIT_DONE) {
8115                         struct sge_rxq *rxq;
8116                         struct sge_txq *txq;
8117                         struct sge_wrq *wrq;
8118
8119                         for_each_rxq(pi, i, rxq) {
8120 #if defined(INET) || defined(INET6)
8121                                 rxq->lro.lro_queued = 0;
8122                                 rxq->lro.lro_flushed = 0;
8123 #endif
8124                                 rxq->rxcsum = 0;
8125                                 rxq->vlan_extraction = 0;
8126                         }
8127
8128                         for_each_txq(pi, i, txq) {
8129                                 txq->txcsum = 0;
8130                                 txq->tso_wrs = 0;
8131                                 txq->vlan_insertion = 0;
8132                                 txq->imm_wrs = 0;
8133                                 txq->sgl_wrs = 0;
8134                                 txq->txpkt_wrs = 0;
8135                                 txq->txpkts0_wrs = 0;
8136                                 txq->txpkts1_wrs = 0;
8137                                 txq->txpkts0_pkts = 0;
8138                                 txq->txpkts1_pkts = 0;
8139                                 mp_ring_reset_stats(txq->r);
8140                         }
8141
8142 #ifdef TCP_OFFLOAD
8143                         /* nothing to clear for each ofld_rxq */
8144
8145                         for_each_ofld_txq(pi, i, wrq) {
8146                                 wrq->tx_wrs_direct = 0;
8147                                 wrq->tx_wrs_copied = 0;
8148                         }
8149 #endif
8150                         wrq = &sc->sge.ctrlq[pi->port_id];
8151                         wrq->tx_wrs_direct = 0;
8152                         wrq->tx_wrs_copied = 0;
8153                 }
8154                 break;
8155         }
8156         case CHELSIO_T4_SCHED_CLASS:
8157                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8158                 break;
8159         case CHELSIO_T4_SCHED_QUEUE:
8160                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8161                 break;
8162         case CHELSIO_T4_GET_TRACER:
8163                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8164                 break;
8165         case CHELSIO_T4_SET_TRACER:
8166                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8167                 break;
8168         default:
8169                 rc = EINVAL;
8170         }
8171
8172         return (rc);
8173 }
8174
8175 #ifdef TCP_OFFLOAD
8176 void
8177 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8178     const unsigned int *pgsz_order)
8179 {
8180         struct port_info *pi = ifp->if_softc;
8181         struct adapter *sc = pi->adapter;
8182
8183         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8184         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8185                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8186                 V_HPZ3(pgsz_order[3]));
8187 }
8188
8189 static int
8190 toe_capability(struct port_info *pi, int enable)
8191 {
8192         int rc;
8193         struct adapter *sc = pi->adapter;
8194
8195         ASSERT_SYNCHRONIZED_OP(sc);
8196
8197         if (!is_offload(sc))
8198                 return (ENODEV);
8199
8200         if (enable) {
8201                 /*
8202                  * We need the port's queues around so that we're able to send
8203                  * and receive CPLs to/from the TOE even if the ifnet for this
8204                  * port has never been UP'd administratively.
8205                  */
8206                 if (!(pi->flags & PORT_INIT_DONE)) {
8207                         rc = cxgbe_init_synchronized(pi);
8208                         if (rc)
8209                                 return (rc);
8210                 }
8211
8212                 if (isset(&sc->offload_map, pi->port_id))
8213                         return (0);
8214
8215                 if (!uld_active(sc, ULD_TOM)) {
8216                         rc = t4_activate_uld(sc, ULD_TOM);
8217                         if (rc == EAGAIN) {
8218                                 log(LOG_WARNING,
8219                                     "You must kldload t4_tom.ko before trying "
8220                                     "to enable TOE on a cxgbe interface.\n");
8221                         }
8222                         if (rc != 0)
8223                                 return (rc);
8224                         KASSERT(sc->tom_softc != NULL,
8225                             ("%s: TOM activated but softc NULL", __func__));
8226                         KASSERT(uld_active(sc, ULD_TOM),
8227                             ("%s: TOM activated but flag not set", __func__));
8228                 }
8229
8230                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8231                 if (!uld_active(sc, ULD_IWARP))
8232                         (void) t4_activate_uld(sc, ULD_IWARP);
8233                 if (!uld_active(sc, ULD_ISCSI))
8234                         (void) t4_activate_uld(sc, ULD_ISCSI);
8235
8236                 setbit(&sc->offload_map, pi->port_id);
8237         } else {
8238                 if (!isset(&sc->offload_map, pi->port_id))
8239                         return (0);
8240
8241                 KASSERT(uld_active(sc, ULD_TOM),
8242                     ("%s: TOM never initialized?", __func__));
8243                 clrbit(&sc->offload_map, pi->port_id);
8244         }
8245
8246         return (0);
8247 }
8248
8249 /*
8250  * Add an upper layer driver to the global list.
8251  */
8252 int
8253 t4_register_uld(struct uld_info *ui)
8254 {
8255         int rc = 0;
8256         struct uld_info *u;
8257
8258         sx_xlock(&t4_uld_list_lock);
8259         SLIST_FOREACH(u, &t4_uld_list, link) {
8260             if (u->uld_id == ui->uld_id) {
8261                     rc = EEXIST;
8262                     goto done;
8263             }
8264         }
8265
8266         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8267         ui->refcount = 0;
8268 done:
8269         sx_xunlock(&t4_uld_list_lock);
8270         return (rc);
8271 }
8272
8273 int
8274 t4_unregister_uld(struct uld_info *ui)
8275 {
8276         int rc = EINVAL;
8277         struct uld_info *u;
8278
8279         sx_xlock(&t4_uld_list_lock);
8280
8281         SLIST_FOREACH(u, &t4_uld_list, link) {
8282             if (u == ui) {
8283                     if (ui->refcount > 0) {
8284                             rc = EBUSY;
8285                             goto done;
8286                     }
8287
8288                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8289                     rc = 0;
8290                     goto done;
8291             }
8292         }
8293 done:
8294         sx_xunlock(&t4_uld_list_lock);
8295         return (rc);
8296 }
8297
8298 int
8299 t4_activate_uld(struct adapter *sc, int id)
8300 {
8301         int rc;
8302         struct uld_info *ui;
8303
8304         ASSERT_SYNCHRONIZED_OP(sc);
8305
8306         if (id < 0 || id > ULD_MAX)
8307                 return (EINVAL);
8308         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8309
8310         sx_slock(&t4_uld_list_lock);
8311
8312         SLIST_FOREACH(ui, &t4_uld_list, link) {
8313                 if (ui->uld_id == id) {
8314                         if (!(sc->flags & FULL_INIT_DONE)) {
8315                                 rc = adapter_full_init(sc);
8316                                 if (rc != 0)
8317                                         break;
8318                         }
8319
8320                         rc = ui->activate(sc);
8321                         if (rc == 0) {
8322                                 setbit(&sc->active_ulds, id);
8323                                 ui->refcount++;
8324                         }
8325                         break;
8326                 }
8327         }
8328
8329         sx_sunlock(&t4_uld_list_lock);
8330
8331         return (rc);
8332 }
8333
8334 int
8335 t4_deactivate_uld(struct adapter *sc, int id)
8336 {
8337         int rc;
8338         struct uld_info *ui;
8339
8340         ASSERT_SYNCHRONIZED_OP(sc);
8341
8342         if (id < 0 || id > ULD_MAX)
8343                 return (EINVAL);
8344         rc = ENXIO;
8345
8346         sx_slock(&t4_uld_list_lock);
8347
8348         SLIST_FOREACH(ui, &t4_uld_list, link) {
8349                 if (ui->uld_id == id) {
8350                         rc = ui->deactivate(sc);
8351                         if (rc == 0) {
8352                                 clrbit(&sc->active_ulds, id);
8353                                 ui->refcount--;
8354                         }
8355                         break;
8356                 }
8357         }
8358
8359         sx_sunlock(&t4_uld_list_lock);
8360
8361         return (rc);
8362 }
8363
8364 int
8365 uld_active(struct adapter *sc, int uld_id)
8366 {
8367
8368         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8369
8370         return (isset(&sc->active_ulds, uld_id));
8371 }
8372 #endif
8373
8374 /*
8375  * Come up with reasonable defaults for some of the tunables, provided they're
8376  * not set by the user (in which case we'll use the values as is).
8377  */
8378 static void
8379 tweak_tunables(void)
8380 {
8381         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8382
8383         if (t4_ntxq10g < 1)
8384                 t4_ntxq10g = min(nc, NTXQ_10G);
8385
8386         if (t4_ntxq1g < 1)
8387                 t4_ntxq1g = min(nc, NTXQ_1G);
8388
8389         if (t4_nrxq10g < 1)
8390                 t4_nrxq10g = min(nc, NRXQ_10G);
8391
8392         if (t4_nrxq1g < 1)
8393                 t4_nrxq1g = min(nc, NRXQ_1G);
8394
8395 #ifdef TCP_OFFLOAD
8396         if (t4_nofldtxq10g < 1)
8397                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8398
8399         if (t4_nofldtxq1g < 1)
8400                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8401
8402         if (t4_nofldrxq10g < 1)
8403                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8404
8405         if (t4_nofldrxq1g < 1)
8406                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8407
8408         if (t4_toecaps_allowed == -1)
8409                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8410 #else
8411         if (t4_toecaps_allowed == -1)
8412                 t4_toecaps_allowed = 0;
8413 #endif
8414
8415 #ifdef DEV_NETMAP
8416         if (t4_nnmtxq10g < 1)
8417                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8418
8419         if (t4_nnmtxq1g < 1)
8420                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8421
8422         if (t4_nnmrxq10g < 1)
8423                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8424
8425         if (t4_nnmrxq1g < 1)
8426                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8427 #endif
8428
8429         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8430                 t4_tmr_idx_10g = TMR_IDX_10G;
8431
8432         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8433                 t4_pktc_idx_10g = PKTC_IDX_10G;
8434
8435         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8436                 t4_tmr_idx_1g = TMR_IDX_1G;
8437
8438         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8439                 t4_pktc_idx_1g = PKTC_IDX_1G;
8440
8441         if (t4_qsize_txq < 128)
8442                 t4_qsize_txq = 128;
8443
8444         if (t4_qsize_rxq < 128)
8445                 t4_qsize_rxq = 128;
8446         while (t4_qsize_rxq & 7)
8447                 t4_qsize_rxq++;
8448
8449         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8450 }
8451
8452 static struct sx mlu;   /* mod load unload */
8453 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8454
8455 static int
8456 mod_event(module_t mod, int cmd, void *arg)
8457 {
8458         int rc = 0;
8459         static int loaded = 0;
8460
8461         switch (cmd) {
8462         case MOD_LOAD:
8463                 sx_xlock(&mlu);
8464                 if (loaded++ == 0) {
8465                         t4_sge_modload();
8466                         sx_init(&t4_list_lock, "T4/T5 adapters");
8467                         SLIST_INIT(&t4_list);
8468 #ifdef TCP_OFFLOAD
8469                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8470                         SLIST_INIT(&t4_uld_list);
8471 #endif
8472                         t4_tracer_modload();
8473                         tweak_tunables();
8474                 }
8475                 sx_xunlock(&mlu);
8476                 break;
8477
8478         case MOD_UNLOAD:
8479                 sx_xlock(&mlu);
8480                 if (--loaded == 0) {
8481                         int tries;
8482
8483                         sx_slock(&t4_list_lock);
8484                         if (!SLIST_EMPTY(&t4_list)) {
8485                                 rc = EBUSY;
8486                                 sx_sunlock(&t4_list_lock);
8487                                 goto done_unload;
8488                         }
8489 #ifdef TCP_OFFLOAD
8490                         sx_slock(&t4_uld_list_lock);
8491                         if (!SLIST_EMPTY(&t4_uld_list)) {
8492                                 rc = EBUSY;
8493                                 sx_sunlock(&t4_uld_list_lock);
8494                                 sx_sunlock(&t4_list_lock);
8495                                 goto done_unload;
8496                         }
8497 #endif
8498                         tries = 0;
8499                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8500                                 uprintf("%ju clusters with custom free routine "
8501                                     "still is use.\n", t4_sge_extfree_refs());
8502                                 pause("t4unload", 2 * hz);
8503                         }
8504 #ifdef TCP_OFFLOAD
8505                         sx_sunlock(&t4_uld_list_lock);
8506 #endif
8507                         sx_sunlock(&t4_list_lock);
8508
8509                         if (t4_sge_extfree_refs() == 0) {
8510                                 t4_tracer_modunload();
8511 #ifdef TCP_OFFLOAD
8512                                 sx_destroy(&t4_uld_list_lock);
8513 #endif
8514                                 sx_destroy(&t4_list_lock);
8515                                 t4_sge_modunload();
8516                                 loaded = 0;
8517                         } else {
8518                                 rc = EBUSY;
8519                                 loaded++;       /* undo earlier decrement */
8520                         }
8521                 }
8522 done_unload:
8523                 sx_xunlock(&mlu);
8524                 break;
8525         }
8526
8527         return (rc);
8528 }
8529
8530 static devclass_t t4_devclass, t5_devclass;
8531 static devclass_t cxgbe_devclass, cxl_devclass;
8532
8533 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8534 MODULE_VERSION(t4nex, 1);
8535 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8536 #ifdef DEV_NETMAP
8537 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
8538 #endif /* DEV_NETMAP */
8539
8540
8541 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8542 MODULE_VERSION(t5nex, 1);
8543 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8544 #ifdef DEV_NETMAP
8545 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
8546 #endif /* DEV_NETMAP */
8547
8548 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8549 MODULE_VERSION(cxgbe, 1);
8550
8551 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8552 MODULE_VERSION(cxl, 1);