sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69 #include "t4_mp_ring.h"
  70
  71 /* T4 bus driver interface */
  72 static int t4_probe(device_t);
  73 static int t4_attach(device_t);
  74 static int t4_detach(device_t);
  75 static device_method_t t4_methods[] = {
  76         DEVMETHOD(device_probe,         t4_probe),
  77         DEVMETHOD(device_attach,        t4_attach),
  78         DEVMETHOD(device_detach,        t4_detach),
  79
  80         DEVMETHOD_END
  81 };
  82 static driver_t t4_driver = {
  83         "t4nex",
  84         t4_methods,
  85         sizeof(struct adapter)
  86 };
  87
  88
  89 /* T4 port (cxgbe) interface */
  90 static int cxgbe_probe(device_t);
  91 static int cxgbe_attach(device_t);
  92 static int cxgbe_detach(device_t);
  93 static device_method_t cxgbe_methods[] = {
  94         DEVMETHOD(device_probe,         cxgbe_probe),
  95         DEVMETHOD(device_attach,        cxgbe_attach),
  96         DEVMETHOD(device_detach,        cxgbe_detach),
  97         { 0, 0 }
  98 };
  99 static driver_t cxgbe_driver = {
 100         "cxgbe",
 101         cxgbe_methods,
 102         sizeof(struct port_info)
 103 };
 104
 105 static d_ioctl_t t4_ioctl;
 106 static d_open_t t4_open;
 107 static d_close_t t4_close;
 108
 109 static struct cdevsw t4_cdevsw = {
 110        .d_version = D_VERSION,
 111        .d_flags = 0,
 112        .d_open = t4_open,
 113        .d_close = t4_close,
 114        .d_ioctl = t4_ioctl,
 115        .d_name = "t4nex",
 116 };
 117
 118 /* T5 bus driver interface */
 119 static int t5_probe(device_t);
 120 static device_method_t t5_methods[] = {
 121         DEVMETHOD(device_probe,         t5_probe),
 122         DEVMETHOD(device_attach,        t4_attach),
 123         DEVMETHOD(device_detach,        t4_detach),
 124
 125         DEVMETHOD_END
 126 };
 127 static driver_t t5_driver = {
 128         "t5nex",
 129         t5_methods,
 130         sizeof(struct adapter)
 131 };
 132
 133
 134 /* T5 port (cxl) interface */
 135 static driver_t cxl_driver = {
 136         "cxl",
 137         cxgbe_methods,
 138         sizeof(struct port_info)
 139 };
 140
 141 static struct cdevsw t5_cdevsw = {
 142        .d_version = D_VERSION,
 143        .d_flags = 0,
 144        .d_open = t4_open,
 145        .d_close = t4_close,
 146        .d_ioctl = t4_ioctl,
 147        .d_name = "t5nex",
 148 };
 149
 150 /* ifnet + media interface */
 151 static void cxgbe_init(void *);
 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 154 static void cxgbe_qflush(struct ifnet *);
 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 156 static int cxgbe_media_change(struct ifnet *);
 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 158
 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 160
 161 /*
 162  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 163  * then ADAPTER_LOCK, then t4_uld_list_lock.
 164  */
 165 static struct sx t4_list_lock;
 166 SLIST_HEAD(, adapter) t4_list;
 167 #ifdef TCP_OFFLOAD
 168 static struct sx t4_uld_list_lock;
 169 SLIST_HEAD(, uld_info) t4_uld_list;
 170 #endif
 171
 172 /*
 173  * Tunables.  See tweak_tunables() too.
 174  *
 175  * Each tunable is set to a default value here if it's known at compile-time.
 176  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 177  * provide a reasonable default when the driver is loaded.
 178  *
 179  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 180  * T5 are under hw.cxl.
 181  */
 182
 183 /*
 184  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 185  */
 186 #define NTXQ_10G 16
 187 static int t4_ntxq10g = -1;
 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 189
 190 #define NRXQ_10G 8
 191 static int t4_nrxq10g = -1;
 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 193
 194 #define NTXQ_1G 4
 195 static int t4_ntxq1g = -1;
 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 197
 198 #define NRXQ_1G 2
 199 static int t4_nrxq1g = -1;
 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 201
 202 static int t4_rsrv_noflowq = 0;
 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 204
 205 #ifdef TCP_OFFLOAD
 206 #define NOFLDTXQ_10G 8
 207 static int t4_nofldtxq10g = -1;
 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 209
 210 #define NOFLDRXQ_10G 2
 211 static int t4_nofldrxq10g = -1;
 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 213
 214 #define NOFLDTXQ_1G 2
 215 static int t4_nofldtxq1g = -1;
 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 217
 218 #define NOFLDRXQ_1G 1
 219 static int t4_nofldrxq1g = -1;
 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 221 #endif
 222
 223 #ifdef DEV_NETMAP
 224 #define NNMTXQ_10G 2
 225 static int t4_nnmtxq10g = -1;
 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 227
 228 #define NNMRXQ_10G 2
 229 static int t4_nnmrxq10g = -1;
 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 231
 232 #define NNMTXQ_1G 1
 233 static int t4_nnmtxq1g = -1;
 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 235
 236 #define NNMRXQ_1G 1
 237 static int t4_nnmrxq1g = -1;
 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 239 #endif
 240
 241 /*
 242  * Holdoff parameters for 10G and 1G ports.
 243  */
 244 #define TMR_IDX_10G 1
 245 static int t4_tmr_idx_10g = TMR_IDX_10G;
 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 247
 248 #define PKTC_IDX_10G (-1)
 249 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 251
 252 #define TMR_IDX_1G 1
 253 static int t4_tmr_idx_1g = TMR_IDX_1G;
 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 255
 256 #define PKTC_IDX_1G (-1)
 257 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 259
 260 /*
 261  * Size (# of entries) of each tx and rx queue.
 262  */
 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 265
 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 268
 269 /*
 270  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 271  */
 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 274
 275 /*
 276  * Configuration file.
 277  */
 278 #define DEFAULT_CF      "default"
 279 #define FLASH_CF        "flash"
 280 #define UWIRE_CF        "uwire"
 281 #define FPGA_CF         "fpga"
 282 static char t4_cfg_file[32] = DEFAULT_CF;
 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 284
 285 /*
 286  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 287  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 288  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 289  *            mark or when signalled to do so, 0 to never emit PAUSE.
 290  */
 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 293
 294 /*
 295  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 296  * encouraged respectively).
 297  */
 298 static unsigned int t4_fw_install = 1;
 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 300
 301 /*
 302  * ASIC features that will be used.  Disable the ones you don't want so that the
 303  * chip resources aren't wasted on features that will not be used.
 304  */
 305 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 307
 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 310
 311 static int t4_toecaps_allowed = -1;
 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 313
 314 static int t4_rdmacaps_allowed = 0;
 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 316
 317 static int t4_iscsicaps_allowed = 0;
 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 319
 320 static int t4_fcoecaps_allowed = 0;
 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 322
 323 static int t5_write_combine = 0;
 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 325
 326 struct intrs_and_queues {
 327         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 328         uint16_t nirq;          /* Total # of vectors */
 329         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 330         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 331         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 332         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 333         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 334         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 335         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 336 #ifdef TCP_OFFLOAD
 337         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 338         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 339         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 340         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 341 #endif
 342 #ifdef DEV_NETMAP
 343         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 344         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 345         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 346         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 347 #endif
 348 };
 349
 350 struct filter_entry {
 351         uint32_t valid:1;       /* filter allocated and valid */
 352         uint32_t locked:1;      /* filter is administratively locked */
 353         uint32_t pending:1;     /* filter action is pending firmware reply */
 354         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 355         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 356
 357         struct t4_filter_specification fs;
 358 };
 359
 360 static int map_bars_0_and_4(struct adapter *);
 361 static int map_bar_2(struct adapter *);
 362 static void setup_memwin(struct adapter *);
 363 static int validate_mem_range(struct adapter *, uint32_t, int);
 364 static int fwmtype_to_hwmtype(int);
 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 366     uint32_t *);
 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 368 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 369 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 370     struct intrs_and_queues *);
 371 static int prep_firmware(struct adapter *);
 372 static int partition_resources(struct adapter *, const struct firmware *,
 373     const char *);
 374 static int get_params__pre_init(struct adapter *);
 375 static int get_params__post_init(struct adapter *);
 376 static int set_params__post_init(struct adapter *);
 377 static void t4_set_desc(struct adapter *);
 378 static void build_medialist(struct port_info *, struct ifmedia *);
 379 static int cxgbe_init_synchronized(struct port_info *);
 380 static int cxgbe_uninit_synchronized(struct port_info *);
 381 static int setup_intr_handlers(struct adapter *);
 382 static void quiesce_txq(struct adapter *, struct sge_txq *);
 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 384 static void quiesce_iq(struct adapter *, struct sge_iq *);
 385 static void quiesce_fl(struct adapter *, struct sge_fl *);
 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 387     driver_intr_t *, void *, char *);
 388 static int t4_free_irq(struct adapter *, struct irq *);
 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 390     unsigned int);
 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 393 static void cxgbe_tick(void *);
 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 396     struct mbuf *);
 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 398 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 399 static int t4_sysctls(struct adapter *);
 400 static int cxgbe_sysctls(struct port_info *);
 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 412 #ifdef SBUF_DRAIN
 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 438 #endif
 439 static uint32_t fconf_to_mode(uint32_t);
 440 static uint32_t mode_to_fconf(uint32_t);
 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 442 static int get_filter_mode(struct adapter *, uint32_t *);
 443 static int set_filter_mode(struct adapter *, uint32_t);
 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 445 static int get_filter(struct adapter *, struct t4_filter *);
 446 static int set_filter(struct adapter *, struct t4_filter *);
 447 static int del_filter(struct adapter *, struct t4_filter *);
 448 static void clear_filter(struct filter_entry *);
 449 static int set_filter_wr(struct adapter *, int);
 450 static int del_filter_wr(struct adapter *, int);
 451 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 452 static int load_fw(struct adapter *, struct t4_data *);
 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 454 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 455 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 457 #ifdef TCP_OFFLOAD
 458 static int toe_capability(struct port_info *, int);
 459 #endif
 460 static int mod_event(module_t, int, void *);
 461
 462 struct {
 463         uint16_t device;
 464         char *desc;
 465 } t4_pciids[] = {
 466         {0xa000, "Chelsio Terminator 4 FPGA"},
 467         {0x4400, "Chelsio T440-dbg"},
 468         {0x4401, "Chelsio T420-CR"},
 469         {0x4402, "Chelsio T422-CR"},
 470         {0x4403, "Chelsio T440-CR"},
 471         {0x4404, "Chelsio T420-BCH"},
 472         {0x4405, "Chelsio T440-BCH"},
 473         {0x4406, "Chelsio T440-CH"},
 474         {0x4407, "Chelsio T420-SO"},
 475         {0x4408, "Chelsio T420-CX"},
 476         {0x4409, "Chelsio T420-BT"},
 477         {0x440a, "Chelsio T404-BT"},
 478         {0x440e, "Chelsio T440-LP-CR"},
 479 }, t5_pciids[] = {
 480         {0xb000, "Chelsio Terminator 5 FPGA"},
 481         {0x5400, "Chelsio T580-dbg"},
 482         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 483         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 484         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 485         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 486         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 487         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 488         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 489         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 490         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 491         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 492         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 493         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 494         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 495 #ifdef notyet
 496         {0x5404,  "Chelsio T520-BCH"},
 497         {0x5405,  "Chelsio T540-BCH"},
 498         {0x5406,  "Chelsio T540-CH"},
 499         {0x5408,  "Chelsio T520-CX"},
 500         {0x540b,  "Chelsio B520-SR"},
 501         {0x540c,  "Chelsio B504-BT"},
 502         {0x540f,  "Chelsio Amsterdam"},
 503         {0x5413,  "Chelsio T580-CHR"},
 504 #endif
 505 };
 506
 507 #ifdef TCP_OFFLOAD
 508 /*
 509  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 510  * exactly the same for both rxq and ofld_rxq.
 511  */
 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 514 #endif
 515
 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 519
 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 521
 522 static int
 523 t4_probe(device_t dev)
 524 {
 525         int i;
 526         uint16_t v = pci_get_vendor(dev);
 527         uint16_t d = pci_get_device(dev);
 528         uint8_t f = pci_get_function(dev);
 529
 530         if (v != PCI_VENDOR_ID_CHELSIO)
 531                 return (ENXIO);
 532
 533         /* Attach only to PF0 of the FPGA */
 534         if (d == 0xa000 && f != 0)
 535                 return (ENXIO);
 536
 537         for (i = 0; i < nitems(t4_pciids); i++) {
 538                 if (d == t4_pciids[i].device) {
 539                         device_set_desc(dev, t4_pciids[i].desc);
 540                         return (BUS_PROBE_DEFAULT);
 541                 }
 542         }
 543
 544         return (ENXIO);
 545 }
 546
 547 static int
 548 t5_probe(device_t dev)
 549 {
 550         int i;
 551         uint16_t v = pci_get_vendor(dev);
 552         uint16_t d = pci_get_device(dev);
 553         uint8_t f = pci_get_function(dev);
 554
 555         if (v != PCI_VENDOR_ID_CHELSIO)
 556                 return (ENXIO);
 557
 558         /* Attach only to PF0 of the FPGA */
 559         if (d == 0xb000 && f != 0)
 560                 return (ENXIO);
 561
 562         for (i = 0; i < nitems(t5_pciids); i++) {
 563                 if (d == t5_pciids[i].device) {
 564                         device_set_desc(dev, t5_pciids[i].desc);
 565                         return (BUS_PROBE_DEFAULT);
 566                 }
 567         }
 568
 569         return (ENXIO);
 570 }
 571
 572 static int
 573 t4_attach(device_t dev)
 574 {
 575         struct adapter *sc;
 576         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 577         struct intrs_and_queues iaq;
 578         struct sge *s;
 579 #ifdef TCP_OFFLOAD
 580         int ofld_rqidx, ofld_tqidx;
 581 #endif
 582 #ifdef DEV_NETMAP
 583         int nm_rqidx, nm_tqidx;
 584 #endif
 585         const char *pcie_ts;
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589
 590         pci_enable_busmaster(dev);
 591         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 592                 uint32_t v;
 593
 594                 pci_set_max_read_req(dev, 4096);
 595                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 596                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 597                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 598
 599                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 600         }
 601
 602         sc->traceq = -1;
 603         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 604         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 605             device_get_nameunit(dev));
 606
 607         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 608             device_get_nameunit(dev));
 609         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 610         sx_xlock(&t4_list_lock);
 611         SLIST_INSERT_HEAD(&t4_list, sc, link);
 612         sx_xunlock(&t4_list_lock);
 613
 614         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 615         TAILQ_INIT(&sc->sfl);
 616         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 617
 618         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 619
 620         rc = map_bars_0_and_4(sc);
 621         if (rc != 0)
 622                 goto done; /* error message displayed already */
 623
 624         /*
 625          * This is the real PF# to which we're attaching.  Works from within PCI
 626          * passthrough environments too, where pci_get_function() could return a
 627          * different PF# depending on the passthrough configuration.  We need to
 628          * use the real PF# in all our communication with the firmware.
 629          */
 630         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 631         sc->mbox = sc->pf;
 632
 633         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 634         sc->an_handler = an_not_handled;
 635         for (i = 0; i < nitems(sc->cpl_handler); i++)
 636                 sc->cpl_handler[i] = cpl_not_handled;
 637         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 638                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 639         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 642         t4_init_sge_cpl_handlers(sc);
 643
 644         /* Prepare the adapter for operation */
 645         rc = -t4_prep_adapter(sc);
 646         if (rc != 0) {
 647                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 648                 goto done;
 649         }
 650
 651         /*
 652          * Do this really early, with the memory windows set up even before the
 653          * character device.  The userland tool's register i/o and mem read
 654          * will work even in "recovery mode".
 655          */
 656         setup_memwin(sc);
 657         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 658             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 659             device_get_nameunit(dev));
 660         if (sc->cdev == NULL)
 661                 device_printf(dev, "failed to create nexus char device.\n");
 662         else
 663                 sc->cdev->si_drv1 = sc;
 664
 665         /* Go no further if recovery mode has been requested. */
 666         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 667                 device_printf(dev, "recovery mode.\n");
 668                 goto done;
 669         }
 670
 671 #if defined(__i386__)
 672         if ((cpu_feature & CPUID_CX8) == 0) {
 673                 device_printf(dev, "64 bit atomics not available.\n");
 674                 rc = ENOTSUP;
 675                 goto done;
 676         }
 677 #endif
 678
 679         /* Prepare the firmware for operation */
 680         rc = prep_firmware(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         rc = get_params__post_init(sc);
 685         if (rc != 0)
 686                 goto done; /* error message displayed already */
 687
 688         rc = set_params__post_init(sc);
 689         if (rc != 0)
 690                 goto done; /* error message displayed already */
 691
 692         rc = map_bar_2(sc);
 693         if (rc != 0)
 694                 goto done; /* error message displayed already */
 695
 696         rc = t4_create_dma_tag(sc);
 697         if (rc != 0)
 698                 goto done; /* error message displayed already */
 699
 700         /*
 701          * First pass over all the ports - allocate VIs and initialize some
 702          * basic parameters like mac address, port type, etc.  We also figure
 703          * out whether a port is 10G or 1G and use that information when
 704          * calculating how many interrupts to attempt to allocate.
 705          */
 706         n10g = n1g = 0;
 707         for_each_port(sc, i) {
 708                 struct port_info *pi;
 709
 710                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 711                 sc->port[i] = pi;
 712
 713                 /* These must be set before t4_port_init */
 714                 pi->adapter = sc;
 715                 pi->port_id = i;
 716
 717                 /* Allocate the vi and initialize parameters like mac addr */
 718                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 719                 if (rc != 0) {
 720                         device_printf(dev, "unable to initialize port %d: %d\n",
 721                             i, rc);
 722                         free(pi, M_CXGBE);
 723                         sc->port[i] = NULL;
 724                         goto done;
 725                 }
 726
 727                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 728                 pi->link_cfg.requested_fc |= t4_pause_settings;
 729                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 730                 pi->link_cfg.fc |= t4_pause_settings;
 731
 732                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 733                 if (rc != 0) {
 734                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 735                         free(pi, M_CXGBE);
 736                         sc->port[i] = NULL;
 737                         goto done;
 738                 }
 739
 740                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 741                     device_get_nameunit(dev), i);
 742                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 743                 sc->chan_map[pi->tx_chan] = i;
 744
 745                 if (is_10G_port(pi) || is_40G_port(pi)) {
 746                         n10g++;
 747                         pi->tmr_idx = t4_tmr_idx_10g;
 748                         pi->pktc_idx = t4_pktc_idx_10g;
 749                 } else {
 750                         n1g++;
 751                         pi->tmr_idx = t4_tmr_idx_1g;
 752                         pi->pktc_idx = t4_pktc_idx_1g;
 753                 }
 754
 755                 pi->xact_addr_filt = -1;
 756                 pi->linkdnrc = -1;
 757
 758                 pi->qsize_rxq = t4_qsize_rxq;
 759                 pi->qsize_txq = t4_qsize_txq;
 760
 761                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 762                 if (pi->dev == NULL) {
 763                         device_printf(dev,
 764                             "failed to add device for port %d.\n", i);
 765                         rc = ENXIO;
 766                         goto done;
 767                 }
 768                 device_set_softc(pi->dev, pi);
 769         }
 770
 771         /*
 772          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 773          */
 774         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 775         if (rc != 0)
 776                 goto done; /* error message displayed already */
 777
 778         sc->intr_type = iaq.intr_type;
 779         sc->intr_count = iaq.nirq;
 780
 781         s = &sc->sge;
 782         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 783         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 784         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 785         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 786         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 787 #ifdef TCP_OFFLOAD
 788         if (is_offload(sc)) {
 789                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 790                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 791                 s->neq += s->nofldtxq + s->nofldrxq;
 792                 s->niq += s->nofldrxq;
 793
 794                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 795                     M_CXGBE, M_ZERO | M_WAITOK);
 796                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 797                     M_CXGBE, M_ZERO | M_WAITOK);
 798         }
 799 #endif
 800 #ifdef DEV_NETMAP
 801         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 802         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 803         s->neq += s->nnmtxq + s->nnmrxq;
 804         s->niq += s->nnmrxq;
 805
 806         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 807             M_CXGBE, M_ZERO | M_WAITOK);
 808         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 809             M_CXGBE, M_ZERO | M_WAITOK);
 810 #endif
 811
 812         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 813             M_ZERO | M_WAITOK);
 814         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 815             M_ZERO | M_WAITOK);
 816         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 817             M_ZERO | M_WAITOK);
 818         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 819             M_ZERO | M_WAITOK);
 820         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 821             M_ZERO | M_WAITOK);
 822
 823         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 824             M_ZERO | M_WAITOK);
 825
 826         t4_init_l2t(sc, M_WAITOK);
 827
 828         /*
 829          * Second pass over the ports.  This time we know the number of rx and
 830          * tx queues that each port should get.
 831          */
 832         rqidx = tqidx = 0;
 833 #ifdef TCP_OFFLOAD
 834         ofld_rqidx = ofld_tqidx = 0;
 835 #endif
 836 #ifdef DEV_NETMAP
 837         nm_rqidx = nm_tqidx = 0;
 838 #endif
 839         for_each_port(sc, i) {
 840                 struct port_info *pi = sc->port[i];
 841
 842                 if (pi == NULL)
 843                         continue;
 844
 845                 pi->first_rxq = rqidx;
 846                 pi->first_txq = tqidx;
 847                 if (is_10G_port(pi) || is_40G_port(pi)) {
 848                         pi->flags |= iaq.intr_flags_10g;
 849                         pi->nrxq = iaq.nrxq10g;
 850                         pi->ntxq = iaq.ntxq10g;
 851                 } else {
 852                         pi->flags |= iaq.intr_flags_1g;
 853                         pi->nrxq = iaq.nrxq1g;
 854                         pi->ntxq = iaq.ntxq1g;
 855                 }
 856
 857                 if (pi->ntxq > 1)
 858                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 859                 else
 860                         pi->rsrv_noflowq = 0;
 861
 862                 rqidx += pi->nrxq;
 863                 tqidx += pi->ntxq;
 864 #ifdef TCP_OFFLOAD
 865                 if (is_offload(sc)) {
 866                         pi->first_ofld_rxq = ofld_rqidx;
 867                         pi->first_ofld_txq = ofld_tqidx;
 868                         if (is_10G_port(pi) || is_40G_port(pi)) {
 869                                 pi->nofldrxq = iaq.nofldrxq10g;
 870                                 pi->nofldtxq = iaq.nofldtxq10g;
 871                         } else {
 872                                 pi->nofldrxq = iaq.nofldrxq1g;
 873                                 pi->nofldtxq = iaq.nofldtxq1g;
 874                         }
 875                         ofld_rqidx += pi->nofldrxq;
 876                         ofld_tqidx += pi->nofldtxq;
 877                 }
 878 #endif
 879 #ifdef DEV_NETMAP
 880                 pi->first_nm_rxq = nm_rqidx;
 881                 pi->first_nm_txq = nm_tqidx;
 882                 if (is_10G_port(pi) || is_40G_port(pi)) {
 883                         pi->nnmrxq = iaq.nnmrxq10g;
 884                         pi->nnmtxq = iaq.nnmtxq10g;
 885                 } else {
 886                         pi->nnmrxq = iaq.nnmrxq1g;
 887                         pi->nnmtxq = iaq.nnmtxq1g;
 888                 }
 889                 nm_rqidx += pi->nnmrxq;
 890                 nm_tqidx += pi->nnmtxq;
 891 #endif
 892         }
 893
 894         rc = setup_intr_handlers(sc);
 895         if (rc != 0) {
 896                 device_printf(dev,
 897                     "failed to setup interrupt handlers: %d\n", rc);
 898                 goto done;
 899         }
 900
 901         rc = bus_generic_attach(dev);
 902         if (rc != 0) {
 903                 device_printf(dev,
 904                     "failed to attach all child ports: %d\n", rc);
 905                 goto done;
 906         }
 907
 908         switch (sc->params.pci.speed) {
 909                 case 0x1:
 910                         pcie_ts = "2.5";
 911                         break;
 912                 case 0x2:
 913                         pcie_ts = "5.0";
 914                         break;
 915                 case 0x3:
 916                         pcie_ts = "8.0";
 917                         break;
 918                 default:
 919                         pcie_ts = "??";
 920                         break;
 921         }
 922         device_printf(dev,
 923             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 924             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 925             sc->params.nports, sc->intr_count,
 926             sc->intr_type == INTR_MSIX ? "MSI-X" :
 927             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 928             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 929
 930         t4_set_desc(sc);
 931
 932 done:
 933         if (rc != 0 && sc->cdev) {
 934                 /* cdev was created and so cxgbetool works; recover that way. */
 935                 device_printf(dev,
 936                     "error during attach, adapter is now in recovery mode.\n");
 937                 rc = 0;
 938         }
 939
 940         if (rc != 0)
 941                 t4_detach(dev);
 942         else
 943                 t4_sysctls(sc);
 944
 945         return (rc);
 946 }
 947
 948 /*
 949  * Idempotent
 950  */
 951 static int
 952 t4_detach(device_t dev)
 953 {
 954         struct adapter *sc;
 955         struct port_info *pi;
 956         int i, rc;
 957
 958         sc = device_get_softc(dev);
 959
 960         if (sc->flags & FULL_INIT_DONE)
 961                 t4_intr_disable(sc);
 962
 963         if (sc->cdev) {
 964                 destroy_dev(sc->cdev);
 965                 sc->cdev = NULL;
 966         }
 967
 968         rc = bus_generic_detach(dev);
 969         if (rc) {
 970                 device_printf(dev,
 971                     "failed to detach child devices: %d\n", rc);
 972                 return (rc);
 973         }
 974
 975         for (i = 0; i < sc->intr_count; i++)
 976                 t4_free_irq(sc, &sc->irq[i]);
 977
 978         for (i = 0; i < MAX_NPORTS; i++) {
 979                 pi = sc->port[i];
 980                 if (pi) {
 981                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 982                         if (pi->dev)
 983                                 device_delete_child(dev, pi->dev);
 984
 985                         mtx_destroy(&pi->pi_lock);
 986                         free(pi, M_CXGBE);
 987                 }
 988         }
 989
 990         if (sc->flags & FULL_INIT_DONE)
 991                 adapter_full_uninit(sc);
 992
 993         if (sc->flags & FW_OK)
 994                 t4_fw_bye(sc, sc->mbox);
 995
 996         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 997                 pci_release_msi(dev);
 998
 999         if (sc->regs_res)
1000                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1001                     sc->regs_res);
1002
1003         if (sc->udbs_res)
1004                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1005                     sc->udbs_res);
1006
1007         if (sc->msix_res)
1008                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1009                     sc->msix_res);
1010
1011         if (sc->l2t)
1012                 t4_free_l2t(sc->l2t);
1013
1014 #ifdef TCP_OFFLOAD
1015         free(sc->sge.ofld_rxq, M_CXGBE);
1016         free(sc->sge.ofld_txq, M_CXGBE);
1017 #endif
1018 #ifdef DEV_NETMAP
1019         free(sc->sge.nm_rxq, M_CXGBE);
1020         free(sc->sge.nm_txq, M_CXGBE);
1021 #endif
1022         free(sc->irq, M_CXGBE);
1023         free(sc->sge.rxq, M_CXGBE);
1024         free(sc->sge.txq, M_CXGBE);
1025         free(sc->sge.ctrlq, M_CXGBE);
1026         free(sc->sge.iqmap, M_CXGBE);
1027         free(sc->sge.eqmap, M_CXGBE);
1028         free(sc->tids.ftid_tab, M_CXGBE);
1029         t4_destroy_dma_tag(sc);
1030         if (mtx_initialized(&sc->sc_lock)) {
1031                 sx_xlock(&t4_list_lock);
1032                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1033                 sx_xunlock(&t4_list_lock);
1034                 mtx_destroy(&sc->sc_lock);
1035         }
1036
1037         if (mtx_initialized(&sc->tids.ftid_lock))
1038                 mtx_destroy(&sc->tids.ftid_lock);
1039         if (mtx_initialized(&sc->sfl_lock))
1040                 mtx_destroy(&sc->sfl_lock);
1041         if (mtx_initialized(&sc->ifp_lock))
1042                 mtx_destroy(&sc->ifp_lock);
1043         if (mtx_initialized(&sc->regwin_lock))
1044                 mtx_destroy(&sc->regwin_lock);
1045
1046         bzero(sc, sizeof(*sc));
1047
1048         return (0);
1049 }
1050
1051 static int
1052 cxgbe_probe(device_t dev)
1053 {
1054         char buf[128];
1055         struct port_info *pi = device_get_softc(dev);
1056
1057         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1058         device_set_desc_copy(dev, buf);
1059
1060         return (BUS_PROBE_DEFAULT);
1061 }
1062
1063 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1064     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1065     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1066 #define T4_CAP_ENABLE (T4_CAP)
1067
1068 static int
1069 cxgbe_attach(device_t dev)
1070 {
1071         struct port_info *pi = device_get_softc(dev);
1072         struct ifnet *ifp;
1073         char *s;
1074         int n, o;
1075
1076         /* Allocate an ifnet and set it up */
1077         ifp = if_alloc(IFT_ETHER);
1078         if (ifp == NULL) {
1079                 device_printf(dev, "Cannot allocate ifnet\n");
1080                 return (ENOMEM);
1081         }
1082         pi->ifp = ifp;
1083         ifp->if_softc = pi;
1084
1085         callout_init(&pi->tick, CALLOUT_MPSAFE);
1086
1087         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1088         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1089
1090         ifp->if_init = cxgbe_init;
1091         ifp->if_ioctl = cxgbe_ioctl;
1092         ifp->if_transmit = cxgbe_transmit;
1093         ifp->if_qflush = cxgbe_qflush;
1094         ifp->if_get_counter = cxgbe_get_counter;
1095
1096         ifp->if_capabilities = T4_CAP;
1097 #ifdef TCP_OFFLOAD
1098         if (is_offload(pi->adapter))
1099                 ifp->if_capabilities |= IFCAP_TOE;
1100 #endif
1101         ifp->if_capenable = T4_CAP_ENABLE;
1102         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1103             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1104
1105         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1106         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1107         ifp->if_hw_tsomaxsegsize = 65536;
1108
1109         /* Initialize ifmedia for this port */
1110         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1111             cxgbe_media_status);
1112         build_medialist(pi, &pi->media);
1113
1114         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1115             EVENTHANDLER_PRI_ANY);
1116
1117         ether_ifattach(ifp, pi->hw_addr);
1118
1119         n = 128;
1120         s = malloc(n, M_CXGBE, M_WAITOK);
1121         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1122         MPASS(n > o);
1123 #ifdef TCP_OFFLOAD
1124         if (is_offload(pi->adapter)) {
1125                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1126                     pi->nofldtxq, pi->nofldrxq);
1127                 MPASS(n > o);
1128         }
1129 #endif
1130 #ifdef DEV_NETMAP
1131         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1132             pi->nnmrxq);
1133         MPASS(n > o);
1134 #endif
1135         device_printf(dev, "%s\n", s);
1136         free(s, M_CXGBE);
1137
1138 #ifdef DEV_NETMAP
1139         /* nm_media handled here to keep implementation private to this file */
1140         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1141             cxgbe_media_status);
1142         build_medialist(pi, &pi->nm_media);
1143         create_netmap_ifnet(pi);        /* logs errors it something fails */
1144 #endif
1145         cxgbe_sysctls(pi);
1146
1147         return (0);
1148 }
1149
1150 static int
1151 cxgbe_detach(device_t dev)
1152 {
1153         struct port_info *pi = device_get_softc(dev);
1154         struct adapter *sc = pi->adapter;
1155         struct ifnet *ifp = pi->ifp;
1156
1157         /* Tell if_ioctl and if_init that the port is going away */
1158         ADAPTER_LOCK(sc);
1159         SET_DOOMED(pi);
1160         wakeup(&sc->flags);
1161         while (IS_BUSY(sc))
1162                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1163         SET_BUSY(sc);
1164 #ifdef INVARIANTS
1165         sc->last_op = "t4detach";
1166         sc->last_op_thr = curthread;
1167 #endif
1168         ADAPTER_UNLOCK(sc);
1169
1170         if (pi->flags & HAS_TRACEQ) {
1171                 sc->traceq = -1;        /* cloner should not create ifnet */
1172                 t4_tracer_port_detach(sc);
1173         }
1174
1175         if (pi->vlan_c)
1176                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1177
1178         PORT_LOCK(pi);
1179         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1180         callout_stop(&pi->tick);
1181         PORT_UNLOCK(pi);
1182         callout_drain(&pi->tick);
1183
1184         /* Let detach proceed even if these fail. */
1185         cxgbe_uninit_synchronized(pi);
1186         port_full_uninit(pi);
1187
1188         ifmedia_removeall(&pi->media);
1189         ether_ifdetach(pi->ifp);
1190         if_free(pi->ifp);
1191
1192 #ifdef DEV_NETMAP
1193         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1194         destroy_netmap_ifnet(pi);
1195 #endif
1196
1197         ADAPTER_LOCK(sc);
1198         CLR_BUSY(sc);
1199         wakeup(&sc->flags);
1200         ADAPTER_UNLOCK(sc);
1201
1202         return (0);
1203 }
1204
1205 static void
1206 cxgbe_init(void *arg)
1207 {
1208         struct port_info *pi = arg;
1209         struct adapter *sc = pi->adapter;
1210
1211         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1212                 return;
1213         cxgbe_init_synchronized(pi);
1214         end_synchronized_op(sc, 0);
1215 }
1216
1217 static int
1218 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1219 {
1220         int rc = 0, mtu, flags, can_sleep;
1221         struct port_info *pi = ifp->if_softc;
1222         struct adapter *sc = pi->adapter;
1223         struct ifreq *ifr = (struct ifreq *)data;
1224         uint32_t mask;
1225
1226         switch (cmd) {
1227         case SIOCSIFMTU:
1228                 mtu = ifr->ifr_mtu;
1229                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1230                         return (EINVAL);
1231
1232                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1233                 if (rc)
1234                         return (rc);
1235                 ifp->if_mtu = mtu;
1236                 if (pi->flags & PORT_INIT_DONE) {
1237                         t4_update_fl_bufsize(ifp);
1238                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1239                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1240                 }
1241                 end_synchronized_op(sc, 0);
1242                 break;
1243
1244         case SIOCSIFFLAGS:
1245                 can_sleep = 0;
1246 redo_sifflags:
1247                 rc = begin_synchronized_op(sc, pi,
1248                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1249                 if (rc)
1250                         return (rc);
1251
1252                 if (ifp->if_flags & IFF_UP) {
1253                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1254                                 flags = pi->if_flags;
1255                                 if ((ifp->if_flags ^ flags) &
1256                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1257                                         if (can_sleep == 1) {
1258                                                 end_synchronized_op(sc, 0);
1259                                                 can_sleep = 0;
1260                                                 goto redo_sifflags;
1261                                         }
1262                                         rc = update_mac_settings(ifp,
1263                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1264                                 }
1265                         } else {
1266                                 if (can_sleep == 0) {
1267                                         end_synchronized_op(sc, LOCK_HELD);
1268                                         can_sleep = 1;
1269                                         goto redo_sifflags;
1270                                 }
1271                                 rc = cxgbe_init_synchronized(pi);
1272                         }
1273                         pi->if_flags = ifp->if_flags;
1274                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1275                         if (can_sleep == 0) {
1276                                 end_synchronized_op(sc, LOCK_HELD);
1277                                 can_sleep = 1;
1278                                 goto redo_sifflags;
1279                         }
1280                         rc = cxgbe_uninit_synchronized(pi);
1281                 }
1282                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1283                 break;
1284
1285         case SIOCADDMULTI:
1286         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1287                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1288                 if (rc)
1289                         return (rc);
1290                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1291                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1292                 end_synchronized_op(sc, LOCK_HELD);
1293                 break;
1294
1295         case SIOCSIFCAP:
1296                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1297                 if (rc)
1298                         return (rc);
1299
1300                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1301                 if (mask & IFCAP_TXCSUM) {
1302                         ifp->if_capenable ^= IFCAP_TXCSUM;
1303                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1304
1305                         if (IFCAP_TSO4 & ifp->if_capenable &&
1306                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1307                                 ifp->if_capenable &= ~IFCAP_TSO4;
1308                                 if_printf(ifp,
1309                                     "tso4 disabled due to -txcsum.\n");
1310                         }
1311                 }
1312                 if (mask & IFCAP_TXCSUM_IPV6) {
1313                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1314                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1315
1316                         if (IFCAP_TSO6 & ifp->if_capenable &&
1317                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1318                                 ifp->if_capenable &= ~IFCAP_TSO6;
1319                                 if_printf(ifp,
1320                                     "tso6 disabled due to -txcsum6.\n");
1321                         }
1322                 }
1323                 if (mask & IFCAP_RXCSUM)
1324                         ifp->if_capenable ^= IFCAP_RXCSUM;
1325                 if (mask & IFCAP_RXCSUM_IPV6)
1326                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1327
1328                 /*
1329                  * Note that we leave CSUM_TSO alone (it is always set).  The
1330                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1331                  * sending a TSO request our way, so it's sufficient to toggle
1332                  * IFCAP_TSOx only.
1333                  */
1334                 if (mask & IFCAP_TSO4) {
1335                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1336                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1337                                 if_printf(ifp, "enable txcsum first.\n");
1338                                 rc = EAGAIN;
1339                                 goto fail;
1340                         }
1341                         ifp->if_capenable ^= IFCAP_TSO4;
1342                 }
1343                 if (mask & IFCAP_TSO6) {
1344                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1345                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1346                                 if_printf(ifp, "enable txcsum6 first.\n");
1347                                 rc = EAGAIN;
1348                                 goto fail;
1349                         }
1350                         ifp->if_capenable ^= IFCAP_TSO6;
1351                 }
1352                 if (mask & IFCAP_LRO) {
1353 #if defined(INET) || defined(INET6)
1354                         int i;
1355                         struct sge_rxq *rxq;
1356
1357                         ifp->if_capenable ^= IFCAP_LRO;
1358                         for_each_rxq(pi, i, rxq) {
1359                                 if (ifp->if_capenable & IFCAP_LRO)
1360                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1361                                 else
1362                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1363                         }
1364 #endif
1365                 }
1366 #ifdef TCP_OFFLOAD
1367                 if (mask & IFCAP_TOE) {
1368                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1369
1370                         rc = toe_capability(pi, enable);
1371                         if (rc != 0)
1372                                 goto fail;
1373
1374                         ifp->if_capenable ^= mask;
1375                 }
1376 #endif
1377                 if (mask & IFCAP_VLAN_HWTAGGING) {
1378                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1379                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1380                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1381                 }
1382                 if (mask & IFCAP_VLAN_MTU) {
1383                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1384
1385                         /* Need to find out how to disable auto-mtu-inflation */
1386                 }
1387                 if (mask & IFCAP_VLAN_HWTSO)
1388                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1389                 if (mask & IFCAP_VLAN_HWCSUM)
1390                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1391
1392 #ifdef VLAN_CAPABILITIES
1393                 VLAN_CAPABILITIES(ifp);
1394 #endif
1395 fail:
1396                 end_synchronized_op(sc, 0);
1397                 break;
1398
1399         case SIOCSIFMEDIA:
1400         case SIOCGIFMEDIA:
1401                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1402                 break;
1403
1404         case SIOCGI2C: {
1405                 struct ifi2creq i2c;
1406
1407                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1408                 if (rc != 0)
1409                         break;
1410                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1411                         rc = EPERM;
1412                         break;
1413                 }
1414                 if (i2c.len > sizeof(i2c.data)) {
1415                         rc = EINVAL;
1416                         break;
1417                 }
1418                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1419                 if (rc)
1420                         return (rc);
1421                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1422                     i2c.offset, i2c.len, &i2c.data[0]);
1423                 end_synchronized_op(sc, 0);
1424                 if (rc == 0)
1425                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1426                 break;
1427         }
1428
1429         default:
1430                 rc = ether_ioctl(ifp, cmd, data);
1431         }
1432
1433         return (rc);
1434 }
1435
1436 static int
1437 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1438 {
1439         struct port_info *pi = ifp->if_softc;
1440         struct adapter *sc = pi->adapter;
1441         struct sge_txq *txq;
1442         void *items[1];
1443         int rc;
1444
1445         M_ASSERTPKTHDR(m);
1446         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1447
1448         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1449                 m_freem(m);
1450                 return (ENETDOWN);
1451         }
1452
1453         rc = parse_pkt(&m);
1454         if (__predict_false(rc != 0)) {
1455                 MPASS(m == NULL);                       /* was freed already */
1456                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1457                 return (rc);
1458         }
1459
1460         /* Select a txq. */
1461         txq = &sc->sge.txq[pi->first_txq];
1462         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1463                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1464                     pi->rsrv_noflowq);
1465
1466         items[0] = m;
1467         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1468         if (__predict_false(rc != 0))
1469                 m_freem(m);
1470
1471         return (rc);
1472 }
1473
1474 static void
1475 cxgbe_qflush(struct ifnet *ifp)
1476 {
1477         struct port_info *pi = ifp->if_softc;
1478         struct sge_txq *txq;
1479         int i;
1480
1481         /* queues do not exist if !PORT_INIT_DONE. */
1482         if (pi->flags & PORT_INIT_DONE) {
1483                 for_each_txq(pi, i, txq) {
1484                         TXQ_LOCK(txq);
1485                         txq->eq.flags &= ~EQ_ENABLED;
1486                         TXQ_UNLOCK(txq);
1487                         while (!mp_ring_is_idle(txq->r)) {
1488                                 mp_ring_check_drainage(txq->r, 0);
1489                                 pause("qflush", 1);
1490                         }
1491                 }
1492         }
1493         if_qflush(ifp);
1494 }
1495
1496 static uint64_t
1497 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1498 {
1499         struct port_info *pi = ifp->if_softc;
1500         struct adapter *sc = pi->adapter;
1501         struct port_stats *s = &pi->stats;
1502
1503         cxgbe_refresh_stats(sc, pi);
1504
1505         switch (c) {
1506         case IFCOUNTER_IPACKETS:
1507                 return (s->rx_frames - s->rx_pause);
1508
1509         case IFCOUNTER_IERRORS:
1510                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1511                     s->rx_fcs_err + s->rx_len_err);
1512
1513         case IFCOUNTER_OPACKETS:
1514                 return (s->tx_frames - s->tx_pause);
1515
1516         case IFCOUNTER_OERRORS:
1517                 return (s->tx_error_frames);
1518
1519         case IFCOUNTER_IBYTES:
1520                 return (s->rx_octets - s->rx_pause * 64);
1521
1522         case IFCOUNTER_OBYTES:
1523                 return (s->tx_octets - s->tx_pause * 64);
1524
1525         case IFCOUNTER_IMCASTS:
1526                 return (s->rx_mcast_frames - s->rx_pause);
1527
1528         case IFCOUNTER_OMCASTS:
1529                 return (s->tx_mcast_frames - s->tx_pause);
1530
1531         case IFCOUNTER_IQDROPS:
1532                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1533                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1534                     s->rx_trunc3 + pi->tnl_cong_drops);
1535
1536         case IFCOUNTER_OQDROPS: {
1537                 uint64_t drops;
1538
1539                 drops = s->tx_drop;
1540                 if (pi->flags & PORT_INIT_DONE) {
1541                         int i;
1542                         struct sge_txq *txq;
1543
1544                         for_each_txq(pi, i, txq)
1545                                 drops += counter_u64_fetch(txq->r->drops);
1546                 }
1547
1548                 return (drops);
1549
1550         }
1551
1552         default:
1553                 return (if_get_counter_default(ifp, c));
1554         }
1555 }
1556
1557 static int
1558 cxgbe_media_change(struct ifnet *ifp)
1559 {
1560         struct port_info *pi = ifp->if_softc;
1561
1562         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1563
1564         return (EOPNOTSUPP);
1565 }
1566
1567 static void
1568 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1569 {
1570         struct port_info *pi = ifp->if_softc;
1571         struct ifmedia *media = NULL;
1572         struct ifmedia_entry *cur;
1573         int speed = pi->link_cfg.speed;
1574
1575         if (ifp == pi->ifp)
1576                 media = &pi->media;
1577 #ifdef DEV_NETMAP
1578         else if (ifp == pi->nm_ifp)
1579                 media = &pi->nm_media;
1580 #endif
1581         MPASS(media != NULL);
1582
1583         cur = media->ifm_cur;
1584
1585         ifmr->ifm_status = IFM_AVALID;
1586         if (!pi->link_cfg.link_ok)
1587                 return;
1588
1589         ifmr->ifm_status |= IFM_ACTIVE;
1590
1591         /* active and current will differ iff current media is autoselect. */
1592         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1593                 return;
1594
1595         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1596         if (speed == SPEED_10000)
1597                 ifmr->ifm_active |= IFM_10G_T;
1598         else if (speed == SPEED_1000)
1599                 ifmr->ifm_active |= IFM_1000_T;
1600         else if (speed == SPEED_100)
1601                 ifmr->ifm_active |= IFM_100_TX;
1602         else if (speed == SPEED_10)
1603                 ifmr->ifm_active |= IFM_10_T;
1604         else
1605                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1606                             speed));
1607 }
1608
1609 void
1610 t4_fatal_err(struct adapter *sc)
1611 {
1612         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1613         t4_intr_disable(sc);
1614         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1615             device_get_nameunit(sc->dev));
1616 }
1617
1618 static int
1619 map_bars_0_and_4(struct adapter *sc)
1620 {
1621         sc->regs_rid = PCIR_BAR(0);
1622         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1623             &sc->regs_rid, RF_ACTIVE);
1624         if (sc->regs_res == NULL) {
1625                 device_printf(sc->dev, "cannot map registers.\n");
1626                 return (ENXIO);
1627         }
1628         sc->bt = rman_get_bustag(sc->regs_res);
1629         sc->bh = rman_get_bushandle(sc->regs_res);
1630         sc->mmio_len = rman_get_size(sc->regs_res);
1631         setbit(&sc->doorbells, DOORBELL_KDB);
1632
1633         sc->msix_rid = PCIR_BAR(4);
1634         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1635             &sc->msix_rid, RF_ACTIVE);
1636         if (sc->msix_res == NULL) {
1637                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1638                 return (ENXIO);
1639         }
1640
1641         return (0);
1642 }
1643
1644 static int
1645 map_bar_2(struct adapter *sc)
1646 {
1647
1648         /*
1649          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1650          * to map it if RDMA is disabled.
1651          */
1652         if (is_t4(sc) && sc->rdmacaps == 0)
1653                 return (0);
1654
1655         sc->udbs_rid = PCIR_BAR(2);
1656         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1657             &sc->udbs_rid, RF_ACTIVE);
1658         if (sc->udbs_res == NULL) {
1659                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1660                 return (ENXIO);
1661         }
1662         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1663
1664         if (is_t5(sc)) {
1665                 setbit(&sc->doorbells, DOORBELL_UDB);
1666 #if defined(__i386__) || defined(__amd64__)
1667                 if (t5_write_combine) {
1668                         int rc;
1669
1670                         /*
1671                          * Enable write combining on BAR2.  This is the
1672                          * userspace doorbell BAR and is split into 128B
1673                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1674                          * with an egress queue.  The first 64B has the doorbell
1675                          * and the second 64B can be used to submit a tx work
1676                          * request with an implicit doorbell.
1677                          */
1678
1679                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1680                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1681                         if (rc == 0) {
1682                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1683                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1684                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1685                         } else {
1686                                 device_printf(sc->dev,
1687                                     "couldn't enable write combining: %d\n",
1688                                     rc);
1689                         }
1690
1691                         t4_write_reg(sc, A_SGE_STAT_CFG,
1692                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1693                 }
1694 #endif
1695         }
1696
1697         return (0);
1698 }
1699
1700 static const struct memwin t4_memwin[] = {
1701         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1702         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1703         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1704 };
1705
1706 static const struct memwin t5_memwin[] = {
1707         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1708         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1709         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1710 };
1711
1712 static void
1713 setup_memwin(struct adapter *sc)
1714 {
1715         const struct memwin *mw;
1716         int i, n;
1717         uint32_t bar0;
1718
1719         if (is_t4(sc)) {
1720                 /*
1721                  * Read low 32b of bar0 indirectly via the hardware backdoor
1722                  * mechanism.  Works from within PCI passthrough environments
1723                  * too, where rman_get_start() can return a different value.  We
1724                  * need to program the T4 memory window decoders with the actual
1725                  * addresses that will be coming across the PCIe link.
1726                  */
1727                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1728                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1729
1730                 mw = &t4_memwin[0];
1731                 n = nitems(t4_memwin);
1732         } else {
1733                 /* T5 uses the relative offset inside the PCIe BAR */
1734                 bar0 = 0;
1735
1736                 mw = &t5_memwin[0];
1737                 n = nitems(t5_memwin);
1738         }
1739
1740         for (i = 0; i < n; i++, mw++) {
1741                 t4_write_reg(sc,
1742                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1743                     (mw->base + bar0) | V_BIR(0) |
1744                     V_WINDOW(ilog2(mw->aperture) - 10));
1745         }
1746
1747         /* flush */
1748         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1749 }
1750
1751 /*
1752  * Verify that the memory range specified by the addr/len pair is valid and lies
1753  * entirely within a single region (EDCx or MCx).
1754  */
1755 static int
1756 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1757 {
1758         uint32_t em, addr_len, maddr, mlen;
1759
1760         /* Memory can only be accessed in naturally aligned 4 byte units */
1761         if (addr & 3 || len & 3 || len == 0)
1762                 return (EINVAL);
1763
1764         /* Enabled memories */
1765         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1766         if (em & F_EDRAM0_ENABLE) {
1767                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1768                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1769                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1770                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1771                     addr + len <= maddr + mlen)
1772                         return (0);
1773         }
1774         if (em & F_EDRAM1_ENABLE) {
1775                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1776                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1777                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1778                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1779                     addr + len <= maddr + mlen)
1780                         return (0);
1781         }
1782         if (em & F_EXT_MEM_ENABLE) {
1783                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1784                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1785                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1786                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1787                     addr + len <= maddr + mlen)
1788                         return (0);
1789         }
1790         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1791                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1792                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1793                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1794                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1795                     addr + len <= maddr + mlen)
1796                         return (0);
1797         }
1798
1799         return (EFAULT);
1800 }
1801
1802 static int
1803 fwmtype_to_hwmtype(int mtype)
1804 {
1805
1806         switch (mtype) {
1807         case FW_MEMTYPE_EDC0:
1808                 return (MEM_EDC0);
1809         case FW_MEMTYPE_EDC1:
1810                 return (MEM_EDC1);
1811         case FW_MEMTYPE_EXTMEM:
1812                 return (MEM_MC0);
1813         case FW_MEMTYPE_EXTMEM1:
1814                 return (MEM_MC1);
1815         default:
1816                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1817         }
1818 }
1819
1820 /*
1821  * Verify that the memory range specified by the memtype/offset/len pair is
1822  * valid and lies entirely within the memtype specified.  The global address of
1823  * the start of the range is returned in addr.
1824  */
1825 static int
1826 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1827     uint32_t *addr)
1828 {
1829         uint32_t em, addr_len, maddr, mlen;
1830
1831         /* Memory can only be accessed in naturally aligned 4 byte units */
1832         if (off & 3 || len & 3 || len == 0)
1833                 return (EINVAL);
1834
1835         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1836         switch (fwmtype_to_hwmtype(mtype)) {
1837         case MEM_EDC0:
1838                 if (!(em & F_EDRAM0_ENABLE))
1839                         return (EINVAL);
1840                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1841                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1842                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1843                 break;
1844         case MEM_EDC1:
1845                 if (!(em & F_EDRAM1_ENABLE))
1846                         return (EINVAL);
1847                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1848                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1849                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1850                 break;
1851         case MEM_MC:
1852                 if (!(em & F_EXT_MEM_ENABLE))
1853                         return (EINVAL);
1854                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1855                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1856                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1857                 break;
1858         case MEM_MC1:
1859                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1860                         return (EINVAL);
1861                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1862                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1863                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1864                 break;
1865         default:
1866                 return (EINVAL);
1867         }
1868
1869         if (mlen > 0 && off < mlen && off + len <= mlen) {
1870                 *addr = maddr + off;    /* global address */
1871                 return (0);
1872         }
1873
1874         return (EFAULT);
1875 }
1876
1877 static void
1878 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1879 {
1880         const struct memwin *mw;
1881
1882         if (is_t4(sc)) {
1883                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1884                     ("%s: incorrect memwin# (%d)", __func__, win));
1885                 mw = &t4_memwin[win];
1886         } else {
1887                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1888                     ("%s: incorrect memwin# (%d)", __func__, win));
1889                 mw = &t5_memwin[win];
1890         }
1891
1892         if (base != NULL)
1893                 *base = mw->base;
1894         if (aperture != NULL)
1895                 *aperture = mw->aperture;
1896 }
1897
1898 /*
1899  * Positions the memory window such that it can be used to access the specified
1900  * address in the chip's address space.  The return value is the offset of addr
1901  * from the start of the window.
1902  */
1903 static uint32_t
1904 position_memwin(struct adapter *sc, int n, uint32_t addr)
1905 {
1906         uint32_t start, pf;
1907         uint32_t reg;
1908
1909         KASSERT(n >= 0 && n <= 3,
1910             ("%s: invalid window %d.", __func__, n));
1911         KASSERT((addr & 3) == 0,
1912             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1913
1914         if (is_t4(sc)) {
1915                 pf = 0;
1916                 start = addr & ~0xf;    /* start must be 16B aligned */
1917         } else {
1918                 pf = V_PFNUM(sc->pf);
1919                 start = addr & ~0x7f;   /* start must be 128B aligned */
1920         }
1921         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1922
1923         t4_write_reg(sc, reg, start | pf);
1924         t4_read_reg(sc, reg);
1925
1926         return (addr - start);
1927 }
1928
1929 static int
1930 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1931     struct intrs_and_queues *iaq)
1932 {
1933         int rc, itype, navail, nrxq10g, nrxq1g, n;
1934         int nofldrxq10g = 0, nofldrxq1g = 0;
1935         int nnmrxq10g = 0, nnmrxq1g = 0;
1936
1937         bzero(iaq, sizeof(*iaq));
1938
1939         iaq->ntxq10g = t4_ntxq10g;
1940         iaq->ntxq1g = t4_ntxq1g;
1941         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1942         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1943         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1944 #ifdef TCP_OFFLOAD
1945         if (is_offload(sc)) {
1946                 iaq->nofldtxq10g = t4_nofldtxq10g;
1947                 iaq->nofldtxq1g = t4_nofldtxq1g;
1948                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1949                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1950         }
1951 #endif
1952 #ifdef DEV_NETMAP
1953         iaq->nnmtxq10g = t4_nnmtxq10g;
1954         iaq->nnmtxq1g = t4_nnmtxq1g;
1955         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1956         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1957 #endif
1958
1959         for (itype = INTR_MSIX; itype; itype >>= 1) {
1960
1961                 if ((itype & t4_intr_types) == 0)
1962                         continue;       /* not allowed */
1963
1964                 if (itype == INTR_MSIX)
1965                         navail = pci_msix_count(sc->dev);
1966                 else if (itype == INTR_MSI)
1967                         navail = pci_msi_count(sc->dev);
1968                 else
1969                         navail = 1;
1970 restart:
1971                 if (navail == 0)
1972                         continue;
1973
1974                 iaq->intr_type = itype;
1975                 iaq->intr_flags_10g = 0;
1976                 iaq->intr_flags_1g = 0;
1977
1978                 /*
1979                  * Best option: an interrupt vector for errors, one for the
1980                  * firmware event queue, and one for every rxq (NIC, TOE, and
1981                  * netmap).
1982                  */
1983                 iaq->nirq = T4_EXTRA_INTR;
1984                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1985                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1986                 if (iaq->nirq <= navail &&
1987                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1988                         iaq->intr_flags_10g = INTR_ALL;
1989                         iaq->intr_flags_1g = INTR_ALL;
1990                         goto allocate;
1991                 }
1992
1993                 /*
1994                  * Second best option: a vector for errors, one for the firmware
1995                  * event queue, and vectors for either all the NIC rx queues or
1996                  * all the TOE rx queues.  The queues that don't get vectors
1997                  * will forward their interrupts to those that do.
1998                  *
1999                  * Note: netmap rx queues cannot be created early and so they
2000                  * can't be setup to receive forwarded interrupts for others.
2001                  */
2002                 iaq->nirq = T4_EXTRA_INTR;
2003                 if (nrxq10g >= nofldrxq10g) {
2004                         iaq->intr_flags_10g = INTR_RXQ;
2005                         iaq->nirq += n10g * nrxq10g;
2006 #ifdef DEV_NETMAP
2007                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2008 #endif
2009                 } else {
2010                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2011                         iaq->nirq += n10g * nofldrxq10g;
2012 #ifdef DEV_NETMAP
2013                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2014 #endif
2015                 }
2016                 if (nrxq1g >= nofldrxq1g) {
2017                         iaq->intr_flags_1g = INTR_RXQ;
2018                         iaq->nirq += n1g * nrxq1g;
2019 #ifdef DEV_NETMAP
2020                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2021 #endif
2022                 } else {
2023                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2024                         iaq->nirq += n1g * nofldrxq1g;
2025 #ifdef DEV_NETMAP
2026                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2027 #endif
2028                 }
2029                 if (iaq->nirq <= navail &&
2030                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2031                         goto allocate;
2032
2033                 /*
2034                  * Next best option: an interrupt vector for errors, one for the
2035                  * firmware event queue, and at least one per port.  At this
2036                  * point we know we'll have to downsize nrxq and/or nofldrxq
2037                  * and/or nnmrxq to fit what's available to us.
2038                  */
2039                 iaq->nirq = T4_EXTRA_INTR;
2040                 iaq->nirq += n10g + n1g;
2041                 if (iaq->nirq <= navail) {
2042                         int leftover = navail - iaq->nirq;
2043
2044                         if (n10g > 0) {
2045                                 int target = max(nrxq10g, nofldrxq10g);
2046
2047                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2048                                     INTR_RXQ : INTR_OFLD_RXQ;
2049
2050                                 n = 1;
2051                                 while (n < target && leftover >= n10g) {
2052                                         leftover -= n10g;
2053                                         iaq->nirq += n10g;
2054                                         n++;
2055                                 }
2056                                 iaq->nrxq10g = min(n, nrxq10g);
2057 #ifdef TCP_OFFLOAD
2058                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2059 #endif
2060 #ifdef DEV_NETMAP
2061                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2062 #endif
2063                         }
2064
2065                         if (n1g > 0) {
2066                                 int target = max(nrxq1g, nofldrxq1g);
2067
2068                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2069                                     INTR_RXQ : INTR_OFLD_RXQ;
2070
2071                                 n = 1;
2072                                 while (n < target && leftover >= n1g) {
2073                                         leftover -= n1g;
2074                                         iaq->nirq += n1g;
2075                                         n++;
2076                                 }
2077                                 iaq->nrxq1g = min(n, nrxq1g);
2078 #ifdef TCP_OFFLOAD
2079                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2080 #endif
2081 #ifdef DEV_NETMAP
2082                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2083 #endif
2084                         }
2085
2086                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2087                                 goto allocate;
2088                 }
2089
2090                 /*
2091                  * Least desirable option: one interrupt vector for everything.
2092                  */
2093                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2094                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2095 #ifdef TCP_OFFLOAD
2096                 if (is_offload(sc))
2097                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2098 #endif
2099 #ifdef DEV_NETMAP
2100                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2101 #endif
2102
2103 allocate:
2104                 navail = iaq->nirq;
2105                 rc = 0;
2106                 if (itype == INTR_MSIX)
2107                         rc = pci_alloc_msix(sc->dev, &navail);
2108                 else if (itype == INTR_MSI)
2109                         rc = pci_alloc_msi(sc->dev, &navail);
2110
2111                 if (rc == 0) {
2112                         if (navail == iaq->nirq)
2113                                 return (0);
2114
2115                         /*
2116                          * Didn't get the number requested.  Use whatever number
2117                          * the kernel is willing to allocate (it's in navail).
2118                          */
2119                         device_printf(sc->dev, "fewer vectors than requested, "
2120                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2121                             itype, iaq->nirq, navail);
2122                         pci_release_msi(sc->dev);
2123                         goto restart;
2124                 }
2125
2126                 device_printf(sc->dev,
2127                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2128                     itype, rc, iaq->nirq, navail);
2129         }
2130
2131         device_printf(sc->dev,
2132             "failed to find a usable interrupt type.  "
2133             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2134             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2135
2136         return (ENXIO);
2137 }
2138
2139 #define FW_VERSION(chip) ( \
2140     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2141     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2142     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2143     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2144 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2145
2146 struct fw_info {
2147         uint8_t chip;
2148         char *kld_name;
2149         char *fw_mod_name;
2150         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2151 } fw_info[] = {
2152         {
2153                 .chip = CHELSIO_T4,
2154                 .kld_name = "t4fw_cfg",
2155                 .fw_mod_name = "t4fw",
2156                 .fw_hdr = {
2157                         .chip = FW_HDR_CHIP_T4,
2158                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2159                         .intfver_nic = FW_INTFVER(T4, NIC),
2160                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2161                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2162                         .intfver_ri = FW_INTFVER(T4, RI),
2163                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2164                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2165                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2166                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2167                 },
2168         }, {
2169                 .chip = CHELSIO_T5,
2170                 .kld_name = "t5fw_cfg",
2171                 .fw_mod_name = "t5fw",
2172                 .fw_hdr = {
2173                         .chip = FW_HDR_CHIP_T5,
2174                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2175                         .intfver_nic = FW_INTFVER(T5, NIC),
2176                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2177                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2178                         .intfver_ri = FW_INTFVER(T5, RI),
2179                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2180                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2181                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2182                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2183                 },
2184         }
2185 };
2186
2187 static struct fw_info *
2188 find_fw_info(int chip)
2189 {
2190         int i;
2191
2192         for (i = 0; i < nitems(fw_info); i++) {
2193                 if (fw_info[i].chip == chip)
2194                         return (&fw_info[i]);
2195         }
2196         return (NULL);
2197 }
2198
2199 /*
2200  * Is the given firmware API compatible with the one the driver was compiled
2201  * with?
2202  */
2203 static int
2204 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2205 {
2206
2207         /* short circuit if it's the exact same firmware version */
2208         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2209                 return (1);
2210
2211         /*
2212          * XXX: Is this too conservative?  Perhaps I should limit this to the
2213          * features that are supported in the driver.
2214          */
2215 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2216         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2217             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2218             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2219                 return (1);
2220 #undef SAME_INTF
2221
2222         return (0);
2223 }
2224
2225 /*
2226  * The firmware in the KLD is usable, but should it be installed?  This routine
2227  * explains itself in detail if it indicates the KLD firmware should be
2228  * installed.
2229  */
2230 static int
2231 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2232 {
2233         const char *reason;
2234
2235         if (!card_fw_usable) {
2236                 reason = "incompatible or unusable";
2237                 goto install;
2238         }
2239
2240         if (k > c) {
2241                 reason = "older than the version bundled with this driver";
2242                 goto install;
2243         }
2244
2245         if (t4_fw_install == 2 && k != c) {
2246                 reason = "different than the version bundled with this driver";
2247                 goto install;
2248         }
2249
2250         return (0);
2251
2252 install:
2253         if (t4_fw_install == 0) {
2254                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2255                     "but the driver is prohibited from installing a different "
2256                     "firmware on the card.\n",
2257                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2258                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2259
2260                 return (0);
2261         }
2262
2263         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2264             "installing firmware %u.%u.%u.%u on card.\n",
2265             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2266             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2267             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2268             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2269
2270         return (1);
2271 }
2272 /*
2273  * Establish contact with the firmware and determine if we are the master driver
2274  * or not, and whether we are responsible for chip initialization.
2275  */
2276 static int
2277 prep_firmware(struct adapter *sc)
2278 {
2279         const struct firmware *fw = NULL, *default_cfg;
2280         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2281         enum dev_state state;
2282         struct fw_info *fw_info;
2283         struct fw_hdr *card_fw;         /* fw on the card */
2284         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2285         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2286                                            against */
2287
2288         /* Contact firmware. */
2289         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2290         if (rc < 0 || state == DEV_STATE_ERR) {
2291                 rc = -rc;
2292                 device_printf(sc->dev,
2293                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2294                 return (rc);
2295         }
2296         pf = rc;
2297         if (pf == sc->mbox)
2298                 sc->flags |= MASTER_PF;
2299         else if (state == DEV_STATE_UNINIT) {
2300                 /*
2301                  * We didn't get to be the master so we definitely won't be
2302                  * configuring the chip.  It's a bug if someone else hasn't
2303                  * configured it already.
2304                  */
2305                 device_printf(sc->dev, "couldn't be master(%d), "
2306                     "device not already initialized either(%d).\n", rc, state);
2307                 return (EDOOFUS);
2308         }
2309
2310         /* This is the firmware whose headers the driver was compiled against */
2311         fw_info = find_fw_info(chip_id(sc));
2312         if (fw_info == NULL) {
2313                 device_printf(sc->dev,
2314                     "unable to look up firmware information for chip %d.\n",
2315                     chip_id(sc));
2316                 return (EINVAL);
2317         }
2318         drv_fw = &fw_info->fw_hdr;
2319
2320         /*
2321          * The firmware KLD contains many modules.  The KLD name is also the
2322          * name of the module that contains the default config file.
2323          */
2324         default_cfg = firmware_get(fw_info->kld_name);
2325
2326         /* Read the header of the firmware on the card */
2327         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2328         rc = -t4_read_flash(sc, FLASH_FW_START,
2329             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2330         if (rc == 0)
2331                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2332         else {
2333                 device_printf(sc->dev,
2334                     "Unable to read card's firmware header: %d\n", rc);
2335                 card_fw_usable = 0;
2336         }
2337
2338         /* This is the firmware in the KLD */
2339         fw = firmware_get(fw_info->fw_mod_name);
2340         if (fw != NULL) {
2341                 kld_fw = (const void *)fw->data;
2342                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2343         } else {
2344                 kld_fw = NULL;
2345                 kld_fw_usable = 0;
2346         }
2347
2348         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2349             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2350                 /*
2351                  * Common case: the firmware on the card is an exact match and
2352                  * the KLD is an exact match too, or the KLD is
2353                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2354                  * here -- use cxgbetool loadfw if you want to reinstall the
2355                  * same firmware as the one on the card.
2356                  */
2357         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2358             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2359             be32toh(card_fw->fw_ver))) {
2360
2361                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2362                 if (rc != 0) {
2363                         device_printf(sc->dev,
2364                             "failed to install firmware: %d\n", rc);
2365                         goto done;
2366                 }
2367
2368                 /* Installed successfully, update the cached header too. */
2369                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2370                 card_fw_usable = 1;
2371                 need_fw_reset = 0;      /* already reset as part of load_fw */
2372         }
2373
2374         if (!card_fw_usable) {
2375                 uint32_t d, c, k;
2376
2377                 d = ntohl(drv_fw->fw_ver);
2378                 c = ntohl(card_fw->fw_ver);
2379                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2380
2381                 device_printf(sc->dev, "Cannot find a usable firmware: "
2382                     "fw_install %d, chip state %d, "
2383                     "driver compiled with %d.%d.%d.%d, "
2384                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2385                     t4_fw_install, state,
2386                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2387                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2388                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2389                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2390                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2391                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2392                 rc = EINVAL;
2393                 goto done;
2394         }
2395
2396         /* We're using whatever's on the card and it's known to be good. */
2397         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2398         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2399             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2400             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2401             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2402             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2403         t4_get_tp_version(sc, &sc->params.tp_vers);
2404
2405         /* Reset device */
2406         if (need_fw_reset &&
2407             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2408                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2409                 if (rc != ETIMEDOUT && rc != EIO)
2410                         t4_fw_bye(sc, sc->mbox);
2411                 goto done;
2412         }
2413         sc->flags |= FW_OK;
2414
2415         rc = get_params__pre_init(sc);
2416         if (rc != 0)
2417                 goto done; /* error message displayed already */
2418
2419         /* Partition adapter resources as specified in the config file. */
2420         if (state == DEV_STATE_UNINIT) {
2421
2422                 KASSERT(sc->flags & MASTER_PF,
2423                     ("%s: trying to change chip settings when not master.",
2424                     __func__));
2425
2426                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2427                 if (rc != 0)
2428                         goto done;      /* error message displayed already */
2429
2430                 t4_tweak_chip_settings(sc);
2431
2432                 /* get basic stuff going */
2433                 rc = -t4_fw_initialize(sc, sc->mbox);
2434                 if (rc != 0) {
2435                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2436                         goto done;
2437                 }
2438         } else {
2439                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2440                 sc->cfcsum = 0;
2441         }
2442
2443 done:
2444         free(card_fw, M_CXGBE);
2445         if (fw != NULL)
2446                 firmware_put(fw, FIRMWARE_UNLOAD);
2447         if (default_cfg != NULL)
2448                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2449
2450         return (rc);
2451 }
2452
2453 #define FW_PARAM_DEV(param) \
2454         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2455          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2456 #define FW_PARAM_PFVF(param) \
2457         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2458          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2459
2460 /*
2461  * Partition chip resources for use between various PFs, VFs, etc.
2462  */
2463 static int
2464 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2465     const char *name_prefix)
2466 {
2467         const struct firmware *cfg = NULL;
2468         int rc = 0;
2469         struct fw_caps_config_cmd caps;
2470         uint32_t mtype, moff, finicsum, cfcsum;
2471
2472         /*
2473          * Figure out what configuration file to use.  Pick the default config
2474          * file for the card if the user hasn't specified one explicitly.
2475          */
2476         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2477         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2478                 /* Card specific overrides go here. */
2479                 if (pci_get_device(sc->dev) == 0x440a)
2480                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2481                 if (is_fpga(sc))
2482                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2483         }
2484
2485         /*
2486          * We need to load another module if the profile is anything except
2487          * "default" or "flash".
2488          */
2489         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2490             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2491                 char s[32];
2492
2493                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2494                 cfg = firmware_get(s);
2495                 if (cfg == NULL) {
2496                         if (default_cfg != NULL) {
2497                                 device_printf(sc->dev,
2498                                     "unable to load module \"%s\" for "
2499                                     "configuration profile \"%s\", will use "
2500                                     "the default config file instead.\n",
2501                                     s, sc->cfg_file);
2502                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2503                                     "%s", DEFAULT_CF);
2504                         } else {
2505                                 device_printf(sc->dev,
2506                                     "unable to load module \"%s\" for "
2507                                     "configuration profile \"%s\", will use "
2508                                     "the config file on the card's flash "
2509                                     "instead.\n", s, sc->cfg_file);
2510                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2511                                     "%s", FLASH_CF);
2512                         }
2513                 }
2514         }
2515
2516         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2517             default_cfg == NULL) {
2518                 device_printf(sc->dev,
2519                     "default config file not available, will use the config "
2520                     "file on the card's flash instead.\n");
2521                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2522         }
2523
2524         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2525                 u_int cflen, i, n;
2526                 const uint32_t *cfdata;
2527                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2528
2529                 KASSERT(cfg != NULL || default_cfg != NULL,
2530                     ("%s: no config to upload", __func__));
2531
2532                 /*
2533                  * Ask the firmware where it wants us to upload the config file.
2534                  */
2535                 param = FW_PARAM_DEV(CF);
2536                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2537                 if (rc != 0) {
2538                         /* No support for config file?  Shouldn't happen. */
2539                         device_printf(sc->dev,
2540                             "failed to query config file location: %d.\n", rc);
2541                         goto done;
2542                 }
2543                 mtype = G_FW_PARAMS_PARAM_Y(val);
2544                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2545
2546                 /*
2547                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2548                  * useless stuffing/comments at the end of the config file so
2549                  * it's ok to simply throw away the last remaining bytes when
2550                  * the config file is not an exact multiple of 4.  This also
2551                  * helps with the validate_mt_off_len check.
2552                  */
2553                 if (cfg != NULL) {
2554                         cflen = cfg->datasize & ~3;
2555                         cfdata = cfg->data;
2556                 } else {
2557                         cflen = default_cfg->datasize & ~3;
2558                         cfdata = default_cfg->data;
2559                 }
2560
2561                 if (cflen > FLASH_CFG_MAX_SIZE) {
2562                         device_printf(sc->dev,
2563                             "config file too long (%d, max allowed is %d).  "
2564                             "Will try to use the config on the card, if any.\n",
2565                             cflen, FLASH_CFG_MAX_SIZE);
2566                         goto use_config_on_flash;
2567                 }
2568
2569                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2570                 if (rc != 0) {
2571                         device_printf(sc->dev,
2572                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2573                             "Will try to use the config on the card, if any.\n",
2574                             __func__, mtype, moff, cflen, rc);
2575                         goto use_config_on_flash;
2576                 }
2577
2578                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2579                 while (cflen) {
2580                         off = position_memwin(sc, 2, addr);
2581                         n = min(cflen, mw_aperture - off);
2582                         for (i = 0; i < n; i += 4)
2583                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2584                         cflen -= n;
2585                         addr += n;
2586                 }
2587         } else {
2588 use_config_on_flash:
2589                 mtype = FW_MEMTYPE_FLASH;
2590                 moff = t4_flash_cfg_addr(sc);
2591         }
2592
2593         bzero(&caps, sizeof(caps));
2594         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2595             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2596         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2597             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2598             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2599         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2600         if (rc != 0) {
2601                 device_printf(sc->dev,
2602                     "failed to pre-process config file: %d "
2603                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2604                 goto done;
2605         }
2606
2607         finicsum = be32toh(caps.finicsum);
2608         cfcsum = be32toh(caps.cfcsum);
2609         if (finicsum != cfcsum) {
2610                 device_printf(sc->dev,
2611                     "WARNING: config file checksum mismatch: %08x %08x\n",
2612                     finicsum, cfcsum);
2613         }
2614         sc->cfcsum = cfcsum;
2615
2616 #define LIMIT_CAPS(x) do { \
2617         caps.x &= htobe16(t4_##x##_allowed); \
2618 } while (0)
2619
2620         /*
2621          * Let the firmware know what features will (not) be used so it can tune
2622          * things accordingly.
2623          */
2624         LIMIT_CAPS(linkcaps);
2625         LIMIT_CAPS(niccaps);
2626         LIMIT_CAPS(toecaps);
2627         LIMIT_CAPS(rdmacaps);
2628         LIMIT_CAPS(iscsicaps);
2629         LIMIT_CAPS(fcoecaps);
2630 #undef LIMIT_CAPS
2631
2632         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2633             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2634         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2635         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2636         if (rc != 0) {
2637                 device_printf(sc->dev,
2638                     "failed to process config file: %d.\n", rc);
2639         }
2640 done:
2641         if (cfg != NULL)
2642                 firmware_put(cfg, FIRMWARE_UNLOAD);
2643         return (rc);
2644 }
2645
2646 /*
2647  * Retrieve parameters that are needed (or nice to have) very early.
2648  */
2649 static int
2650 get_params__pre_init(struct adapter *sc)
2651 {
2652         int rc;
2653         uint32_t param[2], val[2];
2654         struct fw_devlog_cmd cmd;
2655         struct devlog_params *dlog = &sc->params.devlog;
2656
2657         param[0] = FW_PARAM_DEV(PORTVEC);
2658         param[1] = FW_PARAM_DEV(CCLK);
2659         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2660         if (rc != 0) {
2661                 device_printf(sc->dev,
2662                     "failed to query parameters (pre_init): %d.\n", rc);
2663                 return (rc);
2664         }
2665
2666         sc->params.portvec = val[0];
2667         sc->params.nports = bitcount32(val[0]);
2668         sc->params.vpd.cclk = val[1];
2669
2670         /* Read device log parameters. */
2671         bzero(&cmd, sizeof(cmd));
2672         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2673             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2674         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2675         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2676         if (rc != 0) {
2677                 device_printf(sc->dev,
2678                     "failed to get devlog parameters: %d.\n", rc);
2679                 bzero(dlog, sizeof (*dlog));
2680                 rc = 0; /* devlog isn't critical for device operation */
2681         } else {
2682                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2683                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2684                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2685                 dlog->size = be32toh(cmd.memsize_devlog);
2686         }
2687
2688         return (rc);
2689 }
2690
2691 /*
2692  * Retrieve various parameters that are of interest to the driver.  The device
2693  * has been initialized by the firmware at this point.
2694  */
2695 static int
2696 get_params__post_init(struct adapter *sc)
2697 {
2698         int rc;
2699         uint32_t param[7], val[7];
2700         struct fw_caps_config_cmd caps;
2701
2702         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2703         param[1] = FW_PARAM_PFVF(EQ_START);
2704         param[2] = FW_PARAM_PFVF(FILTER_START);
2705         param[3] = FW_PARAM_PFVF(FILTER_END);
2706         param[4] = FW_PARAM_PFVF(L2T_START);
2707         param[5] = FW_PARAM_PFVF(L2T_END);
2708         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2709         if (rc != 0) {
2710                 device_printf(sc->dev,
2711                     "failed to query parameters (post_init): %d.\n", rc);
2712                 return (rc);
2713         }
2714
2715         sc->sge.iq_start = val[0];
2716         sc->sge.eq_start = val[1];
2717         sc->tids.ftid_base = val[2];
2718         sc->tids.nftids = val[3] - val[2] + 1;
2719         sc->params.ftid_min = val[2];
2720         sc->params.ftid_max = val[3];
2721         sc->vres.l2t.start = val[4];
2722         sc->vres.l2t.size = val[5] - val[4] + 1;
2723         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2724             ("%s: L2 table size (%u) larger than expected (%u)",
2725             __func__, sc->vres.l2t.size, L2T_SIZE));
2726
2727         /* get capabilites */
2728         bzero(&caps, sizeof(caps));
2729         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2730             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2731         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2732         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2733         if (rc != 0) {
2734                 device_printf(sc->dev,
2735                     "failed to get card capabilities: %d.\n", rc);
2736                 return (rc);
2737         }
2738
2739 #define READ_CAPS(x) do { \
2740         sc->x = htobe16(caps.x); \
2741 } while (0)
2742         READ_CAPS(linkcaps);
2743         READ_CAPS(niccaps);
2744         READ_CAPS(toecaps);
2745         READ_CAPS(rdmacaps);
2746         READ_CAPS(iscsicaps);
2747         READ_CAPS(fcoecaps);
2748
2749         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2750                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2751                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2752                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2753                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2754                 if (rc != 0) {
2755                         device_printf(sc->dev,
2756                             "failed to query NIC parameters: %d.\n", rc);
2757                         return (rc);
2758                 }
2759                 sc->tids.etid_base = val[0];
2760                 sc->params.etid_min = val[0];
2761                 sc->tids.netids = val[1] - val[0] + 1;
2762                 sc->params.netids = sc->tids.netids;
2763                 sc->params.eo_wr_cred = val[2];
2764                 sc->params.ethoffload = 1;
2765         }
2766
2767         if (sc->toecaps) {
2768                 /* query offload-related parameters */
2769                 param[0] = FW_PARAM_DEV(NTID);
2770                 param[1] = FW_PARAM_PFVF(SERVER_START);
2771                 param[2] = FW_PARAM_PFVF(SERVER_END);
2772                 param[3] = FW_PARAM_PFVF(TDDP_START);
2773                 param[4] = FW_PARAM_PFVF(TDDP_END);
2774                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2775                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2776                 if (rc != 0) {
2777                         device_printf(sc->dev,
2778                             "failed to query TOE parameters: %d.\n", rc);
2779                         return (rc);
2780                 }
2781                 sc->tids.ntids = val[0];
2782                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2783                 sc->tids.stid_base = val[1];
2784                 sc->tids.nstids = val[2] - val[1] + 1;
2785                 sc->vres.ddp.start = val[3];
2786                 sc->vres.ddp.size = val[4] - val[3] + 1;
2787                 sc->params.ofldq_wr_cred = val[5];
2788                 sc->params.offload = 1;
2789         }
2790         if (sc->rdmacaps) {
2791                 param[0] = FW_PARAM_PFVF(STAG_START);
2792                 param[1] = FW_PARAM_PFVF(STAG_END);
2793                 param[2] = FW_PARAM_PFVF(RQ_START);
2794                 param[3] = FW_PARAM_PFVF(RQ_END);
2795                 param[4] = FW_PARAM_PFVF(PBL_START);
2796                 param[5] = FW_PARAM_PFVF(PBL_END);
2797                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2798                 if (rc != 0) {
2799                         device_printf(sc->dev,
2800                             "failed to query RDMA parameters(1): %d.\n", rc);
2801                         return (rc);
2802                 }
2803                 sc->vres.stag.start = val[0];
2804                 sc->vres.stag.size = val[1] - val[0] + 1;
2805                 sc->vres.rq.start = val[2];
2806                 sc->vres.rq.size = val[3] - val[2] + 1;
2807                 sc->vres.pbl.start = val[4];
2808                 sc->vres.pbl.size = val[5] - val[4] + 1;
2809
2810                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2811                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2812                 param[2] = FW_PARAM_PFVF(CQ_START);
2813                 param[3] = FW_PARAM_PFVF(CQ_END);
2814                 param[4] = FW_PARAM_PFVF(OCQ_START);
2815                 param[5] = FW_PARAM_PFVF(OCQ_END);
2816                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2817                 if (rc != 0) {
2818                         device_printf(sc->dev,
2819                             "failed to query RDMA parameters(2): %d.\n", rc);
2820                         return (rc);
2821                 }
2822                 sc->vres.qp.start = val[0];
2823                 sc->vres.qp.size = val[1] - val[0] + 1;
2824                 sc->vres.cq.start = val[2];
2825                 sc->vres.cq.size = val[3] - val[2] + 1;
2826                 sc->vres.ocq.start = val[4];
2827                 sc->vres.ocq.size = val[5] - val[4] + 1;
2828         }
2829         if (sc->iscsicaps) {
2830                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2831                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2832                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2833                 if (rc != 0) {
2834                         device_printf(sc->dev,
2835                             "failed to query iSCSI parameters: %d.\n", rc);
2836                         return (rc);
2837                 }
2838                 sc->vres.iscsi.start = val[0];
2839                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2840         }
2841
2842         /*
2843          * We've got the params we wanted to query via the firmware.  Now grab
2844          * some others directly from the chip.
2845          */
2846         rc = t4_read_chip_settings(sc);
2847
2848         return (rc);
2849 }
2850
2851 static int
2852 set_params__post_init(struct adapter *sc)
2853 {
2854         uint32_t param, val;
2855
2856         /* ask for encapsulated CPLs */
2857         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2858         val = 1;
2859         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2860
2861         return (0);
2862 }
2863
2864 #undef FW_PARAM_PFVF
2865 #undef FW_PARAM_DEV
2866
2867 static void
2868 t4_set_desc(struct adapter *sc)
2869 {
2870         char buf[128];
2871         struct adapter_params *p = &sc->params;
2872
2873         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2874             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2875             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2876
2877         device_set_desc_copy(sc->dev, buf);
2878 }
2879
2880 static void
2881 build_medialist(struct port_info *pi, struct ifmedia *media)
2882 {
2883         int m;
2884
2885         PORT_LOCK(pi);
2886
2887         ifmedia_removeall(media);
2888
2889         m = IFM_ETHER | IFM_FDX;
2890
2891         switch(pi->port_type) {
2892         case FW_PORT_TYPE_BT_XFI:
2893         case FW_PORT_TYPE_BT_XAUI:
2894                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
2895                 /* fall through */
2896
2897         case FW_PORT_TYPE_BT_SGMII:
2898                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
2899                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
2900                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
2901                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2902                 break;
2903
2904         case FW_PORT_TYPE_CX4:
2905                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
2906                 ifmedia_set(media, m | IFM_10G_CX4);
2907                 break;
2908
2909         case FW_PORT_TYPE_QSFP_10G:
2910         case FW_PORT_TYPE_SFP:
2911         case FW_PORT_TYPE_FIBER_XFI:
2912         case FW_PORT_TYPE_FIBER_XAUI:
2913                 switch (pi->mod_type) {
2914
2915                 case FW_PORT_MOD_TYPE_LR:
2916                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
2917                         ifmedia_set(media, m | IFM_10G_LR);
2918                         break;
2919
2920                 case FW_PORT_MOD_TYPE_SR:
2921                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
2922                         ifmedia_set(media, m | IFM_10G_SR);
2923                         break;
2924
2925                 case FW_PORT_MOD_TYPE_LRM:
2926                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
2927                         ifmedia_set(media, m | IFM_10G_LRM);
2928                         break;
2929
2930                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2931                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2932                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
2933                         ifmedia_set(media, m | IFM_10G_TWINAX);
2934                         break;
2935
2936                 case FW_PORT_MOD_TYPE_NONE:
2937                         m &= ~IFM_FDX;
2938                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2939                         ifmedia_set(media, m | IFM_NONE);
2940                         break;
2941
2942                 case FW_PORT_MOD_TYPE_NA:
2943                 case FW_PORT_MOD_TYPE_ER:
2944                 default:
2945                         device_printf(pi->dev,
2946                             "unknown port_type (%d), mod_type (%d)\n",
2947                             pi->port_type, pi->mod_type);
2948                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2949                         ifmedia_set(media, m | IFM_UNKNOWN);
2950                         break;
2951                 }
2952                 break;
2953
2954         case FW_PORT_TYPE_QSFP:
2955                 switch (pi->mod_type) {
2956
2957                 case FW_PORT_MOD_TYPE_LR:
2958                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
2959                         ifmedia_set(media, m | IFM_40G_LR4);
2960                         break;
2961
2962                 case FW_PORT_MOD_TYPE_SR:
2963                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
2964                         ifmedia_set(media, m | IFM_40G_SR4);
2965                         break;
2966
2967                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2968                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2969                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
2970                         ifmedia_set(media, m | IFM_40G_CR4);
2971                         break;
2972
2973                 case FW_PORT_MOD_TYPE_NONE:
2974                         m &= ~IFM_FDX;
2975                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2976                         ifmedia_set(media, m | IFM_NONE);
2977                         break;
2978
2979                 default:
2980                         device_printf(pi->dev,
2981                             "unknown port_type (%d), mod_type (%d)\n",
2982                             pi->port_type, pi->mod_type);
2983                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2984                         ifmedia_set(media, m | IFM_UNKNOWN);
2985                         break;
2986                 }
2987                 break;
2988
2989         default:
2990                 device_printf(pi->dev,
2991                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2992                     pi->mod_type);
2993                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2994                 ifmedia_set(media, m | IFM_UNKNOWN);
2995                 break;
2996         }
2997
2998         PORT_UNLOCK(pi);
2999 }
3000
3001 #define FW_MAC_EXACT_CHUNK      7
3002
3003 /*
3004  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3005  * indicates which parameters should be programmed (the rest are left alone).
3006  */
3007 int
3008 update_mac_settings(struct ifnet *ifp, int flags)
3009 {
3010         int rc = 0;
3011         struct port_info *pi = ifp->if_softc;
3012         struct adapter *sc = pi->adapter;
3013         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3014         uint16_t viid = 0xffff;
3015         int16_t *xact_addr_filt = NULL;
3016
3017         ASSERT_SYNCHRONIZED_OP(sc);
3018         KASSERT(flags, ("%s: not told what to update.", __func__));
3019
3020         if (ifp == pi->ifp) {
3021                 viid = pi->viid;
3022                 xact_addr_filt = &pi->xact_addr_filt;
3023         }
3024 #ifdef DEV_NETMAP
3025         else if (ifp == pi->nm_ifp) {
3026                 viid = pi->nm_viid;
3027                 xact_addr_filt = &pi->nm_xact_addr_filt;
3028         }
3029 #endif
3030         if (flags & XGMAC_MTU)
3031                 mtu = ifp->if_mtu;
3032
3033         if (flags & XGMAC_PROMISC)
3034                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3035
3036         if (flags & XGMAC_ALLMULTI)
3037                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3038
3039         if (flags & XGMAC_VLANEX)
3040                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3041
3042         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3043                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3044                     1, vlanex, false);
3045                 if (rc) {
3046                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3047                             rc);
3048                         return (rc);
3049                 }
3050         }
3051
3052         if (flags & XGMAC_UCADDR) {
3053                 uint8_t ucaddr[ETHER_ADDR_LEN];
3054
3055                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3056                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3057                     true, true);
3058                 if (rc < 0) {
3059                         rc = -rc;
3060                         if_printf(ifp, "change_mac failed: %d\n", rc);
3061                         return (rc);
3062                 } else {
3063                         *xact_addr_filt = rc;
3064                         rc = 0;
3065                 }
3066         }
3067
3068         if (flags & XGMAC_MCADDRS) {
3069                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3070                 int del = 1;
3071                 uint64_t hash = 0;
3072                 struct ifmultiaddr *ifma;
3073                 int i = 0, j;
3074
3075                 if_maddr_rlock(ifp);
3076                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3077                         if (ifma->ifma_addr->sa_family != AF_LINK)
3078                                 continue;
3079                         mcaddr[i] =
3080                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3081                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3082                         i++;
3083
3084                         if (i == FW_MAC_EXACT_CHUNK) {
3085                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3086                                     i, mcaddr, NULL, &hash, 0);
3087                                 if (rc < 0) {
3088                                         rc = -rc;
3089                                         for (j = 0; j < i; j++) {
3090                                                 if_printf(ifp,
3091                                                     "failed to add mc address"
3092                                                     " %02x:%02x:%02x:"
3093                                                     "%02x:%02x:%02x rc=%d\n",
3094                                                     mcaddr[j][0], mcaddr[j][1],
3095                                                     mcaddr[j][2], mcaddr[j][3],
3096                                                     mcaddr[j][4], mcaddr[j][5],
3097                                                     rc);
3098                                         }
3099                                         goto mcfail;
3100                                 }
3101                                 del = 0;
3102                                 i = 0;
3103                         }
3104                 }
3105                 if (i > 0) {
3106                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3107                             mcaddr, NULL, &hash, 0);
3108                         if (rc < 0) {
3109                                 rc = -rc;
3110                                 for (j = 0; j < i; j++) {
3111                                         if_printf(ifp,
3112                                             "failed to add mc address"
3113                                             " %02x:%02x:%02x:"
3114                                             "%02x:%02x:%02x rc=%d\n",
3115                                             mcaddr[j][0], mcaddr[j][1],
3116                                             mcaddr[j][2], mcaddr[j][3],
3117                                             mcaddr[j][4], mcaddr[j][5],
3118                                             rc);
3119                                 }
3120                                 goto mcfail;
3121                         }
3122                 }
3123
3124                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3125                 if (rc != 0)
3126                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3127 mcfail:
3128                 if_maddr_runlock(ifp);
3129         }
3130
3131         return (rc);
3132 }
3133
3134 /*
3135  * {begin|end}_synchronized_op must be called from the same thread.
3136  */
3137 int
3138 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3139     char *wmesg)
3140 {
3141         int rc, pri;
3142
3143 #ifdef WITNESS
3144         /* the caller thinks it's ok to sleep, but is it really? */
3145         if (flags & SLEEP_OK)
3146                 pause("t4slptst", 1);
3147 #endif
3148
3149         if (INTR_OK)
3150                 pri = PCATCH;
3151         else
3152                 pri = 0;
3153
3154         ADAPTER_LOCK(sc);
3155         for (;;) {
3156
3157                 if (pi && IS_DOOMED(pi)) {
3158                         rc = ENXIO;
3159                         goto done;
3160                 }
3161
3162                 if (!IS_BUSY(sc)) {
3163                         rc = 0;
3164                         break;
3165                 }
3166
3167                 if (!(flags & SLEEP_OK)) {
3168                         rc = EBUSY;
3169                         goto done;
3170                 }
3171
3172                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3173                         rc = EINTR;
3174                         goto done;
3175                 }
3176         }
3177
3178         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3179         SET_BUSY(sc);
3180 #ifdef INVARIANTS
3181         sc->last_op = wmesg;
3182         sc->last_op_thr = curthread;
3183 #endif
3184
3185 done:
3186         if (!(flags & HOLD_LOCK) || rc)
3187                 ADAPTER_UNLOCK(sc);
3188
3189         return (rc);
3190 }
3191
3192 /*
3193  * {begin|end}_synchronized_op must be called from the same thread.
3194  */
3195 void
3196 end_synchronized_op(struct adapter *sc, int flags)
3197 {
3198
3199         if (flags & LOCK_HELD)
3200                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3201         else
3202                 ADAPTER_LOCK(sc);
3203
3204         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3205         CLR_BUSY(sc);
3206         wakeup(&sc->flags);
3207         ADAPTER_UNLOCK(sc);
3208 }
3209
3210 static int
3211 cxgbe_init_synchronized(struct port_info *pi)
3212 {
3213         struct adapter *sc = pi->adapter;
3214         struct ifnet *ifp = pi->ifp;
3215         int rc = 0, i;
3216         struct sge_txq *txq;
3217
3218         ASSERT_SYNCHRONIZED_OP(sc);
3219
3220         if (isset(&sc->open_device_map, pi->port_id)) {
3221                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3222                     ("mismatch between open_device_map and if_drv_flags"));
3223                 return (0);     /* already running */
3224         }
3225
3226         if (!(sc->flags & FULL_INIT_DONE) &&
3227             ((rc = adapter_full_init(sc)) != 0))
3228                 return (rc);    /* error message displayed already */
3229
3230         if (!(pi->flags & PORT_INIT_DONE) &&
3231             ((rc = port_full_init(pi)) != 0))
3232                 return (rc); /* error message displayed already */
3233
3234         rc = update_mac_settings(ifp, XGMAC_ALL);
3235         if (rc)
3236                 goto done;      /* error message displayed already */
3237
3238         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3239         if (rc != 0) {
3240                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3241                 goto done;
3242         }
3243
3244         /*
3245          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3246          * if this changes.
3247          */
3248
3249         for_each_txq(pi, i, txq) {
3250                 TXQ_LOCK(txq);
3251                 txq->eq.flags |= EQ_ENABLED;
3252                 TXQ_UNLOCK(txq);
3253         }
3254
3255         /*
3256          * The first iq of the first port to come up is used for tracing.
3257          */
3258         if (sc->traceq < 0) {
3259                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3260                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3261                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3262                     V_QUEUENUMBER(sc->traceq));
3263                 pi->flags |= HAS_TRACEQ;
3264         }
3265
3266         /* all ok */
3267         setbit(&sc->open_device_map, pi->port_id);
3268         PORT_LOCK(pi);
3269         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3270         PORT_UNLOCK(pi);
3271
3272         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3273 done:
3274         if (rc != 0)
3275                 cxgbe_uninit_synchronized(pi);
3276
3277         return (rc);
3278 }
3279
3280 /*
3281  * Idempotent.
3282  */
3283 static int
3284 cxgbe_uninit_synchronized(struct port_info *pi)
3285 {
3286         struct adapter *sc = pi->adapter;
3287         struct ifnet *ifp = pi->ifp;
3288         int rc, i;
3289         struct sge_txq *txq;
3290
3291         ASSERT_SYNCHRONIZED_OP(sc);
3292
3293         if (!(pi->flags & PORT_INIT_DONE)) {
3294                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3295                     ("uninited port is running"));
3296                 return (0);
3297         }
3298
3299         /*
3300          * Disable the VI so that all its data in either direction is discarded
3301          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3302          * tick) intact as the TP can deliver negative advice or data that it's
3303          * holding in its RAM (for an offloaded connection) even after the VI is
3304          * disabled.
3305          */
3306         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3307         if (rc) {
3308                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3309                 return (rc);
3310         }
3311
3312         for_each_txq(pi, i, txq) {
3313                 TXQ_LOCK(txq);
3314                 txq->eq.flags &= ~EQ_ENABLED;
3315                 TXQ_UNLOCK(txq);
3316         }
3317
3318         clrbit(&sc->open_device_map, pi->port_id);
3319         PORT_LOCK(pi);
3320         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3321         PORT_UNLOCK(pi);
3322
3323         pi->link_cfg.link_ok = 0;
3324         pi->link_cfg.speed = 0;
3325         pi->linkdnrc = -1;
3326         t4_os_link_changed(sc, pi->port_id, 0, -1);
3327
3328         return (0);
3329 }
3330
3331 /*
3332  * It is ok for this function to fail midway and return right away.  t4_detach
3333  * will walk the entire sc->irq list and clean up whatever is valid.
3334  */
3335 static int
3336 setup_intr_handlers(struct adapter *sc)
3337 {
3338         int rc, rid, p, q;
3339         char s[8];
3340         struct irq *irq;
3341         struct port_info *pi;
3342         struct sge_rxq *rxq;
3343 #ifdef TCP_OFFLOAD
3344         struct sge_ofld_rxq *ofld_rxq;
3345 #endif
3346 #ifdef DEV_NETMAP
3347         struct sge_nm_rxq *nm_rxq;
3348 #endif
3349
3350         /*
3351          * Setup interrupts.
3352          */
3353         irq = &sc->irq[0];
3354         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3355         if (sc->intr_count == 1)
3356                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3357
3358         /* Multiple interrupts. */
3359         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3360             ("%s: too few intr.", __func__));
3361
3362         /* The first one is always error intr */
3363         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3364         if (rc != 0)
3365                 return (rc);
3366         irq++;
3367         rid++;
3368
3369         /* The second one is always the firmware event queue */
3370         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3371         if (rc != 0)
3372                 return (rc);
3373         irq++;
3374         rid++;
3375
3376         for_each_port(sc, p) {
3377                 pi = sc->port[p];
3378
3379                 if (pi->flags & INTR_RXQ) {
3380                         for_each_rxq(pi, q, rxq) {
3381                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3382                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3383                                     s);
3384                                 if (rc != 0)
3385                                         return (rc);
3386                                 irq++;
3387                                 rid++;
3388                         }
3389                 }
3390 #ifdef TCP_OFFLOAD
3391                 if (pi->flags & INTR_OFLD_RXQ) {
3392                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3393                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3394                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3395                                     ofld_rxq, s);
3396                                 if (rc != 0)
3397                                         return (rc);
3398                                 irq++;
3399                                 rid++;
3400                         }
3401                 }
3402 #endif
3403 #ifdef DEV_NETMAP
3404                 if (pi->flags & INTR_NM_RXQ) {
3405                         for_each_nm_rxq(pi, q, nm_rxq) {
3406                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3407                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3408                                     nm_rxq, s);
3409                                 if (rc != 0)
3410                                         return (rc);
3411                                 irq++;
3412                                 rid++;
3413                         }
3414                 }
3415 #endif
3416         }
3417         MPASS(irq == &sc->irq[sc->intr_count]);
3418
3419         return (0);
3420 }
3421
3422 int
3423 adapter_full_init(struct adapter *sc)
3424 {
3425         int rc, i;
3426
3427         ASSERT_SYNCHRONIZED_OP(sc);
3428         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3429         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3430             ("%s: FULL_INIT_DONE already", __func__));
3431
3432         /*
3433          * queues that belong to the adapter (not any particular port).
3434          */
3435         rc = t4_setup_adapter_queues(sc);
3436         if (rc != 0)
3437                 goto done;
3438
3439         for (i = 0; i < nitems(sc->tq); i++) {
3440                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3441                     taskqueue_thread_enqueue, &sc->tq[i]);
3442                 if (sc->tq[i] == NULL) {
3443                         device_printf(sc->dev,
3444                             "failed to allocate task queue %d\n", i);
3445                         rc = ENOMEM;
3446                         goto done;
3447                 }
3448                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3449                     device_get_nameunit(sc->dev), i);
3450         }
3451
3452         t4_intr_enable(sc);
3453         sc->flags |= FULL_INIT_DONE;
3454 done:
3455         if (rc != 0)
3456                 adapter_full_uninit(sc);
3457
3458         return (rc);
3459 }
3460
3461 int
3462 adapter_full_uninit(struct adapter *sc)
3463 {
3464         int i;
3465
3466         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3467
3468         t4_teardown_adapter_queues(sc);
3469
3470         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3471                 taskqueue_free(sc->tq[i]);
3472                 sc->tq[i] = NULL;
3473         }
3474
3475         sc->flags &= ~FULL_INIT_DONE;
3476
3477         return (0);
3478 }
3479
3480 int
3481 port_full_init(struct port_info *pi)
3482 {
3483         struct adapter *sc = pi->adapter;
3484         struct ifnet *ifp = pi->ifp;
3485         uint16_t *rss;
3486         struct sge_rxq *rxq;
3487         int rc, i, j;
3488
3489         ASSERT_SYNCHRONIZED_OP(sc);
3490         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3491             ("%s: PORT_INIT_DONE already", __func__));
3492
3493         sysctl_ctx_init(&pi->ctx);
3494         pi->flags |= PORT_SYSCTL_CTX;
3495
3496         /*
3497          * Allocate tx/rx/fl queues for this port.
3498          */
3499         rc = t4_setup_port_queues(pi);
3500         if (rc != 0)
3501                 goto done;      /* error message displayed already */
3502
3503         /*
3504          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3505          */
3506         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3507         for (i = 0; i < pi->rss_size;) {
3508                 for_each_rxq(pi, j, rxq) {
3509                         rss[i++] = rxq->iq.abs_id;
3510                         if (i == pi->rss_size)
3511                                 break;
3512                 }
3513         }
3514
3515         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3516             pi->rss_size);
3517         if (rc != 0) {
3518                 if_printf(ifp, "rss_config failed: %d\n", rc);
3519                 goto done;
3520         }
3521
3522         pi->rss = rss;
3523         pi->flags |= PORT_INIT_DONE;
3524 done:
3525         if (rc != 0)
3526                 port_full_uninit(pi);
3527
3528         return (rc);
3529 }
3530
3531 /*
3532  * Idempotent.
3533  */
3534 int
3535 port_full_uninit(struct port_info *pi)
3536 {
3537         struct adapter *sc = pi->adapter;
3538         int i;
3539         struct sge_rxq *rxq;
3540         struct sge_txq *txq;
3541 #ifdef TCP_OFFLOAD
3542         struct sge_ofld_rxq *ofld_rxq;
3543         struct sge_wrq *ofld_txq;
3544 #endif
3545
3546         if (pi->flags & PORT_INIT_DONE) {
3547
3548                 /* Need to quiesce queues.  */
3549
3550                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3551
3552                 for_each_txq(pi, i, txq) {
3553                         quiesce_txq(sc, txq);
3554                 }
3555
3556 #ifdef TCP_OFFLOAD
3557                 for_each_ofld_txq(pi, i, ofld_txq) {
3558                         quiesce_wrq(sc, ofld_txq);
3559                 }
3560 #endif
3561
3562                 for_each_rxq(pi, i, rxq) {
3563                         quiesce_iq(sc, &rxq->iq);
3564                         quiesce_fl(sc, &rxq->fl);
3565                 }
3566
3567 #ifdef TCP_OFFLOAD
3568                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3569                         quiesce_iq(sc, &ofld_rxq->iq);
3570                         quiesce_fl(sc, &ofld_rxq->fl);
3571                 }
3572 #endif
3573                 free(pi->rss, M_CXGBE);
3574         }
3575
3576         t4_teardown_port_queues(pi);
3577         pi->flags &= ~PORT_INIT_DONE;
3578
3579         return (0);
3580 }
3581
3582 static void
3583 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3584 {
3585         struct sge_eq *eq = &txq->eq;
3586         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3587
3588         (void) sc;      /* unused */
3589
3590 #ifdef INVARIANTS
3591         TXQ_LOCK(txq);
3592         MPASS((eq->flags & EQ_ENABLED) == 0);
3593         TXQ_UNLOCK(txq);
3594 #endif
3595
3596         /* Wait for the mp_ring to empty. */
3597         while (!mp_ring_is_idle(txq->r)) {
3598                 mp_ring_check_drainage(txq->r, 0);
3599                 pause("rquiesce", 1);
3600         }
3601
3602         /* Then wait for the hardware to finish. */
3603         while (spg->cidx != htobe16(eq->pidx))
3604                 pause("equiesce", 1);
3605
3606         /* Finally, wait for the driver to reclaim all descriptors. */
3607         while (eq->cidx != eq->pidx)
3608                 pause("dquiesce", 1);
3609 }
3610
3611 static void
3612 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3613 {
3614
3615         /* XXXTX */
3616 }
3617
3618 static void
3619 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3620 {
3621         (void) sc;      /* unused */
3622
3623         /* Synchronize with the interrupt handler */
3624         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3625                 pause("iqfree", 1);
3626 }
3627
3628 static void
3629 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3630 {
3631         mtx_lock(&sc->sfl_lock);
3632         FL_LOCK(fl);
3633         fl->flags |= FL_DOOMED;
3634         FL_UNLOCK(fl);
3635         mtx_unlock(&sc->sfl_lock);
3636
3637         callout_drain(&sc->sfl_callout);
3638         KASSERT((fl->flags & FL_STARVING) == 0,
3639             ("%s: still starving", __func__));
3640 }
3641
3642 static int
3643 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3644     driver_intr_t *handler, void *arg, char *name)
3645 {
3646         int rc;
3647
3648         irq->rid = rid;
3649         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3650             RF_SHAREABLE | RF_ACTIVE);
3651         if (irq->res == NULL) {
3652                 device_printf(sc->dev,
3653                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3654                 return (ENOMEM);
3655         }
3656
3657         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3658             NULL, handler, arg, &irq->tag);
3659         if (rc != 0) {
3660                 device_printf(sc->dev,
3661                     "failed to setup interrupt for rid %d, name %s: %d\n",
3662                     rid, name, rc);
3663         } else if (name)
3664                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3665
3666         return (rc);
3667 }
3668
3669 static int
3670 t4_free_irq(struct adapter *sc, struct irq *irq)
3671 {
3672         if (irq->tag)
3673                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3674         if (irq->res)
3675                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3676
3677         bzero(irq, sizeof(*irq));
3678
3679         return (0);
3680 }
3681
3682 static void
3683 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3684     unsigned int end)
3685 {
3686         uint32_t *p = (uint32_t *)(buf + start);
3687
3688         for ( ; start <= end; start += sizeof(uint32_t))
3689                 *p++ = t4_read_reg(sc, start);
3690 }
3691
3692 static void
3693 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3694 {
3695         int i, n;
3696         const unsigned int *reg_ranges;
3697         static const unsigned int t4_reg_ranges[] = {
3698                 0x1008, 0x1108,
3699                 0x1180, 0x11b4,
3700                 0x11fc, 0x123c,
3701                 0x1300, 0x173c,
3702                 0x1800, 0x18fc,
3703                 0x3000, 0x30d8,
3704                 0x30e0, 0x5924,
3705                 0x5960, 0x59d4,
3706                 0x5a00, 0x5af8,
3707                 0x6000, 0x6098,
3708                 0x6100, 0x6150,
3709                 0x6200, 0x6208,
3710                 0x6240, 0x6248,
3711                 0x6280, 0x6338,
3712                 0x6370, 0x638c,
3713                 0x6400, 0x643c,
3714                 0x6500, 0x6524,
3715                 0x6a00, 0x6a38,
3716                 0x6a60, 0x6a78,
3717                 0x6b00, 0x6b84,
3718                 0x6bf0, 0x6c84,
3719                 0x6cf0, 0x6d84,
3720                 0x6df0, 0x6e84,
3721                 0x6ef0, 0x6f84,
3722                 0x6ff0, 0x7084,
3723                 0x70f0, 0x7184,
3724                 0x71f0, 0x7284,
3725                 0x72f0, 0x7384,
3726                 0x73f0, 0x7450,
3727                 0x7500, 0x7530,
3728                 0x7600, 0x761c,
3729                 0x7680, 0x76cc,
3730                 0x7700, 0x7798,
3731                 0x77c0, 0x77fc,
3732                 0x7900, 0x79fc,
3733                 0x7b00, 0x7c38,
3734                 0x7d00, 0x7efc,
3735                 0x8dc0, 0x8e1c,
3736                 0x8e30, 0x8e78,
3737                 0x8ea0, 0x8f6c,
3738                 0x8fc0, 0x9074,
3739                 0x90fc, 0x90fc,
3740                 0x9400, 0x9458,
3741                 0x9600, 0x96bc,
3742                 0x9800, 0x9808,
3743                 0x9820, 0x983c,
3744                 0x9850, 0x9864,
3745                 0x9c00, 0x9c6c,
3746                 0x9c80, 0x9cec,
3747                 0x9d00, 0x9d6c,
3748                 0x9d80, 0x9dec,
3749                 0x9e00, 0x9e6c,
3750                 0x9e80, 0x9eec,
3751                 0x9f00, 0x9f6c,
3752                 0x9f80, 0x9fec,
3753                 0xd004, 0xd03c,
3754                 0xdfc0, 0xdfe0,
3755                 0xe000, 0xea7c,
3756                 0xf000, 0x11110,
3757                 0x11118, 0x11190,
3758                 0x19040, 0x1906c,
3759                 0x19078, 0x19080,
3760                 0x1908c, 0x19124,
3761                 0x19150, 0x191b0,
3762                 0x191d0, 0x191e8,
3763                 0x19238, 0x1924c,
3764                 0x193f8, 0x19474,
3765                 0x19490, 0x194f8,
3766                 0x19800, 0x19f30,
3767                 0x1a000, 0x1a06c,
3768                 0x1a0b0, 0x1a120,
3769                 0x1a128, 0x1a138,
3770                 0x1a190, 0x1a1c4,
3771                 0x1a1fc, 0x1a1fc,
3772                 0x1e040, 0x1e04c,
3773                 0x1e284, 0x1e28c,
3774                 0x1e2c0, 0x1e2c0,
3775                 0x1e2e0, 0x1e2e0,
3776                 0x1e300, 0x1e384,
3777                 0x1e3c0, 0x1e3c8,
3778                 0x1e440, 0x1e44c,
3779                 0x1e684, 0x1e68c,
3780                 0x1e6c0, 0x1e6c0,
3781                 0x1e6e0, 0x1e6e0,
3782                 0x1e700, 0x1e784,
3783                 0x1e7c0, 0x1e7c8,
3784                 0x1e840, 0x1e84c,
3785                 0x1ea84, 0x1ea8c,
3786                 0x1eac0, 0x1eac0,
3787                 0x1eae0, 0x1eae0,
3788                 0x1eb00, 0x1eb84,
3789                 0x1ebc0, 0x1ebc8,
3790                 0x1ec40, 0x1ec4c,
3791                 0x1ee84, 0x1ee8c,
3792                 0x1eec0, 0x1eec0,
3793                 0x1eee0, 0x1eee0,
3794                 0x1ef00, 0x1ef84,
3795                 0x1efc0, 0x1efc8,
3796                 0x1f040, 0x1f04c,
3797                 0x1f284, 0x1f28c,
3798                 0x1f2c0, 0x1f2c0,
3799                 0x1f2e0, 0x1f2e0,
3800                 0x1f300, 0x1f384,
3801                 0x1f3c0, 0x1f3c8,
3802                 0x1f440, 0x1f44c,
3803                 0x1f684, 0x1f68c,
3804                 0x1f6c0, 0x1f6c0,
3805                 0x1f6e0, 0x1f6e0,
3806                 0x1f700, 0x1f784,
3807                 0x1f7c0, 0x1f7c8,
3808                 0x1f840, 0x1f84c,
3809                 0x1fa84, 0x1fa8c,
3810                 0x1fac0, 0x1fac0,
3811                 0x1fae0, 0x1fae0,
3812                 0x1fb00, 0x1fb84,
3813                 0x1fbc0, 0x1fbc8,
3814                 0x1fc40, 0x1fc4c,
3815                 0x1fe84, 0x1fe8c,
3816                 0x1fec0, 0x1fec0,
3817                 0x1fee0, 0x1fee0,
3818                 0x1ff00, 0x1ff84,
3819                 0x1ffc0, 0x1ffc8,
3820                 0x20000, 0x2002c,
3821                 0x20100, 0x2013c,
3822                 0x20190, 0x201c8,
3823                 0x20200, 0x20318,
3824                 0x20400, 0x20528,
3825                 0x20540, 0x20614,
3826                 0x21000, 0x21040,
3827                 0x2104c, 0x21060,
3828                 0x210c0, 0x210ec,
3829                 0x21200, 0x21268,
3830                 0x21270, 0x21284,
3831                 0x212fc, 0x21388,
3832                 0x21400, 0x21404,
3833                 0x21500, 0x21518,
3834                 0x2152c, 0x2153c,
3835                 0x21550, 0x21554,
3836                 0x21600, 0x21600,
3837                 0x21608, 0x21628,
3838                 0x21630, 0x2163c,
3839                 0x21700, 0x2171c,
3840                 0x21780, 0x2178c,
3841                 0x21800, 0x21c38,
3842                 0x21c80, 0x21d7c,
3843                 0x21e00, 0x21e04,
3844                 0x22000, 0x2202c,
3845                 0x22100, 0x2213c,
3846                 0x22190, 0x221c8,
3847                 0x22200, 0x22318,
3848                 0x22400, 0x22528,
3849                 0x22540, 0x22614,
3850                 0x23000, 0x23040,
3851                 0x2304c, 0x23060,
3852                 0x230c0, 0x230ec,
3853                 0x23200, 0x23268,
3854                 0x23270, 0x23284,
3855                 0x232fc, 0x23388,
3856                 0x23400, 0x23404,
3857                 0x23500, 0x23518,
3858                 0x2352c, 0x2353c,
3859                 0x23550, 0x23554,
3860                 0x23600, 0x23600,
3861                 0x23608, 0x23628,
3862                 0x23630, 0x2363c,
3863                 0x23700, 0x2371c,
3864                 0x23780, 0x2378c,
3865                 0x23800, 0x23c38,
3866                 0x23c80, 0x23d7c,
3867                 0x23e00, 0x23e04,
3868                 0x24000, 0x2402c,
3869                 0x24100, 0x2413c,
3870                 0x24190, 0x241c8,
3871                 0x24200, 0x24318,
3872                 0x24400, 0x24528,
3873                 0x24540, 0x24614,
3874                 0x25000, 0x25040,
3875                 0x2504c, 0x25060,
3876                 0x250c0, 0x250ec,
3877                 0x25200, 0x25268,
3878                 0x25270, 0x25284,
3879                 0x252fc, 0x25388,
3880                 0x25400, 0x25404,
3881                 0x25500, 0x25518,
3882                 0x2552c, 0x2553c,
3883                 0x25550, 0x25554,
3884                 0x25600, 0x25600,
3885                 0x25608, 0x25628,
3886                 0x25630, 0x2563c,
3887                 0x25700, 0x2571c,
3888                 0x25780, 0x2578c,
3889                 0x25800, 0x25c38,
3890                 0x25c80, 0x25d7c,
3891                 0x25e00, 0x25e04,
3892                 0x26000, 0x2602c,
3893                 0x26100, 0x2613c,
3894                 0x26190, 0x261c8,
3895                 0x26200, 0x26318,
3896                 0x26400, 0x26528,
3897                 0x26540, 0x26614,
3898                 0x27000, 0x27040,
3899                 0x2704c, 0x27060,
3900                 0x270c0, 0x270ec,
3901                 0x27200, 0x27268,
3902                 0x27270, 0x27284,
3903                 0x272fc, 0x27388,
3904                 0x27400, 0x27404,
3905                 0x27500, 0x27518,
3906                 0x2752c, 0x2753c,
3907                 0x27550, 0x27554,
3908                 0x27600, 0x27600,
3909                 0x27608, 0x27628,
3910                 0x27630, 0x2763c,
3911                 0x27700, 0x2771c,
3912                 0x27780, 0x2778c,
3913                 0x27800, 0x27c38,
3914                 0x27c80, 0x27d7c,
3915                 0x27e00, 0x27e04
3916         };
3917         static const unsigned int t5_reg_ranges[] = {
3918                 0x1008, 0x1148,
3919                 0x1180, 0x11b4,
3920                 0x11fc, 0x123c,
3921                 0x1280, 0x173c,
3922                 0x1800, 0x18fc,
3923                 0x3000, 0x3028,
3924                 0x3060, 0x30d8,
3925                 0x30e0, 0x30fc,
3926                 0x3140, 0x357c,
3927                 0x35a8, 0x35cc,
3928                 0x35ec, 0x35ec,
3929                 0x3600, 0x5624,
3930                 0x56cc, 0x575c,
3931                 0x580c, 0x5814,
3932                 0x5890, 0x58bc,
3933                 0x5940, 0x59dc,
3934                 0x59fc, 0x5a18,
3935                 0x5a60, 0x5a9c,
3936                 0x5b94, 0x5bfc,
3937                 0x6000, 0x6040,
3938                 0x6058, 0x614c,
3939                 0x7700, 0x7798,
3940                 0x77c0, 0x78fc,
3941                 0x7b00, 0x7c54,
3942                 0x7d00, 0x7efc,
3943                 0x8dc0, 0x8de0,
3944                 0x8df8, 0x8e84,
3945                 0x8ea0, 0x8f84,
3946                 0x8fc0, 0x90f8,
3947                 0x9400, 0x9470,
3948                 0x9600, 0x96f4,
3949                 0x9800, 0x9808,
3950                 0x9820, 0x983c,
3951                 0x9850, 0x9864,
3952                 0x9c00, 0x9c6c,
3953                 0x9c80, 0x9cec,
3954                 0x9d00, 0x9d6c,
3955                 0x9d80, 0x9dec,
3956                 0x9e00, 0x9e6c,
3957                 0x9e80, 0x9eec,
3958                 0x9f00, 0x9f6c,
3959                 0x9f80, 0xa020,
3960                 0xd004, 0xd03c,
3961                 0xdfc0, 0xdfe0,
3962                 0xe000, 0x11088,
3963                 0x1109c, 0x11110,
3964                 0x11118, 0x1117c,
3965                 0x11190, 0x11204,
3966                 0x19040, 0x1906c,
3967                 0x19078, 0x19080,
3968                 0x1908c, 0x19124,
3969                 0x19150, 0x191b0,
3970                 0x191d0, 0x191e8,
3971                 0x19238, 0x19290,
3972                 0x193f8, 0x19474,
3973                 0x19490, 0x194cc,
3974                 0x194f0, 0x194f8,
3975                 0x19c00, 0x19c60,
3976                 0x19c94, 0x19e10,
3977                 0x19e50, 0x19f34,
3978                 0x19f40, 0x19f50,
3979                 0x19f90, 0x19fe4,
3980                 0x1a000, 0x1a06c,
3981                 0x1a0b0, 0x1a120,
3982                 0x1a128, 0x1a138,
3983                 0x1a190, 0x1a1c4,
3984                 0x1a1fc, 0x1a1fc,
3985                 0x1e008, 0x1e00c,
3986                 0x1e040, 0x1e04c,
3987                 0x1e284, 0x1e290,
3988                 0x1e2c0, 0x1e2c0,
3989                 0x1e2e0, 0x1e2e0,
3990                 0x1e300, 0x1e384,
3991                 0x1e3c0, 0x1e3c8,
3992                 0x1e408, 0x1e40c,
3993                 0x1e440, 0x1e44c,
3994                 0x1e684, 0x1e690,
3995                 0x1e6c0, 0x1e6c0,
3996                 0x1e6e0, 0x1e6e0,
3997                 0x1e700, 0x1e784,
3998                 0x1e7c0, 0x1e7c8,
3999                 0x1e808, 0x1e80c,
4000                 0x1e840, 0x1e84c,
4001                 0x1ea84, 0x1ea90,
4002                 0x1eac0, 0x1eac0,
4003                 0x1eae0, 0x1eae0,
4004                 0x1eb00, 0x1eb84,
4005                 0x1ebc0, 0x1ebc8,
4006                 0x1ec08, 0x1ec0c,
4007                 0x1ec40, 0x1ec4c,
4008                 0x1ee84, 0x1ee90,
4009                 0x1eec0, 0x1eec0,
4010                 0x1eee0, 0x1eee0,
4011                 0x1ef00, 0x1ef84,
4012                 0x1efc0, 0x1efc8,
4013                 0x1f008, 0x1f00c,
4014                 0x1f040, 0x1f04c,
4015                 0x1f284, 0x1f290,
4016                 0x1f2c0, 0x1f2c0,
4017                 0x1f2e0, 0x1f2e0,
4018                 0x1f300, 0x1f384,
4019                 0x1f3c0, 0x1f3c8,
4020                 0x1f408, 0x1f40c,
4021                 0x1f440, 0x1f44c,
4022                 0x1f684, 0x1f690,
4023                 0x1f6c0, 0x1f6c0,
4024                 0x1f6e0, 0x1f6e0,
4025                 0x1f700, 0x1f784,
4026                 0x1f7c0, 0x1f7c8,
4027                 0x1f808, 0x1f80c,
4028                 0x1f840, 0x1f84c,
4029                 0x1fa84, 0x1fa90,
4030                 0x1fac0, 0x1fac0,
4031                 0x1fae0, 0x1fae0,
4032                 0x1fb00, 0x1fb84,
4033                 0x1fbc0, 0x1fbc8,
4034                 0x1fc08, 0x1fc0c,
4035                 0x1fc40, 0x1fc4c,
4036                 0x1fe84, 0x1fe90,
4037                 0x1fec0, 0x1fec0,
4038                 0x1fee0, 0x1fee0,
4039                 0x1ff00, 0x1ff84,
4040                 0x1ffc0, 0x1ffc8,
4041                 0x30000, 0x30030,
4042                 0x30100, 0x30144,
4043                 0x30190, 0x301d0,
4044                 0x30200, 0x30318,
4045                 0x30400, 0x3052c,
4046                 0x30540, 0x3061c,
4047                 0x30800, 0x30834,
4048                 0x308c0, 0x30908,
4049                 0x30910, 0x309ac,
4050                 0x30a00, 0x30a2c,
4051                 0x30a44, 0x30a50,
4052                 0x30a74, 0x30c24,
4053                 0x30d00, 0x30d00,
4054                 0x30d08, 0x30d14,
4055                 0x30d1c, 0x30d20,
4056                 0x30d3c, 0x30d50,
4057                 0x31200, 0x3120c,
4058                 0x31220, 0x31220,
4059                 0x31240, 0x31240,
4060                 0x31600, 0x3160c,
4061                 0x31a00, 0x31a1c,
4062                 0x31e00, 0x31e20,
4063                 0x31e38, 0x31e3c,
4064                 0x31e80, 0x31e80,
4065                 0x31e88, 0x31ea8,
4066                 0x31eb0, 0x31eb4,
4067                 0x31ec8, 0x31ed4,
4068                 0x31fb8, 0x32004,
4069                 0x32200, 0x32200,
4070                 0x32208, 0x32240,
4071                 0x32248, 0x32280,
4072                 0x32288, 0x322c0,
4073                 0x322c8, 0x322fc,
4074                 0x32600, 0x32630,
4075                 0x32a00, 0x32abc,
4076                 0x32b00, 0x32b70,
4077                 0x33000, 0x33048,
4078                 0x33060, 0x3309c,
4079                 0x330f0, 0x33148,
4080                 0x33160, 0x3319c,
4081                 0x331f0, 0x332e4,
4082                 0x332f8, 0x333e4,
4083                 0x333f8, 0x33448,
4084                 0x33460, 0x3349c,
4085                 0x334f0, 0x33548,
4086                 0x33560, 0x3359c,
4087                 0x335f0, 0x336e4,
4088                 0x336f8, 0x337e4,
4089                 0x337f8, 0x337fc,
4090                 0x33814, 0x33814,
4091                 0x3382c, 0x3382c,
4092                 0x33880, 0x3388c,
4093                 0x338e8, 0x338ec,
4094                 0x33900, 0x33948,
4095                 0x33960, 0x3399c,
4096                 0x339f0, 0x33ae4,
4097                 0x33af8, 0x33b10,
4098                 0x33b28, 0x33b28,
4099                 0x33b3c, 0x33b50,
4100                 0x33bf0, 0x33c10,
4101                 0x33c28, 0x33c28,
4102                 0x33c3c, 0x33c50,
4103                 0x33cf0, 0x33cfc,
4104                 0x34000, 0x34030,
4105                 0x34100, 0x34144,
4106                 0x34190, 0x341d0,
4107                 0x34200, 0x34318,
4108                 0x34400, 0x3452c,
4109                 0x34540, 0x3461c,
4110                 0x34800, 0x34834,
4111                 0x348c0, 0x34908,
4112                 0x34910, 0x349ac,
4113                 0x34a00, 0x34a2c,
4114                 0x34a44, 0x34a50,
4115                 0x34a74, 0x34c24,
4116                 0x34d00, 0x34d00,
4117                 0x34d08, 0x34d14,
4118                 0x34d1c, 0x34d20,
4119                 0x34d3c, 0x34d50,
4120                 0x35200, 0x3520c,
4121                 0x35220, 0x35220,
4122                 0x35240, 0x35240,
4123                 0x35600, 0x3560c,
4124                 0x35a00, 0x35a1c,
4125                 0x35e00, 0x35e20,
4126                 0x35e38, 0x35e3c,
4127                 0x35e80, 0x35e80,
4128                 0x35e88, 0x35ea8,
4129                 0x35eb0, 0x35eb4,
4130                 0x35ec8, 0x35ed4,
4131                 0x35fb8, 0x36004,
4132                 0x36200, 0x36200,
4133                 0x36208, 0x36240,
4134                 0x36248, 0x36280,
4135                 0x36288, 0x362c0,
4136                 0x362c8, 0x362fc,
4137                 0x36600, 0x36630,
4138                 0x36a00, 0x36abc,
4139                 0x36b00, 0x36b70,
4140                 0x37000, 0x37048,
4141                 0x37060, 0x3709c,
4142                 0x370f0, 0x37148,
4143                 0x37160, 0x3719c,
4144                 0x371f0, 0x372e4,
4145                 0x372f8, 0x373e4,
4146                 0x373f8, 0x37448,
4147                 0x37460, 0x3749c,
4148                 0x374f0, 0x37548,
4149                 0x37560, 0x3759c,
4150                 0x375f0, 0x376e4,
4151                 0x376f8, 0x377e4,
4152                 0x377f8, 0x377fc,
4153                 0x37814, 0x37814,
4154                 0x3782c, 0x3782c,
4155                 0x37880, 0x3788c,
4156                 0x378e8, 0x378ec,
4157                 0x37900, 0x37948,
4158                 0x37960, 0x3799c,
4159                 0x379f0, 0x37ae4,
4160                 0x37af8, 0x37b10,
4161                 0x37b28, 0x37b28,
4162                 0x37b3c, 0x37b50,
4163                 0x37bf0, 0x37c10,
4164                 0x37c28, 0x37c28,
4165                 0x37c3c, 0x37c50,
4166                 0x37cf0, 0x37cfc,
4167                 0x38000, 0x38030,
4168                 0x38100, 0x38144,
4169                 0x38190, 0x381d0,
4170                 0x38200, 0x38318,
4171                 0x38400, 0x3852c,
4172                 0x38540, 0x3861c,
4173                 0x38800, 0x38834,
4174                 0x388c0, 0x38908,
4175                 0x38910, 0x389ac,
4176                 0x38a00, 0x38a2c,
4177                 0x38a44, 0x38a50,
4178                 0x38a74, 0x38c24,
4179                 0x38d00, 0x38d00,
4180                 0x38d08, 0x38d14,
4181                 0x38d1c, 0x38d20,
4182                 0x38d3c, 0x38d50,
4183                 0x39200, 0x3920c,
4184                 0x39220, 0x39220,
4185                 0x39240, 0x39240,
4186                 0x39600, 0x3960c,
4187                 0x39a00, 0x39a1c,
4188                 0x39e00, 0x39e20,
4189                 0x39e38, 0x39e3c,
4190                 0x39e80, 0x39e80,
4191                 0x39e88, 0x39ea8,
4192                 0x39eb0, 0x39eb4,
4193                 0x39ec8, 0x39ed4,
4194                 0x39fb8, 0x3a004,
4195                 0x3a200, 0x3a200,
4196                 0x3a208, 0x3a240,
4197                 0x3a248, 0x3a280,
4198                 0x3a288, 0x3a2c0,
4199                 0x3a2c8, 0x3a2fc,
4200                 0x3a600, 0x3a630,
4201                 0x3aa00, 0x3aabc,
4202                 0x3ab00, 0x3ab70,
4203                 0x3b000, 0x3b048,
4204                 0x3b060, 0x3b09c,
4205                 0x3b0f0, 0x3b148,
4206                 0x3b160, 0x3b19c,
4207                 0x3b1f0, 0x3b2e4,
4208                 0x3b2f8, 0x3b3e4,
4209                 0x3b3f8, 0x3b448,
4210                 0x3b460, 0x3b49c,
4211                 0x3b4f0, 0x3b548,
4212                 0x3b560, 0x3b59c,
4213                 0x3b5f0, 0x3b6e4,
4214                 0x3b6f8, 0x3b7e4,
4215                 0x3b7f8, 0x3b7fc,
4216                 0x3b814, 0x3b814,
4217                 0x3b82c, 0x3b82c,
4218                 0x3b880, 0x3b88c,
4219                 0x3b8e8, 0x3b8ec,
4220                 0x3b900, 0x3b948,
4221                 0x3b960, 0x3b99c,
4222                 0x3b9f0, 0x3bae4,
4223                 0x3baf8, 0x3bb10,
4224                 0x3bb28, 0x3bb28,
4225                 0x3bb3c, 0x3bb50,
4226                 0x3bbf0, 0x3bc10,
4227                 0x3bc28, 0x3bc28,
4228                 0x3bc3c, 0x3bc50,
4229                 0x3bcf0, 0x3bcfc,
4230                 0x3c000, 0x3c030,
4231                 0x3c100, 0x3c144,
4232                 0x3c190, 0x3c1d0,
4233                 0x3c200, 0x3c318,
4234                 0x3c400, 0x3c52c,
4235                 0x3c540, 0x3c61c,
4236                 0x3c800, 0x3c834,
4237                 0x3c8c0, 0x3c908,
4238                 0x3c910, 0x3c9ac,
4239                 0x3ca00, 0x3ca2c,
4240                 0x3ca44, 0x3ca50,
4241                 0x3ca74, 0x3cc24,
4242                 0x3cd00, 0x3cd00,
4243                 0x3cd08, 0x3cd14,
4244                 0x3cd1c, 0x3cd20,
4245                 0x3cd3c, 0x3cd50,
4246                 0x3d200, 0x3d20c,
4247                 0x3d220, 0x3d220,
4248                 0x3d240, 0x3d240,
4249                 0x3d600, 0x3d60c,
4250                 0x3da00, 0x3da1c,
4251                 0x3de00, 0x3de20,
4252                 0x3de38, 0x3de3c,
4253                 0x3de80, 0x3de80,
4254                 0x3de88, 0x3dea8,
4255                 0x3deb0, 0x3deb4,
4256                 0x3dec8, 0x3ded4,
4257                 0x3dfb8, 0x3e004,
4258                 0x3e200, 0x3e200,
4259                 0x3e208, 0x3e240,
4260                 0x3e248, 0x3e280,
4261                 0x3e288, 0x3e2c0,
4262                 0x3e2c8, 0x3e2fc,
4263                 0x3e600, 0x3e630,
4264                 0x3ea00, 0x3eabc,
4265                 0x3eb00, 0x3eb70,
4266                 0x3f000, 0x3f048,
4267                 0x3f060, 0x3f09c,
4268                 0x3f0f0, 0x3f148,
4269                 0x3f160, 0x3f19c,
4270                 0x3f1f0, 0x3f2e4,
4271                 0x3f2f8, 0x3f3e4,
4272                 0x3f3f8, 0x3f448,
4273                 0x3f460, 0x3f49c,
4274                 0x3f4f0, 0x3f548,
4275                 0x3f560, 0x3f59c,
4276                 0x3f5f0, 0x3f6e4,
4277                 0x3f6f8, 0x3f7e4,
4278                 0x3f7f8, 0x3f7fc,
4279                 0x3f814, 0x3f814,
4280                 0x3f82c, 0x3f82c,
4281                 0x3f880, 0x3f88c,
4282                 0x3f8e8, 0x3f8ec,
4283                 0x3f900, 0x3f948,
4284                 0x3f960, 0x3f99c,
4285                 0x3f9f0, 0x3fae4,
4286                 0x3faf8, 0x3fb10,
4287                 0x3fb28, 0x3fb28,
4288                 0x3fb3c, 0x3fb50,
4289                 0x3fbf0, 0x3fc10,
4290                 0x3fc28, 0x3fc28,
4291                 0x3fc3c, 0x3fc50,
4292                 0x3fcf0, 0x3fcfc,
4293                 0x40000, 0x4000c,
4294                 0x40040, 0x40068,
4295                 0x4007c, 0x40144,
4296                 0x40180, 0x4018c,
4297                 0x40200, 0x40298,
4298                 0x402ac, 0x4033c,
4299                 0x403f8, 0x403fc,
4300                 0x41304, 0x413c4,
4301                 0x41400, 0x4141c,
4302                 0x41480, 0x414d0,
4303                 0x44000, 0x44078,
4304                 0x440c0, 0x44278,
4305                 0x442c0, 0x44478,
4306                 0x444c0, 0x44678,
4307                 0x446c0, 0x44878,
4308                 0x448c0, 0x449fc,
4309                 0x45000, 0x45068,
4310                 0x45080, 0x45084,
4311                 0x450a0, 0x450b0,
4312                 0x45200, 0x45268,
4313                 0x45280, 0x45284,
4314                 0x452a0, 0x452b0,
4315                 0x460c0, 0x460e4,
4316                 0x47000, 0x4708c,
4317                 0x47200, 0x47250,
4318                 0x47400, 0x47420,
4319                 0x47600, 0x47618,
4320                 0x47800, 0x47814,
4321                 0x48000, 0x4800c,
4322                 0x48040, 0x48068,
4323                 0x4807c, 0x48144,
4324                 0x48180, 0x4818c,
4325                 0x48200, 0x48298,
4326                 0x482ac, 0x4833c,
4327                 0x483f8, 0x483fc,
4328                 0x49304, 0x493c4,
4329                 0x49400, 0x4941c,
4330                 0x49480, 0x494d0,
4331                 0x4c000, 0x4c078,
4332                 0x4c0c0, 0x4c278,
4333                 0x4c2c0, 0x4c478,
4334                 0x4c4c0, 0x4c678,
4335                 0x4c6c0, 0x4c878,
4336                 0x4c8c0, 0x4c9fc,
4337                 0x4d000, 0x4d068,
4338                 0x4d080, 0x4d084,
4339                 0x4d0a0, 0x4d0b0,
4340                 0x4d200, 0x4d268,
4341                 0x4d280, 0x4d284,
4342                 0x4d2a0, 0x4d2b0,
4343                 0x4e0c0, 0x4e0e4,
4344                 0x4f000, 0x4f08c,
4345                 0x4f200, 0x4f250,
4346                 0x4f400, 0x4f420,
4347                 0x4f600, 0x4f618,
4348                 0x4f800, 0x4f814,
4349                 0x50000, 0x500cc,
4350                 0x50400, 0x50400,
4351                 0x50800, 0x508cc,
4352                 0x50c00, 0x50c00,
4353                 0x51000, 0x5101c,
4354                 0x51300, 0x51308,
4355         };
4356
4357         if (is_t4(sc)) {
4358                 reg_ranges = &t4_reg_ranges[0];
4359                 n = nitems(t4_reg_ranges);
4360         } else {
4361                 reg_ranges = &t5_reg_ranges[0];
4362                 n = nitems(t5_reg_ranges);
4363         }
4364
4365         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4366         for (i = 0; i < n; i += 2)
4367                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4368 }
4369
4370 static void
4371 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4372 {
4373         int i;
4374         u_int v, tnl_cong_drops;
4375         struct timeval tv;
4376         const struct timeval interval = {0, 250000};    /* 250ms */
4377
4378         getmicrotime(&tv);
4379         timevalsub(&tv, &interval);
4380         if (timevalcmp(&tv, &pi->last_refreshed, <))
4381                 return;
4382
4383         tnl_cong_drops = 0;
4384         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4385         for (i = 0; i < NCHAN; i++) {
4386                 if (pi->rx_chan_map & (1 << i)) {
4387                         mtx_lock(&sc->regwin_lock);
4388                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4389                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4390                         mtx_unlock(&sc->regwin_lock);
4391                         tnl_cong_drops += v;
4392                 }
4393         }
4394         pi->tnl_cong_drops = tnl_cong_drops;
4395         getmicrotime(&pi->last_refreshed);
4396 }
4397
4398 static void
4399 cxgbe_tick(void *arg)
4400 {
4401         struct port_info *pi = arg;
4402         struct adapter *sc = pi->adapter;
4403         struct ifnet *ifp = pi->ifp;
4404
4405         PORT_LOCK(pi);
4406         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4407                 PORT_UNLOCK(pi);
4408                 return; /* without scheduling another callout */
4409         }
4410
4411         cxgbe_refresh_stats(sc, pi);
4412
4413         callout_schedule(&pi->tick, hz);
4414         PORT_UNLOCK(pi);
4415 }
4416
4417 static void
4418 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4419 {
4420         struct ifnet *vlan;
4421
4422         if (arg != ifp || ifp->if_type != IFT_ETHER)
4423                 return;
4424
4425         vlan = VLAN_DEVAT(ifp, vid);
4426         VLAN_SETCOOKIE(vlan, ifp);
4427 }
4428
4429 static int
4430 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4431 {
4432
4433 #ifdef INVARIANTS
4434         panic("%s: opcode 0x%02x on iq %p with payload %p",
4435             __func__, rss->opcode, iq, m);
4436 #else
4437         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4438             __func__, rss->opcode, iq, m);
4439         m_freem(m);
4440 #endif
4441         return (EDOOFUS);
4442 }
4443
4444 int
4445 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4446 {
4447         uintptr_t *loc, new;
4448
4449         if (opcode >= nitems(sc->cpl_handler))
4450                 return (EINVAL);
4451
4452         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4453         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4454         atomic_store_rel_ptr(loc, new);
4455
4456         return (0);
4457 }
4458
4459 static int
4460 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4461 {
4462
4463 #ifdef INVARIANTS
4464         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4465 #else
4466         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4467             __func__, iq, ctrl);
4468 #endif
4469         return (EDOOFUS);
4470 }
4471
4472 int
4473 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4474 {
4475         uintptr_t *loc, new;
4476
4477         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4478         loc = (uintptr_t *) &sc->an_handler;
4479         atomic_store_rel_ptr(loc, new);
4480
4481         return (0);
4482 }
4483
4484 static int
4485 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4486 {
4487         const struct cpl_fw6_msg *cpl =
4488             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4489
4490 #ifdef INVARIANTS
4491         panic("%s: fw_msg type %d", __func__, cpl->type);
4492 #else
4493         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4494 #endif
4495         return (EDOOFUS);
4496 }
4497
4498 int
4499 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4500 {
4501         uintptr_t *loc, new;
4502
4503         if (type >= nitems(sc->fw_msg_handler))
4504                 return (EINVAL);
4505
4506         /*
4507          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4508          * handler dispatch table.  Reject any attempt to install a handler for
4509          * this subtype.
4510          */
4511         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4512                 return (EINVAL);
4513
4514         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4515         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4516         atomic_store_rel_ptr(loc, new);
4517
4518         return (0);
4519 }
4520
4521 static int
4522 t4_sysctls(struct adapter *sc)
4523 {
4524         struct sysctl_ctx_list *ctx;
4525         struct sysctl_oid *oid;
4526         struct sysctl_oid_list *children, *c0;
4527         static char *caps[] = {
4528                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4529                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4530                     "\6HASHFILTER\7ETHOFLD",
4531                 "\20\1TOE",                             /* caps[2] toecaps */
4532                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4533                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4534                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4535                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4536                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4537                     "\4PO_INITIAOR\5PO_TARGET"
4538         };
4539         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4540
4541         ctx = device_get_sysctl_ctx(sc->dev);
4542
4543         /*
4544          * dev.t4nex.X.
4545          */
4546         oid = device_get_sysctl_tree(sc->dev);
4547         c0 = children = SYSCTL_CHILDREN(oid);
4548
4549         sc->sc_do_rxcopy = 1;
4550         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4551             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4552
4553         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4554             sc->params.nports, "# of ports");
4555
4556         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4557             NULL, chip_rev(sc), "chip hardware revision");
4558
4559         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4560             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4561
4562         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4563             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4564
4565         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4566             sc->cfcsum, "config file checksum");
4567
4568         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4569             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4570             sysctl_bitfield, "A", "available doorbells");
4571
4572         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4573             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4574             sysctl_bitfield, "A", "available link capabilities");
4575
4576         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4577             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4578             sysctl_bitfield, "A", "available NIC capabilities");
4579
4580         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4581             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4582             sysctl_bitfield, "A", "available TCP offload capabilities");
4583
4584         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4585             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4586             sysctl_bitfield, "A", "available RDMA capabilities");
4587
4588         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4589             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4590             sysctl_bitfield, "A", "available iSCSI capabilities");
4591
4592         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4593             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4594             sysctl_bitfield, "A", "available FCoE capabilities");
4595
4596         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4597             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4598
4599         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4600             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4601             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4602             "interrupt holdoff timer values (us)");
4603
4604         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4605             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4606             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4607             "interrupt holdoff packet counter values");
4608
4609         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4610             NULL, sc->tids.nftids, "number of filters");
4611
4612         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4613             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4614             "chip temperature (in Celsius)");
4615
4616         t4_sge_sysctls(sc, ctx, children);
4617
4618         sc->lro_timeout = 100;
4619         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4620             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4621
4622 #ifdef SBUF_DRAIN
4623         /*
4624          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4625          */
4626         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4627             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4628             "logs and miscellaneous information");
4629         children = SYSCTL_CHILDREN(oid);
4630
4631         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4632             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4633             sysctl_cctrl, "A", "congestion control");
4634
4635         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4636             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4637             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4638
4639         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4640             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4641             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4642
4643         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4644             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4645             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4646
4647         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4648             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4649             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4650
4651         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4652             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4653             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4654
4655         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4656             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4657             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4658
4659         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4660             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4661             sysctl_cim_la, "A", "CIM logic analyzer");
4662
4663         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4664             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4665             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4666
4667         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4668             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4669             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4670
4671         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4672             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4673             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4674
4675         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4676             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4677             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4678
4679         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4680             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4681             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4682
4683         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4684             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4685             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4686
4687         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4688             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4689             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4690
4691         if (is_t5(sc)) {
4692                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4693                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4694                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4695
4696                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4697                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4698                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4699         }
4700
4701         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4702             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4703             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4704
4705         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4706             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4707             sysctl_cim_qcfg, "A", "CIM queue configuration");
4708
4709         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4710             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4711             sysctl_cpl_stats, "A", "CPL statistics");
4712
4713         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4714             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4715             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4716
4717         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4718             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4719             sysctl_devlog, "A", "firmware's device log");
4720
4721         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4722             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4723             sysctl_fcoe_stats, "A", "FCoE statistics");
4724
4725         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4726             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4727             sysctl_hw_sched, "A", "hardware scheduler ");
4728
4729         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4730             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4731             sysctl_l2t, "A", "hardware L2 table");
4732
4733         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4734             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4735             sysctl_lb_stats, "A", "loopback statistics");
4736
4737         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4738             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4739             sysctl_meminfo, "A", "memory regions");
4740
4741         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4742             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4743             sysctl_mps_tcam, "A", "MPS TCAM entries");
4744
4745         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4746             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4747             sysctl_path_mtus, "A", "path MTUs");
4748
4749         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4750             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4751             sysctl_pm_stats, "A", "PM statistics");
4752
4753         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4754             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4755             sysctl_rdma_stats, "A", "RDMA statistics");
4756
4757         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4758             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4759             sysctl_tcp_stats, "A", "TCP statistics");
4760
4761         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4762             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4763             sysctl_tids, "A", "TID information");
4764
4765         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4766             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4767             sysctl_tp_err_stats, "A", "TP error statistics");
4768
4769         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4770             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4771             sysctl_tp_la, "A", "TP logic analyzer");
4772
4773         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4774             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4775             sysctl_tx_rate, "A", "Tx rate");
4776
4777         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4778             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4779             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4780
4781         if (is_t5(sc)) {
4782                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4783                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4784                     sysctl_wcwr_stats, "A", "write combined work requests");
4785         }
4786 #endif
4787
4788 #ifdef TCP_OFFLOAD
4789         if (is_offload(sc)) {
4790                 /*
4791                  * dev.t4nex.X.toe.
4792                  */
4793                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4794                     NULL, "TOE parameters");
4795                 children = SYSCTL_CHILDREN(oid);
4796
4797                 sc->tt.sndbuf = 256 * 1024;
4798                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4799                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4800
4801                 sc->tt.ddp = 0;
4802                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4803                     &sc->tt.ddp, 0, "DDP allowed");
4804
4805                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4806                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4807                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4808
4809                 sc->tt.ddp_thres =
4810                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4811                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4812                     &sc->tt.ddp_thres, 0, "DDP threshold");
4813
4814                 sc->tt.rx_coalesce = 1;
4815                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4816                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4817
4818                 sc->tt.tx_align = 1;
4819                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4820                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4821         }
4822 #endif
4823
4824
4825         return (0);
4826 }
4827
4828 static int
4829 cxgbe_sysctls(struct port_info *pi)
4830 {
4831         struct sysctl_ctx_list *ctx;
4832         struct sysctl_oid *oid;
4833         struct sysctl_oid_list *children;
4834         struct adapter *sc = pi->adapter;
4835
4836         ctx = device_get_sysctl_ctx(pi->dev);
4837
4838         /*
4839          * dev.cxgbe.X.
4840          */
4841         oid = device_get_sysctl_tree(pi->dev);
4842         children = SYSCTL_CHILDREN(oid);
4843
4844         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4845            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4846         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4847                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4848                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4849                     "PHY temperature (in Celsius)");
4850                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4851                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4852                     "PHY firmware version");
4853         }
4854         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4855             &pi->nrxq, 0, "# of rx queues");
4856         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4857             &pi->ntxq, 0, "# of tx queues");
4858         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4859             &pi->first_rxq, 0, "index of first rx queue");
4860         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4861             &pi->first_txq, 0, "index of first tx queue");
4862         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4863             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4864             "Reserve queue 0 for non-flowid packets");
4865
4866 #ifdef TCP_OFFLOAD
4867         if (is_offload(sc)) {
4868                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4869                     &pi->nofldrxq, 0,
4870                     "# of rx queues for offloaded TCP connections");
4871                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4872                     &pi->nofldtxq, 0,
4873                     "# of tx queues for offloaded TCP connections");
4874                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4875                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4876                     "index of first TOE rx queue");
4877                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4878                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4879                     "index of first TOE tx queue");
4880         }
4881 #endif
4882 #ifdef DEV_NETMAP
4883         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4884             &pi->nnmrxq, 0, "# of rx queues for netmap");
4885         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4886             &pi->nnmtxq, 0, "# of tx queues for netmap");
4887         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4888             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4889             "index of first netmap rx queue");
4890         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4891             CTLFLAG_RD, &pi->first_nm_txq, 0,
4892             "index of first netmap tx queue");
4893 #endif
4894
4895         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4896             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4897             "holdoff timer index");
4898         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4899             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4900             "holdoff packet counter index");
4901
4902         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4903             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4904             "rx queue size");
4905         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4906             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4907             "tx queue size");
4908
4909         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4910             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4911             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4912
4913         /*
4914          * dev.cxgbe.X.stats.
4915          */
4916         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4917             NULL, "port statistics");
4918         children = SYSCTL_CHILDREN(oid);
4919         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4920             &pi->tx_parse_error, 0,
4921             "# of tx packets with invalid length or # of segments");
4922
4923 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4924         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4925             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4926             sysctl_handle_t4_reg64, "QU", desc)
4927
4928         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4929             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4930         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4931             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4932         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4934         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4936         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4938         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4940         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4941             "# of tx frames in this range",
4942             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4943         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4944             "# of tx frames in this range",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4946         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4947             "# of tx frames in this range",
4948             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4949         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4950             "# of tx frames in this range",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4952         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4953             "# of tx frames in this range",
4954             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4955         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4956             "# of tx frames in this range",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4958         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4959             "# of tx frames in this range",
4960             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4961         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4962             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4963         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4964             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4965         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4966             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4967         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4968             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4969         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4971         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4972             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4973         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4974             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4975         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4976             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4977         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4979         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4980             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4981
4982         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4983             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4984         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4985             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4986         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4987             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4988         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4989             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4990         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4991             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4992         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4993             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4994         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4995             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4996         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4997             "# of frames received with bad FCS",
4998             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4999         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5000             "# of frames received with length error",
5001             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5002         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5003             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5004         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5005             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5006         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5007             "# of rx frames in this range",
5008             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5009         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5010             "# of rx frames in this range",
5011             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5012         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5013             "# of rx frames in this range",
5014             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5015         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5016             "# of rx frames in this range",
5017             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5018         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5019             "# of rx frames in this range",
5020             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5021         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5022             "# of rx frames in this range",
5023             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5024         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5025             "# of rx frames in this range",
5026             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5027         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5028             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5029         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5030             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5031         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5032             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5033         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5034             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5035         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5036             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5037         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5038             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5039         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5040             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5041         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5042             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5043         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5044             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5045
5046 #undef SYSCTL_ADD_T4_REG64
5047
5048 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5049         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5050             &pi->stats.name, desc)
5051
5052         /* We get these from port_stats and they may be stale by upto 1s */
5053         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5054             "# drops due to buffer-group 0 overflows");
5055         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5056             "# drops due to buffer-group 1 overflows");
5057         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5058             "# drops due to buffer-group 2 overflows");
5059         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5060             "# drops due to buffer-group 3 overflows");
5061         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5062             "# of buffer-group 0 truncated packets");
5063         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5064             "# of buffer-group 1 truncated packets");
5065         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5066             "# of buffer-group 2 truncated packets");
5067         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5068             "# of buffer-group 3 truncated packets");
5069
5070 #undef SYSCTL_ADD_T4_PORTSTAT
5071
5072         return (0);
5073 }
5074
5075 static int
5076 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5077 {
5078         int rc, *i, space = 0;
5079         struct sbuf sb;
5080
5081         sbuf_new_for_sysctl(&sb, NULL, 64, req);
5082         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5083                 if (space)
5084                         sbuf_printf(&sb, " ");
5085                 sbuf_printf(&sb, "%d", *i);
5086                 space = 1;
5087         }
5088         rc = sbuf_finish(&sb);
5089         sbuf_delete(&sb);
5090         return (rc);
5091 }
5092
5093 static int
5094 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5095 {
5096         int rc;
5097         struct sbuf *sb;
5098
5099         rc = sysctl_wire_old_buffer(req, 0);
5100         if (rc != 0)
5101                 return(rc);
5102
5103         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5104         if (sb == NULL)
5105                 return (ENOMEM);
5106
5107         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5108         rc = sbuf_finish(sb);
5109         sbuf_delete(sb);
5110
5111         return (rc);
5112 }
5113
5114 static int
5115 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5116 {
5117         struct port_info *pi = arg1;
5118         int op = arg2;
5119         struct adapter *sc = pi->adapter;
5120         u_int v;
5121         int rc;
5122
5123         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5124         if (rc)
5125                 return (rc);
5126         /* XXX: magic numbers */
5127         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5128             &v);
5129         end_synchronized_op(sc, 0);
5130         if (rc)
5131                 return (rc);
5132         if (op == 0)
5133                 v /= 256;
5134
5135         rc = sysctl_handle_int(oidp, &v, 0, req);
5136         return (rc);
5137 }
5138
5139 static int
5140 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5141 {
5142         struct port_info *pi = arg1;
5143         int rc, val;
5144
5145         val = pi->rsrv_noflowq;
5146         rc = sysctl_handle_int(oidp, &val, 0, req);
5147         if (rc != 0 || req->newptr == NULL)
5148                 return (rc);
5149
5150         if ((val >= 1) && (pi->ntxq > 1))
5151                 pi->rsrv_noflowq = 1;
5152         else
5153                 pi->rsrv_noflowq = 0;
5154
5155         return (rc);
5156 }
5157
5158 static int
5159 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5160 {
5161         struct port_info *pi = arg1;
5162         struct adapter *sc = pi->adapter;
5163         int idx, rc, i;
5164         struct sge_rxq *rxq;
5165 #ifdef TCP_OFFLOAD
5166         struct sge_ofld_rxq *ofld_rxq;
5167 #endif
5168         uint8_t v;
5169
5170         idx = pi->tmr_idx;
5171
5172         rc = sysctl_handle_int(oidp, &idx, 0, req);
5173         if (rc != 0 || req->newptr == NULL)
5174                 return (rc);
5175
5176         if (idx < 0 || idx >= SGE_NTIMERS)
5177                 return (EINVAL);
5178
5179         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5180             "t4tmr");
5181         if (rc)
5182                 return (rc);
5183
5184         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5185         for_each_rxq(pi, i, rxq) {
5186 #ifdef atomic_store_rel_8
5187                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5188 #else
5189                 rxq->iq.intr_params = v;
5190 #endif
5191         }
5192 #ifdef TCP_OFFLOAD
5193         for_each_ofld_rxq(pi, i, ofld_rxq) {
5194 #ifdef atomic_store_rel_8
5195                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5196 #else
5197                 ofld_rxq->iq.intr_params = v;
5198 #endif
5199         }
5200 #endif
5201         pi->tmr_idx = idx;
5202
5203         end_synchronized_op(sc, LOCK_HELD);
5204         return (0);
5205 }
5206
5207 static int
5208 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5209 {
5210         struct port_info *pi = arg1;
5211         struct adapter *sc = pi->adapter;
5212         int idx, rc;
5213
5214         idx = pi->pktc_idx;
5215
5216         rc = sysctl_handle_int(oidp, &idx, 0, req);
5217         if (rc != 0 || req->newptr == NULL)
5218                 return (rc);
5219
5220         if (idx < -1 || idx >= SGE_NCOUNTERS)
5221                 return (EINVAL);
5222
5223         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5224             "t4pktc");
5225         if (rc)
5226                 return (rc);
5227
5228         if (pi->flags & PORT_INIT_DONE)
5229                 rc = EBUSY; /* cannot be changed once the queues are created */
5230         else
5231                 pi->pktc_idx = idx;
5232
5233         end_synchronized_op(sc, LOCK_HELD);
5234         return (rc);
5235 }
5236
5237 static int
5238 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5239 {
5240         struct port_info *pi = arg1;
5241         struct adapter *sc = pi->adapter;
5242         int qsize, rc;
5243
5244         qsize = pi->qsize_rxq;
5245
5246         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5247         if (rc != 0 || req->newptr == NULL)
5248                 return (rc);
5249
5250         if (qsize < 128 || (qsize & 7))
5251                 return (EINVAL);
5252
5253         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5254             "t4rxqs");
5255         if (rc)
5256                 return (rc);
5257
5258         if (pi->flags & PORT_INIT_DONE)
5259                 rc = EBUSY; /* cannot be changed once the queues are created */
5260         else
5261                 pi->qsize_rxq = qsize;
5262
5263         end_synchronized_op(sc, LOCK_HELD);
5264         return (rc);
5265 }
5266
5267 static int
5268 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5269 {
5270         struct port_info *pi = arg1;
5271         struct adapter *sc = pi->adapter;
5272         int qsize, rc;
5273
5274         qsize = pi->qsize_txq;
5275
5276         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5277         if (rc != 0 || req->newptr == NULL)
5278                 return (rc);
5279
5280         if (qsize < 128 || qsize > 65536)
5281                 return (EINVAL);
5282
5283         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5284             "t4txqs");
5285         if (rc)
5286                 return (rc);
5287
5288         if (pi->flags & PORT_INIT_DONE)
5289                 rc = EBUSY; /* cannot be changed once the queues are created */
5290         else
5291                 pi->qsize_txq = qsize;
5292
5293         end_synchronized_op(sc, LOCK_HELD);
5294         return (rc);
5295 }
5296
5297 static int
5298 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5299 {
5300         struct port_info *pi = arg1;
5301         struct adapter *sc = pi->adapter;
5302         struct link_config *lc = &pi->link_cfg;
5303         int rc;
5304
5305         if (req->newptr == NULL) {
5306                 struct sbuf *sb;
5307                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5308
5309                 rc = sysctl_wire_old_buffer(req, 0);
5310                 if (rc != 0)
5311                         return(rc);
5312
5313                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5314                 if (sb == NULL)
5315                         return (ENOMEM);
5316
5317                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5318                 rc = sbuf_finish(sb);
5319                 sbuf_delete(sb);
5320         } else {
5321                 char s[2];
5322                 int n;
5323
5324                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5325                 s[1] = 0;
5326
5327                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5328                 if (rc != 0)
5329                         return(rc);
5330
5331                 if (s[1] != 0)
5332                         return (EINVAL);
5333                 if (s[0] < '0' || s[0] > '9')
5334                         return (EINVAL);        /* not a number */
5335                 n = s[0] - '0';
5336                 if (n & ~(PAUSE_TX | PAUSE_RX))
5337                         return (EINVAL);        /* some other bit is set too */
5338
5339                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5340                 if (rc)
5341                         return (rc);
5342                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5343                         int link_ok = lc->link_ok;
5344
5345                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5346                         lc->requested_fc |= n;
5347                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5348                         lc->link_ok = link_ok;  /* restore */
5349                 }
5350                 end_synchronized_op(sc, 0);
5351         }
5352
5353         return (rc);
5354 }
5355
5356 static int
5357 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5358 {
5359         struct adapter *sc = arg1;
5360         int reg = arg2;
5361         uint64_t val;
5362
5363         val = t4_read_reg64(sc, reg);
5364
5365         return (sysctl_handle_64(oidp, &val, 0, req));
5366 }
5367
5368 static int
5369 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5370 {
5371         struct adapter *sc = arg1;
5372         int rc, t;
5373         uint32_t param, val;
5374
5375         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5376         if (rc)
5377                 return (rc);
5378         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5379             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5380             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5381         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5382         end_synchronized_op(sc, 0);
5383         if (rc)
5384                 return (rc);
5385
5386         /* unknown is returned as 0 but we display -1 in that case */
5387         t = val == 0 ? -1 : val;
5388
5389         rc = sysctl_handle_int(oidp, &t, 0, req);
5390         return (rc);
5391 }
5392
5393 #ifdef SBUF_DRAIN
5394 static int
5395 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5396 {
5397         struct adapter *sc = arg1;
5398         struct sbuf *sb;
5399         int rc, i;
5400         uint16_t incr[NMTUS][NCCTRL_WIN];
5401         static const char *dec_fac[] = {
5402                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5403                 "0.9375"
5404         };
5405
5406         rc = sysctl_wire_old_buffer(req, 0);
5407         if (rc != 0)
5408                 return (rc);
5409
5410         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5411         if (sb == NULL)
5412                 return (ENOMEM);
5413
5414         t4_read_cong_tbl(sc, incr);
5415
5416         for (i = 0; i < NCCTRL_WIN; ++i) {
5417                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5418                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5419                     incr[5][i], incr[6][i], incr[7][i]);
5420                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5421                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5422                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5423                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5424         }
5425
5426         rc = sbuf_finish(sb);
5427         sbuf_delete(sb);
5428
5429         return (rc);
5430 }
5431
5432 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5433         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5434         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5435         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5436 };
5437
5438 static int
5439 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5440 {
5441         struct adapter *sc = arg1;
5442         struct sbuf *sb;
5443         int rc, i, n, qid = arg2;
5444         uint32_t *buf, *p;
5445         char *qtype;
5446         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5447
5448         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5449             ("%s: bad qid %d\n", __func__, qid));
5450
5451         if (qid < CIM_NUM_IBQ) {
5452                 /* inbound queue */
5453                 qtype = "IBQ";
5454                 n = 4 * CIM_IBQ_SIZE;
5455                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5456                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5457         } else {
5458                 /* outbound queue */
5459                 qtype = "OBQ";
5460                 qid -= CIM_NUM_IBQ;
5461                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5462                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5463                 rc = t4_read_cim_obq(sc, qid, buf, n);
5464         }
5465
5466         if (rc < 0) {
5467                 rc = -rc;
5468                 goto done;
5469         }
5470         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5471
5472         rc = sysctl_wire_old_buffer(req, 0);
5473         if (rc != 0)
5474                 goto done;
5475
5476         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5477         if (sb == NULL) {
5478                 rc = ENOMEM;
5479                 goto done;
5480         }
5481
5482         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5483         for (i = 0, p = buf; i < n; i += 16, p += 4)
5484                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5485                     p[2], p[3]);
5486
5487         rc = sbuf_finish(sb);
5488         sbuf_delete(sb);
5489 done:
5490         free(buf, M_CXGBE);
5491         return (rc);
5492 }
5493
5494 static int
5495 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5496 {
5497         struct adapter *sc = arg1;
5498         u_int cfg;
5499         struct sbuf *sb;
5500         uint32_t *buf, *p;
5501         int rc;
5502
5503         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5504         if (rc != 0)
5505                 return (rc);
5506
5507         rc = sysctl_wire_old_buffer(req, 0);
5508         if (rc != 0)
5509                 return (rc);
5510
5511         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5512         if (sb == NULL)
5513                 return (ENOMEM);
5514
5515         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5516             M_ZERO | M_WAITOK);
5517
5518         rc = -t4_cim_read_la(sc, buf, NULL);
5519         if (rc != 0)
5520                 goto done;
5521
5522         sbuf_printf(sb, "Status   Data      PC%s",
5523             cfg & F_UPDBGLACAPTPCONLY ? "" :
5524             "     LS0Stat  LS0Addr             LS0Data");
5525
5526         KASSERT((sc->params.cim_la_size & 7) == 0,
5527             ("%s: p will walk off the end of buf", __func__));
5528
5529         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5530                 if (cfg & F_UPDBGLACAPTPCONLY) {
5531                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5532                             p[6], p[7]);
5533                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5534                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5535                             p[4] & 0xff, p[5] >> 8);
5536                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5537                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5538                             p[1] & 0xf, p[2] >> 4);
5539                 } else {
5540                         sbuf_printf(sb,
5541                             "\n  %02x   %x%07x %x%07x %08x %08x "
5542                             "%08x%08x%08x%08x",
5543                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5544                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5545                             p[6], p[7]);
5546                 }
5547         }
5548
5549         rc = sbuf_finish(sb);
5550         sbuf_delete(sb);
5551 done:
5552         free(buf, M_CXGBE);
5553         return (rc);
5554 }
5555
5556 static int
5557 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5558 {
5559         struct adapter *sc = arg1;
5560         u_int i;
5561         struct sbuf *sb;
5562         uint32_t *buf, *p;
5563         int rc;
5564
5565         rc = sysctl_wire_old_buffer(req, 0);
5566         if (rc != 0)
5567                 return (rc);
5568
5569         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5570         if (sb == NULL)
5571                 return (ENOMEM);
5572
5573         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5574             M_ZERO | M_WAITOK);
5575
5576         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5577         p = buf;
5578
5579         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5580                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5581                     p[1], p[0]);
5582         }
5583
5584         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5585         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5586                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5587                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5588                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5589                     (p[1] >> 2) | ((p[2] & 3) << 30),
5590                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5591                     p[0] & 1);
5592         }
5593
5594         rc = sbuf_finish(sb);
5595         sbuf_delete(sb);
5596         free(buf, M_CXGBE);
5597         return (rc);
5598 }
5599
5600 static int
5601 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5602 {
5603         struct adapter *sc = arg1;
5604         u_int i;
5605         struct sbuf *sb;
5606         uint32_t *buf, *p;
5607         int rc;
5608
5609         rc = sysctl_wire_old_buffer(req, 0);
5610         if (rc != 0)
5611                 return (rc);
5612
5613         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5614         if (sb == NULL)
5615                 return (ENOMEM);
5616
5617         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5618             M_ZERO | M_WAITOK);
5619
5620         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5621         p = buf;
5622
5623         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5624         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5625                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5626                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5627                     p[4], p[3], p[2], p[1], p[0]);
5628         }
5629
5630         sbuf_printf(sb, "\n\nCntl ID               Data");
5631         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5632                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5633                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5634         }
5635
5636         rc = sbuf_finish(sb);
5637         sbuf_delete(sb);
5638         free(buf, M_CXGBE);
5639         return (rc);
5640 }
5641
5642 static int
5643 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5644 {
5645         struct adapter *sc = arg1;
5646         struct sbuf *sb;
5647         int rc, i;
5648         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5649         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5650         uint16_t thres[CIM_NUM_IBQ];
5651         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5652         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5653         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5654
5655         if (is_t4(sc)) {
5656                 cim_num_obq = CIM_NUM_OBQ;
5657                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5658                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5659         } else {
5660                 cim_num_obq = CIM_NUM_OBQ_T5;
5661                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5662                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5663         }
5664         nq = CIM_NUM_IBQ + cim_num_obq;
5665
5666         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5667         if (rc == 0)
5668                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5669         if (rc != 0)
5670                 return (rc);
5671
5672         t4_read_cimq_cfg(sc, base, size, thres);
5673
5674         rc = sysctl_wire_old_buffer(req, 0);
5675         if (rc != 0)
5676                 return (rc);
5677
5678         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5679         if (sb == NULL)
5680                 return (ENOMEM);
5681
5682         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5683
5684         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5685                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5686                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5687                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5688                     G_QUEREMFLITS(p[2]) * 16);
5689         for ( ; i < nq; i++, p += 4, wr += 2)
5690                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5691                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5692                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5693                     G_QUEREMFLITS(p[2]) * 16);
5694
5695         rc = sbuf_finish(sb);
5696         sbuf_delete(sb);
5697
5698         return (rc);
5699 }
5700
5701 static int
5702 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5703 {
5704         struct adapter *sc = arg1;
5705         struct sbuf *sb;
5706         int rc;
5707         struct tp_cpl_stats stats;
5708
5709         rc = sysctl_wire_old_buffer(req, 0);
5710         if (rc != 0)
5711                 return (rc);
5712
5713         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5714         if (sb == NULL)
5715                 return (ENOMEM);
5716
5717         t4_tp_get_cpl_stats(sc, &stats);
5718
5719         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5720             "channel 3\n");
5721         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5722                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5723         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5724                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5725
5726         rc = sbuf_finish(sb);
5727         sbuf_delete(sb);
5728
5729         return (rc);
5730 }
5731
5732 static int
5733 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5734 {
5735         struct adapter *sc = arg1;
5736         struct sbuf *sb;
5737         int rc;
5738         struct tp_usm_stats stats;
5739
5740         rc = sysctl_wire_old_buffer(req, 0);
5741         if (rc != 0)
5742                 return(rc);
5743
5744         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5745         if (sb == NULL)
5746                 return (ENOMEM);
5747
5748         t4_get_usm_stats(sc, &stats);
5749
5750         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5751         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5752         sbuf_printf(sb, "Drops:  %u", stats.drops);
5753
5754         rc = sbuf_finish(sb);
5755         sbuf_delete(sb);
5756
5757         return (rc);
5758 }
5759
5760 const char *devlog_level_strings[] = {
5761         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5762         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5763         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5764         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5765         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5766         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5767 };
5768
5769 const char *devlog_facility_strings[] = {
5770         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5771         [FW_DEVLOG_FACILITY_CF]         = "CF",
5772         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5773         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5774         [FW_DEVLOG_FACILITY_RES]        = "RES",
5775         [FW_DEVLOG_FACILITY_HW]         = "HW",
5776         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5777         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5778         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5779         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5780         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5781         [FW_DEVLOG_FACILITY_VI]         = "VI",
5782         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5783         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5784         [FW_DEVLOG_FACILITY_TM]         = "TM",
5785         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5786         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5787         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5788         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5789         [FW_DEVLOG_FACILITY_RI]         = "RI",
5790         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5791         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5792         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5793         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5794 };
5795
5796 static int
5797 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5798 {
5799         struct adapter *sc = arg1;
5800         struct devlog_params *dparams = &sc->params.devlog;
5801         struct fw_devlog_e *buf, *e;
5802         int i, j, rc, nentries, first = 0, m;
5803         struct sbuf *sb;
5804         uint64_t ftstamp = UINT64_MAX;
5805
5806         if (dparams->start == 0) {
5807                 dparams->memtype = FW_MEMTYPE_EDC0;
5808                 dparams->start = 0x84000;
5809                 dparams->size = 32768;
5810         }
5811
5812         nentries = dparams->size / sizeof(struct fw_devlog_e);
5813
5814         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5815         if (buf == NULL)
5816                 return (ENOMEM);
5817
5818         m = fwmtype_to_hwmtype(dparams->memtype);
5819         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5820         if (rc != 0)
5821                 goto done;
5822
5823         for (i = 0; i < nentries; i++) {
5824                 e = &buf[i];
5825
5826                 if (e->timestamp == 0)
5827                         break;  /* end */
5828
5829                 e->timestamp = be64toh(e->timestamp);
5830                 e->seqno = be32toh(e->seqno);
5831                 for (j = 0; j < 8; j++)
5832                         e->params[j] = be32toh(e->params[j]);
5833
5834                 if (e->timestamp < ftstamp) {
5835                         ftstamp = e->timestamp;
5836                         first = i;
5837                 }
5838         }
5839
5840         if (buf[first].timestamp == 0)
5841                 goto done;      /* nothing in the log */
5842
5843         rc = sysctl_wire_old_buffer(req, 0);
5844         if (rc != 0)
5845                 goto done;
5846
5847         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5848         if (sb == NULL) {
5849                 rc = ENOMEM;
5850                 goto done;
5851         }
5852         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5853             "Seq#", "Tstamp", "Level", "Facility", "Message");
5854
5855         i = first;
5856         do {
5857                 e = &buf[i];
5858                 if (e->timestamp == 0)
5859                         break;  /* end */
5860
5861                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5862                     e->seqno, e->timestamp,
5863                     (e->level < nitems(devlog_level_strings) ?
5864                         devlog_level_strings[e->level] : "UNKNOWN"),
5865                     (e->facility < nitems(devlog_facility_strings) ?
5866                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5867                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5868                     e->params[2], e->params[3], e->params[4],
5869                     e->params[5], e->params[6], e->params[7]);
5870
5871                 if (++i == nentries)
5872                         i = 0;
5873         } while (i != first);
5874
5875         rc = sbuf_finish(sb);
5876         sbuf_delete(sb);
5877 done:
5878         free(buf, M_CXGBE);
5879         return (rc);
5880 }
5881
5882 static int
5883 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5884 {
5885         struct adapter *sc = arg1;
5886         struct sbuf *sb;
5887         int rc;
5888         struct tp_fcoe_stats stats[4];
5889
5890         rc = sysctl_wire_old_buffer(req, 0);
5891         if (rc != 0)
5892                 return (rc);
5893
5894         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5895         if (sb == NULL)
5896                 return (ENOMEM);
5897
5898         t4_get_fcoe_stats(sc, 0, &stats[0]);
5899         t4_get_fcoe_stats(sc, 1, &stats[1]);
5900         t4_get_fcoe_stats(sc, 2, &stats[2]);
5901         t4_get_fcoe_stats(sc, 3, &stats[3]);
5902
5903         sbuf_printf(sb, "                   channel 0        channel 1        "
5904             "channel 2        channel 3\n");
5905         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5906             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5907             stats[3].octetsDDP);
5908         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5909             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5910         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5911             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5912             stats[3].framesDrop);
5913
5914         rc = sbuf_finish(sb);
5915         sbuf_delete(sb);
5916
5917         return (rc);
5918 }
5919
5920 static int
5921 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5922 {
5923         struct adapter *sc = arg1;
5924         struct sbuf *sb;
5925         int rc, i;
5926         unsigned int map, kbps, ipg, mode;
5927         unsigned int pace_tab[NTX_SCHED];
5928
5929         rc = sysctl_wire_old_buffer(req, 0);
5930         if (rc != 0)
5931                 return (rc);
5932
5933         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5934         if (sb == NULL)
5935                 return (ENOMEM);
5936
5937         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5938         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5939         t4_read_pace_tbl(sc, pace_tab);
5940
5941         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5942             "Class IPG (0.1 ns)   Flow IPG (us)");
5943
5944         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5945                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5946                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5947                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5948                 if (kbps)
5949                         sbuf_printf(sb, "%9u     ", kbps);
5950                 else
5951                         sbuf_printf(sb, " disabled     ");
5952
5953                 if (ipg)
5954                         sbuf_printf(sb, "%13u        ", ipg);
5955                 else
5956                         sbuf_printf(sb, "     disabled        ");
5957
5958                 if (pace_tab[i])
5959                         sbuf_printf(sb, "%10u", pace_tab[i]);
5960                 else
5961                         sbuf_printf(sb, "  disabled");
5962         }
5963
5964         rc = sbuf_finish(sb);
5965         sbuf_delete(sb);
5966
5967         return (rc);
5968 }
5969
5970 static int
5971 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5972 {
5973         struct adapter *sc = arg1;
5974         struct sbuf *sb;
5975         int rc, i, j;
5976         uint64_t *p0, *p1;
5977         struct lb_port_stats s[2];
5978         static const char *stat_name[] = {
5979                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5980                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5981                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5982                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5983                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5984                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5985                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5986         };
5987
5988         rc = sysctl_wire_old_buffer(req, 0);
5989         if (rc != 0)
5990                 return (rc);
5991
5992         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5993         if (sb == NULL)
5994                 return (ENOMEM);
5995
5996         memset(s, 0, sizeof(s));
5997
5998         for (i = 0; i < 4; i += 2) {
5999                 t4_get_lb_stats(sc, i, &s[0]);
6000                 t4_get_lb_stats(sc, i + 1, &s[1]);
6001
6002                 p0 = &s[0].octets;
6003                 p1 = &s[1].octets;
6004                 sbuf_printf(sb, "%s                       Loopback %u"
6005                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6006
6007                 for (j = 0; j < nitems(stat_name); j++)
6008                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6009                                    *p0++, *p1++);
6010         }
6011
6012         rc = sbuf_finish(sb);
6013         sbuf_delete(sb);
6014
6015         return (rc);
6016 }
6017
6018 static int
6019 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6020 {
6021         int rc = 0;
6022         struct port_info *pi = arg1;
6023         struct sbuf *sb;
6024         static const char *linkdnreasons[] = {
6025                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6026                 "PHY overheated", "unknown", "rx los", "reserved7"
6027         };
6028
6029         rc = sysctl_wire_old_buffer(req, 0);
6030         if (rc != 0)
6031                 return(rc);
6032         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6033         if (sb == NULL)
6034                 return (ENOMEM);
6035
6036         if (pi->linkdnrc < 0)
6037                 sbuf_printf(sb, "n/a");
6038         else if (pi->linkdnrc < nitems(linkdnreasons))
6039                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6040         else
6041                 sbuf_printf(sb, "%d", pi->linkdnrc);
6042
6043         rc = sbuf_finish(sb);
6044         sbuf_delete(sb);
6045
6046         return (rc);
6047 }
6048
6049 struct mem_desc {
6050         unsigned int base;
6051         unsigned int limit;
6052         unsigned int idx;
6053 };
6054
6055 static int
6056 mem_desc_cmp(const void *a, const void *b)
6057 {
6058         return ((const struct mem_desc *)a)->base -
6059                ((const struct mem_desc *)b)->base;
6060 }
6061
6062 static void
6063 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6064     unsigned int to)
6065 {
6066         unsigned int size;
6067
6068         size = to - from + 1;
6069         if (size == 0)
6070                 return;
6071
6072         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6073         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6074 }
6075
6076 static int
6077 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6078 {
6079         struct adapter *sc = arg1;
6080         struct sbuf *sb;
6081         int rc, i, n;
6082         uint32_t lo, hi, used, alloc;
6083         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6084         static const char *region[] = {
6085                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6086                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6087                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6088                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6089                 "RQUDP region:", "PBL region:", "TXPBL region:",
6090                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6091                 "On-chip queues:"
6092         };
6093         struct mem_desc avail[4];
6094         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6095         struct mem_desc *md = mem;
6096
6097         rc = sysctl_wire_old_buffer(req, 0);
6098         if (rc != 0)
6099                 return (rc);
6100
6101         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6102         if (sb == NULL)
6103                 return (ENOMEM);
6104
6105         for (i = 0; i < nitems(mem); i++) {
6106                 mem[i].limit = 0;
6107                 mem[i].idx = i;
6108         }
6109
6110         /* Find and sort the populated memory ranges */
6111         i = 0;
6112         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6113         if (lo & F_EDRAM0_ENABLE) {
6114                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6115                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6116                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6117                 avail[i].idx = 0;
6118                 i++;
6119         }
6120         if (lo & F_EDRAM1_ENABLE) {
6121                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6122                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6123                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6124                 avail[i].idx = 1;
6125                 i++;
6126         }
6127         if (lo & F_EXT_MEM_ENABLE) {
6128                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6129                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6130                 avail[i].limit = avail[i].base +
6131                     (G_EXT_MEM_SIZE(hi) << 20);
6132                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6133                 i++;
6134         }
6135         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6136                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6137                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6138                 avail[i].limit = avail[i].base +
6139                     (G_EXT_MEM1_SIZE(hi) << 20);
6140                 avail[i].idx = 4;
6141                 i++;
6142         }
6143         if (!i)                                    /* no memory available */
6144                 return 0;
6145         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6146
6147         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6148         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6149         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6150         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6151         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6152         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6153         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6154         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6155         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6156
6157         /* the next few have explicit upper bounds */
6158         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6159         md->limit = md->base - 1 +
6160                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6161                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6162         md++;
6163
6164         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6165         md->limit = md->base - 1 +
6166                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6167                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6168         md++;
6169
6170         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6171                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6172                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6173                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6174         } else {
6175                 md->base = 0;
6176                 md->idx = nitems(region);  /* hide it */
6177         }
6178         md++;
6179
6180 #define ulp_region(reg) \
6181         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6182         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6183
6184         ulp_region(RX_ISCSI);
6185         ulp_region(RX_TDDP);
6186         ulp_region(TX_TPT);
6187         ulp_region(RX_STAG);
6188         ulp_region(RX_RQ);
6189         ulp_region(RX_RQUDP);
6190         ulp_region(RX_PBL);
6191         ulp_region(TX_PBL);
6192 #undef ulp_region
6193
6194         md->base = 0;
6195         md->idx = nitems(region);
6196         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6197                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6198                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6199                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6200         }
6201         md++;
6202
6203         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6204         md->limit = md->base + sc->tids.ntids - 1;
6205         md++;
6206         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6207         md->limit = md->base + sc->tids.ntids - 1;
6208         md++;
6209
6210         md->base = sc->vres.ocq.start;
6211         if (sc->vres.ocq.size)
6212                 md->limit = md->base + sc->vres.ocq.size - 1;
6213         else
6214                 md->idx = nitems(region);  /* hide it */
6215         md++;
6216
6217         /* add any address-space holes, there can be up to 3 */
6218         for (n = 0; n < i - 1; n++)
6219                 if (avail[n].limit < avail[n + 1].base)
6220                         (md++)->base = avail[n].limit;
6221         if (avail[n].limit)
6222                 (md++)->base = avail[n].limit;
6223
6224         n = md - mem;
6225         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6226
6227         for (lo = 0; lo < i; lo++)
6228                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6229                                 avail[lo].limit - 1);
6230
6231         sbuf_printf(sb, "\n");
6232         for (i = 0; i < n; i++) {
6233                 if (mem[i].idx >= nitems(region))
6234                         continue;                        /* skip holes */
6235                 if (!mem[i].limit)
6236                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6237                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6238                                 mem[i].limit);
6239         }
6240
6241         sbuf_printf(sb, "\n");
6242         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6243         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6244         mem_region_show(sb, "uP RAM:", lo, hi);
6245
6246         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6247         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6248         mem_region_show(sb, "uP Extmem2:", lo, hi);
6249
6250         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6251         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6252                    G_PMRXMAXPAGE(lo),
6253                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6254                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6255
6256         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6257         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6258         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6259                    G_PMTXMAXPAGE(lo),
6260                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6261                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6262         sbuf_printf(sb, "%u p-structs\n",
6263                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6264
6265         for (i = 0; i < 4; i++) {
6266                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6267                 if (is_t4(sc)) {
6268                         used = G_USED(lo);
6269                         alloc = G_ALLOC(lo);
6270                 } else {
6271                         used = G_T5_USED(lo);
6272                         alloc = G_T5_ALLOC(lo);
6273                 }
6274                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6275                            i, used, alloc);
6276         }
6277         for (i = 0; i < 4; i++) {
6278                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6279                 if (is_t4(sc)) {
6280                         used = G_USED(lo);
6281                         alloc = G_ALLOC(lo);
6282                 } else {
6283                         used = G_T5_USED(lo);
6284                         alloc = G_T5_ALLOC(lo);
6285                 }
6286                 sbuf_printf(sb,
6287                            "\nLoopback %d using %u pages out of %u allocated",
6288                            i, used, alloc);
6289         }
6290
6291         rc = sbuf_finish(sb);
6292         sbuf_delete(sb);
6293
6294         return (rc);
6295 }
6296
6297 static inline void
6298 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6299 {
6300         *mask = x | y;
6301         y = htobe64(y);
6302         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6303 }
6304
6305 static int
6306 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6307 {
6308         struct adapter *sc = arg1;
6309         struct sbuf *sb;
6310         int rc, i, n;
6311
6312         rc = sysctl_wire_old_buffer(req, 0);
6313         if (rc != 0)
6314                 return (rc);
6315
6316         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6317         if (sb == NULL)
6318                 return (ENOMEM);
6319
6320         sbuf_printf(sb,
6321             "Idx  Ethernet address     Mask     Vld Ports PF"
6322             "  VF              Replication             P0 P1 P2 P3  ML");
6323         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6324             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6325         for (i = 0; i < n; i++) {
6326                 uint64_t tcamx, tcamy, mask;
6327                 uint32_t cls_lo, cls_hi;
6328                 uint8_t addr[ETHER_ADDR_LEN];
6329
6330                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6331                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6332                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6333                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6334
6335                 if (tcamx & tcamy)
6336                         continue;
6337
6338                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6339                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6340                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6341                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6342                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6343                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6344                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6345
6346                 if (cls_lo & F_REPLICATE) {
6347                         struct fw_ldst_cmd ldst_cmd;
6348
6349                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6350                         ldst_cmd.op_to_addrspace =
6351                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6352                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6353                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6354                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6355                         ldst_cmd.u.mps.fid_ctl =
6356                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6357                                 V_FW_LDST_CMD_CTL(i));
6358
6359                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6360                             "t4mps");
6361                         if (rc)
6362                                 break;
6363                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6364                             sizeof(ldst_cmd), &ldst_cmd);
6365                         end_synchronized_op(sc, 0);
6366
6367                         if (rc != 0) {
6368                                 sbuf_printf(sb,
6369                                     " ------------ error %3u ------------", rc);
6370                                 rc = 0;
6371                         } else {
6372                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6373                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6374                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6375                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6376                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6377                         }
6378                 } else
6379                         sbuf_printf(sb, "%36s", "");
6380
6381                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6382                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6383                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6384         }
6385
6386         if (rc)
6387                 (void) sbuf_finish(sb);
6388         else
6389                 rc = sbuf_finish(sb);
6390         sbuf_delete(sb);
6391
6392         return (rc);
6393 }
6394
6395 static int
6396 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6397 {
6398         struct adapter *sc = arg1;
6399         struct sbuf *sb;
6400         int rc;
6401         uint16_t mtus[NMTUS];
6402
6403         rc = sysctl_wire_old_buffer(req, 0);
6404         if (rc != 0)
6405                 return (rc);
6406
6407         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6408         if (sb == NULL)
6409                 return (ENOMEM);
6410
6411         t4_read_mtu_tbl(sc, mtus, NULL);
6412
6413         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6414             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6415             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6416             mtus[14], mtus[15]);
6417
6418         rc = sbuf_finish(sb);
6419         sbuf_delete(sb);
6420
6421         return (rc);
6422 }
6423
6424 static int
6425 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6426 {
6427         struct adapter *sc = arg1;
6428         struct sbuf *sb;
6429         int rc, i;
6430         uint32_t cnt[PM_NSTATS];
6431         uint64_t cyc[PM_NSTATS];
6432         static const char *rx_stats[] = {
6433                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6434         };
6435         static const char *tx_stats[] = {
6436                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6437         };
6438
6439         rc = sysctl_wire_old_buffer(req, 0);
6440         if (rc != 0)
6441                 return (rc);
6442
6443         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6444         if (sb == NULL)
6445                 return (ENOMEM);
6446
6447         t4_pmtx_get_stats(sc, cnt, cyc);
6448         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6449         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6450                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6451                     cyc[i]);
6452
6453         t4_pmrx_get_stats(sc, cnt, cyc);
6454         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6455         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6456                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6457                     cyc[i]);
6458
6459         rc = sbuf_finish(sb);
6460         sbuf_delete(sb);
6461
6462         return (rc);
6463 }
6464
6465 static int
6466 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6467 {
6468         struct adapter *sc = arg1;
6469         struct sbuf *sb;
6470         int rc;
6471         struct tp_rdma_stats stats;
6472
6473         rc = sysctl_wire_old_buffer(req, 0);
6474         if (rc != 0)
6475                 return (rc);
6476
6477         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6478         if (sb == NULL)
6479                 return (ENOMEM);
6480
6481         t4_tp_get_rdma_stats(sc, &stats);
6482         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6483         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6484
6485         rc = sbuf_finish(sb);
6486         sbuf_delete(sb);
6487
6488         return (rc);
6489 }
6490
6491 static int
6492 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6493 {
6494         struct adapter *sc = arg1;
6495         struct sbuf *sb;
6496         int rc;
6497         struct tp_tcp_stats v4, v6;
6498
6499         rc = sysctl_wire_old_buffer(req, 0);
6500         if (rc != 0)
6501                 return (rc);
6502
6503         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6504         if (sb == NULL)
6505                 return (ENOMEM);
6506
6507         t4_tp_get_tcp_stats(sc, &v4, &v6);
6508         sbuf_printf(sb,
6509             "                                IP                 IPv6\n");
6510         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6511             v4.tcpOutRsts, v6.tcpOutRsts);
6512         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6513             v4.tcpInSegs, v6.tcpInSegs);
6514         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6515             v4.tcpOutSegs, v6.tcpOutSegs);
6516         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6517             v4.tcpRetransSegs, v6.tcpRetransSegs);
6518
6519         rc = sbuf_finish(sb);
6520         sbuf_delete(sb);
6521
6522         return (rc);
6523 }
6524
6525 static int
6526 sysctl_tids(SYSCTL_HANDLER_ARGS)
6527 {
6528         struct adapter *sc = arg1;
6529         struct sbuf *sb;
6530         int rc;
6531         struct tid_info *t = &sc->tids;
6532
6533         rc = sysctl_wire_old_buffer(req, 0);
6534         if (rc != 0)
6535                 return (rc);
6536
6537         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6538         if (sb == NULL)
6539                 return (ENOMEM);
6540
6541         if (t->natids) {
6542                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6543                     t->atids_in_use);
6544         }
6545
6546         if (t->ntids) {
6547                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6548                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6549
6550                         if (b) {
6551                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6552                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6553                                     t->ntids - 1);
6554                         } else {
6555                                 sbuf_printf(sb, "TID range: %u-%u",
6556                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6557                                     t->ntids - 1);
6558                         }
6559                 } else
6560                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6561                 sbuf_printf(sb, ", in use: %u\n",
6562                     atomic_load_acq_int(&t->tids_in_use));
6563         }
6564
6565         if (t->nstids) {
6566                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6567                     t->stid_base + t->nstids - 1, t->stids_in_use);
6568         }
6569
6570         if (t->nftids) {
6571                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6572                     t->ftid_base + t->nftids - 1);
6573         }
6574
6575         if (t->netids) {
6576                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6577                     t->etid_base + t->netids - 1);
6578         }
6579
6580         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6581             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6582             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6583
6584         rc = sbuf_finish(sb);
6585         sbuf_delete(sb);
6586
6587         return (rc);
6588 }
6589
6590 static int
6591 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6592 {
6593         struct adapter *sc = arg1;
6594         struct sbuf *sb;
6595         int rc;
6596         struct tp_err_stats stats;
6597
6598         rc = sysctl_wire_old_buffer(req, 0);
6599         if (rc != 0)
6600                 return (rc);
6601
6602         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6603         if (sb == NULL)
6604                 return (ENOMEM);
6605
6606         t4_tp_get_err_stats(sc, &stats);
6607
6608         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6609                       "channel 3\n");
6610         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6611             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6612             stats.macInErrs[3]);
6613         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6614             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6615             stats.hdrInErrs[3]);
6616         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6617             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6618             stats.tcpInErrs[3]);
6619         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6620             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6621             stats.tcp6InErrs[3]);
6622         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6623             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6624             stats.tnlCongDrops[3]);
6625         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6626             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6627             stats.tnlTxDrops[3]);
6628         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6629             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6630             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6631         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6632             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6633             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6634         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6635             stats.ofldNoNeigh, stats.ofldCongDefer);
6636
6637         rc = sbuf_finish(sb);
6638         sbuf_delete(sb);
6639
6640         return (rc);
6641 }
6642
6643 struct field_desc {
6644         const char *name;
6645         u_int start;
6646         u_int width;
6647 };
6648
6649 static void
6650 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6651 {
6652         char buf[32];
6653         int line_size = 0;
6654
6655         while (f->name) {
6656                 uint64_t mask = (1ULL << f->width) - 1;
6657                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6658                     ((uintmax_t)v >> f->start) & mask);
6659
6660                 if (line_size + len >= 79) {
6661                         line_size = 8;
6662                         sbuf_printf(sb, "\n        ");
6663                 }
6664                 sbuf_printf(sb, "%s ", buf);
6665                 line_size += len + 1;
6666                 f++;
6667         }
6668         sbuf_printf(sb, "\n");
6669 }
6670
6671 static struct field_desc tp_la0[] = {
6672         { "RcfOpCodeOut", 60, 4 },
6673         { "State", 56, 4 },
6674         { "WcfState", 52, 4 },
6675         { "RcfOpcSrcOut", 50, 2 },
6676         { "CRxError", 49, 1 },
6677         { "ERxError", 48, 1 },
6678         { "SanityFailed", 47, 1 },
6679         { "SpuriousMsg", 46, 1 },
6680         { "FlushInputMsg", 45, 1 },
6681         { "FlushInputCpl", 44, 1 },
6682         { "RssUpBit", 43, 1 },
6683         { "RssFilterHit", 42, 1 },
6684         { "Tid", 32, 10 },
6685         { "InitTcb", 31, 1 },
6686         { "LineNumber", 24, 7 },
6687         { "Emsg", 23, 1 },
6688         { "EdataOut", 22, 1 },
6689         { "Cmsg", 21, 1 },
6690         { "CdataOut", 20, 1 },
6691         { "EreadPdu", 19, 1 },
6692         { "CreadPdu", 18, 1 },
6693         { "TunnelPkt", 17, 1 },
6694         { "RcfPeerFin", 16, 1 },
6695         { "RcfReasonOut", 12, 4 },
6696         { "TxCchannel", 10, 2 },
6697         { "RcfTxChannel", 8, 2 },
6698         { "RxEchannel", 6, 2 },
6699         { "RcfRxChannel", 5, 1 },
6700         { "RcfDataOutSrdy", 4, 1 },
6701         { "RxDvld", 3, 1 },
6702         { "RxOoDvld", 2, 1 },
6703         { "RxCongestion", 1, 1 },
6704         { "TxCongestion", 0, 1 },
6705         { NULL }
6706 };
6707
6708 static struct field_desc tp_la1[] = {
6709         { "CplCmdIn", 56, 8 },
6710         { "CplCmdOut", 48, 8 },
6711         { "ESynOut", 47, 1 },
6712         { "EAckOut", 46, 1 },
6713         { "EFinOut", 45, 1 },
6714         { "ERstOut", 44, 1 },
6715         { "SynIn", 43, 1 },
6716         { "AckIn", 42, 1 },
6717         { "FinIn", 41, 1 },
6718         { "RstIn", 40, 1 },
6719         { "DataIn", 39, 1 },
6720         { "DataInVld", 38, 1 },
6721         { "PadIn", 37, 1 },
6722         { "RxBufEmpty", 36, 1 },
6723         { "RxDdp", 35, 1 },
6724         { "RxFbCongestion", 34, 1 },
6725         { "TxFbCongestion", 33, 1 },
6726         { "TxPktSumSrdy", 32, 1 },
6727         { "RcfUlpType", 28, 4 },
6728         { "Eread", 27, 1 },
6729         { "Ebypass", 26, 1 },
6730         { "Esave", 25, 1 },
6731         { "Static0", 24, 1 },
6732         { "Cread", 23, 1 },
6733         { "Cbypass", 22, 1 },
6734         { "Csave", 21, 1 },
6735         { "CPktOut", 20, 1 },
6736         { "RxPagePoolFull", 18, 2 },
6737         { "RxLpbkPkt", 17, 1 },
6738         { "TxLpbkPkt", 16, 1 },
6739         { "RxVfValid", 15, 1 },
6740         { "SynLearned", 14, 1 },
6741         { "SetDelEntry", 13, 1 },
6742         { "SetInvEntry", 12, 1 },
6743         { "CpcmdDvld", 11, 1 },
6744         { "CpcmdSave", 10, 1 },
6745         { "RxPstructsFull", 8, 2 },
6746         { "EpcmdDvld", 7, 1 },
6747         { "EpcmdFlush", 6, 1 },
6748         { "EpcmdTrimPrefix", 5, 1 },
6749         { "EpcmdTrimPostfix", 4, 1 },
6750         { "ERssIp4Pkt", 3, 1 },
6751         { "ERssIp6Pkt", 2, 1 },
6752         { "ERssTcpUdpPkt", 1, 1 },
6753         { "ERssFceFipPkt", 0, 1 },
6754         { NULL }
6755 };
6756
6757 static struct field_desc tp_la2[] = {
6758         { "CplCmdIn", 56, 8 },
6759         { "MpsVfVld", 55, 1 },
6760         { "MpsPf", 52, 3 },
6761         { "MpsVf", 44, 8 },
6762         { "SynIn", 43, 1 },
6763         { "AckIn", 42, 1 },
6764         { "FinIn", 41, 1 },
6765         { "RstIn", 40, 1 },
6766         { "DataIn", 39, 1 },
6767         { "DataInVld", 38, 1 },
6768         { "PadIn", 37, 1 },
6769         { "RxBufEmpty", 36, 1 },
6770         { "RxDdp", 35, 1 },
6771         { "RxFbCongestion", 34, 1 },
6772         { "TxFbCongestion", 33, 1 },
6773         { "TxPktSumSrdy", 32, 1 },
6774         { "RcfUlpType", 28, 4 },
6775         { "Eread", 27, 1 },
6776         { "Ebypass", 26, 1 },
6777         { "Esave", 25, 1 },
6778         { "Static0", 24, 1 },
6779         { "Cread", 23, 1 },
6780         { "Cbypass", 22, 1 },
6781         { "Csave", 21, 1 },
6782         { "CPktOut", 20, 1 },
6783         { "RxPagePoolFull", 18, 2 },
6784         { "RxLpbkPkt", 17, 1 },
6785         { "TxLpbkPkt", 16, 1 },
6786         { "RxVfValid", 15, 1 },
6787         { "SynLearned", 14, 1 },
6788         { "SetDelEntry", 13, 1 },
6789         { "SetInvEntry", 12, 1 },
6790         { "CpcmdDvld", 11, 1 },
6791         { "CpcmdSave", 10, 1 },
6792         { "RxPstructsFull", 8, 2 },
6793         { "EpcmdDvld", 7, 1 },
6794         { "EpcmdFlush", 6, 1 },
6795         { "EpcmdTrimPrefix", 5, 1 },
6796         { "EpcmdTrimPostfix", 4, 1 },
6797         { "ERssIp4Pkt", 3, 1 },
6798         { "ERssIp6Pkt", 2, 1 },
6799         { "ERssTcpUdpPkt", 1, 1 },
6800         { "ERssFceFipPkt", 0, 1 },
6801         { NULL }
6802 };
6803
6804 static void
6805 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6806 {
6807
6808         field_desc_show(sb, *p, tp_la0);
6809 }
6810
6811 static void
6812 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6813 {
6814
6815         if (idx)
6816                 sbuf_printf(sb, "\n");
6817         field_desc_show(sb, p[0], tp_la0);
6818         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6819                 field_desc_show(sb, p[1], tp_la0);
6820 }
6821
6822 static void
6823 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6824 {
6825
6826         if (idx)
6827                 sbuf_printf(sb, "\n");
6828         field_desc_show(sb, p[0], tp_la0);
6829         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6830                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6831 }
6832
6833 static int
6834 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6835 {
6836         struct adapter *sc = arg1;
6837         struct sbuf *sb;
6838         uint64_t *buf, *p;
6839         int rc;
6840         u_int i, inc;
6841         void (*show_func)(struct sbuf *, uint64_t *, int);
6842
6843         rc = sysctl_wire_old_buffer(req, 0);
6844         if (rc != 0)
6845                 return (rc);
6846
6847         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6848         if (sb == NULL)
6849                 return (ENOMEM);
6850
6851         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6852
6853         t4_tp_read_la(sc, buf, NULL);
6854         p = buf;
6855
6856         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6857         case 2:
6858                 inc = 2;
6859                 show_func = tp_la_show2;
6860                 break;
6861         case 3:
6862                 inc = 2;
6863                 show_func = tp_la_show3;
6864                 break;
6865         default:
6866                 inc = 1;
6867                 show_func = tp_la_show;
6868         }
6869
6870         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6871                 (*show_func)(sb, p, i);
6872
6873         rc = sbuf_finish(sb);
6874         sbuf_delete(sb);
6875         free(buf, M_CXGBE);
6876         return (rc);
6877 }
6878
6879 static int
6880 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6881 {
6882         struct adapter *sc = arg1;
6883         struct sbuf *sb;
6884         int rc;
6885         u64 nrate[NCHAN], orate[NCHAN];
6886
6887         rc = sysctl_wire_old_buffer(req, 0);
6888         if (rc != 0)
6889                 return (rc);
6890
6891         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6892         if (sb == NULL)
6893                 return (ENOMEM);
6894
6895         t4_get_chan_txrate(sc, nrate, orate);
6896         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6897                  "channel 3\n");
6898         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6899             nrate[0], nrate[1], nrate[2], nrate[3]);
6900         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6901             orate[0], orate[1], orate[2], orate[3]);
6902
6903         rc = sbuf_finish(sb);
6904         sbuf_delete(sb);
6905
6906         return (rc);
6907 }
6908
6909 static int
6910 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6911 {
6912         struct adapter *sc = arg1;
6913         struct sbuf *sb;
6914         uint32_t *buf, *p;
6915         int rc, i;
6916
6917         rc = sysctl_wire_old_buffer(req, 0);
6918         if (rc != 0)
6919                 return (rc);
6920
6921         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6922         if (sb == NULL)
6923                 return (ENOMEM);
6924
6925         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6926             M_ZERO | M_WAITOK);
6927
6928         t4_ulprx_read_la(sc, buf);
6929         p = buf;
6930
6931         sbuf_printf(sb, "      Pcmd        Type   Message"
6932             "                Data");
6933         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6934                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6935                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6936         }
6937
6938         rc = sbuf_finish(sb);
6939         sbuf_delete(sb);
6940         free(buf, M_CXGBE);
6941         return (rc);
6942 }
6943
6944 static int
6945 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6946 {
6947         struct adapter *sc = arg1;
6948         struct sbuf *sb;
6949         int rc, v;
6950
6951         rc = sysctl_wire_old_buffer(req, 0);
6952         if (rc != 0)
6953                 return (rc);
6954
6955         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6956         if (sb == NULL)
6957                 return (ENOMEM);
6958
6959         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6960         if (G_STATSOURCE_T5(v) == 7) {
6961                 if (G_STATMODE(v) == 0) {
6962                         sbuf_printf(sb, "total %d, incomplete %d",
6963                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6964                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6965                 } else if (G_STATMODE(v) == 1) {
6966                         sbuf_printf(sb, "total %d, data overflow %d",
6967                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6968                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6969                 }
6970         }
6971         rc = sbuf_finish(sb);
6972         sbuf_delete(sb);
6973
6974         return (rc);
6975 }
6976 #endif
6977
6978 static uint32_t
6979 fconf_to_mode(uint32_t fconf)
6980 {
6981         uint32_t mode;
6982
6983         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6984             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6985
6986         if (fconf & F_FRAGMENTATION)
6987                 mode |= T4_FILTER_IP_FRAGMENT;
6988
6989         if (fconf & F_MPSHITTYPE)
6990                 mode |= T4_FILTER_MPS_HIT_TYPE;
6991
6992         if (fconf & F_MACMATCH)
6993                 mode |= T4_FILTER_MAC_IDX;
6994
6995         if (fconf & F_ETHERTYPE)
6996                 mode |= T4_FILTER_ETH_TYPE;
6997
6998         if (fconf & F_PROTOCOL)
6999                 mode |= T4_FILTER_IP_PROTO;
7000
7001         if (fconf & F_TOS)
7002                 mode |= T4_FILTER_IP_TOS;
7003
7004         if (fconf & F_VLAN)
7005                 mode |= T4_FILTER_VLAN;
7006
7007         if (fconf & F_VNIC_ID)
7008                 mode |= T4_FILTER_VNIC;
7009
7010         if (fconf & F_PORT)
7011                 mode |= T4_FILTER_PORT;
7012
7013         if (fconf & F_FCOE)
7014                 mode |= T4_FILTER_FCoE;
7015
7016         return (mode);
7017 }
7018
7019 static uint32_t
7020 mode_to_fconf(uint32_t mode)
7021 {
7022         uint32_t fconf = 0;
7023
7024         if (mode & T4_FILTER_IP_FRAGMENT)
7025                 fconf |= F_FRAGMENTATION;
7026
7027         if (mode & T4_FILTER_MPS_HIT_TYPE)
7028                 fconf |= F_MPSHITTYPE;
7029
7030         if (mode & T4_FILTER_MAC_IDX)
7031                 fconf |= F_MACMATCH;
7032
7033         if (mode & T4_FILTER_ETH_TYPE)
7034                 fconf |= F_ETHERTYPE;
7035
7036         if (mode & T4_FILTER_IP_PROTO)
7037                 fconf |= F_PROTOCOL;
7038
7039         if (mode & T4_FILTER_IP_TOS)
7040                 fconf |= F_TOS;
7041
7042         if (mode & T4_FILTER_VLAN)
7043                 fconf |= F_VLAN;
7044
7045         if (mode & T4_FILTER_VNIC)
7046                 fconf |= F_VNIC_ID;
7047
7048         if (mode & T4_FILTER_PORT)
7049                 fconf |= F_PORT;
7050
7051         if (mode & T4_FILTER_FCoE)
7052                 fconf |= F_FCOE;
7053
7054         return (fconf);
7055 }
7056
7057 static uint32_t
7058 fspec_to_fconf(struct t4_filter_specification *fs)
7059 {
7060         uint32_t fconf = 0;
7061
7062         if (fs->val.frag || fs->mask.frag)
7063                 fconf |= F_FRAGMENTATION;
7064
7065         if (fs->val.matchtype || fs->mask.matchtype)
7066                 fconf |= F_MPSHITTYPE;
7067
7068         if (fs->val.macidx || fs->mask.macidx)
7069                 fconf |= F_MACMATCH;
7070
7071         if (fs->val.ethtype || fs->mask.ethtype)
7072                 fconf |= F_ETHERTYPE;
7073
7074         if (fs->val.proto || fs->mask.proto)
7075                 fconf |= F_PROTOCOL;
7076
7077         if (fs->val.tos || fs->mask.tos)
7078                 fconf |= F_TOS;
7079
7080         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7081                 fconf |= F_VLAN;
7082
7083         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7084                 fconf |= F_VNIC_ID;
7085
7086         if (fs->val.iport || fs->mask.iport)
7087                 fconf |= F_PORT;
7088
7089         if (fs->val.fcoe || fs->mask.fcoe)
7090                 fconf |= F_FCOE;
7091
7092         return (fconf);
7093 }
7094
7095 static int
7096 get_filter_mode(struct adapter *sc, uint32_t *mode)
7097 {
7098         int rc;
7099         uint32_t fconf;
7100
7101         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7102             "t4getfm");
7103         if (rc)
7104                 return (rc);
7105
7106         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7107             A_TP_VLAN_PRI_MAP);
7108
7109         if (sc->params.tp.vlan_pri_map != fconf) {
7110                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7111                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7112                     fconf);
7113         }
7114
7115         *mode = fconf_to_mode(fconf);
7116
7117         end_synchronized_op(sc, LOCK_HELD);
7118         return (0);
7119 }
7120
7121 static int
7122 set_filter_mode(struct adapter *sc, uint32_t mode)
7123 {
7124         uint32_t fconf;
7125         int rc;
7126
7127         fconf = mode_to_fconf(mode);
7128
7129         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7130             "t4setfm");
7131         if (rc)
7132                 return (rc);
7133
7134         if (sc->tids.ftids_in_use > 0) {
7135                 rc = EBUSY;
7136                 goto done;
7137         }
7138
7139 #ifdef TCP_OFFLOAD
7140         if (uld_active(sc, ULD_TOM)) {
7141                 rc = EBUSY;
7142                 goto done;
7143         }
7144 #endif
7145
7146         rc = -t4_set_filter_mode(sc, fconf);
7147 done:
7148         end_synchronized_op(sc, LOCK_HELD);
7149         return (rc);
7150 }
7151
7152 static inline uint64_t
7153 get_filter_hits(struct adapter *sc, uint32_t fid)
7154 {
7155         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7156         uint64_t hits;
7157
7158         memwin_info(sc, 0, &mw_base, NULL);
7159         off = position_memwin(sc, 0,
7160             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7161         if (is_t4(sc)) {
7162                 hits = t4_read_reg64(sc, mw_base + off + 16);
7163                 hits = be64toh(hits);
7164         } else {
7165                 hits = t4_read_reg(sc, mw_base + off + 24);
7166                 hits = be32toh(hits);
7167         }
7168
7169         return (hits);
7170 }
7171
7172 static int
7173 get_filter(struct adapter *sc, struct t4_filter *t)
7174 {
7175         int i, rc, nfilters = sc->tids.nftids;
7176         struct filter_entry *f;
7177
7178         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7179             "t4getf");
7180         if (rc)
7181                 return (rc);
7182
7183         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7184             t->idx >= nfilters) {
7185                 t->idx = 0xffffffff;
7186                 goto done;
7187         }
7188
7189         f = &sc->tids.ftid_tab[t->idx];
7190         for (i = t->idx; i < nfilters; i++, f++) {
7191                 if (f->valid) {
7192                         t->idx = i;
7193                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7194                         t->smtidx = f->smtidx;
7195                         if (f->fs.hitcnts)
7196                                 t->hits = get_filter_hits(sc, t->idx);
7197                         else
7198                                 t->hits = UINT64_MAX;
7199                         t->fs = f->fs;
7200
7201                         goto done;
7202                 }
7203         }
7204
7205         t->idx = 0xffffffff;
7206 done:
7207         end_synchronized_op(sc, LOCK_HELD);
7208         return (0);
7209 }
7210
7211 static int
7212 set_filter(struct adapter *sc, struct t4_filter *t)
7213 {
7214         unsigned int nfilters, nports;
7215         struct filter_entry *f;
7216         int i, rc;
7217
7218         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7219         if (rc)
7220                 return (rc);
7221
7222         nfilters = sc->tids.nftids;
7223         nports = sc->params.nports;
7224
7225         if (nfilters == 0) {
7226                 rc = ENOTSUP;
7227                 goto done;
7228         }
7229
7230         if (!(sc->flags & FULL_INIT_DONE)) {
7231                 rc = EAGAIN;
7232                 goto done;
7233         }
7234
7235         if (t->idx >= nfilters) {
7236                 rc = EINVAL;
7237                 goto done;
7238         }
7239
7240         /* Validate against the global filter mode */
7241         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7242             sc->params.tp.vlan_pri_map) {
7243                 rc = E2BIG;
7244                 goto done;
7245         }
7246
7247         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7248                 rc = EINVAL;
7249                 goto done;
7250         }
7251
7252         if (t->fs.val.iport >= nports) {
7253                 rc = EINVAL;
7254                 goto done;
7255         }
7256
7257         /* Can't specify an iq if not steering to it */
7258         if (!t->fs.dirsteer && t->fs.iq) {
7259                 rc = EINVAL;
7260                 goto done;
7261         }
7262
7263         /* IPv6 filter idx must be 4 aligned */
7264         if (t->fs.type == 1 &&
7265             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7266                 rc = EINVAL;
7267                 goto done;
7268         }
7269
7270         if (sc->tids.ftid_tab == NULL) {
7271                 KASSERT(sc->tids.ftids_in_use == 0,
7272                     ("%s: no memory allocated but filters_in_use > 0",
7273                     __func__));
7274
7275                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7276                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7277                 if (sc->tids.ftid_tab == NULL) {
7278                         rc = ENOMEM;
7279                         goto done;
7280                 }
7281                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7282         }
7283
7284         for (i = 0; i < 4; i++) {
7285                 f = &sc->tids.ftid_tab[t->idx + i];
7286
7287                 if (f->pending || f->valid) {
7288                         rc = EBUSY;
7289                         goto done;
7290                 }
7291                 if (f->locked) {
7292                         rc = EPERM;
7293                         goto done;
7294                 }
7295
7296                 if (t->fs.type == 0)
7297                         break;
7298         }
7299
7300         f = &sc->tids.ftid_tab[t->idx];
7301         f->fs = t->fs;
7302
7303         rc = set_filter_wr(sc, t->idx);
7304 done:
7305         end_synchronized_op(sc, 0);
7306
7307         if (rc == 0) {
7308                 mtx_lock(&sc->tids.ftid_lock);
7309                 for (;;) {
7310                         if (f->pending == 0) {
7311                                 rc = f->valid ? 0 : EIO;
7312                                 break;
7313                         }
7314
7315                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7316                             PCATCH, "t4setfw", 0)) {
7317                                 rc = EINPROGRESS;
7318                                 break;
7319                         }
7320                 }
7321                 mtx_unlock(&sc->tids.ftid_lock);
7322         }
7323         return (rc);
7324 }
7325
7326 static int
7327 del_filter(struct adapter *sc, struct t4_filter *t)
7328 {
7329         unsigned int nfilters;
7330         struct filter_entry *f;
7331         int rc;
7332
7333         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7334         if (rc)
7335                 return (rc);
7336
7337         nfilters = sc->tids.nftids;
7338
7339         if (nfilters == 0) {
7340                 rc = ENOTSUP;
7341                 goto done;
7342         }
7343
7344         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7345             t->idx >= nfilters) {
7346                 rc = EINVAL;
7347                 goto done;
7348         }
7349
7350         if (!(sc->flags & FULL_INIT_DONE)) {
7351                 rc = EAGAIN;
7352                 goto done;
7353         }
7354
7355         f = &sc->tids.ftid_tab[t->idx];
7356
7357         if (f->pending) {
7358                 rc = EBUSY;
7359                 goto done;
7360         }
7361         if (f->locked) {
7362                 rc = EPERM;
7363                 goto done;
7364         }
7365
7366         if (f->valid) {
7367                 t->fs = f->fs;  /* extra info for the caller */
7368                 rc = del_filter_wr(sc, t->idx);
7369         }
7370
7371 done:
7372         end_synchronized_op(sc, 0);
7373
7374         if (rc == 0) {
7375                 mtx_lock(&sc->tids.ftid_lock);
7376                 for (;;) {
7377                         if (f->pending == 0) {
7378                                 rc = f->valid ? EIO : 0;
7379                                 break;
7380                         }
7381
7382                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7383                             PCATCH, "t4delfw", 0)) {
7384                                 rc = EINPROGRESS;
7385                                 break;
7386                         }
7387                 }
7388                 mtx_unlock(&sc->tids.ftid_lock);
7389         }
7390
7391         return (rc);
7392 }
7393
7394 static void
7395 clear_filter(struct filter_entry *f)
7396 {
7397         if (f->l2t)
7398                 t4_l2t_release(f->l2t);
7399
7400         bzero(f, sizeof (*f));
7401 }
7402
7403 static int
7404 set_filter_wr(struct adapter *sc, int fidx)
7405 {
7406         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7407         struct fw_filter_wr *fwr;
7408         unsigned int ftid;
7409         struct wrq_cookie cookie;
7410
7411         ASSERT_SYNCHRONIZED_OP(sc);
7412
7413         if (f->fs.newdmac || f->fs.newvlan) {
7414                 /* This filter needs an L2T entry; allocate one. */
7415                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7416                 if (f->l2t == NULL)
7417                         return (EAGAIN);
7418                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7419                     f->fs.dmac)) {
7420                         t4_l2t_release(f->l2t);
7421                         f->l2t = NULL;
7422                         return (ENOMEM);
7423                 }
7424         }
7425
7426         ftid = sc->tids.ftid_base + fidx;
7427
7428         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7429         if (fwr == NULL)
7430                 return (ENOMEM);
7431         bzero(fwr, sizeof(*fwr));
7432
7433         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7434         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7435         fwr->tid_to_iq =
7436             htobe32(V_FW_FILTER_WR_TID(ftid) |
7437                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7438                 V_FW_FILTER_WR_NOREPLY(0) |
7439                 V_FW_FILTER_WR_IQ(f->fs.iq));
7440         fwr->del_filter_to_l2tix =
7441             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7442                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7443                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7444                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7445                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7446                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7447                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7448                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7449                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7450                     f->fs.newvlan == VLAN_REWRITE) |
7451                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7452                     f->fs.newvlan == VLAN_REWRITE) |
7453                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7454                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7455                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7456                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7457         fwr->ethtype = htobe16(f->fs.val.ethtype);
7458         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7459         fwr->frag_to_ovlan_vldm =
7460             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7461                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7462                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7463                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7464                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7465                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7466         fwr->smac_sel = 0;
7467         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7468             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7469         fwr->maci_to_matchtypem =
7470             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7471                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7472                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7473                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7474                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7475                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7476                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7477                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7478         fwr->ptcl = f->fs.val.proto;
7479         fwr->ptclm = f->fs.mask.proto;
7480         fwr->ttyp = f->fs.val.tos;
7481         fwr->ttypm = f->fs.mask.tos;
7482         fwr->ivlan = htobe16(f->fs.val.vlan);
7483         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7484         fwr->ovlan = htobe16(f->fs.val.vnic);
7485         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7486         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7487         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7488         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7489         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7490         fwr->lp = htobe16(f->fs.val.dport);
7491         fwr->lpm = htobe16(f->fs.mask.dport);
7492         fwr->fp = htobe16(f->fs.val.sport);
7493         fwr->fpm = htobe16(f->fs.mask.sport);
7494         if (f->fs.newsmac)
7495                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7496
7497         f->pending = 1;
7498         sc->tids.ftids_in_use++;
7499
7500         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7501         return (0);
7502 }
7503
7504 static int
7505 del_filter_wr(struct adapter *sc, int fidx)
7506 {
7507         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7508         struct fw_filter_wr *fwr;
7509         unsigned int ftid;
7510         struct wrq_cookie cookie;
7511
7512         ftid = sc->tids.ftid_base + fidx;
7513
7514         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7515         if (fwr == NULL)
7516                 return (ENOMEM);
7517         bzero(fwr, sizeof (*fwr));
7518
7519         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7520
7521         f->pending = 1;
7522         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7523         return (0);
7524 }
7525
7526 int
7527 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7528 {
7529         struct adapter *sc = iq->adapter;
7530         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7531         unsigned int idx = GET_TID(rpl);
7532         unsigned int rc;
7533         struct filter_entry *f;
7534
7535         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7536             rss->opcode));
7537
7538         if (is_ftid(sc, idx)) {
7539
7540                 idx -= sc->tids.ftid_base;
7541                 f = &sc->tids.ftid_tab[idx];
7542                 rc = G_COOKIE(rpl->cookie);
7543
7544                 mtx_lock(&sc->tids.ftid_lock);
7545                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7546                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7547                             __func__, idx));
7548                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7549                         f->pending = 0;  /* asynchronous setup completed */
7550                         f->valid = 1;
7551                 } else {
7552                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7553                                 /* Add or delete failed, display an error */
7554                                 log(LOG_ERR,
7555                                     "filter %u setup failed with error %u\n",
7556                                     idx, rc);
7557                         }
7558
7559                         clear_filter(f);
7560                         sc->tids.ftids_in_use--;
7561                 }
7562                 wakeup(&sc->tids.ftid_tab);
7563                 mtx_unlock(&sc->tids.ftid_lock);
7564         }
7565
7566         return (0);
7567 }
7568
7569 static int
7570 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7571 {
7572         int rc;
7573
7574         if (cntxt->cid > M_CTXTQID)
7575                 return (EINVAL);
7576
7577         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7578             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7579                 return (EINVAL);
7580
7581         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7582         if (rc)
7583                 return (rc);
7584
7585         if (sc->flags & FW_OK) {
7586                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7587                     &cntxt->data[0]);
7588                 if (rc == 0)
7589                         goto done;
7590         }
7591
7592         /*
7593          * Read via firmware failed or wasn't even attempted.  Read directly via
7594          * the backdoor.
7595          */
7596         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7597 done:
7598         end_synchronized_op(sc, 0);
7599         return (rc);
7600 }
7601
7602 static int
7603 load_fw(struct adapter *sc, struct t4_data *fw)
7604 {
7605         int rc;
7606         uint8_t *fw_data;
7607
7608         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7609         if (rc)
7610                 return (rc);
7611
7612         if (sc->flags & FULL_INIT_DONE) {
7613                 rc = EBUSY;
7614                 goto done;
7615         }
7616
7617         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7618         if (fw_data == NULL) {
7619                 rc = ENOMEM;
7620                 goto done;
7621         }
7622
7623         rc = copyin(fw->data, fw_data, fw->len);
7624         if (rc == 0)
7625                 rc = -t4_load_fw(sc, fw_data, fw->len);
7626
7627         free(fw_data, M_CXGBE);
7628 done:
7629         end_synchronized_op(sc, 0);
7630         return (rc);
7631 }
7632
7633 static int
7634 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7635 {
7636         uint32_t addr, off, remaining, i, n;
7637         uint32_t *buf, *b;
7638         uint32_t mw_base, mw_aperture;
7639         int rc;
7640         uint8_t *dst;
7641
7642         rc = validate_mem_range(sc, mr->addr, mr->len);
7643         if (rc != 0)
7644                 return (rc);
7645
7646         memwin_info(sc, win, &mw_base, &mw_aperture);
7647         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7648         addr = mr->addr;
7649         remaining = mr->len;
7650         dst = (void *)mr->data;
7651
7652         while (remaining) {
7653                 off = position_memwin(sc, win, addr);
7654
7655                 /* number of bytes that we'll copy in the inner loop */
7656                 n = min(remaining, mw_aperture - off);
7657                 for (i = 0; i < n; i += 4)
7658                         *b++ = t4_read_reg(sc, mw_base + off + i);
7659
7660                 rc = copyout(buf, dst, n);
7661                 if (rc != 0)
7662                         break;
7663
7664                 b = buf;
7665                 dst += n;
7666                 remaining -= n;
7667                 addr += n;
7668         }
7669
7670         free(buf, M_CXGBE);
7671         return (rc);
7672 }
7673
7674 static int
7675 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7676 {
7677         int rc;
7678
7679         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7680                 return (EINVAL);
7681
7682         if (i2cd->len > sizeof(i2cd->data))
7683                 return (EFBIG);
7684
7685         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7686         if (rc)
7687                 return (rc);
7688         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7689             i2cd->offset, i2cd->len, &i2cd->data[0]);
7690         end_synchronized_op(sc, 0);
7691
7692         return (rc);
7693 }
7694
7695 static int
7696 in_range(int val, int lo, int hi)
7697 {
7698
7699         return (val < 0 || (val <= hi && val >= lo));
7700 }
7701
7702 static int
7703 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7704 {
7705         int fw_subcmd, fw_type, rc;
7706
7707         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7708         if (rc)
7709                 return (rc);
7710
7711         if (!(sc->flags & FULL_INIT_DONE)) {
7712                 rc = EAGAIN;
7713                 goto done;
7714         }
7715
7716         /*
7717          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7718          * sub-command and type are in common locations.)
7719          */
7720         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7721                 fw_subcmd = FW_SCHED_SC_CONFIG;
7722         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7723                 fw_subcmd = FW_SCHED_SC_PARAMS;
7724         else {
7725                 rc = EINVAL;
7726                 goto done;
7727         }
7728         if (p->type == SCHED_CLASS_TYPE_PACKET)
7729                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7730         else {
7731                 rc = EINVAL;
7732                 goto done;
7733         }
7734
7735         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7736                 /* Vet our parameters ..*/
7737                 if (p->u.config.minmax < 0) {
7738                         rc = EINVAL;
7739                         goto done;
7740                 }
7741
7742                 /* And pass the request to the firmware ...*/
7743                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7744                 goto done;
7745         }
7746
7747         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7748                 int fw_level;
7749                 int fw_mode;
7750                 int fw_rateunit;
7751                 int fw_ratemode;
7752
7753                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7754                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7755                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7756                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7757                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7758                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7759                 else {
7760                         rc = EINVAL;
7761                         goto done;
7762                 }
7763
7764                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7765                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7766                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7767                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7768                 else {
7769                         rc = EINVAL;
7770                         goto done;
7771                 }
7772
7773                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7774                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7775                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7776                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7777                 else {
7778                         rc = EINVAL;
7779                         goto done;
7780                 }
7781
7782                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7783                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7784                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7785                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7786                 else {
7787                         rc = EINVAL;
7788                         goto done;
7789                 }
7790
7791                 /* Vet our parameters ... */
7792                 if (!in_range(p->u.params.channel, 0, 3) ||
7793                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7794                     !in_range(p->u.params.minrate, 0, 10000000) ||
7795                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7796                     !in_range(p->u.params.weight, 0, 100)) {
7797                         rc = ERANGE;
7798                         goto done;
7799                 }
7800
7801                 /*
7802                  * Translate any unset parameters into the firmware's
7803                  * nomenclature and/or fail the call if the parameters
7804                  * are required ...
7805                  */
7806                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7807                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7808                         rc = EINVAL;
7809                         goto done;
7810                 }
7811                 if (p->u.params.minrate < 0)
7812                         p->u.params.minrate = 0;
7813                 if (p->u.params.maxrate < 0) {
7814                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7815                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7816                                 rc = EINVAL;
7817                                 goto done;
7818                         } else
7819                                 p->u.params.maxrate = 0;
7820                 }
7821                 if (p->u.params.weight < 0) {
7822                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7823                                 rc = EINVAL;
7824                                 goto done;
7825                         } else
7826                                 p->u.params.weight = 0;
7827                 }
7828                 if (p->u.params.pktsize < 0) {
7829                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7830                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7831                                 rc = EINVAL;
7832                                 goto done;
7833                         } else
7834                                 p->u.params.pktsize = 0;
7835                 }
7836
7837                 /* See what the firmware thinks of the request ... */
7838                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7839                     fw_rateunit, fw_ratemode, p->u.params.channel,
7840                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7841                     p->u.params.weight, p->u.params.pktsize, 1);
7842                 goto done;
7843         }
7844
7845         rc = EINVAL;
7846 done:
7847         end_synchronized_op(sc, 0);
7848         return (rc);
7849 }
7850
7851 static int
7852 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7853 {
7854         struct port_info *pi = NULL;
7855         struct sge_txq *txq;
7856         uint32_t fw_mnem, fw_queue, fw_class;
7857         int i, rc;
7858
7859         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7860         if (rc)
7861                 return (rc);
7862
7863         if (!(sc->flags & FULL_INIT_DONE)) {
7864                 rc = EAGAIN;
7865                 goto done;
7866         }
7867
7868         if (p->port >= sc->params.nports) {
7869                 rc = EINVAL;
7870                 goto done;
7871         }
7872
7873         pi = sc->port[p->port];
7874         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7875                 rc = EINVAL;
7876                 goto done;
7877         }
7878
7879         /*
7880          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7881          * Scheduling Class in this case).
7882          */
7883         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7884             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7885         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7886
7887         /*
7888          * If op.queue is non-negative, then we're only changing the scheduling
7889          * on a single specified TX queue.
7890          */
7891         if (p->queue >= 0) {
7892                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7893                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7894                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7895                     &fw_class);
7896                 goto done;
7897         }
7898
7899         /*
7900          * Change the scheduling on all the TX queues for the
7901          * interface.
7902          */
7903         for_each_txq(pi, i, txq) {
7904                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7905                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7906                     &fw_class);
7907                 if (rc)
7908                         goto done;
7909         }
7910
7911         rc = 0;
7912 done:
7913         end_synchronized_op(sc, 0);
7914         return (rc);
7915 }
7916
7917 int
7918 t4_os_find_pci_capability(struct adapter *sc, int cap)
7919 {
7920         int i;
7921
7922         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7923 }
7924
7925 int
7926 t4_os_pci_save_state(struct adapter *sc)
7927 {
7928         device_t dev;
7929         struct pci_devinfo *dinfo;
7930
7931         dev = sc->dev;
7932         dinfo = device_get_ivars(dev);
7933
7934         pci_cfg_save(dev, dinfo, 0);
7935         return (0);
7936 }
7937
7938 int
7939 t4_os_pci_restore_state(struct adapter *sc)
7940 {
7941         device_t dev;
7942         struct pci_devinfo *dinfo;
7943
7944         dev = sc->dev;
7945         dinfo = device_get_ivars(dev);
7946
7947         pci_cfg_restore(dev, dinfo);
7948         return (0);
7949 }
7950
7951 void
7952 t4_os_portmod_changed(const struct adapter *sc, int idx)
7953 {
7954         struct port_info *pi = sc->port[idx];
7955         static const char *mod_str[] = {
7956                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7957         };
7958
7959         build_medialist(pi, &pi->media);
7960 #ifdef DEV_NETMAP
7961         build_medialist(pi, &pi->nm_media);
7962 #endif
7963
7964         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7965                 if_printf(pi->ifp, "transceiver unplugged.\n");
7966         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7967                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7968         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7969                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7970         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7971                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7972                     mod_str[pi->mod_type]);
7973         } else {
7974                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7975                     pi->mod_type);
7976         }
7977 }
7978
7979 void
7980 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7981 {
7982         struct port_info *pi = sc->port[idx];
7983         struct ifnet *ifp = pi->ifp;
7984
7985         if (link_stat) {
7986                 pi->linkdnrc = -1;
7987                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7988                 if_link_state_change(ifp, LINK_STATE_UP);
7989         } else {
7990                 if (reason >= 0)
7991                         pi->linkdnrc = reason;
7992                 if_link_state_change(ifp, LINK_STATE_DOWN);
7993         }
7994 }
7995
7996 void
7997 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7998 {
7999         struct adapter *sc;
8000
8001         sx_slock(&t4_list_lock);
8002         SLIST_FOREACH(sc, &t4_list, link) {
8003                 /*
8004                  * func should not make any assumptions about what state sc is
8005                  * in - the only guarantee is that sc->sc_lock is a valid lock.
8006                  */
8007                 func(sc, arg);
8008         }
8009         sx_sunlock(&t4_list_lock);
8010 }
8011
8012 static int
8013 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8014 {
8015        return (0);
8016 }
8017
8018 static int
8019 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8020 {
8021        return (0);
8022 }
8023
8024 static int
8025 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8026     struct thread *td)
8027 {
8028         int rc;
8029         struct adapter *sc = dev->si_drv1;
8030
8031         rc = priv_check(td, PRIV_DRIVER);
8032         if (rc != 0)
8033                 return (rc);
8034
8035         switch (cmd) {
8036         case CHELSIO_T4_GETREG: {
8037                 struct t4_reg *edata = (struct t4_reg *)data;
8038
8039                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8040                         return (EFAULT);
8041
8042                 if (edata->size == 4)
8043                         edata->val = t4_read_reg(sc, edata->addr);
8044                 else if (edata->size == 8)
8045                         edata->val = t4_read_reg64(sc, edata->addr);
8046                 else
8047                         return (EINVAL);
8048
8049                 break;
8050         }
8051         case CHELSIO_T4_SETREG: {
8052                 struct t4_reg *edata = (struct t4_reg *)data;
8053
8054                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8055                         return (EFAULT);
8056
8057                 if (edata->size == 4) {
8058                         if (edata->val & 0xffffffff00000000)
8059                                 return (EINVAL);
8060                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8061                 } else if (edata->size == 8)
8062                         t4_write_reg64(sc, edata->addr, edata->val);
8063                 else
8064                         return (EINVAL);
8065                 break;
8066         }
8067         case CHELSIO_T4_REGDUMP: {
8068                 struct t4_regdump *regs = (struct t4_regdump *)data;
8069                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8070                 uint8_t *buf;
8071
8072                 if (regs->len < reglen) {
8073                         regs->len = reglen; /* hint to the caller */
8074                         return (ENOBUFS);
8075                 }
8076
8077                 regs->len = reglen;
8078                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8079                 t4_get_regs(sc, regs, buf);
8080                 rc = copyout(buf, regs->data, reglen);
8081                 free(buf, M_CXGBE);
8082                 break;
8083         }
8084         case CHELSIO_T4_GET_FILTER_MODE:
8085                 rc = get_filter_mode(sc, (uint32_t *)data);
8086                 break;
8087         case CHELSIO_T4_SET_FILTER_MODE:
8088                 rc = set_filter_mode(sc, *(uint32_t *)data);
8089                 break;
8090         case CHELSIO_T4_GET_FILTER:
8091                 rc = get_filter(sc, (struct t4_filter *)data);
8092                 break;
8093         case CHELSIO_T4_SET_FILTER:
8094                 rc = set_filter(sc, (struct t4_filter *)data);
8095                 break;
8096         case CHELSIO_T4_DEL_FILTER:
8097                 rc = del_filter(sc, (struct t4_filter *)data);
8098                 break;
8099         case CHELSIO_T4_GET_SGE_CONTEXT:
8100                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8101                 break;
8102         case CHELSIO_T4_LOAD_FW:
8103                 rc = load_fw(sc, (struct t4_data *)data);
8104                 break;
8105         case CHELSIO_T4_GET_MEM:
8106                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8107                 break;
8108         case CHELSIO_T4_GET_I2C:
8109                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8110                 break;
8111         case CHELSIO_T4_CLEAR_STATS: {
8112                 int i;
8113                 u_int port_id = *(uint32_t *)data;
8114                 struct port_info *pi;
8115
8116                 if (port_id >= sc->params.nports)
8117                         return (EINVAL);
8118                 pi = sc->port[port_id];
8119
8120                 /* MAC stats */
8121                 t4_clr_port_stats(sc, pi->tx_chan);
8122                 pi->tx_parse_error = 0;
8123
8124                 if (pi->flags & PORT_INIT_DONE) {
8125                         struct sge_rxq *rxq;
8126                         struct sge_txq *txq;
8127                         struct sge_wrq *wrq;
8128
8129                         for_each_rxq(pi, i, rxq) {
8130 #if defined(INET) || defined(INET6)
8131                                 rxq->lro.lro_queued = 0;
8132                                 rxq->lro.lro_flushed = 0;
8133 #endif
8134                                 rxq->rxcsum = 0;
8135                                 rxq->vlan_extraction = 0;
8136                         }
8137
8138                         for_each_txq(pi, i, txq) {
8139                                 txq->txcsum = 0;
8140                                 txq->tso_wrs = 0;
8141                                 txq->vlan_insertion = 0;
8142                                 txq->imm_wrs = 0;
8143                                 txq->sgl_wrs = 0;
8144                                 txq->txpkt_wrs = 0;
8145                                 txq->txpkts0_wrs = 0;
8146                                 txq->txpkts1_wrs = 0;
8147                                 txq->txpkts0_pkts = 0;
8148                                 txq->txpkts1_pkts = 0;
8149                                 mp_ring_reset_stats(txq->r);
8150                         }
8151
8152 #ifdef TCP_OFFLOAD
8153                         /* nothing to clear for each ofld_rxq */
8154
8155                         for_each_ofld_txq(pi, i, wrq) {
8156                                 wrq->tx_wrs_direct = 0;
8157                                 wrq->tx_wrs_copied = 0;
8158                         }
8159 #endif
8160                         wrq = &sc->sge.ctrlq[pi->port_id];
8161                         wrq->tx_wrs_direct = 0;
8162                         wrq->tx_wrs_copied = 0;
8163                 }
8164                 break;
8165         }
8166         case CHELSIO_T4_SCHED_CLASS:
8167                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8168                 break;
8169         case CHELSIO_T4_SCHED_QUEUE:
8170                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8171                 break;
8172         case CHELSIO_T4_GET_TRACER:
8173                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8174                 break;
8175         case CHELSIO_T4_SET_TRACER:
8176                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8177                 break;
8178         default:
8179                 rc = EINVAL;
8180         }
8181
8182         return (rc);
8183 }
8184
8185 #ifdef TCP_OFFLOAD
8186 void
8187 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8188     const unsigned int *pgsz_order)
8189 {
8190         struct port_info *pi = ifp->if_softc;
8191         struct adapter *sc = pi->adapter;
8192
8193         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8194         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8195                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8196                 V_HPZ3(pgsz_order[3]));
8197 }
8198
8199 static int
8200 toe_capability(struct port_info *pi, int enable)
8201 {
8202         int rc;
8203         struct adapter *sc = pi->adapter;
8204
8205         ASSERT_SYNCHRONIZED_OP(sc);
8206
8207         if (!is_offload(sc))
8208                 return (ENODEV);
8209
8210         if (enable) {
8211                 /*
8212                  * We need the port's queues around so that we're able to send
8213                  * and receive CPLs to/from the TOE even if the ifnet for this
8214                  * port has never been UP'd administratively.
8215                  */
8216                 if (!(pi->flags & PORT_INIT_DONE)) {
8217                         rc = cxgbe_init_synchronized(pi);
8218                         if (rc)
8219                                 return (rc);
8220                 }
8221
8222                 if (isset(&sc->offload_map, pi->port_id))
8223                         return (0);
8224
8225                 if (!uld_active(sc, ULD_TOM)) {
8226                         rc = t4_activate_uld(sc, ULD_TOM);
8227                         if (rc == EAGAIN) {
8228                                 log(LOG_WARNING,
8229                                     "You must kldload t4_tom.ko before trying "
8230                                     "to enable TOE on a cxgbe interface.\n");
8231                         }
8232                         if (rc != 0)
8233                                 return (rc);
8234                         KASSERT(sc->tom_softc != NULL,
8235                             ("%s: TOM activated but softc NULL", __func__));
8236                         KASSERT(uld_active(sc, ULD_TOM),
8237                             ("%s: TOM activated but flag not set", __func__));
8238                 }
8239
8240                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8241                 if (!uld_active(sc, ULD_IWARP))
8242                         (void) t4_activate_uld(sc, ULD_IWARP);
8243                 if (!uld_active(sc, ULD_ISCSI))
8244                         (void) t4_activate_uld(sc, ULD_ISCSI);
8245
8246                 setbit(&sc->offload_map, pi->port_id);
8247         } else {
8248                 if (!isset(&sc->offload_map, pi->port_id))
8249                         return (0);
8250
8251                 KASSERT(uld_active(sc, ULD_TOM),
8252                     ("%s: TOM never initialized?", __func__));
8253                 clrbit(&sc->offload_map, pi->port_id);
8254         }
8255
8256         return (0);
8257 }
8258
8259 /*
8260  * Add an upper layer driver to the global list.
8261  */
8262 int
8263 t4_register_uld(struct uld_info *ui)
8264 {
8265         int rc = 0;
8266         struct uld_info *u;
8267
8268         sx_xlock(&t4_uld_list_lock);
8269         SLIST_FOREACH(u, &t4_uld_list, link) {
8270             if (u->uld_id == ui->uld_id) {
8271                     rc = EEXIST;
8272                     goto done;
8273             }
8274         }
8275
8276         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8277         ui->refcount = 0;
8278 done:
8279         sx_xunlock(&t4_uld_list_lock);
8280         return (rc);
8281 }
8282
8283 int
8284 t4_unregister_uld(struct uld_info *ui)
8285 {
8286         int rc = EINVAL;
8287         struct uld_info *u;
8288
8289         sx_xlock(&t4_uld_list_lock);
8290
8291         SLIST_FOREACH(u, &t4_uld_list, link) {
8292             if (u == ui) {
8293                     if (ui->refcount > 0) {
8294                             rc = EBUSY;
8295                             goto done;
8296                     }
8297
8298                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8299                     rc = 0;
8300                     goto done;
8301             }
8302         }
8303 done:
8304         sx_xunlock(&t4_uld_list_lock);
8305         return (rc);
8306 }
8307
8308 int
8309 t4_activate_uld(struct adapter *sc, int id)
8310 {
8311         int rc;
8312         struct uld_info *ui;
8313
8314         ASSERT_SYNCHRONIZED_OP(sc);
8315
8316         if (id < 0 || id > ULD_MAX)
8317                 return (EINVAL);
8318         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8319
8320         sx_slock(&t4_uld_list_lock);
8321
8322         SLIST_FOREACH(ui, &t4_uld_list, link) {
8323                 if (ui->uld_id == id) {
8324                         if (!(sc->flags & FULL_INIT_DONE)) {
8325                                 rc = adapter_full_init(sc);
8326                                 if (rc != 0)
8327                                         break;
8328                         }
8329
8330                         rc = ui->activate(sc);
8331                         if (rc == 0) {
8332                                 setbit(&sc->active_ulds, id);
8333                                 ui->refcount++;
8334                         }
8335                         break;
8336                 }
8337         }
8338
8339         sx_sunlock(&t4_uld_list_lock);
8340
8341         return (rc);
8342 }
8343
8344 int
8345 t4_deactivate_uld(struct adapter *sc, int id)
8346 {
8347         int rc;
8348         struct uld_info *ui;
8349
8350         ASSERT_SYNCHRONIZED_OP(sc);
8351
8352         if (id < 0 || id > ULD_MAX)
8353                 return (EINVAL);
8354         rc = ENXIO;
8355
8356         sx_slock(&t4_uld_list_lock);
8357
8358         SLIST_FOREACH(ui, &t4_uld_list, link) {
8359                 if (ui->uld_id == id) {
8360                         rc = ui->deactivate(sc);
8361                         if (rc == 0) {
8362                                 clrbit(&sc->active_ulds, id);
8363                                 ui->refcount--;
8364                         }
8365                         break;
8366                 }
8367         }
8368
8369         sx_sunlock(&t4_uld_list_lock);
8370
8371         return (rc);
8372 }
8373
8374 int
8375 uld_active(struct adapter *sc, int uld_id)
8376 {
8377
8378         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8379
8380         return (isset(&sc->active_ulds, uld_id));
8381 }
8382 #endif
8383
8384 /*
8385  * Come up with reasonable defaults for some of the tunables, provided they're
8386  * not set by the user (in which case we'll use the values as is).
8387  */
8388 static void
8389 tweak_tunables(void)
8390 {
8391         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8392
8393         if (t4_ntxq10g < 1)
8394                 t4_ntxq10g = min(nc, NTXQ_10G);
8395
8396         if (t4_ntxq1g < 1)
8397                 t4_ntxq1g = min(nc, NTXQ_1G);
8398
8399         if (t4_nrxq10g < 1)
8400                 t4_nrxq10g = min(nc, NRXQ_10G);
8401
8402         if (t4_nrxq1g < 1)
8403                 t4_nrxq1g = min(nc, NRXQ_1G);
8404
8405 #ifdef TCP_OFFLOAD
8406         if (t4_nofldtxq10g < 1)
8407                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8408
8409         if (t4_nofldtxq1g < 1)
8410                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8411
8412         if (t4_nofldrxq10g < 1)
8413                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8414
8415         if (t4_nofldrxq1g < 1)
8416                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8417
8418         if (t4_toecaps_allowed == -1)
8419                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8420 #else
8421         if (t4_toecaps_allowed == -1)
8422                 t4_toecaps_allowed = 0;
8423 #endif
8424
8425 #ifdef DEV_NETMAP
8426         if (t4_nnmtxq10g < 1)
8427                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8428
8429         if (t4_nnmtxq1g < 1)
8430                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8431
8432         if (t4_nnmrxq10g < 1)
8433                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8434
8435         if (t4_nnmrxq1g < 1)
8436                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8437 #endif
8438
8439         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8440                 t4_tmr_idx_10g = TMR_IDX_10G;
8441
8442         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8443                 t4_pktc_idx_10g = PKTC_IDX_10G;
8444
8445         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8446                 t4_tmr_idx_1g = TMR_IDX_1G;
8447
8448         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8449                 t4_pktc_idx_1g = PKTC_IDX_1G;
8450
8451         if (t4_qsize_txq < 128)
8452                 t4_qsize_txq = 128;
8453
8454         if (t4_qsize_rxq < 128)
8455                 t4_qsize_rxq = 128;
8456         while (t4_qsize_rxq & 7)
8457                 t4_qsize_rxq++;
8458
8459         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8460 }
8461
8462 static struct sx mlu;   /* mod load unload */
8463 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8464
8465 static int
8466 mod_event(module_t mod, int cmd, void *arg)
8467 {
8468         int rc = 0;
8469         static int loaded = 0;
8470
8471         switch (cmd) {
8472         case MOD_LOAD:
8473                 sx_xlock(&mlu);
8474                 if (loaded++ == 0) {
8475                         t4_sge_modload();
8476                         sx_init(&t4_list_lock, "T4/T5 adapters");
8477                         SLIST_INIT(&t4_list);
8478 #ifdef TCP_OFFLOAD
8479                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8480                         SLIST_INIT(&t4_uld_list);
8481 #endif
8482                         t4_tracer_modload();
8483                         tweak_tunables();
8484                 }
8485                 sx_xunlock(&mlu);
8486                 break;
8487
8488         case MOD_UNLOAD:
8489                 sx_xlock(&mlu);
8490                 if (--loaded == 0) {
8491                         int tries;
8492
8493                         sx_slock(&t4_list_lock);
8494                         if (!SLIST_EMPTY(&t4_list)) {
8495                                 rc = EBUSY;
8496                                 sx_sunlock(&t4_list_lock);
8497                                 goto done_unload;
8498                         }
8499 #ifdef TCP_OFFLOAD
8500                         sx_slock(&t4_uld_list_lock);
8501                         if (!SLIST_EMPTY(&t4_uld_list)) {
8502                                 rc = EBUSY;
8503                                 sx_sunlock(&t4_uld_list_lock);
8504                                 sx_sunlock(&t4_list_lock);
8505                                 goto done_unload;
8506                         }
8507 #endif
8508                         tries = 0;
8509                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8510                                 uprintf("%ju clusters with custom free routine "
8511                                     "still is use.\n", t4_sge_extfree_refs());
8512                                 pause("t4unload", 2 * hz);
8513                         }
8514 #ifdef TCP_OFFLOAD
8515                         sx_sunlock(&t4_uld_list_lock);
8516 #endif
8517                         sx_sunlock(&t4_list_lock);
8518
8519                         if (t4_sge_extfree_refs() == 0) {
8520                                 t4_tracer_modunload();
8521 #ifdef TCP_OFFLOAD
8522                                 sx_destroy(&t4_uld_list_lock);
8523 #endif
8524                                 sx_destroy(&t4_list_lock);
8525                                 t4_sge_modunload();
8526                                 loaded = 0;
8527                         } else {
8528                                 rc = EBUSY;
8529                                 loaded++;       /* undo earlier decrement */
8530                         }
8531                 }
8532 done_unload:
8533                 sx_xunlock(&mlu);
8534                 break;
8535         }
8536
8537         return (rc);
8538 }
8539
8540 static devclass_t t4_devclass, t5_devclass;
8541 static devclass_t cxgbe_devclass, cxl_devclass;
8542
8543 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8544 MODULE_VERSION(t4nex, 1);
8545 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8546
8547 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8548 MODULE_VERSION(t5nex, 1);
8549 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8550
8551 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8552 MODULE_VERSION(cxgbe, 1);
8553
8554 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8555 MODULE_VERSION(cxl, 1);