sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_ddb.h"
  32 #include "opt_inet.h"
  33 #include "opt_inet6.h"
  34 #include "opt_rss.h"
  35
  36 #include <sys/param.h>
  37 #include <sys/conf.h>
  38 #include <sys/priv.h>
  39 #include <sys/kernel.h>
  40 #include <sys/bus.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #ifdef RSS
  61 #include <net/rss_config.h>
  62 #endif
  63 #if defined(__i386__) || defined(__amd64__)
  64 #include <vm/vm.h>
  65 #include <vm/pmap.h>
  66 #endif
  67 #ifdef DDB
  68 #include <ddb/ddb.h>
  69 #include <ddb/db_lex.h>
  70 #endif
  71
  72 #include "common/common.h"
  73 #include "common/t4_msg.h"
  74 #include "common/t4_regs.h"
  75 #include "common/t4_regs_values.h"
  76 #include "t4_ioctl.h"
  77 #include "t4_l2t.h"
  78 #include "t4_mp_ring.h"
  79 #include "t4_if.h"
  80
  81 /* T4 bus driver interface */
  82 static int t4_probe(device_t);
  83 static int t4_attach(device_t);
  84 static int t4_detach(device_t);
  85 static int t4_ready(device_t);
  86 static int t4_read_port_device(device_t, int, device_t *);
  87 static device_method_t t4_methods[] = {
  88         DEVMETHOD(device_probe,         t4_probe),
  89         DEVMETHOD(device_attach,        t4_attach),
  90         DEVMETHOD(device_detach,        t4_detach),
  91
  92         DEVMETHOD(t4_is_main_ready,     t4_ready),
  93         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
  94
  95         DEVMETHOD_END
  96 };
  97 static driver_t t4_driver = {
  98         "t4nex",
  99         t4_methods,
 100         sizeof(struct adapter)
 101 };
 102
 103
 104 /* T4 port (cxgbe) interface */
 105 static int cxgbe_probe(device_t);
 106 static int cxgbe_attach(device_t);
 107 static int cxgbe_detach(device_t);
 108 device_method_t cxgbe_methods[] = {
 109         DEVMETHOD(device_probe,         cxgbe_probe),
 110         DEVMETHOD(device_attach,        cxgbe_attach),
 111         DEVMETHOD(device_detach,        cxgbe_detach),
 112         { 0, 0 }
 113 };
 114 static driver_t cxgbe_driver = {
 115         "cxgbe",
 116         cxgbe_methods,
 117         sizeof(struct port_info)
 118 };
 119
 120 /* T4 VI (vcxgbe) interface */
 121 static int vcxgbe_probe(device_t);
 122 static int vcxgbe_attach(device_t);
 123 static int vcxgbe_detach(device_t);
 124 static device_method_t vcxgbe_methods[] = {
 125         DEVMETHOD(device_probe,         vcxgbe_probe),
 126         DEVMETHOD(device_attach,        vcxgbe_attach),
 127         DEVMETHOD(device_detach,        vcxgbe_detach),
 128         { 0, 0 }
 129 };
 130 static driver_t vcxgbe_driver = {
 131         "vcxgbe",
 132         vcxgbe_methods,
 133         sizeof(struct vi_info)
 134 };
 135
 136 static d_ioctl_t t4_ioctl;
 137
 138 static struct cdevsw t4_cdevsw = {
 139        .d_version = D_VERSION,
 140        .d_ioctl = t4_ioctl,
 141        .d_name = "t4nex",
 142 };
 143
 144 /* T5 bus driver interface */
 145 static int t5_probe(device_t);
 146 static device_method_t t5_methods[] = {
 147         DEVMETHOD(device_probe,         t5_probe),
 148         DEVMETHOD(device_attach,        t4_attach),
 149         DEVMETHOD(device_detach,        t4_detach),
 150
 151         DEVMETHOD(t4_is_main_ready,     t4_ready),
 152         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
 153
 154         DEVMETHOD_END
 155 };
 156 static driver_t t5_driver = {
 157         "t5nex",
 158         t5_methods,
 159         sizeof(struct adapter)
 160 };
 161
 162
 163 /* T5 port (cxl) interface */
 164 static driver_t cxl_driver = {
 165         "cxl",
 166         cxgbe_methods,
 167         sizeof(struct port_info)
 168 };
 169
 170 /* T5 VI (vcxl) interface */
 171 static driver_t vcxl_driver = {
 172         "vcxl",
 173         vcxgbe_methods,
 174         sizeof(struct vi_info)
 175 };
 176
 177 /* T6 bus driver interface */
 178 static int t6_probe(device_t);
 179 static device_method_t t6_methods[] = {
 180         DEVMETHOD(device_probe,         t6_probe),
 181         DEVMETHOD(device_attach,        t4_attach),
 182         DEVMETHOD(device_detach,        t4_detach),
 183
 184         DEVMETHOD(t4_is_main_ready,     t4_ready),
 185         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
 186
 187         DEVMETHOD_END
 188 };
 189 static driver_t t6_driver = {
 190         "t6nex",
 191         t6_methods,
 192         sizeof(struct adapter)
 193 };
 194
 195
 196 /* T6 port (cc) interface */
 197 static driver_t cc_driver = {
 198         "cc",
 199         cxgbe_methods,
 200         sizeof(struct port_info)
 201 };
 202
 203 /* T6 VI (vcc) interface */
 204 static driver_t vcc_driver = {
 205         "vcc",
 206         vcxgbe_methods,
 207         sizeof(struct vi_info)
 208 };
 209
 210 /* ifnet + media interface */
 211 static void cxgbe_init(void *);
 212 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 213 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 214 static void cxgbe_qflush(struct ifnet *);
 215 static int cxgbe_media_change(struct ifnet *);
 216 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 217
 218 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 219
 220 /*
 221  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 222  * then ADAPTER_LOCK, then t4_uld_list_lock.
 223  */
 224 static struct sx t4_list_lock;
 225 SLIST_HEAD(, adapter) t4_list;
 226 #ifdef TCP_OFFLOAD
 227 static struct sx t4_uld_list_lock;
 228 SLIST_HEAD(, uld_info) t4_uld_list;
 229 #endif
 230
 231 /*
 232  * Tunables.  See tweak_tunables() too.
 233  *
 234  * Each tunable is set to a default value here if it's known at compile-time.
 235  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 236  * provide a reasonable default when the driver is loaded.
 237  *
 238  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 239  * T5 are under hw.cxl.
 240  */
 241
 242 /*
 243  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 244  */
 245 #define NTXQ_10G 16
 246 int t4_ntxq10g = -1;
 247 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 248
 249 #define NRXQ_10G 8
 250 int t4_nrxq10g = -1;
 251 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 252
 253 #define NTXQ_1G 4
 254 int t4_ntxq1g = -1;
 255 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 256
 257 #define NRXQ_1G 2
 258 int t4_nrxq1g = -1;
 259 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 260
 261 #define NTXQ_VI 1
 262 static int t4_ntxq_vi = -1;
 263 TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi);
 264
 265 #define NRXQ_VI 1
 266 static int t4_nrxq_vi = -1;
 267 TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi);
 268
 269 static int t4_rsrv_noflowq = 0;
 270 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 271
 272 #ifdef TCP_OFFLOAD
 273 #define NOFLDTXQ_10G 8
 274 static int t4_nofldtxq10g = -1;
 275 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 276
 277 #define NOFLDRXQ_10G 2
 278 static int t4_nofldrxq10g = -1;
 279 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 280
 281 #define NOFLDTXQ_1G 2
 282 static int t4_nofldtxq1g = -1;
 283 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 284
 285 #define NOFLDRXQ_1G 1
 286 static int t4_nofldrxq1g = -1;
 287 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 288
 289 #define NOFLDTXQ_VI 1
 290 static int t4_nofldtxq_vi = -1;
 291 TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi);
 292
 293 #define NOFLDRXQ_VI 1
 294 static int t4_nofldrxq_vi = -1;
 295 TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi);
 296 #endif
 297
 298 #ifdef DEV_NETMAP
 299 #define NNMTXQ_VI 2
 300 static int t4_nnmtxq_vi = -1;
 301 TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi);
 302
 303 #define NNMRXQ_VI 2
 304 static int t4_nnmrxq_vi = -1;
 305 TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi);
 306 #endif
 307
 308 /*
 309  * Holdoff parameters for 10G and 1G ports.
 310  */
 311 #define TMR_IDX_10G 1
 312 int t4_tmr_idx_10g = TMR_IDX_10G;
 313 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 314
 315 #define PKTC_IDX_10G (-1)
 316 int t4_pktc_idx_10g = PKTC_IDX_10G;
 317 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 318
 319 #define TMR_IDX_1G 1
 320 int t4_tmr_idx_1g = TMR_IDX_1G;
 321 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 322
 323 #define PKTC_IDX_1G (-1)
 324 int t4_pktc_idx_1g = PKTC_IDX_1G;
 325 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 326
 327 /*
 328  * Size (# of entries) of each tx and rx queue.
 329  */
 330 unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 331 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 332
 333 unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 334 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 335
 336 /*
 337  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 338  */
 339 int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 340 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 341
 342 /*
 343  * Configuration file.
 344  */
 345 #define DEFAULT_CF      "default"
 346 #define FLASH_CF        "flash"
 347 #define UWIRE_CF        "uwire"
 348 #define FPGA_CF         "fpga"
 349 static char t4_cfg_file[32] = DEFAULT_CF;
 350 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 351
 352 /*
 353  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 354  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 355  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 356  *            mark or when signalled to do so, 0 to never emit PAUSE.
 357  */
 358 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 359 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 360
 361 /*
 362  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 363  * encouraged respectively).
 364  */
 365 static unsigned int t4_fw_install = 1;
 366 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 367
 368 /*
 369  * ASIC features that will be used.  Disable the ones you don't want so that the
 370  * chip resources aren't wasted on features that will not be used.
 371  */
 372 static int t4_nbmcaps_allowed = 0;
 373 TUNABLE_INT("hw.cxgbe.nbmcaps_allowed", &t4_nbmcaps_allowed);
 374
 375 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 376 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 377
 378 static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS |
 379     FW_CAPS_CONFIG_SWITCH_EGRESS;
 380 TUNABLE_INT("hw.cxgbe.switchcaps_allowed", &t4_switchcaps_allowed);
 381
 382 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 383 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 384
 385 static int t4_toecaps_allowed = -1;
 386 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 387
 388 static int t4_rdmacaps_allowed = -1;
 389 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 390
 391 static int t4_cryptocaps_allowed = 0;
 392 TUNABLE_INT("hw.cxgbe.cryptocaps_allowed", &t4_cryptocaps_allowed);
 393
 394 static int t4_iscsicaps_allowed = -1;
 395 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 396
 397 static int t4_fcoecaps_allowed = 0;
 398 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 399
 400 static int t5_write_combine = 0;
 401 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 402
 403 static int t4_num_vis = 1;
 404 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis);
 405
 406 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */
 407 static int vi_mac_funcs[] = {
 408         FW_VI_FUNC_OFLD,
 409         FW_VI_FUNC_IWARP,
 410         FW_VI_FUNC_OPENISCSI,
 411         FW_VI_FUNC_OPENFCOE,
 412         FW_VI_FUNC_FOISCSI,
 413         FW_VI_FUNC_FOFCOE,
 414 };
 415
 416 struct intrs_and_queues {
 417         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 418         uint16_t nirq;          /* Total # of vectors */
 419         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 420         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 421         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 422         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 423         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 424         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 425         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 426         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 427         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 428         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 429         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 430
 431         /* The vcxgbe/vcxl interfaces use these and not the ones above. */
 432         uint16_t ntxq_vi;       /* # of NIC txq's */
 433         uint16_t nrxq_vi;       /* # of NIC rxq's */
 434         uint16_t nofldtxq_vi;   /* # of TOE txq's */
 435         uint16_t nofldrxq_vi;   /* # of TOE rxq's */
 436         uint16_t nnmtxq_vi;     /* # of netmap txq's */
 437         uint16_t nnmrxq_vi;     /* # of netmap rxq's */
 438 };
 439
 440 struct filter_entry {
 441         uint32_t valid:1;       /* filter allocated and valid */
 442         uint32_t locked:1;      /* filter is administratively locked */
 443         uint32_t pending:1;     /* filter action is pending firmware reply */
 444         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 445         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 446
 447         struct t4_filter_specification fs;
 448 };
 449
 450 static void setup_memwin(struct adapter *);
 451 static void position_memwin(struct adapter *, int, uint32_t);
 452 static int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int);
 453 static inline int read_via_memwin(struct adapter *, int, uint32_t, uint32_t *,
 454     int);
 455 static inline int write_via_memwin(struct adapter *, int, uint32_t,
 456     const uint32_t *, int);
 457 static int validate_mem_range(struct adapter *, uint32_t, int);
 458 static int fwmtype_to_hwmtype(int);
 459 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 460     uint32_t *);
 461 static int fixup_devlog_params(struct adapter *);
 462 static int cfg_itype_and_nqueues(struct adapter *, int, int, int,
 463     struct intrs_and_queues *);
 464 static int prep_firmware(struct adapter *);
 465 static int partition_resources(struct adapter *, const struct firmware *,
 466     const char *);
 467 static int get_params__pre_init(struct adapter *);
 468 static int get_params__post_init(struct adapter *);
 469 static int set_params__post_init(struct adapter *);
 470 static void t4_set_desc(struct adapter *);
 471 static void build_medialist(struct port_info *, struct ifmedia *);
 472 static int cxgbe_init_synchronized(struct vi_info *);
 473 static int cxgbe_uninit_synchronized(struct vi_info *);
 474 static void quiesce_txq(struct adapter *, struct sge_txq *);
 475 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 476 static void quiesce_iq(struct adapter *, struct sge_iq *);
 477 static void quiesce_fl(struct adapter *, struct sge_fl *);
 478 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 479     driver_intr_t *, void *, char *);
 480 static int t4_free_irq(struct adapter *, struct irq *);
 481 static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 482 static void vi_refresh_stats(struct adapter *, struct vi_info *);
 483 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 484 static void cxgbe_tick(void *);
 485 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 486 static void cxgbe_sysctls(struct port_info *);
 487 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 488 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 489 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 490 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 491 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 492 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 493 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 494 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 495 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 496 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 497 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 498 #ifdef SBUF_DRAIN
 499 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 500 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 501 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 502 static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS);
 503 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 504 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 505 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 506 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 507 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 508 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 509 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 510 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 511 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 512 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 513 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 514 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 515 static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS);
 516 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 517 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 518 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 519 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 520 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 521 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 522 static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS);
 523 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 524 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 525 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 526 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 527 static int sysctl_tc_params(SYSCTL_HANDLER_ARGS);
 528 #endif
 529 #ifdef TCP_OFFLOAD
 530 static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
 531 static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
 532 static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
 533 #endif
 534 static uint32_t fconf_iconf_to_mode(uint32_t, uint32_t);
 535 static uint32_t mode_to_fconf(uint32_t);
 536 static uint32_t mode_to_iconf(uint32_t);
 537 static int check_fspec_against_fconf_iconf(struct adapter *,
 538     struct t4_filter_specification *);
 539 static int get_filter_mode(struct adapter *, uint32_t *);
 540 static int set_filter_mode(struct adapter *, uint32_t);
 541 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 542 static int get_filter(struct adapter *, struct t4_filter *);
 543 static int set_filter(struct adapter *, struct t4_filter *);
 544 static int del_filter(struct adapter *, struct t4_filter *);
 545 static void clear_filter(struct filter_entry *);
 546 static int set_filter_wr(struct adapter *, int);
 547 static int del_filter_wr(struct adapter *, int);
 548 static int set_tcb_rpl(struct sge_iq *, const struct rss_header *,
 549     struct mbuf *);
 550 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 551 static int load_fw(struct adapter *, struct t4_data *);
 552 static int load_cfg(struct adapter *, struct t4_data *);
 553 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 554 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 555 #ifdef TCP_OFFLOAD
 556 static int toe_capability(struct vi_info *, int);
 557 #endif
 558 static int mod_event(module_t, int, void *);
 559 static int notify_siblings(device_t, int);
 560
 561 struct {
 562         uint16_t device;
 563         char *desc;
 564 } t4_pciids[] = {
 565         {0xa000, "Chelsio Terminator 4 FPGA"},
 566         {0x4400, "Chelsio T440-dbg"},
 567         {0x4401, "Chelsio T420-CR"},
 568         {0x4402, "Chelsio T422-CR"},
 569         {0x4403, "Chelsio T440-CR"},
 570         {0x4404, "Chelsio T420-BCH"},
 571         {0x4405, "Chelsio T440-BCH"},
 572         {0x4406, "Chelsio T440-CH"},
 573         {0x4407, "Chelsio T420-SO"},
 574         {0x4408, "Chelsio T420-CX"},
 575         {0x4409, "Chelsio T420-BT"},
 576         {0x440a, "Chelsio T404-BT"},
 577         {0x440e, "Chelsio T440-LP-CR"},
 578 }, t5_pciids[] = {
 579         {0xb000, "Chelsio Terminator 5 FPGA"},
 580         {0x5400, "Chelsio T580-dbg"},
 581         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 582         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 583         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 584         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 585         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 586         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 587         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 588         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 589         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 590         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 591         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 592         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 593         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 594 #ifdef notyet
 595         {0x5404,  "Chelsio T520-BCH"},
 596         {0x5405,  "Chelsio T540-BCH"},
 597         {0x5406,  "Chelsio T540-CH"},
 598         {0x5408,  "Chelsio T520-CX"},
 599         {0x540b,  "Chelsio B520-SR"},
 600         {0x540c,  "Chelsio B504-BT"},
 601         {0x540f,  "Chelsio Amsterdam"},
 602         {0x5413,  "Chelsio T580-CHR"},
 603 #endif
 604 }, t6_pciids[] = {
 605         {0xc006, "Chelsio Terminator 6 FPGA"},  /* T6 PE10K6 FPGA (PF0) */
 606         {0x6401, "Chelsio T6225-CR"},           /* 2 x 10/25G */
 607         {0x6402, "Chelsio T6225-SO-CR"},        /* 2 x 10/25G, nomem */
 608         {0x6407, "Chelsio T62100-LP-CR"},       /* 2 x 40/50/100G */
 609         {0x6408, "Chelsio T62100-SO-CR"},       /* 2 x 40/50/100G, nomem */
 610         {0x640d, "Chelsio T62100-CR"},          /* 2 x 40/50/100G */
 611         {0x6410, "Chelsio T62100-DBG"},         /* 2 x 40/50/100G, debug */
 612 };
 613
 614 #ifdef TCP_OFFLOAD
 615 /*
 616  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 617  * exactly the same for both rxq and ofld_rxq.
 618  */
 619 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 620 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 621 #endif
 622 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 623
 624 static int
 625 t4_probe(device_t dev)
 626 {
 627         int i;
 628         uint16_t v = pci_get_vendor(dev);
 629         uint16_t d = pci_get_device(dev);
 630         uint8_t f = pci_get_function(dev);
 631
 632         if (v != PCI_VENDOR_ID_CHELSIO)
 633                 return (ENXIO);
 634
 635         /* Attach only to PF0 of the FPGA */
 636         if (d == 0xa000 && f != 0)
 637                 return (ENXIO);
 638
 639         for (i = 0; i < nitems(t4_pciids); i++) {
 640                 if (d == t4_pciids[i].device) {
 641                         device_set_desc(dev, t4_pciids[i].desc);
 642                         return (BUS_PROBE_DEFAULT);
 643                 }
 644         }
 645
 646         return (ENXIO);
 647 }
 648
 649 static int
 650 t5_probe(device_t dev)
 651 {
 652         int i;
 653         uint16_t v = pci_get_vendor(dev);
 654         uint16_t d = pci_get_device(dev);
 655         uint8_t f = pci_get_function(dev);
 656
 657         if (v != PCI_VENDOR_ID_CHELSIO)
 658                 return (ENXIO);
 659
 660         /* Attach only to PF0 of the FPGA */
 661         if (d == 0xb000 && f != 0)
 662                 return (ENXIO);
 663
 664         for (i = 0; i < nitems(t5_pciids); i++) {
 665                 if (d == t5_pciids[i].device) {
 666                         device_set_desc(dev, t5_pciids[i].desc);
 667                         return (BUS_PROBE_DEFAULT);
 668                 }
 669         }
 670
 671         return (ENXIO);
 672 }
 673
 674 static int
 675 t6_probe(device_t dev)
 676 {
 677         int i;
 678         uint16_t v = pci_get_vendor(dev);
 679         uint16_t d = pci_get_device(dev);
 680
 681         if (v != PCI_VENDOR_ID_CHELSIO)
 682                 return (ENXIO);
 683
 684         for (i = 0; i < nitems(t6_pciids); i++) {
 685                 if (d == t6_pciids[i].device) {
 686                         device_set_desc(dev, t6_pciids[i].desc);
 687                         return (BUS_PROBE_DEFAULT);
 688                 }
 689         }
 690
 691         return (ENXIO);
 692 }
 693
 694 static void
 695 t5_attribute_workaround(device_t dev)
 696 {
 697         device_t root_port;
 698         uint32_t v;
 699
 700         /*
 701          * The T5 chips do not properly echo the No Snoop and Relaxed
 702          * Ordering attributes when replying to a TLP from a Root
 703          * Port.  As a workaround, find the parent Root Port and
 704          * disable No Snoop and Relaxed Ordering.  Note that this
 705          * affects all devices under this root port.
 706          */
 707         root_port = pci_find_pcie_root_port(dev);
 708         if (root_port == NULL) {
 709                 device_printf(dev, "Unable to find parent root port\n");
 710                 return;
 711         }
 712
 713         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 714             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 715         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 716             0)
 717                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 718                     device_get_nameunit(root_port));
 719 }
 720
 721 static const struct devnames devnames[] = {
 722         {
 723                 .nexus_name = "t4nex",
 724                 .ifnet_name = "cxgbe",
 725                 .vi_ifnet_name = "vcxgbe",
 726                 .pf03_drv_name = "t4iov",
 727                 .vf_nexus_name = "t4vf",
 728                 .vf_ifnet_name = "cxgbev"
 729         }, {
 730                 .nexus_name = "t5nex",
 731                 .ifnet_name = "cxl",
 732                 .vi_ifnet_name = "vcxl",
 733                 .pf03_drv_name = "t5iov",
 734                 .vf_nexus_name = "t5vf",
 735                 .vf_ifnet_name = "cxlv"
 736         }, {
 737                 .nexus_name = "t6nex",
 738                 .ifnet_name = "cc",
 739                 .vi_ifnet_name = "vcc",
 740                 .pf03_drv_name = "t6iov",
 741                 .vf_nexus_name = "t6vf",
 742                 .vf_ifnet_name = "ccv"
 743         }
 744 };
 745
 746 void
 747 t4_init_devnames(struct adapter *sc)
 748 {
 749         int id;
 750
 751         id = chip_id(sc);
 752         if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames))
 753                 sc->names = &devnames[id - CHELSIO_T4];
 754         else {
 755                 device_printf(sc->dev, "chip id %d is not supported.\n", id);
 756                 sc->names = NULL;
 757         }
 758 }
 759
 760 static int
 761 t4_attach(device_t dev)
 762 {
 763         struct adapter *sc;
 764         int rc = 0, i, j, n10g, n1g, rqidx, tqidx;
 765         struct make_dev_args mda;
 766         struct intrs_and_queues iaq;
 767         struct sge *s;
 768         uint8_t *buf;
 769 #ifdef TCP_OFFLOAD
 770         int ofld_rqidx, ofld_tqidx;
 771 #endif
 772 #ifdef DEV_NETMAP
 773         int nm_rqidx, nm_tqidx;
 774 #endif
 775         int num_vis;
 776
 777         sc = device_get_softc(dev);
 778         sc->dev = dev;
 779         TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
 780
 781         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 782                 t5_attribute_workaround(dev);
 783         pci_enable_busmaster(dev);
 784         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 785                 uint32_t v;
 786
 787                 pci_set_max_read_req(dev, 4096);
 788                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 789                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 790                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 791
 792                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 793         }
 794
 795         sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
 796         sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
 797         sc->traceq = -1;
 798         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 799         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 800             device_get_nameunit(dev));
 801
 802         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 803             device_get_nameunit(dev));
 804         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 805         t4_add_adapter(sc);
 806
 807         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 808         TAILQ_INIT(&sc->sfl);
 809         callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
 810
 811         mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
 812
 813         rc = t4_map_bars_0_and_4(sc);
 814         if (rc != 0)
 815                 goto done; /* error message displayed already */
 816
 817         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 818
 819         /* Prepare the adapter for operation. */
 820         buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK);
 821         rc = -t4_prep_adapter(sc, buf);
 822         free(buf, M_CXGBE);
 823         if (rc != 0) {
 824                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 825                 goto done;
 826         }
 827
 828         /*
 829          * This is the real PF# to which we're attaching.  Works from within PCI
 830          * passthrough environments too, where pci_get_function() could return a
 831          * different PF# depending on the passthrough configuration.  We need to
 832          * use the real PF# in all our communication with the firmware.
 833          */
 834         j = t4_read_reg(sc, A_PL_WHOAMI);
 835         sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
 836         sc->mbox = sc->pf;
 837
 838         t4_init_devnames(sc);
 839         if (sc->names == NULL) {
 840                 rc = ENOTSUP;
 841                 goto done; /* error message displayed already */
 842         }
 843
 844         /*
 845          * Do this really early, with the memory windows set up even before the
 846          * character device.  The userland tool's register i/o and mem read
 847          * will work even in "recovery mode".
 848          */
 849         setup_memwin(sc);
 850         if (t4_init_devlog_params(sc, 0) == 0)
 851                 fixup_devlog_params(sc);
 852         make_dev_args_init(&mda);
 853         mda.mda_devsw = &t4_cdevsw;
 854         mda.mda_uid = UID_ROOT;
 855         mda.mda_gid = GID_WHEEL;
 856         mda.mda_mode = 0600;
 857         mda.mda_si_drv1 = sc;
 858         rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
 859         if (rc != 0)
 860                 device_printf(dev, "failed to create nexus char device: %d.\n",
 861                     rc);
 862
 863         /* Go no further if recovery mode has been requested. */
 864         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 865                 device_printf(dev, "recovery mode.\n");
 866                 goto done;
 867         }
 868
 869 #if defined(__i386__)
 870         if ((cpu_feature & CPUID_CX8) == 0) {
 871                 device_printf(dev, "64 bit atomics not available.\n");
 872                 rc = ENOTSUP;
 873                 goto done;
 874         }
 875 #endif
 876
 877         /* Prepare the firmware for operation */
 878         rc = prep_firmware(sc);
 879         if (rc != 0)
 880                 goto done; /* error message displayed already */
 881
 882         rc = get_params__post_init(sc);
 883         if (rc != 0)
 884                 goto done; /* error message displayed already */
 885
 886         rc = set_params__post_init(sc);
 887         if (rc != 0)
 888                 goto done; /* error message displayed already */
 889
 890         rc = t4_map_bar_2(sc);
 891         if (rc != 0)
 892                 goto done; /* error message displayed already */
 893
 894         rc = t4_create_dma_tag(sc);
 895         if (rc != 0)
 896                 goto done; /* error message displayed already */
 897
 898         /*
 899          * Number of VIs to create per-port.  The first VI is the "main" regular
 900          * VI for the port.  The rest are additional virtual interfaces on the
 901          * same physical port.  Note that the main VI does not have native
 902          * netmap support but the extra VIs do.
 903          *
 904          * Limit the number of VIs per port to the number of available
 905          * MAC addresses per port.
 906          */
 907         if (t4_num_vis >= 1)
 908                 num_vis = t4_num_vis;
 909         else
 910                 num_vis = 1;
 911         if (num_vis > nitems(vi_mac_funcs)) {
 912                 num_vis = nitems(vi_mac_funcs);
 913                 device_printf(dev, "Number of VIs limited to %d\n", num_vis);
 914         }
 915
 916         /*
 917          * First pass over all the ports - allocate VIs and initialize some
 918          * basic parameters like mac address, port type, etc.  We also figure
 919          * out whether a port is 10G or 1G and use that information when
 920          * calculating how many interrupts to attempt to allocate.
 921          */
 922         n10g = n1g = 0;
 923         for_each_port(sc, i) {
 924                 struct port_info *pi;
 925
 926                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 927                 sc->port[i] = pi;
 928
 929                 /* These must be set before t4_port_init */
 930                 pi->adapter = sc;
 931                 pi->port_id = i;
 932                 /*
 933                  * XXX: vi[0] is special so we can't delay this allocation until
 934                  * pi->nvi's final value is known.
 935                  */
 936                 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE,
 937                     M_ZERO | M_WAITOK);
 938
 939                 /*
 940                  * Allocate the "main" VI and initialize parameters
 941                  * like mac addr.
 942                  */
 943                 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
 944                 if (rc != 0) {
 945                         device_printf(dev, "unable to initialize port %d: %d\n",
 946                             i, rc);
 947                         free(pi->vi, M_CXGBE);
 948                         free(pi, M_CXGBE);
 949                         sc->port[i] = NULL;
 950                         goto done;
 951                 }
 952
 953                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 954                 pi->link_cfg.requested_fc |= t4_pause_settings;
 955                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 956                 pi->link_cfg.fc |= t4_pause_settings;
 957
 958                 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 959                 if (rc != 0) {
 960                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 961                         free(pi->vi, M_CXGBE);
 962                         free(pi, M_CXGBE);
 963                         sc->port[i] = NULL;
 964                         goto done;
 965                 }
 966
 967                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 968                     device_get_nameunit(dev), i);
 969                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 970                 sc->chan_map[pi->tx_chan] = i;
 971
 972                 pi->tc = malloc(sizeof(struct tx_sched_class) *
 973                     sc->chip_params->nsched_cls, M_CXGBE, M_ZERO | M_WAITOK);
 974
 975                 if (port_top_speed(pi) >= 10) {
 976                         n10g++;
 977                 } else {
 978                         n1g++;
 979                 }
 980
 981                 pi->linkdnrc = -1;
 982
 983                 pi->dev = device_add_child(dev, sc->names->ifnet_name, -1);
 984                 if (pi->dev == NULL) {
 985                         device_printf(dev,
 986                             "failed to add device for port %d.\n", i);
 987                         rc = ENXIO;
 988                         goto done;
 989                 }
 990                 pi->vi[0].dev = pi->dev;
 991                 device_set_softc(pi->dev, pi);
 992         }
 993
 994         /*
 995          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 996          */
 997         rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq);
 998         if (rc != 0)
 999                 goto done; /* error message displayed already */
1000         if (iaq.nrxq_vi + iaq.nofldrxq_vi + iaq.nnmrxq_vi == 0)
1001                 num_vis = 1;
1002
1003         sc->intr_type = iaq.intr_type;
1004         sc->intr_count = iaq.nirq;
1005
1006         s = &sc->sge;
1007         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
1008         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
1009         if (num_vis > 1) {
1010                 s->nrxq += (n10g + n1g) * (num_vis - 1) * iaq.nrxq_vi;
1011                 s->ntxq += (n10g + n1g) * (num_vis - 1) * iaq.ntxq_vi;
1012         }
1013         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
1014         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
1015         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
1016 #ifdef TCP_OFFLOAD
1017         if (is_offload(sc)) {
1018                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
1019                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
1020                 if (num_vis > 1) {
1021                         s->nofldrxq += (n10g + n1g) * (num_vis - 1) *
1022                             iaq.nofldrxq_vi;
1023                         s->nofldtxq += (n10g + n1g) * (num_vis - 1) *
1024                             iaq.nofldtxq_vi;
1025                 }
1026                 s->neq += s->nofldtxq + s->nofldrxq;
1027                 s->niq += s->nofldrxq;
1028
1029                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
1030                     M_CXGBE, M_ZERO | M_WAITOK);
1031                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
1032                     M_CXGBE, M_ZERO | M_WAITOK);
1033         }
1034 #endif
1035 #ifdef DEV_NETMAP
1036         if (num_vis > 1) {
1037                 s->nnmrxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmrxq_vi;
1038                 s->nnmtxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmtxq_vi;
1039         }
1040         s->neq += s->nnmtxq + s->nnmrxq;
1041         s->niq += s->nnmrxq;
1042
1043         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
1044             M_CXGBE, M_ZERO | M_WAITOK);
1045         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
1046             M_CXGBE, M_ZERO | M_WAITOK);
1047 #endif
1048
1049         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
1050             M_ZERO | M_WAITOK);
1051         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
1052             M_ZERO | M_WAITOK);
1053         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
1054             M_ZERO | M_WAITOK);
1055         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
1056             M_ZERO | M_WAITOK);
1057         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
1058             M_ZERO | M_WAITOK);
1059
1060         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
1061             M_ZERO | M_WAITOK);
1062
1063         t4_init_l2t(sc, M_WAITOK);
1064
1065         /*
1066          * Second pass over the ports.  This time we know the number of rx and
1067          * tx queues that each port should get.
1068          */
1069         rqidx = tqidx = 0;
1070 #ifdef TCP_OFFLOAD
1071         ofld_rqidx = ofld_tqidx = 0;
1072 #endif
1073 #ifdef DEV_NETMAP
1074         nm_rqidx = nm_tqidx = 0;
1075 #endif
1076         for_each_port(sc, i) {
1077                 struct port_info *pi = sc->port[i];
1078                 struct vi_info *vi;
1079
1080                 if (pi == NULL)
1081                         continue;
1082
1083                 pi->nvi = num_vis;
1084                 for_each_vi(pi, j, vi) {
1085                         vi->pi = pi;
1086                         vi->qsize_rxq = t4_qsize_rxq;
1087                         vi->qsize_txq = t4_qsize_txq;
1088
1089                         vi->first_rxq = rqidx;
1090                         vi->first_txq = tqidx;
1091                         if (port_top_speed(pi) >= 10) {
1092                                 vi->tmr_idx = t4_tmr_idx_10g;
1093                                 vi->pktc_idx = t4_pktc_idx_10g;
1094                                 vi->flags |= iaq.intr_flags_10g & INTR_RXQ;
1095                                 vi->nrxq = j == 0 ? iaq.nrxq10g : iaq.nrxq_vi;
1096                                 vi->ntxq = j == 0 ? iaq.ntxq10g : iaq.ntxq_vi;
1097                         } else {
1098                                 vi->tmr_idx = t4_tmr_idx_1g;
1099                                 vi->pktc_idx = t4_pktc_idx_1g;
1100                                 vi->flags |= iaq.intr_flags_1g & INTR_RXQ;
1101                                 vi->nrxq = j == 0 ? iaq.nrxq1g : iaq.nrxq_vi;
1102                                 vi->ntxq = j == 0 ? iaq.ntxq1g : iaq.ntxq_vi;
1103                         }
1104                         rqidx += vi->nrxq;
1105                         tqidx += vi->ntxq;
1106
1107                         if (j == 0 && vi->ntxq > 1)
1108                                 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
1109                         else
1110                                 vi->rsrv_noflowq = 0;
1111
1112 #ifdef TCP_OFFLOAD
1113                         vi->first_ofld_rxq = ofld_rqidx;
1114                         vi->first_ofld_txq = ofld_tqidx;
1115                         if (port_top_speed(pi) >= 10) {
1116                                 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ;
1117                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g :
1118                                     iaq.nofldrxq_vi;
1119                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g :
1120                                     iaq.nofldtxq_vi;
1121                         } else {
1122                                 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ;
1123                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g :
1124                                     iaq.nofldrxq_vi;
1125                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g :
1126                                     iaq.nofldtxq_vi;
1127                         }
1128                         ofld_rqidx += vi->nofldrxq;
1129                         ofld_tqidx += vi->nofldtxq;
1130 #endif
1131 #ifdef DEV_NETMAP
1132                         if (j > 0) {
1133                                 vi->first_nm_rxq = nm_rqidx;
1134                                 vi->first_nm_txq = nm_tqidx;
1135                                 vi->nnmrxq = iaq.nnmrxq_vi;
1136                                 vi->nnmtxq = iaq.nnmtxq_vi;
1137                                 nm_rqidx += vi->nnmrxq;
1138                                 nm_tqidx += vi->nnmtxq;
1139                         }
1140 #endif
1141                 }
1142         }
1143
1144         rc = t4_setup_intr_handlers(sc);
1145         if (rc != 0) {
1146                 device_printf(dev,
1147                     "failed to setup interrupt handlers: %d\n", rc);
1148                 goto done;
1149         }
1150
1151         rc = bus_generic_attach(dev);
1152         if (rc != 0) {
1153                 device_printf(dev,
1154                     "failed to attach all child ports: %d\n", rc);
1155                 goto done;
1156         }
1157
1158         device_printf(dev,
1159             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
1160             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1161             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1162             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1163             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1164
1165         t4_set_desc(sc);
1166
1167         notify_siblings(dev, 0);
1168
1169 done:
1170         if (rc != 0 && sc->cdev) {
1171                 /* cdev was created and so cxgbetool works; recover that way. */
1172                 device_printf(dev,
1173                     "error during attach, adapter is now in recovery mode.\n");
1174                 rc = 0;
1175         }
1176
1177         if (rc != 0)
1178                 t4_detach_common(dev);
1179         else
1180                 t4_sysctls(sc);
1181
1182         return (rc);
1183 }
1184
1185 static int
1186 t4_ready(device_t dev)
1187 {
1188         struct adapter *sc;
1189
1190         sc = device_get_softc(dev);
1191         if (sc->flags & FW_OK)
1192                 return (0);
1193         return (ENXIO);
1194 }
1195
1196 static int
1197 t4_read_port_device(device_t dev, int port, device_t *child)
1198 {
1199         struct adapter *sc;
1200         struct port_info *pi;
1201
1202         sc = device_get_softc(dev);
1203         if (port < 0 || port >= MAX_NPORTS)
1204                 return (EINVAL);
1205         pi = sc->port[port];
1206         if (pi == NULL || pi->dev == NULL)
1207                 return (ENXIO);
1208         *child = pi->dev;
1209         return (0);
1210 }
1211
1212 static int
1213 notify_siblings(device_t dev, int detaching)
1214 {
1215         device_t sibling;
1216         int error, i;
1217
1218         error = 0;
1219         for (i = 0; i < PCI_FUNCMAX; i++) {
1220                 if (i == pci_get_function(dev))
1221                         continue;
1222                 sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
1223                     pci_get_slot(dev), i);
1224                 if (sibling == NULL || !device_is_attached(sibling))
1225                         continue;
1226                 if (detaching)
1227                         error = T4_DETACH_CHILD(sibling);
1228                 else
1229                         (void)T4_ATTACH_CHILD(sibling);
1230                 if (error)
1231                         break;
1232         }
1233         return (error);
1234 }
1235
1236 /*
1237  * Idempotent
1238  */
1239 static int
1240 t4_detach(device_t dev)
1241 {
1242         struct adapter *sc;
1243         int rc;
1244
1245         sc = device_get_softc(dev);
1246
1247         rc = notify_siblings(dev, 1);
1248         if (rc) {
1249                 device_printf(dev,
1250                     "failed to detach sibling devices: %d\n", rc);
1251                 return (rc);
1252         }
1253
1254         return (t4_detach_common(dev));
1255 }
1256
1257 int
1258 t4_detach_common(device_t dev)
1259 {
1260         struct adapter *sc;
1261         struct port_info *pi;
1262         int i, rc;
1263
1264         sc = device_get_softc(dev);
1265
1266         if (sc->flags & FULL_INIT_DONE) {
1267                 if (!(sc->flags & IS_VF))
1268                         t4_intr_disable(sc);
1269         }
1270
1271         if (sc->cdev) {
1272                 destroy_dev(sc->cdev);
1273                 sc->cdev = NULL;
1274         }
1275
1276         if (device_is_attached(dev)) {
1277                 rc = bus_generic_detach(dev);
1278                 if (rc) {
1279                         device_printf(dev,
1280                             "failed to detach child devices: %d\n", rc);
1281                         return (rc);
1282                 }
1283         }
1284
1285         for (i = 0; i < sc->intr_count; i++)
1286                 t4_free_irq(sc, &sc->irq[i]);
1287
1288         for (i = 0; i < MAX_NPORTS; i++) {
1289                 pi = sc->port[i];
1290                 if (pi) {
1291                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1292                         if (pi->dev)
1293                                 device_delete_child(dev, pi->dev);
1294
1295                         mtx_destroy(&pi->pi_lock);
1296                         free(pi->vi, M_CXGBE);
1297                         free(pi->tc, M_CXGBE);
1298                         free(pi, M_CXGBE);
1299                 }
1300         }
1301
1302         if (sc->flags & FULL_INIT_DONE)
1303                 adapter_full_uninit(sc);
1304
1305         if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1306                 t4_fw_bye(sc, sc->mbox);
1307
1308         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1309                 pci_release_msi(dev);
1310
1311         if (sc->regs_res)
1312                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1313                     sc->regs_res);
1314
1315         if (sc->udbs_res)
1316                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1317                     sc->udbs_res);
1318
1319         if (sc->msix_res)
1320                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1321                     sc->msix_res);
1322
1323         if (sc->l2t)
1324                 t4_free_l2t(sc->l2t);
1325
1326 #ifdef TCP_OFFLOAD
1327         free(sc->sge.ofld_rxq, M_CXGBE);
1328         free(sc->sge.ofld_txq, M_CXGBE);
1329 #endif
1330 #ifdef DEV_NETMAP
1331         free(sc->sge.nm_rxq, M_CXGBE);
1332         free(sc->sge.nm_txq, M_CXGBE);
1333 #endif
1334         free(sc->irq, M_CXGBE);
1335         free(sc->sge.rxq, M_CXGBE);
1336         free(sc->sge.txq, M_CXGBE);
1337         free(sc->sge.ctrlq, M_CXGBE);
1338         free(sc->sge.iqmap, M_CXGBE);
1339         free(sc->sge.eqmap, M_CXGBE);
1340         free(sc->tids.ftid_tab, M_CXGBE);
1341         t4_destroy_dma_tag(sc);
1342         if (mtx_initialized(&sc->sc_lock)) {
1343                 sx_xlock(&t4_list_lock);
1344                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1345                 sx_xunlock(&t4_list_lock);
1346                 mtx_destroy(&sc->sc_lock);
1347         }
1348
1349         callout_drain(&sc->sfl_callout);
1350         if (mtx_initialized(&sc->tids.ftid_lock))
1351                 mtx_destroy(&sc->tids.ftid_lock);
1352         if (mtx_initialized(&sc->sfl_lock))
1353                 mtx_destroy(&sc->sfl_lock);
1354         if (mtx_initialized(&sc->ifp_lock))
1355                 mtx_destroy(&sc->ifp_lock);
1356         if (mtx_initialized(&sc->reg_lock))
1357                 mtx_destroy(&sc->reg_lock);
1358
1359         for (i = 0; i < NUM_MEMWIN; i++) {
1360                 struct memwin *mw = &sc->memwin[i];
1361
1362                 if (rw_initialized(&mw->mw_lock))
1363                         rw_destroy(&mw->mw_lock);
1364         }
1365
1366         bzero(sc, sizeof(*sc));
1367
1368         return (0);
1369 }
1370
1371 static int
1372 cxgbe_probe(device_t dev)
1373 {
1374         char buf[128];
1375         struct port_info *pi = device_get_softc(dev);
1376
1377         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1378         device_set_desc_copy(dev, buf);
1379
1380         return (BUS_PROBE_DEFAULT);
1381 }
1382
1383 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1384     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1385     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1386 #define T4_CAP_ENABLE (T4_CAP)
1387
1388 static int
1389 cxgbe_vi_attach(device_t dev, struct vi_info *vi)
1390 {
1391         struct ifnet *ifp;
1392         struct sbuf *sb;
1393
1394         vi->xact_addr_filt = -1;
1395         callout_init(&vi->tick, 1);
1396
1397         /* Allocate an ifnet and set it up */
1398         ifp = if_alloc(IFT_ETHER);
1399         if (ifp == NULL) {
1400                 device_printf(dev, "Cannot allocate ifnet\n");
1401                 return (ENOMEM);
1402         }
1403         vi->ifp = ifp;
1404         ifp->if_softc = vi;
1405
1406         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1407         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1408
1409         ifp->if_init = cxgbe_init;
1410         ifp->if_ioctl = cxgbe_ioctl;
1411         ifp->if_transmit = cxgbe_transmit;
1412         ifp->if_qflush = cxgbe_qflush;
1413         ifp->if_get_counter = cxgbe_get_counter;
1414
1415         ifp->if_capabilities = T4_CAP;
1416 #ifdef TCP_OFFLOAD
1417         if (vi->nofldrxq != 0)
1418                 ifp->if_capabilities |= IFCAP_TOE;
1419 #endif
1420 #ifdef DEV_NETMAP
1421         if (vi->nnmrxq != 0)
1422                 ifp->if_capabilities |= IFCAP_NETMAP;
1423 #endif
1424         ifp->if_capenable = T4_CAP_ENABLE;
1425         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1426             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1427
1428         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1429         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1430         ifp->if_hw_tsomaxsegsize = 65536;
1431
1432         /* Initialize ifmedia for this VI */
1433         ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change,
1434             cxgbe_media_status);
1435         build_medialist(vi->pi, &vi->media);
1436
1437         vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1438             EVENTHANDLER_PRI_ANY);
1439
1440         ether_ifattach(ifp, vi->hw_addr);
1441 #ifdef DEV_NETMAP
1442         if (ifp->if_capabilities & IFCAP_NETMAP)
1443                 cxgbe_nm_attach(vi);
1444 #endif
1445         sb = sbuf_new_auto();
1446         sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
1447 #ifdef TCP_OFFLOAD
1448         if (ifp->if_capabilities & IFCAP_TOE)
1449                 sbuf_printf(sb, "; %d txq, %d rxq (TOE)",
1450                     vi->nofldtxq, vi->nofldrxq);
1451 #endif
1452 #ifdef DEV_NETMAP
1453         if (ifp->if_capabilities & IFCAP_NETMAP)
1454                 sbuf_printf(sb, "; %d txq, %d rxq (netmap)",
1455                     vi->nnmtxq, vi->nnmrxq);
1456 #endif
1457         sbuf_finish(sb);
1458         device_printf(dev, "%s\n", sbuf_data(sb));
1459         sbuf_delete(sb);
1460
1461         vi_sysctls(vi);
1462
1463         return (0);
1464 }
1465
1466 static int
1467 cxgbe_attach(device_t dev)
1468 {
1469         struct port_info *pi = device_get_softc(dev);
1470         struct adapter *sc = pi->adapter;
1471         struct vi_info *vi;
1472         int i, rc;
1473
1474         callout_init_mtx(&pi->tick, &pi->pi_lock, 0);
1475
1476         rc = cxgbe_vi_attach(dev, &pi->vi[0]);
1477         if (rc)
1478                 return (rc);
1479
1480         for_each_vi(pi, i, vi) {
1481                 if (i == 0)
1482                         continue;
1483                 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1);
1484                 if (vi->dev == NULL) {
1485                         device_printf(dev, "failed to add VI %d\n", i);
1486                         continue;
1487                 }
1488                 device_set_softc(vi->dev, vi);
1489         }
1490
1491         cxgbe_sysctls(pi);
1492
1493         bus_generic_attach(dev);
1494
1495         return (0);
1496 }
1497
1498 static void
1499 cxgbe_vi_detach(struct vi_info *vi)
1500 {
1501         struct ifnet *ifp = vi->ifp;
1502
1503         ether_ifdetach(ifp);
1504
1505         if (vi->vlan_c)
1506                 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c);
1507
1508         /* Let detach proceed even if these fail. */
1509 #ifdef DEV_NETMAP
1510         if (ifp->if_capabilities & IFCAP_NETMAP)
1511                 cxgbe_nm_detach(vi);
1512 #endif
1513         cxgbe_uninit_synchronized(vi);
1514         callout_drain(&vi->tick);
1515         vi_full_uninit(vi);
1516
1517         ifmedia_removeall(&vi->media);
1518         if_free(vi->ifp);
1519         vi->ifp = NULL;
1520 }
1521
1522 static int
1523 cxgbe_detach(device_t dev)
1524 {
1525         struct port_info *pi = device_get_softc(dev);
1526         struct adapter *sc = pi->adapter;
1527         int rc;
1528
1529         /* Detach the extra VIs first. */
1530         rc = bus_generic_detach(dev);
1531         if (rc)
1532                 return (rc);
1533         device_delete_children(dev);
1534
1535         doom_vi(sc, &pi->vi[0]);
1536
1537         if (pi->flags & HAS_TRACEQ) {
1538                 sc->traceq = -1;        /* cloner should not create ifnet */
1539                 t4_tracer_port_detach(sc);
1540         }
1541
1542         cxgbe_vi_detach(&pi->vi[0]);
1543         callout_drain(&pi->tick);
1544
1545         end_synchronized_op(sc, 0);
1546
1547         return (0);
1548 }
1549
1550 static void
1551 cxgbe_init(void *arg)
1552 {
1553         struct vi_info *vi = arg;
1554         struct adapter *sc = vi->pi->adapter;
1555
1556         if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
1557                 return;
1558         cxgbe_init_synchronized(vi);
1559         end_synchronized_op(sc, 0);
1560 }
1561
1562 static int
1563 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1564 {
1565         int rc = 0, mtu, flags, can_sleep;
1566         struct vi_info *vi = ifp->if_softc;
1567         struct adapter *sc = vi->pi->adapter;
1568         struct ifreq *ifr = (struct ifreq *)data;
1569         uint32_t mask;
1570
1571         switch (cmd) {
1572         case SIOCSIFMTU:
1573                 mtu = ifr->ifr_mtu;
1574                 if (mtu < ETHERMIN || mtu > MAX_MTU)
1575                         return (EINVAL);
1576
1577                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
1578                 if (rc)
1579                         return (rc);
1580                 ifp->if_mtu = mtu;
1581                 if (vi->flags & VI_INIT_DONE) {
1582                         t4_update_fl_bufsize(ifp);
1583                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1584                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1585                 }
1586                 end_synchronized_op(sc, 0);
1587                 break;
1588
1589         case SIOCSIFFLAGS:
1590                 can_sleep = 0;
1591 redo_sifflags:
1592                 rc = begin_synchronized_op(sc, vi,
1593                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1594                 if (rc)
1595                         return (rc);
1596
1597                 if (ifp->if_flags & IFF_UP) {
1598                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1599                                 flags = vi->if_flags;
1600                                 if ((ifp->if_flags ^ flags) &
1601                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1602                                         if (can_sleep == 1) {
1603                                                 end_synchronized_op(sc, 0);
1604                                                 can_sleep = 0;
1605                                                 goto redo_sifflags;
1606                                         }
1607                                         rc = update_mac_settings(ifp,
1608                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1609                                 }
1610                         } else {
1611                                 if (can_sleep == 0) {
1612                                         end_synchronized_op(sc, LOCK_HELD);
1613                                         can_sleep = 1;
1614                                         goto redo_sifflags;
1615                                 }
1616                                 rc = cxgbe_init_synchronized(vi);
1617                         }
1618                         vi->if_flags = ifp->if_flags;
1619                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1620                         if (can_sleep == 0) {
1621                                 end_synchronized_op(sc, LOCK_HELD);
1622                                 can_sleep = 1;
1623                                 goto redo_sifflags;
1624                         }
1625                         rc = cxgbe_uninit_synchronized(vi);
1626                 }
1627                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1628                 break;
1629
1630         case SIOCADDMULTI:
1631         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1632                 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi");
1633                 if (rc)
1634                         return (rc);
1635                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1636                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1637                 end_synchronized_op(sc, LOCK_HELD);
1638                 break;
1639
1640         case SIOCSIFCAP:
1641                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
1642                 if (rc)
1643                         return (rc);
1644
1645                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1646                 if (mask & IFCAP_TXCSUM) {
1647                         ifp->if_capenable ^= IFCAP_TXCSUM;
1648                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1649
1650                         if (IFCAP_TSO4 & ifp->if_capenable &&
1651                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1652                                 ifp->if_capenable &= ~IFCAP_TSO4;
1653                                 if_printf(ifp,
1654                                     "tso4 disabled due to -txcsum.\n");
1655                         }
1656                 }
1657                 if (mask & IFCAP_TXCSUM_IPV6) {
1658                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1659                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1660
1661                         if (IFCAP_TSO6 & ifp->if_capenable &&
1662                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1663                                 ifp->if_capenable &= ~IFCAP_TSO6;
1664                                 if_printf(ifp,
1665                                     "tso6 disabled due to -txcsum6.\n");
1666                         }
1667                 }
1668                 if (mask & IFCAP_RXCSUM)
1669                         ifp->if_capenable ^= IFCAP_RXCSUM;
1670                 if (mask & IFCAP_RXCSUM_IPV6)
1671                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1672
1673                 /*
1674                  * Note that we leave CSUM_TSO alone (it is always set).  The
1675                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1676                  * sending a TSO request our way, so it's sufficient to toggle
1677                  * IFCAP_TSOx only.
1678                  */
1679                 if (mask & IFCAP_TSO4) {
1680                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1681                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1682                                 if_printf(ifp, "enable txcsum first.\n");
1683                                 rc = EAGAIN;
1684                                 goto fail;
1685                         }
1686                         ifp->if_capenable ^= IFCAP_TSO4;
1687                 }
1688                 if (mask & IFCAP_TSO6) {
1689                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1690                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1691                                 if_printf(ifp, "enable txcsum6 first.\n");
1692                                 rc = EAGAIN;
1693                                 goto fail;
1694                         }
1695                         ifp->if_capenable ^= IFCAP_TSO6;
1696                 }
1697                 if (mask & IFCAP_LRO) {
1698 #if defined(INET) || defined(INET6)
1699                         int i;
1700                         struct sge_rxq *rxq;
1701
1702                         ifp->if_capenable ^= IFCAP_LRO;
1703                         for_each_rxq(vi, i, rxq) {
1704                                 if (ifp->if_capenable & IFCAP_LRO)
1705                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1706                                 else
1707                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1708                         }
1709 #endif
1710                 }
1711 #ifdef TCP_OFFLOAD
1712                 if (mask & IFCAP_TOE) {
1713                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1714
1715                         rc = toe_capability(vi, enable);
1716                         if (rc != 0)
1717                                 goto fail;
1718
1719                         ifp->if_capenable ^= mask;
1720                 }
1721 #endif
1722                 if (mask & IFCAP_VLAN_HWTAGGING) {
1723                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1724                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1725                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1726                 }
1727                 if (mask & IFCAP_VLAN_MTU) {
1728                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1729
1730                         /* Need to find out how to disable auto-mtu-inflation */
1731                 }
1732                 if (mask & IFCAP_VLAN_HWTSO)
1733                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1734                 if (mask & IFCAP_VLAN_HWCSUM)
1735                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1736
1737 #ifdef VLAN_CAPABILITIES
1738                 VLAN_CAPABILITIES(ifp);
1739 #endif
1740 fail:
1741                 end_synchronized_op(sc, 0);
1742                 break;
1743
1744         case SIOCSIFMEDIA:
1745         case SIOCGIFMEDIA:
1746         case SIOCGIFXMEDIA:
1747                 ifmedia_ioctl(ifp, ifr, &vi->media, cmd);
1748                 break;
1749
1750         case SIOCGI2C: {
1751                 struct ifi2creq i2c;
1752
1753                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1754                 if (rc != 0)
1755                         break;
1756                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1757                         rc = EPERM;
1758                         break;
1759                 }
1760                 if (i2c.len > sizeof(i2c.data)) {
1761                         rc = EINVAL;
1762                         break;
1763                 }
1764                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
1765                 if (rc)
1766                         return (rc);
1767                 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr,
1768                     i2c.offset, i2c.len, &i2c.data[0]);
1769                 end_synchronized_op(sc, 0);
1770                 if (rc == 0)
1771                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1772                 break;
1773         }
1774
1775         default:
1776                 rc = ether_ioctl(ifp, cmd, data);
1777         }
1778
1779         return (rc);
1780 }
1781
1782 static int
1783 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1784 {
1785         struct vi_info *vi = ifp->if_softc;
1786         struct port_info *pi = vi->pi;
1787         struct adapter *sc = pi->adapter;
1788         struct sge_txq *txq;
1789         void *items[1];
1790         int rc;
1791
1792         M_ASSERTPKTHDR(m);
1793         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1794
1795         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1796                 m_freem(m);
1797                 return (ENETDOWN);
1798         }
1799
1800         rc = parse_pkt(sc, &m);
1801         if (__predict_false(rc != 0)) {
1802                 MPASS(m == NULL);                       /* was freed already */
1803                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1804                 return (rc);
1805         }
1806
1807         /* Select a txq. */
1808         txq = &sc->sge.txq[vi->first_txq];
1809         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1810                 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
1811                     vi->rsrv_noflowq);
1812
1813         items[0] = m;
1814         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1815         if (__predict_false(rc != 0))
1816                 m_freem(m);
1817
1818         return (rc);
1819 }
1820
1821 static void
1822 cxgbe_qflush(struct ifnet *ifp)
1823 {
1824         struct vi_info *vi = ifp->if_softc;
1825         struct sge_txq *txq;
1826         int i;
1827
1828         /* queues do not exist if !VI_INIT_DONE. */
1829         if (vi->flags & VI_INIT_DONE) {
1830                 for_each_txq(vi, i, txq) {
1831                         TXQ_LOCK(txq);
1832                         txq->eq.flags &= ~EQ_ENABLED;
1833                         TXQ_UNLOCK(txq);
1834                         while (!mp_ring_is_idle(txq->r)) {
1835                                 mp_ring_check_drainage(txq->r, 0);
1836                                 pause("qflush", 1);
1837                         }
1838                 }
1839         }
1840         if_qflush(ifp);
1841 }
1842
1843 static uint64_t
1844 vi_get_counter(struct ifnet *ifp, ift_counter c)
1845 {
1846         struct vi_info *vi = ifp->if_softc;
1847         struct fw_vi_stats_vf *s = &vi->stats;
1848
1849         vi_refresh_stats(vi->pi->adapter, vi);
1850
1851         switch (c) {
1852         case IFCOUNTER_IPACKETS:
1853                 return (s->rx_bcast_frames + s->rx_mcast_frames +
1854                     s->rx_ucast_frames);
1855         case IFCOUNTER_IERRORS:
1856                 return (s->rx_err_frames);
1857         case IFCOUNTER_OPACKETS:
1858                 return (s->tx_bcast_frames + s->tx_mcast_frames +
1859                     s->tx_ucast_frames + s->tx_offload_frames);
1860         case IFCOUNTER_OERRORS:
1861                 return (s->tx_drop_frames);
1862         case IFCOUNTER_IBYTES:
1863                 return (s->rx_bcast_bytes + s->rx_mcast_bytes +
1864                     s->rx_ucast_bytes);
1865         case IFCOUNTER_OBYTES:
1866                 return (s->tx_bcast_bytes + s->tx_mcast_bytes +
1867                     s->tx_ucast_bytes + s->tx_offload_bytes);
1868         case IFCOUNTER_IMCASTS:
1869                 return (s->rx_mcast_frames);
1870         case IFCOUNTER_OMCASTS:
1871                 return (s->tx_mcast_frames);
1872         case IFCOUNTER_OQDROPS: {
1873                 uint64_t drops;
1874
1875                 drops = 0;
1876                 if (vi->flags & VI_INIT_DONE) {
1877                         int i;
1878                         struct sge_txq *txq;
1879
1880                         for_each_txq(vi, i, txq)
1881                                 drops += counter_u64_fetch(txq->r->drops);
1882                 }
1883
1884                 return (drops);
1885
1886         }
1887
1888         default:
1889                 return (if_get_counter_default(ifp, c));
1890         }
1891 }
1892
1893 uint64_t
1894 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1895 {
1896         struct vi_info *vi = ifp->if_softc;
1897         struct port_info *pi = vi->pi;
1898         struct adapter *sc = pi->adapter;
1899         struct port_stats *s = &pi->stats;
1900
1901         if (pi->nvi > 1 || sc->flags & IS_VF)
1902                 return (vi_get_counter(ifp, c));
1903
1904         cxgbe_refresh_stats(sc, pi);
1905
1906         switch (c) {
1907         case IFCOUNTER_IPACKETS:
1908                 return (s->rx_frames);
1909
1910         case IFCOUNTER_IERRORS:
1911                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1912                     s->rx_fcs_err + s->rx_len_err);
1913
1914         case IFCOUNTER_OPACKETS:
1915                 return (s->tx_frames);
1916
1917         case IFCOUNTER_OERRORS:
1918                 return (s->tx_error_frames);
1919
1920         case IFCOUNTER_IBYTES:
1921                 return (s->rx_octets);
1922
1923         case IFCOUNTER_OBYTES:
1924                 return (s->tx_octets);
1925
1926         case IFCOUNTER_IMCASTS:
1927                 return (s->rx_mcast_frames);
1928
1929         case IFCOUNTER_OMCASTS:
1930                 return (s->tx_mcast_frames);
1931
1932         case IFCOUNTER_IQDROPS:
1933                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1934                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1935                     s->rx_trunc3 + pi->tnl_cong_drops);
1936
1937         case IFCOUNTER_OQDROPS: {
1938                 uint64_t drops;
1939
1940                 drops = s->tx_drop;
1941                 if (vi->flags & VI_INIT_DONE) {
1942                         int i;
1943                         struct sge_txq *txq;
1944
1945                         for_each_txq(vi, i, txq)
1946                                 drops += counter_u64_fetch(txq->r->drops);
1947                 }
1948
1949                 return (drops);
1950
1951         }
1952
1953         default:
1954                 return (if_get_counter_default(ifp, c));
1955         }
1956 }
1957
1958 static int
1959 cxgbe_media_change(struct ifnet *ifp)
1960 {
1961         struct vi_info *vi = ifp->if_softc;
1962
1963         device_printf(vi->dev, "%s unimplemented.\n", __func__);
1964
1965         return (EOPNOTSUPP);
1966 }
1967
1968 static void
1969 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1970 {
1971         struct vi_info *vi = ifp->if_softc;
1972         struct port_info *pi = vi->pi;
1973         struct ifmedia_entry *cur;
1974         int speed = pi->link_cfg.speed;
1975
1976         cur = vi->media.ifm_cur;
1977
1978         ifmr->ifm_status = IFM_AVALID;
1979         if (!pi->link_cfg.link_ok)
1980                 return;
1981
1982         ifmr->ifm_status |= IFM_ACTIVE;
1983
1984         /* active and current will differ iff current media is autoselect. */
1985         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1986                 return;
1987
1988         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1989         if (speed == 10000)
1990                 ifmr->ifm_active |= IFM_10G_T;
1991         else if (speed == 1000)
1992                 ifmr->ifm_active |= IFM_1000_T;
1993         else if (speed == 100)
1994                 ifmr->ifm_active |= IFM_100_TX;
1995         else if (speed == 10)
1996                 ifmr->ifm_active |= IFM_10_T;
1997         else
1998                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1999                             speed));
2000 }
2001
2002 static int
2003 vcxgbe_probe(device_t dev)
2004 {
2005         char buf[128];
2006         struct vi_info *vi = device_get_softc(dev);
2007
2008         snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
2009             vi - vi->pi->vi);
2010         device_set_desc_copy(dev, buf);
2011
2012         return (BUS_PROBE_DEFAULT);
2013 }
2014
2015 static int
2016 vcxgbe_attach(device_t dev)
2017 {
2018         struct vi_info *vi;
2019         struct port_info *pi;
2020         struct adapter *sc;
2021         int func, index, rc;
2022         u32 param, val;
2023
2024         vi = device_get_softc(dev);
2025         pi = vi->pi;
2026         sc = pi->adapter;
2027
2028         index = vi - pi->vi;
2029         KASSERT(index < nitems(vi_mac_funcs),
2030             ("%s: VI %s doesn't have a MAC func", __func__,
2031             device_get_nameunit(dev)));
2032         func = vi_mac_funcs[index];
2033         rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
2034             vi->hw_addr, &vi->rss_size, func, 0);
2035         if (rc < 0) {
2036                 device_printf(dev, "Failed to allocate virtual interface "
2037                     "for port %d: %d\n", pi->port_id, -rc);
2038                 return (-rc);
2039         }
2040         vi->viid = rc;
2041         if (chip_id(sc) <= CHELSIO_T5)
2042                 vi->smt_idx = (rc & 0x7f) << 1;
2043         else
2044                 vi->smt_idx = (rc & 0x7f);
2045
2046         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
2047             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
2048             V_FW_PARAMS_PARAM_YZ(vi->viid);
2049         rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2050         if (rc)
2051                 vi->rss_base = 0xffff;
2052         else {
2053                 /* MPASS((val >> 16) == rss_size); */
2054                 vi->rss_base = val & 0xffff;
2055         }
2056
2057         rc = cxgbe_vi_attach(dev, vi);
2058         if (rc) {
2059                 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
2060                 return (rc);
2061         }
2062         return (0);
2063 }
2064
2065 static int
2066 vcxgbe_detach(device_t dev)
2067 {
2068         struct vi_info *vi;
2069         struct adapter *sc;
2070
2071         vi = device_get_softc(dev);
2072         sc = vi->pi->adapter;
2073
2074         doom_vi(sc, vi);
2075
2076         cxgbe_vi_detach(vi);
2077         t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
2078
2079         end_synchronized_op(sc, 0);
2080
2081         return (0);
2082 }
2083
2084 void
2085 t4_fatal_err(struct adapter *sc)
2086 {
2087         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
2088         t4_intr_disable(sc);
2089         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
2090             device_get_nameunit(sc->dev));
2091 }
2092
2093 void
2094 t4_add_adapter(struct adapter *sc)
2095 {
2096         sx_xlock(&t4_list_lock);
2097         SLIST_INSERT_HEAD(&t4_list, sc, link);
2098         sx_xunlock(&t4_list_lock);
2099 }
2100
2101 int
2102 t4_map_bars_0_and_4(struct adapter *sc)
2103 {
2104         sc->regs_rid = PCIR_BAR(0);
2105         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2106             &sc->regs_rid, RF_ACTIVE);
2107         if (sc->regs_res == NULL) {
2108                 device_printf(sc->dev, "cannot map registers.\n");
2109                 return (ENXIO);
2110         }
2111         sc->bt = rman_get_bustag(sc->regs_res);
2112         sc->bh = rman_get_bushandle(sc->regs_res);
2113         sc->mmio_len = rman_get_size(sc->regs_res);
2114         setbit(&sc->doorbells, DOORBELL_KDB);
2115
2116         sc->msix_rid = PCIR_BAR(4);
2117         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2118             &sc->msix_rid, RF_ACTIVE);
2119         if (sc->msix_res == NULL) {
2120                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
2121                 return (ENXIO);
2122         }
2123
2124         return (0);
2125 }
2126
2127 int
2128 t4_map_bar_2(struct adapter *sc)
2129 {
2130
2131         /*
2132          * T4: only iWARP driver uses the userspace doorbells.  There is no need
2133          * to map it if RDMA is disabled.
2134          */
2135         if (is_t4(sc) && sc->rdmacaps == 0)
2136                 return (0);
2137
2138         sc->udbs_rid = PCIR_BAR(2);
2139         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2140             &sc->udbs_rid, RF_ACTIVE);
2141         if (sc->udbs_res == NULL) {
2142                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
2143                 return (ENXIO);
2144         }
2145         sc->udbs_base = rman_get_virtual(sc->udbs_res);
2146
2147         if (chip_id(sc) >= CHELSIO_T5) {
2148                 setbit(&sc->doorbells, DOORBELL_UDB);
2149 #if defined(__i386__) || defined(__amd64__)
2150                 if (t5_write_combine) {
2151                         int rc, mode;
2152
2153                         /*
2154                          * Enable write combining on BAR2.  This is the
2155                          * userspace doorbell BAR and is split into 128B
2156                          * (UDBS_SEG_SIZE) doorbell regions, each associated
2157                          * with an egress queue.  The first 64B has the doorbell
2158                          * and the second 64B can be used to submit a tx work
2159                          * request with an implicit doorbell.
2160                          */
2161
2162                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
2163                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
2164                         if (rc == 0) {
2165                                 clrbit(&sc->doorbells, DOORBELL_UDB);
2166                                 setbit(&sc->doorbells, DOORBELL_WCWR);
2167                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
2168                         } else {
2169                                 device_printf(sc->dev,
2170                                     "couldn't enable write combining: %d\n",
2171                                     rc);
2172                         }
2173
2174                         mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0);
2175                         t4_write_reg(sc, A_SGE_STAT_CFG,
2176                             V_STATSOURCE_T5(7) | mode);
2177                 }
2178 #endif
2179         }
2180
2181         return (0);
2182 }
2183
2184 struct memwin_init {
2185         uint32_t base;
2186         uint32_t aperture;
2187 };
2188
2189 static const struct memwin_init t4_memwin[NUM_MEMWIN] = {
2190         { MEMWIN0_BASE, MEMWIN0_APERTURE },
2191         { MEMWIN1_BASE, MEMWIN1_APERTURE },
2192         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
2193 };
2194
2195 static const struct memwin_init t5_memwin[NUM_MEMWIN] = {
2196         { MEMWIN0_BASE, MEMWIN0_APERTURE },
2197         { MEMWIN1_BASE, MEMWIN1_APERTURE },
2198         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
2199 };
2200
2201 static void
2202 setup_memwin(struct adapter *sc)
2203 {
2204         const struct memwin_init *mw_init;
2205         struct memwin *mw;
2206         int i;
2207         uint32_t bar0;
2208
2209         if (is_t4(sc)) {
2210                 /*
2211                  * Read low 32b of bar0 indirectly via the hardware backdoor
2212                  * mechanism.  Works from within PCI passthrough environments
2213                  * too, where rman_get_start() can return a different value.  We
2214                  * need to program the T4 memory window decoders with the actual
2215                  * addresses that will be coming across the PCIe link.
2216                  */
2217                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
2218                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
2219
2220                 mw_init = &t4_memwin[0];
2221         } else {
2222                 /* T5+ use the relative offset inside the PCIe BAR */
2223                 bar0 = 0;
2224
2225                 mw_init = &t5_memwin[0];
2226         }
2227
2228         for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
2229                 rw_init(&mw->mw_lock, "memory window access");
2230                 mw->mw_base = mw_init->base;
2231                 mw->mw_aperture = mw_init->aperture;
2232                 mw->mw_curpos = 0;
2233                 t4_write_reg(sc,
2234                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
2235                     (mw->mw_base + bar0) | V_BIR(0) |
2236                     V_WINDOW(ilog2(mw->mw_aperture) - 10));
2237                 rw_wlock(&mw->mw_lock);
2238                 position_memwin(sc, i, 0);
2239                 rw_wunlock(&mw->mw_lock);
2240         }
2241
2242         /* flush */
2243         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
2244 }
2245
2246 /*
2247  * Positions the memory window at the given address in the card's address space.
2248  * There are some alignment requirements and the actual position may be at an
2249  * address prior to the requested address.  mw->mw_curpos always has the actual
2250  * position of the window.
2251  */
2252 static void
2253 position_memwin(struct adapter *sc, int idx, uint32_t addr)
2254 {
2255         struct memwin *mw;
2256         uint32_t pf;
2257         uint32_t reg;
2258
2259         MPASS(idx >= 0 && idx < NUM_MEMWIN);
2260         mw = &sc->memwin[idx];
2261         rw_assert(&mw->mw_lock, RA_WLOCKED);
2262
2263         if (is_t4(sc)) {
2264                 pf = 0;
2265                 mw->mw_curpos = addr & ~0xf;    /* start must be 16B aligned */
2266         } else {
2267                 pf = V_PFNUM(sc->pf);
2268                 mw->mw_curpos = addr & ~0x7f;   /* start must be 128B aligned */
2269         }
2270         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx);
2271         t4_write_reg(sc, reg, mw->mw_curpos | pf);
2272         t4_read_reg(sc, reg);   /* flush */
2273 }
2274
2275 static int
2276 rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
2277     int len, int rw)
2278 {
2279         struct memwin *mw;
2280         uint32_t mw_end, v;
2281
2282         MPASS(idx >= 0 && idx < NUM_MEMWIN);
2283
2284         /* Memory can only be accessed in naturally aligned 4 byte units */
2285         if (addr & 3 || len & 3 || len <= 0)
2286                 return (EINVAL);
2287
2288         mw = &sc->memwin[idx];
2289         while (len > 0) {
2290                 rw_rlock(&mw->mw_lock);
2291                 mw_end = mw->mw_curpos + mw->mw_aperture;
2292                 if (addr >= mw_end || addr < mw->mw_curpos) {
2293                         /* Will need to reposition the window */
2294                         if (!rw_try_upgrade(&mw->mw_lock)) {
2295                                 rw_runlock(&mw->mw_lock);
2296                                 rw_wlock(&mw->mw_lock);
2297                         }
2298                         rw_assert(&mw->mw_lock, RA_WLOCKED);
2299                         position_memwin(sc, idx, addr);
2300                         rw_downgrade(&mw->mw_lock);
2301                         mw_end = mw->mw_curpos + mw->mw_aperture;
2302                 }
2303                 rw_assert(&mw->mw_lock, RA_RLOCKED);
2304                 while (addr < mw_end && len > 0) {
2305                         if (rw == 0) {
2306                                 v = t4_read_reg(sc, mw->mw_base + addr -
2307                                     mw->mw_curpos);
2308                                 *val++ = le32toh(v);
2309                         } else {
2310                                 v = *val++;
2311                                 t4_write_reg(sc, mw->mw_base + addr -
2312                                     mw->mw_curpos, htole32(v));
2313                         }
2314                         addr += 4;
2315                         len -= 4;
2316                 }
2317                 rw_runlock(&mw->mw_lock);
2318         }
2319
2320         return (0);
2321 }
2322
2323 static inline int
2324 read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
2325     int len)
2326 {
2327
2328         return (rw_via_memwin(sc, idx, addr, val, len, 0));
2329 }
2330
2331 static inline int
2332 write_via_memwin(struct adapter *sc, int idx, uint32_t addr,
2333     const uint32_t *val, int len)
2334 {
2335
2336         return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1));
2337 }
2338
2339 static int
2340 t4_range_cmp(const void *a, const void *b)
2341 {
2342         return ((const struct t4_range *)a)->start -
2343                ((const struct t4_range *)b)->start;
2344 }
2345
2346 /*
2347  * Verify that the memory range specified by the addr/len pair is valid within
2348  * the card's address space.
2349  */
2350 static int
2351 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
2352 {
2353         struct t4_range mem_ranges[4], *r, *next;
2354         uint32_t em, addr_len;
2355         int i, n, remaining;
2356
2357         /* Memory can only be accessed in naturally aligned 4 byte units */
2358         if (addr & 3 || len & 3 || len <= 0)
2359                 return (EINVAL);
2360
2361         /* Enabled memories */
2362         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2363
2364         r = &mem_ranges[0];
2365         n = 0;
2366         bzero(r, sizeof(mem_ranges));
2367         if (em & F_EDRAM0_ENABLE) {
2368                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2369                 r->size = G_EDRAM0_SIZE(addr_len) << 20;
2370                 if (r->size > 0) {
2371                         r->start = G_EDRAM0_BASE(addr_len) << 20;
2372                         if (addr >= r->start &&
2373                             addr + len <= r->start + r->size)
2374                                 return (0);
2375                         r++;
2376                         n++;
2377                 }
2378         }
2379         if (em & F_EDRAM1_ENABLE) {
2380                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2381                 r->size = G_EDRAM1_SIZE(addr_len) << 20;
2382                 if (r->size > 0) {
2383                         r->start = G_EDRAM1_BASE(addr_len) << 20;
2384                         if (addr >= r->start &&
2385                             addr + len <= r->start + r->size)
2386                                 return (0);
2387                         r++;
2388                         n++;
2389                 }
2390         }
2391         if (em & F_EXT_MEM_ENABLE) {
2392                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2393                 r->size = G_EXT_MEM_SIZE(addr_len) << 20;
2394                 if (r->size > 0) {
2395                         r->start = G_EXT_MEM_BASE(addr_len) << 20;
2396                         if (addr >= r->start &&
2397                             addr + len <= r->start + r->size)
2398                                 return (0);
2399                         r++;
2400                         n++;
2401                 }
2402         }
2403         if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) {
2404                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2405                 r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
2406                 if (r->size > 0) {
2407                         r->start = G_EXT_MEM1_BASE(addr_len) << 20;
2408                         if (addr >= r->start &&
2409                             addr + len <= r->start + r->size)
2410                                 return (0);
2411                         r++;
2412                         n++;
2413                 }
2414         }
2415         MPASS(n <= nitems(mem_ranges));
2416
2417         if (n > 1) {
2418                 /* Sort and merge the ranges. */
2419                 qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp);
2420
2421                 /* Start from index 0 and examine the next n - 1 entries. */
2422                 r = &mem_ranges[0];
2423                 for (remaining = n - 1; remaining > 0; remaining--, r++) {
2424
2425                         MPASS(r->size > 0);     /* r is a valid entry. */
2426                         next = r + 1;
2427                         MPASS(next->size > 0);  /* and so is the next one. */
2428
2429                         while (r->start + r->size >= next->start) {
2430                                 /* Merge the next one into the current entry. */
2431                                 r->size = max(r->start + r->size,
2432                                     next->start + next->size) - r->start;
2433                                 n--;    /* One fewer entry in total. */
2434                                 if (--remaining == 0)
2435                                         goto done;      /* short circuit */
2436                                 next++;
2437                         }
2438                         if (next != r + 1) {
2439                                 /*
2440                                  * Some entries were merged into r and next
2441                                  * points to the first valid entry that couldn't
2442                                  * be merged.
2443                                  */
2444                                 MPASS(next->size > 0);  /* must be valid */
2445                                 memcpy(r + 1, next, remaining * sizeof(*r));
2446 #ifdef INVARIANTS
2447                                 /*
2448                                  * This so that the foo->size assertion in the
2449                                  * next iteration of the loop do the right
2450                                  * thing for entries that were pulled up and are
2451                                  * no longer valid.
2452                                  */
2453                                 MPASS(n < nitems(mem_ranges));
2454                                 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
2455                                     sizeof(struct t4_range));
2456 #endif
2457                         }
2458                 }
2459 done:
2460                 /* Done merging the ranges. */
2461                 MPASS(n > 0);
2462                 r = &mem_ranges[0];
2463                 for (i = 0; i < n; i++, r++) {
2464                         if (addr >= r->start &&
2465                             addr + len <= r->start + r->size)
2466                                 return (0);
2467                 }
2468         }
2469
2470         return (EFAULT);
2471 }
2472
2473 static int
2474 fwmtype_to_hwmtype(int mtype)
2475 {
2476
2477         switch (mtype) {
2478         case FW_MEMTYPE_EDC0:
2479                 return (MEM_EDC0);
2480         case FW_MEMTYPE_EDC1:
2481                 return (MEM_EDC1);
2482         case FW_MEMTYPE_EXTMEM:
2483                 return (MEM_MC0);
2484         case FW_MEMTYPE_EXTMEM1:
2485                 return (MEM_MC1);
2486         default:
2487                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
2488         }
2489 }
2490
2491 /*
2492  * Verify that the memory range specified by the memtype/offset/len pair is
2493  * valid and lies entirely within the memtype specified.  The global address of
2494  * the start of the range is returned in addr.
2495  */
2496 static int
2497 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
2498     uint32_t *addr)
2499 {
2500         uint32_t em, addr_len, maddr;
2501
2502         /* Memory can only be accessed in naturally aligned 4 byte units */
2503         if (off & 3 || len & 3 || len == 0)
2504                 return (EINVAL);
2505
2506         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2507         switch (fwmtype_to_hwmtype(mtype)) {
2508         case MEM_EDC0:
2509                 if (!(em & F_EDRAM0_ENABLE))
2510                         return (EINVAL);
2511                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2512                 maddr = G_EDRAM0_BASE(addr_len) << 20;
2513                 break;
2514         case MEM_EDC1:
2515                 if (!(em & F_EDRAM1_ENABLE))
2516                         return (EINVAL);
2517                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2518                 maddr = G_EDRAM1_BASE(addr_len) << 20;
2519                 break;
2520         case MEM_MC:
2521                 if (!(em & F_EXT_MEM_ENABLE))
2522                         return (EINVAL);
2523                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2524                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
2525                 break;
2526         case MEM_MC1:
2527                 if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE))
2528                         return (EINVAL);
2529                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2530                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
2531                 break;
2532         default:
2533                 return (EINVAL);
2534         }
2535
2536         *addr = maddr + off;    /* global address */
2537         return (validate_mem_range(sc, *addr, len));
2538 }
2539
2540 static int
2541 fixup_devlog_params(struct adapter *sc)
2542 {
2543         struct devlog_params *dparams = &sc->params.devlog;
2544         int rc;
2545
2546         rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
2547             dparams->size, &dparams->addr);
2548
2549         return (rc);
2550 }
2551
2552 static int
2553 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis,
2554     struct intrs_and_queues *iaq)
2555 {
2556         int rc, itype, navail, nrxq10g, nrxq1g, n;
2557         int nofldrxq10g = 0, nofldrxq1g = 0;
2558
2559         bzero(iaq, sizeof(*iaq));
2560
2561         iaq->ntxq10g = t4_ntxq10g;
2562         iaq->ntxq1g = t4_ntxq1g;
2563         iaq->ntxq_vi = t4_ntxq_vi;
2564         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
2565         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
2566         iaq->nrxq_vi = t4_nrxq_vi;
2567         iaq->rsrv_noflowq = t4_rsrv_noflowq;
2568 #ifdef TCP_OFFLOAD
2569         if (is_offload(sc)) {
2570                 iaq->nofldtxq10g = t4_nofldtxq10g;
2571                 iaq->nofldtxq1g = t4_nofldtxq1g;
2572                 iaq->nofldtxq_vi = t4_nofldtxq_vi;
2573                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
2574                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
2575                 iaq->nofldrxq_vi = t4_nofldrxq_vi;
2576         }
2577 #endif
2578 #ifdef DEV_NETMAP
2579         iaq->nnmtxq_vi = t4_nnmtxq_vi;
2580         iaq->nnmrxq_vi = t4_nnmrxq_vi;
2581 #endif
2582
2583         for (itype = INTR_MSIX; itype; itype >>= 1) {
2584
2585                 if ((itype & t4_intr_types) == 0)
2586                         continue;       /* not allowed */
2587
2588                 if (itype == INTR_MSIX)
2589                         navail = pci_msix_count(sc->dev);
2590                 else if (itype == INTR_MSI)
2591                         navail = pci_msi_count(sc->dev);
2592                 else
2593                         navail = 1;
2594 restart:
2595                 if (navail == 0)
2596                         continue;
2597
2598                 iaq->intr_type = itype;
2599                 iaq->intr_flags_10g = 0;
2600                 iaq->intr_flags_1g = 0;
2601
2602                 /*
2603                  * Best option: an interrupt vector for errors, one for the
2604                  * firmware event queue, and one for every rxq (NIC and TOE) of
2605                  * every VI.  The VIs that support netmap use the same
2606                  * interrupts for the NIC rx queues and the netmap rx queues
2607                  * because only one set of queues is active at a time.
2608                  */
2609                 iaq->nirq = T4_EXTRA_INTR;
2610                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
2611                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
2612                 iaq->nirq += (n10g + n1g) * (num_vis - 1) *
2613                     max(iaq->nrxq_vi, iaq->nnmrxq_vi);  /* See comment above. */
2614                 iaq->nirq += (n10g + n1g) * (num_vis - 1) * iaq->nofldrxq_vi;
2615                 if (iaq->nirq <= navail &&
2616                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
2617                         iaq->intr_flags_10g = INTR_ALL;
2618                         iaq->intr_flags_1g = INTR_ALL;
2619                         goto allocate;
2620                 }
2621
2622                 /* Disable the VIs (and netmap) if there aren't enough intrs */
2623                 if (num_vis > 1) {
2624                         device_printf(sc->dev, "virtual interfaces disabled "
2625                             "because num_vis=%u with current settings "
2626                             "(nrxq10g=%u, nrxq1g=%u, nofldrxq10g=%u, "
2627                             "nofldrxq1g=%u, nrxq_vi=%u nofldrxq_vi=%u, "
2628                             "nnmrxq_vi=%u) would need %u interrupts but "
2629                             "only %u are available.\n", num_vis, nrxq10g,
2630                             nrxq1g, nofldrxq10g, nofldrxq1g, iaq->nrxq_vi,
2631                             iaq->nofldrxq_vi, iaq->nnmrxq_vi, iaq->nirq,
2632                             navail);
2633                         num_vis = 1;
2634                         iaq->ntxq_vi = iaq->nrxq_vi = 0;
2635                         iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
2636                         iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
2637                         goto restart;
2638                 }
2639
2640                 /*
2641                  * Second best option: a vector for errors, one for the firmware
2642                  * event queue, and vectors for either all the NIC rx queues or
2643                  * all the TOE rx queues.  The queues that don't get vectors
2644                  * will forward their interrupts to those that do.
2645                  */
2646                 iaq->nirq = T4_EXTRA_INTR;
2647                 if (nrxq10g >= nofldrxq10g) {
2648                         iaq->intr_flags_10g = INTR_RXQ;
2649                         iaq->nirq += n10g * nrxq10g;
2650                 } else {
2651                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2652                         iaq->nirq += n10g * nofldrxq10g;
2653                 }
2654                 if (nrxq1g >= nofldrxq1g) {
2655                         iaq->intr_flags_1g = INTR_RXQ;
2656                         iaq->nirq += n1g * nrxq1g;
2657                 } else {
2658                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2659                         iaq->nirq += n1g * nofldrxq1g;
2660                 }
2661                 if (iaq->nirq <= navail &&
2662                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2663                         goto allocate;
2664
2665                 /*
2666                  * Next best option: an interrupt vector for errors, one for the
2667                  * firmware event queue, and at least one per main-VI.  At this
2668                  * point we know we'll have to downsize nrxq and/or nofldrxq to
2669                  * fit what's available to us.
2670                  */
2671                 iaq->nirq = T4_EXTRA_INTR;
2672                 iaq->nirq += n10g + n1g;
2673                 if (iaq->nirq <= navail) {
2674                         int leftover = navail - iaq->nirq;
2675
2676                         if (n10g > 0) {
2677                                 int target = max(nrxq10g, nofldrxq10g);
2678
2679                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2680                                     INTR_RXQ : INTR_OFLD_RXQ;
2681
2682                                 n = 1;
2683                                 while (n < target && leftover >= n10g) {
2684                                         leftover -= n10g;
2685                                         iaq->nirq += n10g;
2686                                         n++;
2687                                 }
2688                                 iaq->nrxq10g = min(n, nrxq10g);
2689 #ifdef TCP_OFFLOAD
2690                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2691 #endif
2692                         }
2693
2694                         if (n1g > 0) {
2695                                 int target = max(nrxq1g, nofldrxq1g);
2696
2697                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2698                                     INTR_RXQ : INTR_OFLD_RXQ;
2699
2700                                 n = 1;
2701                                 while (n < target && leftover >= n1g) {
2702                                         leftover -= n1g;
2703                                         iaq->nirq += n1g;
2704                                         n++;
2705                                 }
2706                                 iaq->nrxq1g = min(n, nrxq1g);
2707 #ifdef TCP_OFFLOAD
2708                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2709 #endif
2710                         }
2711
2712                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2713                                 goto allocate;
2714                 }
2715
2716                 /*
2717                  * Least desirable option: one interrupt vector for everything.
2718                  */
2719                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2720                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2721 #ifdef TCP_OFFLOAD
2722                 if (is_offload(sc))
2723                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2724 #endif
2725 allocate:
2726                 navail = iaq->nirq;
2727                 rc = 0;
2728                 if (itype == INTR_MSIX)
2729                         rc = pci_alloc_msix(sc->dev, &navail);
2730                 else if (itype == INTR_MSI)
2731                         rc = pci_alloc_msi(sc->dev, &navail);
2732
2733                 if (rc == 0) {
2734                         if (navail == iaq->nirq)
2735                                 return (0);
2736
2737                         /*
2738                          * Didn't get the number requested.  Use whatever number
2739                          * the kernel is willing to allocate (it's in navail).
2740                          */
2741                         device_printf(sc->dev, "fewer vectors than requested, "
2742                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2743                             itype, iaq->nirq, navail);
2744                         pci_release_msi(sc->dev);
2745                         goto restart;
2746                 }
2747
2748                 device_printf(sc->dev,
2749                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2750                     itype, rc, iaq->nirq, navail);
2751         }
2752
2753         device_printf(sc->dev,
2754             "failed to find a usable interrupt type.  "
2755             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2756             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2757
2758         return (ENXIO);
2759 }
2760
2761 #define FW_VERSION(chip) ( \
2762     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2763     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2764     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2765     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2766 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2767
2768 struct fw_info {
2769         uint8_t chip;
2770         char *kld_name;
2771         char *fw_mod_name;
2772         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2773 } fw_info[] = {
2774         {
2775                 .chip = CHELSIO_T4,
2776                 .kld_name = "t4fw_cfg",
2777                 .fw_mod_name = "t4fw",
2778                 .fw_hdr = {
2779                         .chip = FW_HDR_CHIP_T4,
2780                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2781                         .intfver_nic = FW_INTFVER(T4, NIC),
2782                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2783                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2784                         .intfver_ri = FW_INTFVER(T4, RI),
2785                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2786                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2787                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2788                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2789                 },
2790         }, {
2791                 .chip = CHELSIO_T5,
2792                 .kld_name = "t5fw_cfg",
2793                 .fw_mod_name = "t5fw",
2794                 .fw_hdr = {
2795                         .chip = FW_HDR_CHIP_T5,
2796                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2797                         .intfver_nic = FW_INTFVER(T5, NIC),
2798                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2799                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2800                         .intfver_ri = FW_INTFVER(T5, RI),
2801                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2802                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2803                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2804                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2805                 },
2806         }, {
2807                 .chip = CHELSIO_T6,
2808                 .kld_name = "t6fw_cfg",
2809                 .fw_mod_name = "t6fw",
2810                 .fw_hdr = {
2811                         .chip = FW_HDR_CHIP_T6,
2812                         .fw_ver = htobe32_const(FW_VERSION(T6)),
2813                         .intfver_nic = FW_INTFVER(T6, NIC),
2814                         .intfver_vnic = FW_INTFVER(T6, VNIC),
2815                         .intfver_ofld = FW_INTFVER(T6, OFLD),
2816                         .intfver_ri = FW_INTFVER(T6, RI),
2817                         .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
2818                         .intfver_iscsi = FW_INTFVER(T6, ISCSI),
2819                         .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
2820                         .intfver_fcoe = FW_INTFVER(T6, FCOE),
2821                 },
2822         }
2823 };
2824
2825 static struct fw_info *
2826 find_fw_info(int chip)
2827 {
2828         int i;
2829
2830         for (i = 0; i < nitems(fw_info); i++) {
2831                 if (fw_info[i].chip == chip)
2832                         return (&fw_info[i]);
2833         }
2834         return (NULL);
2835 }
2836
2837 /*
2838  * Is the given firmware API compatible with the one the driver was compiled
2839  * with?
2840  */
2841 static int
2842 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2843 {
2844
2845         /* short circuit if it's the exact same firmware version */
2846         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2847                 return (1);
2848
2849         /*
2850          * XXX: Is this too conservative?  Perhaps I should limit this to the
2851          * features that are supported in the driver.
2852          */
2853 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2854         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2855             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2856             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2857                 return (1);
2858 #undef SAME_INTF
2859
2860         return (0);
2861 }
2862
2863 /*
2864  * The firmware in the KLD is usable, but should it be installed?  This routine
2865  * explains itself in detail if it indicates the KLD firmware should be
2866  * installed.
2867  */
2868 static int
2869 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2870 {
2871         const char *reason;
2872
2873         if (!card_fw_usable) {
2874                 reason = "incompatible or unusable";
2875                 goto install;
2876         }
2877
2878         if (k > c) {
2879                 reason = "older than the version bundled with this driver";
2880                 goto install;
2881         }
2882
2883         if (t4_fw_install == 2 && k != c) {
2884                 reason = "different than the version bundled with this driver";
2885                 goto install;
2886         }
2887
2888         return (0);
2889
2890 install:
2891         if (t4_fw_install == 0) {
2892                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2893                     "but the driver is prohibited from installing a different "
2894                     "firmware on the card.\n",
2895                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2896                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2897
2898                 return (0);
2899         }
2900
2901         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2902             "installing firmware %u.%u.%u.%u on card.\n",
2903             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2904             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2905             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2906             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2907
2908         return (1);
2909 }
2910 /*
2911  * Establish contact with the firmware and determine if we are the master driver
2912  * or not, and whether we are responsible for chip initialization.
2913  */
2914 static int
2915 prep_firmware(struct adapter *sc)
2916 {
2917         const struct firmware *fw = NULL, *default_cfg;
2918         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2919         enum dev_state state;
2920         struct fw_info *fw_info;
2921         struct fw_hdr *card_fw;         /* fw on the card */
2922         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2923         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2924                                            against */
2925
2926         /* Contact firmware. */
2927         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2928         if (rc < 0 || state == DEV_STATE_ERR) {
2929                 rc = -rc;
2930                 device_printf(sc->dev,
2931                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2932                 return (rc);
2933         }
2934         pf = rc;
2935         if (pf == sc->mbox)
2936                 sc->flags |= MASTER_PF;
2937         else if (state == DEV_STATE_UNINIT) {
2938                 /*
2939                  * We didn't get to be the master so we definitely won't be
2940                  * configuring the chip.  It's a bug if someone else hasn't
2941                  * configured it already.
2942                  */
2943                 device_printf(sc->dev, "couldn't be master(%d), "
2944                     "device not already initialized either(%d).\n", rc, state);
2945                 return (EDOOFUS);
2946         }
2947
2948         /* This is the firmware whose headers the driver was compiled against */
2949         fw_info = find_fw_info(chip_id(sc));
2950         if (fw_info == NULL) {
2951                 device_printf(sc->dev,
2952                     "unable to look up firmware information for chip %d.\n",
2953                     chip_id(sc));
2954                 return (EINVAL);
2955         }
2956         drv_fw = &fw_info->fw_hdr;
2957
2958         /*
2959          * The firmware KLD contains many modules.  The KLD name is also the
2960          * name of the module that contains the default config file.
2961          */
2962         default_cfg = firmware_get(fw_info->kld_name);
2963
2964         /* Read the header of the firmware on the card */
2965         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2966         rc = -t4_read_flash(sc, FLASH_FW_START,
2967             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2968         if (rc == 0)
2969                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2970         else {
2971                 device_printf(sc->dev,
2972                     "Unable to read card's firmware header: %d\n", rc);
2973                 card_fw_usable = 0;
2974         }
2975
2976         /* This is the firmware in the KLD */
2977         fw = firmware_get(fw_info->fw_mod_name);
2978         if (fw != NULL) {
2979                 kld_fw = (const void *)fw->data;
2980                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2981         } else {
2982                 kld_fw = NULL;
2983                 kld_fw_usable = 0;
2984         }
2985
2986         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2987             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2988                 /*
2989                  * Common case: the firmware on the card is an exact match and
2990                  * the KLD is an exact match too, or the KLD is
2991                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2992                  * here -- use cxgbetool loadfw if you want to reinstall the
2993                  * same firmware as the one on the card.
2994                  */
2995         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2996             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2997             be32toh(card_fw->fw_ver))) {
2998
2999                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
3000                 if (rc != 0) {
3001                         device_printf(sc->dev,
3002                             "failed to install firmware: %d\n", rc);
3003                         goto done;
3004                 }
3005
3006                 /* Installed successfully, update the cached header too. */
3007                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
3008                 card_fw_usable = 1;
3009                 need_fw_reset = 0;      /* already reset as part of load_fw */
3010         }
3011
3012         if (!card_fw_usable) {
3013                 uint32_t d, c, k;
3014
3015                 d = ntohl(drv_fw->fw_ver);
3016                 c = ntohl(card_fw->fw_ver);
3017                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
3018
3019                 device_printf(sc->dev, "Cannot find a usable firmware: "
3020                     "fw_install %d, chip state %d, "
3021                     "driver compiled with %d.%d.%d.%d, "
3022                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
3023                     t4_fw_install, state,
3024                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
3025                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
3026                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
3027                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
3028                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
3029                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
3030                 rc = EINVAL;
3031                 goto done;
3032         }
3033
3034         /* Reset device */
3035         if (need_fw_reset &&
3036             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
3037                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
3038                 if (rc != ETIMEDOUT && rc != EIO)
3039                         t4_fw_bye(sc, sc->mbox);
3040                 goto done;
3041         }
3042         sc->flags |= FW_OK;
3043
3044         rc = get_params__pre_init(sc);
3045         if (rc != 0)
3046                 goto done; /* error message displayed already */
3047
3048         /* Partition adapter resources as specified in the config file. */
3049         if (state == DEV_STATE_UNINIT) {
3050
3051                 KASSERT(sc->flags & MASTER_PF,
3052                     ("%s: trying to change chip settings when not master.",
3053                     __func__));
3054
3055                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
3056                 if (rc != 0)
3057                         goto done;      /* error message displayed already */
3058
3059                 t4_tweak_chip_settings(sc);
3060
3061                 /* get basic stuff going */
3062                 rc = -t4_fw_initialize(sc, sc->mbox);
3063                 if (rc != 0) {
3064                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
3065                         goto done;
3066                 }
3067         } else {
3068                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
3069                 sc->cfcsum = 0;
3070         }
3071
3072 done:
3073         free(card_fw, M_CXGBE);
3074         if (fw != NULL)
3075                 firmware_put(fw, FIRMWARE_UNLOAD);
3076         if (default_cfg != NULL)
3077                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
3078
3079         return (rc);
3080 }
3081
3082 #define FW_PARAM_DEV(param) \
3083         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3084          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3085 #define FW_PARAM_PFVF(param) \
3086         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3087          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
3088
3089 /*
3090  * Partition chip resources for use between various PFs, VFs, etc.
3091  */
3092 static int
3093 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
3094     const char *name_prefix)
3095 {
3096         const struct firmware *cfg = NULL;
3097         int rc = 0;
3098         struct fw_caps_config_cmd caps;
3099         uint32_t mtype, moff, finicsum, cfcsum;
3100
3101         /*
3102          * Figure out what configuration file to use.  Pick the default config
3103          * file for the card if the user hasn't specified one explicitly.
3104          */
3105         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
3106         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
3107                 /* Card specific overrides go here. */
3108                 if (pci_get_device(sc->dev) == 0x440a)
3109                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
3110                 if (is_fpga(sc))
3111                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
3112         }
3113
3114         /*
3115          * We need to load another module if the profile is anything except
3116          * "default" or "flash".
3117          */
3118         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
3119             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
3120                 char s[32];
3121
3122                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
3123                 cfg = firmware_get(s);
3124                 if (cfg == NULL) {
3125                         if (default_cfg != NULL) {
3126                                 device_printf(sc->dev,
3127                                     "unable to load module \"%s\" for "
3128                                     "configuration profile \"%s\", will use "
3129                                     "the default config file instead.\n",
3130                                     s, sc->cfg_file);
3131                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
3132                                     "%s", DEFAULT_CF);
3133                         } else {
3134                                 device_printf(sc->dev,
3135                                     "unable to load module \"%s\" for "
3136                                     "configuration profile \"%s\", will use "
3137                                     "the config file on the card's flash "
3138                                     "instead.\n", s, sc->cfg_file);
3139                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
3140                                     "%s", FLASH_CF);
3141                         }
3142                 }
3143         }
3144
3145         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
3146             default_cfg == NULL) {
3147                 device_printf(sc->dev,
3148                     "default config file not available, will use the config "
3149                     "file on the card's flash instead.\n");
3150                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
3151         }
3152
3153         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
3154                 u_int cflen;
3155                 const uint32_t *cfdata;
3156                 uint32_t param, val, addr;
3157
3158                 KASSERT(cfg != NULL || default_cfg != NULL,
3159                     ("%s: no config to upload", __func__));
3160
3161                 /*
3162                  * Ask the firmware where it wants us to upload the config file.
3163                  */
3164                 param = FW_PARAM_DEV(CF);
3165                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3166                 if (rc != 0) {
3167                         /* No support for config file?  Shouldn't happen. */
3168                         device_printf(sc->dev,
3169                             "failed to query config file location: %d.\n", rc);
3170                         goto done;
3171                 }
3172                 mtype = G_FW_PARAMS_PARAM_Y(val);
3173                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
3174
3175                 /*
3176                  * XXX: sheer laziness.  We deliberately added 4 bytes of
3177                  * useless stuffing/comments at the end of the config file so
3178                  * it's ok to simply throw away the last remaining bytes when
3179                  * the config file is not an exact multiple of 4.  This also
3180                  * helps with the validate_mt_off_len check.
3181                  */
3182                 if (cfg != NULL) {
3183                         cflen = cfg->datasize & ~3;
3184                         cfdata = cfg->data;
3185                 } else {
3186                         cflen = default_cfg->datasize & ~3;
3187                         cfdata = default_cfg->data;
3188                 }
3189
3190                 if (cflen > FLASH_CFG_MAX_SIZE) {
3191                         device_printf(sc->dev,
3192                             "config file too long (%d, max allowed is %d).  "
3193                             "Will try to use the config on the card, if any.\n",
3194                             cflen, FLASH_CFG_MAX_SIZE);
3195                         goto use_config_on_flash;
3196                 }
3197
3198                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
3199                 if (rc != 0) {
3200                         device_printf(sc->dev,
3201                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
3202                             "Will try to use the config on the card, if any.\n",
3203                             __func__, mtype, moff, cflen, rc);
3204                         goto use_config_on_flash;
3205                 }
3206                 write_via_memwin(sc, 2, addr, cfdata, cflen);
3207         } else {
3208 use_config_on_flash:
3209                 mtype = FW_MEMTYPE_FLASH;
3210                 moff = t4_flash_cfg_addr(sc);
3211         }
3212
3213         bzero(&caps, sizeof(caps));
3214         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3215             F_FW_CMD_REQUEST | F_FW_CMD_READ);
3216         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
3217             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
3218             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
3219         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3220         if (rc != 0) {
3221                 device_printf(sc->dev,
3222                     "failed to pre-process config file: %d "
3223                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
3224                 goto done;
3225         }
3226
3227         finicsum = be32toh(caps.finicsum);
3228         cfcsum = be32toh(caps.cfcsum);
3229         if (finicsum != cfcsum) {
3230                 device_printf(sc->dev,
3231                     "WARNING: config file checksum mismatch: %08x %08x\n",
3232                     finicsum, cfcsum);
3233         }
3234         sc->cfcsum = cfcsum;
3235
3236 #define LIMIT_CAPS(x) do { \
3237         caps.x &= htobe16(t4_##x##_allowed); \
3238 } while (0)
3239
3240         /*
3241          * Let the firmware know what features will (not) be used so it can tune
3242          * things accordingly.
3243          */
3244         LIMIT_CAPS(nbmcaps);
3245         LIMIT_CAPS(linkcaps);
3246         LIMIT_CAPS(switchcaps);
3247         LIMIT_CAPS(niccaps);
3248         LIMIT_CAPS(toecaps);
3249         LIMIT_CAPS(rdmacaps);
3250         LIMIT_CAPS(cryptocaps);
3251         LIMIT_CAPS(iscsicaps);
3252         LIMIT_CAPS(fcoecaps);
3253 #undef LIMIT_CAPS
3254
3255         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3256             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
3257         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
3258         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
3259         if (rc != 0) {
3260                 device_printf(sc->dev,
3261                     "failed to process config file: %d.\n", rc);
3262         }
3263 done:
3264         if (cfg != NULL)
3265                 firmware_put(cfg, FIRMWARE_UNLOAD);
3266         return (rc);
3267 }
3268
3269 /*
3270  * Retrieve parameters that are needed (or nice to have) very early.
3271  */
3272 static int
3273 get_params__pre_init(struct adapter *sc)
3274 {
3275         int rc;
3276         uint32_t param[2], val[2];
3277
3278         t4_get_version_info(sc);
3279
3280         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
3281             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
3282             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
3283             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
3284             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
3285
3286         snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
3287             G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
3288             G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
3289             G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
3290             G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));
3291
3292         snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
3293             G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
3294             G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
3295             G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
3296             G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
3297
3298         snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
3299             G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
3300             G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
3301             G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
3302             G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));
3303
3304         param[0] = FW_PARAM_DEV(PORTVEC);
3305         param[1] = FW_PARAM_DEV(CCLK);
3306         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3307         if (rc != 0) {
3308                 device_printf(sc->dev,
3309                     "failed to query parameters (pre_init): %d.\n", rc);
3310                 return (rc);
3311         }
3312
3313         sc->params.portvec = val[0];
3314         sc->params.nports = bitcount32(val[0]);
3315         sc->params.vpd.cclk = val[1];
3316
3317         /* Read device log parameters. */
3318         rc = -t4_init_devlog_params(sc, 1);
3319         if (rc == 0)
3320                 fixup_devlog_params(sc);
3321         else {
3322                 device_printf(sc->dev,
3323                     "failed to get devlog parameters: %d.\n", rc);
3324                 rc = 0; /* devlog isn't critical for device operation */
3325         }
3326
3327         return (rc);
3328 }
3329
3330 /*
3331  * Retrieve various parameters that are of interest to the driver.  The device
3332  * has been initialized by the firmware at this point.
3333  */
3334 static int
3335 get_params__post_init(struct adapter *sc)
3336 {
3337         int rc;
3338         uint32_t param[7], val[7];
3339         struct fw_caps_config_cmd caps;
3340
3341         param[0] = FW_PARAM_PFVF(IQFLINT_START);
3342         param[1] = FW_PARAM_PFVF(EQ_START);
3343         param[2] = FW_PARAM_PFVF(FILTER_START);
3344         param[3] = FW_PARAM_PFVF(FILTER_END);
3345         param[4] = FW_PARAM_PFVF(L2T_START);
3346         param[5] = FW_PARAM_PFVF(L2T_END);
3347         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3348         if (rc != 0) {
3349                 device_printf(sc->dev,
3350                     "failed to query parameters (post_init): %d.\n", rc);
3351                 return (rc);
3352         }
3353
3354         sc->sge.iq_start = val[0];
3355         sc->sge.eq_start = val[1];
3356         sc->tids.ftid_base = val[2];
3357         sc->tids.nftids = val[3] - val[2] + 1;
3358         sc->params.ftid_min = val[2];
3359         sc->params.ftid_max = val[3];
3360         sc->vres.l2t.start = val[4];
3361         sc->vres.l2t.size = val[5] - val[4] + 1;
3362         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
3363             ("%s: L2 table size (%u) larger than expected (%u)",
3364             __func__, sc->vres.l2t.size, L2T_SIZE));
3365
3366         /* get capabilites */
3367         bzero(&caps, sizeof(caps));
3368         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3369             F_FW_CMD_REQUEST | F_FW_CMD_READ);
3370         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
3371         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3372         if (rc != 0) {
3373                 device_printf(sc->dev,
3374                     "failed to get card capabilities: %d.\n", rc);
3375                 return (rc);
3376         }
3377
3378 #define READ_CAPS(x) do { \
3379         sc->x = htobe16(caps.x); \
3380 } while (0)
3381         READ_CAPS(nbmcaps);
3382         READ_CAPS(linkcaps);
3383         READ_CAPS(switchcaps);
3384         READ_CAPS(niccaps);
3385         READ_CAPS(toecaps);
3386         READ_CAPS(rdmacaps);
3387         READ_CAPS(cryptocaps);
3388         READ_CAPS(iscsicaps);
3389         READ_CAPS(fcoecaps);
3390
3391         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
3392                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
3393                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
3394                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3395                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
3396                 if (rc != 0) {
3397                         device_printf(sc->dev,
3398                             "failed to query NIC parameters: %d.\n", rc);
3399                         return (rc);
3400                 }
3401                 sc->tids.etid_base = val[0];
3402                 sc->params.etid_min = val[0];
3403                 sc->tids.netids = val[1] - val[0] + 1;
3404                 sc->params.netids = sc->tids.netids;
3405                 sc->params.eo_wr_cred = val[2];
3406                 sc->params.ethoffload = 1;
3407         }
3408
3409         if (sc->toecaps) {
3410                 /* query offload-related parameters */
3411                 param[0] = FW_PARAM_DEV(NTID);
3412                 param[1] = FW_PARAM_PFVF(SERVER_START);
3413                 param[2] = FW_PARAM_PFVF(SERVER_END);
3414                 param[3] = FW_PARAM_PFVF(TDDP_START);
3415                 param[4] = FW_PARAM_PFVF(TDDP_END);
3416                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3417                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3418                 if (rc != 0) {
3419                         device_printf(sc->dev,
3420                             "failed to query TOE parameters: %d.\n", rc);
3421                         return (rc);
3422                 }
3423                 sc->tids.ntids = val[0];
3424                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
3425                 sc->tids.stid_base = val[1];
3426                 sc->tids.nstids = val[2] - val[1] + 1;
3427                 sc->vres.ddp.start = val[3];
3428                 sc->vres.ddp.size = val[4] - val[3] + 1;
3429                 sc->params.ofldq_wr_cred = val[5];
3430                 sc->params.offload = 1;
3431         }
3432         if (sc->rdmacaps) {
3433                 param[0] = FW_PARAM_PFVF(STAG_START);
3434                 param[1] = FW_PARAM_PFVF(STAG_END);
3435                 param[2] = FW_PARAM_PFVF(RQ_START);
3436                 param[3] = FW_PARAM_PFVF(RQ_END);
3437                 param[4] = FW_PARAM_PFVF(PBL_START);
3438                 param[5] = FW_PARAM_PFVF(PBL_END);
3439                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3440                 if (rc != 0) {
3441                         device_printf(sc->dev,
3442                             "failed to query RDMA parameters(1): %d.\n", rc);
3443                         return (rc);
3444                 }
3445                 sc->vres.stag.start = val[0];
3446                 sc->vres.stag.size = val[1] - val[0] + 1;
3447                 sc->vres.rq.start = val[2];
3448                 sc->vres.rq.size = val[3] - val[2] + 1;
3449                 sc->vres.pbl.start = val[4];
3450                 sc->vres.pbl.size = val[5] - val[4] + 1;
3451
3452                 param[0] = FW_PARAM_PFVF(SQRQ_START);
3453                 param[1] = FW_PARAM_PFVF(SQRQ_END);
3454                 param[2] = FW_PARAM_PFVF(CQ_START);
3455                 param[3] = FW_PARAM_PFVF(CQ_END);
3456                 param[4] = FW_PARAM_PFVF(OCQ_START);
3457                 param[5] = FW_PARAM_PFVF(OCQ_END);
3458                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3459                 if (rc != 0) {
3460                         device_printf(sc->dev,
3461                             "failed to query RDMA parameters(2): %d.\n", rc);
3462                         return (rc);
3463                 }
3464                 sc->vres.qp.start = val[0];
3465                 sc->vres.qp.size = val[1] - val[0] + 1;
3466                 sc->vres.cq.start = val[2];
3467                 sc->vres.cq.size = val[3] - val[2] + 1;
3468                 sc->vres.ocq.start = val[4];
3469                 sc->vres.ocq.size = val[5] - val[4] + 1;
3470         }
3471         if (sc->iscsicaps) {
3472                 param[0] = FW_PARAM_PFVF(ISCSI_START);
3473                 param[1] = FW_PARAM_PFVF(ISCSI_END);
3474                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3475                 if (rc != 0) {
3476                         device_printf(sc->dev,
3477                             "failed to query iSCSI parameters: %d.\n", rc);
3478                         return (rc);
3479                 }
3480                 sc->vres.iscsi.start = val[0];
3481                 sc->vres.iscsi.size = val[1] - val[0] + 1;
3482         }
3483
3484         t4_init_sge_params(sc);
3485
3486         /*
3487          * We've got the params we wanted to query via the firmware.  Now grab
3488          * some others directly from the chip.
3489          */
3490         rc = t4_read_chip_settings(sc);
3491
3492         return (rc);
3493 }
3494
3495 static int
3496 set_params__post_init(struct adapter *sc)
3497 {
3498         uint32_t param, val;
3499
3500         /* ask for encapsulated CPLs */
3501         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
3502         val = 1;
3503         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3504
3505         return (0);
3506 }
3507
3508 #undef FW_PARAM_PFVF
3509 #undef FW_PARAM_DEV
3510
3511 static void
3512 t4_set_desc(struct adapter *sc)
3513 {
3514         char buf[128];
3515         struct adapter_params *p = &sc->params;
3516
3517         snprintf(buf, sizeof(buf), "Chelsio %s", p->vpd.id);
3518
3519         device_set_desc_copy(sc->dev, buf);
3520 }
3521
3522 static void
3523 build_medialist(struct port_info *pi, struct ifmedia *media)
3524 {
3525         int m;
3526
3527         PORT_LOCK(pi);
3528
3529         ifmedia_removeall(media);
3530
3531         m = IFM_ETHER | IFM_FDX;
3532
3533         switch(pi->port_type) {
3534         case FW_PORT_TYPE_BT_XFI:
3535         case FW_PORT_TYPE_BT_XAUI:
3536                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
3537                 /* fall through */
3538
3539         case FW_PORT_TYPE_BT_SGMII:
3540                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
3541                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
3542                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
3543                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
3544                 break;
3545
3546         case FW_PORT_TYPE_CX4:
3547                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
3548                 ifmedia_set(media, m | IFM_10G_CX4);
3549                 break;
3550
3551         case FW_PORT_TYPE_QSFP_10G:
3552         case FW_PORT_TYPE_SFP:
3553         case FW_PORT_TYPE_FIBER_XFI:
3554         case FW_PORT_TYPE_FIBER_XAUI:
3555                 switch (pi->mod_type) {
3556
3557                 case FW_PORT_MOD_TYPE_LR:
3558                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
3559                         ifmedia_set(media, m | IFM_10G_LR);
3560                         break;
3561
3562                 case FW_PORT_MOD_TYPE_SR:
3563                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
3564                         ifmedia_set(media, m | IFM_10G_SR);
3565                         break;
3566
3567                 case FW_PORT_MOD_TYPE_LRM:
3568                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
3569                         ifmedia_set(media, m | IFM_10G_LRM);
3570                         break;
3571
3572                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3573                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3574                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
3575                         ifmedia_set(media, m | IFM_10G_TWINAX);
3576                         break;
3577
3578                 case FW_PORT_MOD_TYPE_NONE:
3579                         m &= ~IFM_FDX;
3580                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3581                         ifmedia_set(media, m | IFM_NONE);
3582                         break;
3583
3584                 case FW_PORT_MOD_TYPE_NA:
3585                 case FW_PORT_MOD_TYPE_ER:
3586                 default:
3587                         device_printf(pi->dev,
3588                             "unknown port_type (%d), mod_type (%d)\n",
3589                             pi->port_type, pi->mod_type);
3590                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3591                         ifmedia_set(media, m | IFM_UNKNOWN);
3592                         break;
3593                 }
3594                 break;
3595
3596         case FW_PORT_TYPE_CR_QSFP:
3597         case FW_PORT_TYPE_SFP28:
3598         case FW_PORT_TYPE_KR_SFP28:
3599                 switch (pi->mod_type) {
3600
3601                 case FW_PORT_MOD_TYPE_SR:
3602                         ifmedia_add(media, m | IFM_25G_SR, 0, NULL);
3603                         ifmedia_set(media, m | IFM_25G_SR);
3604                         break;
3605
3606                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3607                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3608                         ifmedia_add(media, m | IFM_25G_CR, 0, NULL);
3609                         ifmedia_set(media, m | IFM_25G_CR);
3610                         break;
3611
3612                 case FW_PORT_MOD_TYPE_NONE:
3613                         m &= ~IFM_FDX;
3614                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3615                         ifmedia_set(media, m | IFM_NONE);
3616                         break;
3617
3618                 default:
3619                         device_printf(pi->dev,
3620                             "unknown port_type (%d), mod_type (%d)\n",
3621                             pi->port_type, pi->mod_type);
3622                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3623                         ifmedia_set(media, m | IFM_UNKNOWN);
3624                         break;
3625                 }
3626                 break;
3627
3628         case FW_PORT_TYPE_QSFP:
3629                 switch (pi->mod_type) {
3630
3631                 case FW_PORT_MOD_TYPE_LR:
3632                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
3633                         ifmedia_set(media, m | IFM_40G_LR4);
3634                         break;
3635
3636                 case FW_PORT_MOD_TYPE_SR:
3637                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
3638                         ifmedia_set(media, m | IFM_40G_SR4);
3639                         break;
3640
3641                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3642                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3643                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
3644                         ifmedia_set(media, m | IFM_40G_CR4);
3645                         break;
3646
3647                 case FW_PORT_MOD_TYPE_NONE:
3648                         m &= ~IFM_FDX;
3649                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3650                         ifmedia_set(media, m | IFM_NONE);
3651                         break;
3652
3653                 default:
3654                         device_printf(pi->dev,
3655                             "unknown port_type (%d), mod_type (%d)\n",
3656                             pi->port_type, pi->mod_type);
3657                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3658                         ifmedia_set(media, m | IFM_UNKNOWN);
3659                         break;
3660                 }
3661                 break;
3662
3663         case FW_PORT_TYPE_KR4_100G:
3664         case FW_PORT_TYPE_CR4_QSFP:
3665                 switch (pi->mod_type) {
3666
3667                 case FW_PORT_MOD_TYPE_LR:
3668                         ifmedia_add(media, m | IFM_100G_LR4, 0, NULL);
3669                         ifmedia_set(media, m | IFM_100G_LR4);
3670                         break;
3671
3672                 case FW_PORT_MOD_TYPE_SR:
3673                         ifmedia_add(media, m | IFM_100G_SR4, 0, NULL);
3674                         ifmedia_set(media, m | IFM_100G_SR4);
3675                         break;
3676
3677                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3678                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3679                         ifmedia_add(media, m | IFM_100G_CR4, 0, NULL);
3680                         ifmedia_set(media, m | IFM_100G_CR4);
3681                         break;
3682
3683                 case FW_PORT_MOD_TYPE_NONE:
3684                         m &= ~IFM_FDX;
3685                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3686                         ifmedia_set(media, m | IFM_NONE);
3687                         break;
3688
3689                 default:
3690                         device_printf(pi->dev,
3691                             "unknown port_type (%d), mod_type (%d)\n",
3692                             pi->port_type, pi->mod_type);
3693                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3694                         ifmedia_set(media, m | IFM_UNKNOWN);
3695                         break;
3696                 }
3697                 break;
3698
3699         default:
3700                 device_printf(pi->dev,
3701                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3702                     pi->mod_type);
3703                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3704                 ifmedia_set(media, m | IFM_UNKNOWN);
3705                 break;
3706         }
3707
3708         PORT_UNLOCK(pi);
3709 }
3710
3711 #define FW_MAC_EXACT_CHUNK      7
3712
3713 /*
3714  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3715  * indicates which parameters should be programmed (the rest are left alone).
3716  */
3717 int
3718 update_mac_settings(struct ifnet *ifp, int flags)
3719 {
3720         int rc = 0;
3721         struct vi_info *vi = ifp->if_softc;
3722         struct port_info *pi = vi->pi;
3723         struct adapter *sc = pi->adapter;
3724         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3725
3726         ASSERT_SYNCHRONIZED_OP(sc);
3727         KASSERT(flags, ("%s: not told what to update.", __func__));
3728
3729         if (flags & XGMAC_MTU)
3730                 mtu = ifp->if_mtu;
3731
3732         if (flags & XGMAC_PROMISC)
3733                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3734
3735         if (flags & XGMAC_ALLMULTI)
3736                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3737
3738         if (flags & XGMAC_VLANEX)
3739                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3740
3741         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3742                 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
3743                     allmulti, 1, vlanex, false);
3744                 if (rc) {
3745                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3746                             rc);
3747                         return (rc);
3748                 }
3749         }
3750
3751         if (flags & XGMAC_UCADDR) {
3752                 uint8_t ucaddr[ETHER_ADDR_LEN];
3753
3754                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3755                 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
3756                     ucaddr, true, true);
3757                 if (rc < 0) {
3758                         rc = -rc;
3759                         if_printf(ifp, "change_mac failed: %d\n", rc);
3760                         return (rc);
3761                 } else {
3762                         vi->xact_addr_filt = rc;
3763                         rc = 0;
3764                 }
3765         }
3766
3767         if (flags & XGMAC_MCADDRS) {
3768                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3769                 int del = 1;
3770                 uint64_t hash = 0;
3771                 struct ifmultiaddr *ifma;
3772                 int i = 0, j;
3773
3774                 if_maddr_rlock(ifp);
3775                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3776                         if (ifma->ifma_addr->sa_family != AF_LINK)
3777                                 continue;
3778                         mcaddr[i] =
3779                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3780                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3781                         i++;
3782
3783                         if (i == FW_MAC_EXACT_CHUNK) {
3784                                 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
3785                                     del, i, mcaddr, NULL, &hash, 0);
3786                                 if (rc < 0) {
3787                                         rc = -rc;
3788                                         for (j = 0; j < i; j++) {
3789                                                 if_printf(ifp,
3790                                                     "failed to add mc address"
3791                                                     " %02x:%02x:%02x:"
3792                                                     "%02x:%02x:%02x rc=%d\n",
3793                                                     mcaddr[j][0], mcaddr[j][1],
3794                                                     mcaddr[j][2], mcaddr[j][3],
3795                                                     mcaddr[j][4], mcaddr[j][5],
3796                                                     rc);
3797                                         }
3798                                         goto mcfail;
3799                                 }
3800                                 del = 0;
3801                                 i = 0;
3802                         }
3803                 }
3804                 if (i > 0) {
3805                         rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i,
3806                             mcaddr, NULL, &hash, 0);
3807                         if (rc < 0) {
3808                                 rc = -rc;
3809                                 for (j = 0; j < i; j++) {
3810                                         if_printf(ifp,
3811                                             "failed to add mc address"
3812                                             " %02x:%02x:%02x:"
3813                                             "%02x:%02x:%02x rc=%d\n",
3814                                             mcaddr[j][0], mcaddr[j][1],
3815                                             mcaddr[j][2], mcaddr[j][3],
3816                                             mcaddr[j][4], mcaddr[j][5],
3817                                             rc);
3818                                 }
3819                                 goto mcfail;
3820                         }
3821                 }
3822
3823                 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0);
3824                 if (rc != 0)
3825                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3826 mcfail:
3827                 if_maddr_runlock(ifp);
3828         }
3829
3830         return (rc);
3831 }
3832
3833 /*
3834  * {begin|end}_synchronized_op must be called from the same thread.
3835  */
3836 int
3837 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
3838     char *wmesg)
3839 {
3840         int rc, pri;
3841
3842 #ifdef WITNESS
3843         /* the caller thinks it's ok to sleep, but is it really? */
3844         if (flags & SLEEP_OK)
3845                 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
3846                     "begin_synchronized_op");
3847 #endif
3848
3849         if (INTR_OK)
3850                 pri = PCATCH;
3851         else
3852                 pri = 0;
3853
3854         ADAPTER_LOCK(sc);
3855         for (;;) {
3856
3857                 if (vi && IS_DOOMED(vi)) {
3858                         rc = ENXIO;
3859                         goto done;
3860                 }
3861
3862                 if (!IS_BUSY(sc)) {
3863                         rc = 0;
3864                         break;
3865                 }
3866
3867                 if (!(flags & SLEEP_OK)) {
3868                         rc = EBUSY;
3869                         goto done;
3870                 }
3871
3872                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3873                         rc = EINTR;
3874                         goto done;
3875                 }
3876         }
3877
3878         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3879         SET_BUSY(sc);
3880 #ifdef INVARIANTS
3881         sc->last_op = wmesg;
3882         sc->last_op_thr = curthread;
3883         sc->last_op_flags = flags;
3884 #endif
3885
3886 done:
3887         if (!(flags & HOLD_LOCK) || rc)
3888                 ADAPTER_UNLOCK(sc);
3889
3890         return (rc);
3891 }
3892
3893 /*
3894  * Tell if_ioctl and if_init that the VI is going away.  This is
3895  * special variant of begin_synchronized_op and must be paired with a
3896  * call to end_synchronized_op.
3897  */
3898 void
3899 doom_vi(struct adapter *sc, struct vi_info *vi)
3900 {
3901
3902         ADAPTER_LOCK(sc);
3903         SET_DOOMED(vi);
3904         wakeup(&sc->flags);
3905         while (IS_BUSY(sc))
3906                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
3907         SET_BUSY(sc);
3908 #ifdef INVARIANTS
3909         sc->last_op = "t4detach";
3910         sc->last_op_thr = curthread;
3911         sc->last_op_flags = 0;
3912 #endif
3913         ADAPTER_UNLOCK(sc);
3914 }
3915
3916 /*
3917  * {begin|end}_synchronized_op must be called from the same thread.
3918  */
3919 void
3920 end_synchronized_op(struct adapter *sc, int flags)
3921 {
3922
3923         if (flags & LOCK_HELD)
3924                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3925         else
3926                 ADAPTER_LOCK(sc);
3927
3928         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3929         CLR_BUSY(sc);
3930         wakeup(&sc->flags);
3931         ADAPTER_UNLOCK(sc);
3932 }
3933
3934 static int
3935 cxgbe_init_synchronized(struct vi_info *vi)
3936 {
3937         struct port_info *pi = vi->pi;
3938         struct adapter *sc = pi->adapter;
3939         struct ifnet *ifp = vi->ifp;
3940         int rc = 0, i;
3941         struct sge_txq *txq;
3942
3943         ASSERT_SYNCHRONIZED_OP(sc);
3944
3945         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3946                 return (0);     /* already running */
3947
3948         if (!(sc->flags & FULL_INIT_DONE) &&
3949             ((rc = adapter_full_init(sc)) != 0))
3950                 return (rc);    /* error message displayed already */
3951
3952         if (!(vi->flags & VI_INIT_DONE) &&
3953             ((rc = vi_full_init(vi)) != 0))
3954                 return (rc); /* error message displayed already */
3955
3956         rc = update_mac_settings(ifp, XGMAC_ALL);
3957         if (rc)
3958                 goto done;      /* error message displayed already */
3959
3960         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
3961         if (rc != 0) {
3962                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3963                 goto done;
3964         }
3965
3966         /*
3967          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3968          * if this changes.
3969          */
3970
3971         for_each_txq(vi, i, txq) {
3972                 TXQ_LOCK(txq);
3973                 txq->eq.flags |= EQ_ENABLED;
3974                 TXQ_UNLOCK(txq);
3975         }
3976
3977         /*
3978          * The first iq of the first port to come up is used for tracing.
3979          */
3980         if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
3981                 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
3982                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3983                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3984                     V_QUEUENUMBER(sc->traceq));
3985                 pi->flags |= HAS_TRACEQ;
3986         }
3987
3988         /* all ok */
3989         PORT_LOCK(pi);
3990         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3991         pi->up_vis++;
3992
3993         if (pi->nvi > 1 || sc->flags & IS_VF)
3994                 callout_reset(&vi->tick, hz, vi_tick, vi);
3995         else
3996                 callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3997         PORT_UNLOCK(pi);
3998 done:
3999         if (rc != 0)
4000                 cxgbe_uninit_synchronized(vi);
4001
4002         return (rc);
4003 }
4004
4005 /*
4006  * Idempotent.
4007  */
4008 static int
4009 cxgbe_uninit_synchronized(struct vi_info *vi)
4010 {
4011         struct port_info *pi = vi->pi;
4012         struct adapter *sc = pi->adapter;
4013         struct ifnet *ifp = vi->ifp;
4014         int rc, i;
4015         struct sge_txq *txq;
4016
4017         ASSERT_SYNCHRONIZED_OP(sc);
4018
4019         if (!(vi->flags & VI_INIT_DONE)) {
4020                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
4021                     ("uninited VI is running"));
4022                 return (0);
4023         }
4024
4025         /*
4026          * Disable the VI so that all its data in either direction is discarded
4027          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
4028          * tick) intact as the TP can deliver negative advice or data that it's
4029          * holding in its RAM (for an offloaded connection) even after the VI is
4030          * disabled.
4031          */
4032         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
4033         if (rc) {
4034                 if_printf(ifp, "disable_vi failed: %d\n", rc);
4035                 return (rc);
4036         }
4037
4038         for_each_txq(vi, i, txq) {
4039                 TXQ_LOCK(txq);
4040                 txq->eq.flags &= ~EQ_ENABLED;
4041                 TXQ_UNLOCK(txq);
4042         }
4043
4044         PORT_LOCK(pi);
4045         if (pi->nvi > 1 || sc->flags & IS_VF)
4046                 callout_stop(&vi->tick);
4047         else
4048                 callout_stop(&pi->tick);
4049         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4050                 PORT_UNLOCK(pi);
4051                 return (0);
4052         }
4053         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
4054         pi->up_vis--;
4055         if (pi->up_vis > 0) {
4056                 PORT_UNLOCK(pi);
4057                 return (0);
4058         }
4059         PORT_UNLOCK(pi);
4060
4061         pi->link_cfg.link_ok = 0;
4062         pi->link_cfg.speed = 0;
4063         pi->linkdnrc = -1;
4064         t4_os_link_changed(sc, pi->port_id, 0, -1);
4065
4066         return (0);
4067 }
4068
4069 /*
4070  * It is ok for this function to fail midway and return right away.  t4_detach
4071  * will walk the entire sc->irq list and clean up whatever is valid.
4072  */
4073 int
4074 t4_setup_intr_handlers(struct adapter *sc)
4075 {
4076         int rc, rid, p, q, v;
4077         char s[8];
4078         struct irq *irq;
4079         struct port_info *pi;
4080         struct vi_info *vi;
4081         struct sge *sge = &sc->sge;
4082         struct sge_rxq *rxq;
4083 #ifdef TCP_OFFLOAD
4084         struct sge_ofld_rxq *ofld_rxq;
4085 #endif
4086 #ifdef DEV_NETMAP
4087         struct sge_nm_rxq *nm_rxq;
4088 #endif
4089 #ifdef RSS
4090         int nbuckets = rss_getnumbuckets();
4091 #endif
4092
4093         /*
4094          * Setup interrupts.
4095          */
4096         irq = &sc->irq[0];
4097         rid = sc->intr_type == INTR_INTX ? 0 : 1;
4098         if (sc->intr_count == 1)
4099                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
4100
4101         /* Multiple interrupts. */
4102         if (sc->flags & IS_VF)
4103                 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
4104                     ("%s: too few intr.", __func__));
4105         else
4106                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
4107                     ("%s: too few intr.", __func__));
4108
4109         /* The first one is always error intr on PFs */
4110         if (!(sc->flags & IS_VF)) {
4111                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
4112                 if (rc != 0)
4113                         return (rc);
4114                 irq++;
4115                 rid++;
4116         }
4117
4118         /* The second one is always the firmware event queue (first on VFs) */
4119         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
4120         if (rc != 0)
4121                 return (rc);
4122         irq++;
4123         rid++;
4124
4125         for_each_port(sc, p) {
4126                 pi = sc->port[p];
4127                 for_each_vi(pi, v, vi) {
4128                         vi->first_intr = rid - 1;
4129
4130                         if (vi->nnmrxq > 0) {
4131                                 int n = max(vi->nrxq, vi->nnmrxq);
4132
4133                                 MPASS(vi->flags & INTR_RXQ);
4134
4135                                 rxq = &sge->rxq[vi->first_rxq];
4136 #ifdef DEV_NETMAP
4137                                 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
4138 #endif
4139                                 for (q = 0; q < n; q++) {
4140                                         snprintf(s, sizeof(s), "%x%c%x", p,
4141                                             'a' + v, q);
4142                                         if (q < vi->nrxq)
4143                                                 irq->rxq = rxq++;
4144 #ifdef DEV_NETMAP
4145                                         if (q < vi->nnmrxq)
4146                                                 irq->nm_rxq = nm_rxq++;
4147 #endif
4148                                         rc = t4_alloc_irq(sc, irq, rid,
4149                                             t4_vi_intr, irq, s);
4150                                         if (rc != 0)
4151                                                 return (rc);
4152                                         irq++;
4153                                         rid++;
4154                                         vi->nintr++;
4155                                 }
4156                         } else if (vi->flags & INTR_RXQ) {
4157                                 for_each_rxq(vi, q, rxq) {
4158                                         snprintf(s, sizeof(s), "%x%c%x", p,
4159                                             'a' + v, q);
4160                                         rc = t4_alloc_irq(sc, irq, rid,
4161                                             t4_intr, rxq, s);
4162                                         if (rc != 0)
4163                                                 return (rc);
4164 #ifdef RSS
4165                                         bus_bind_intr(sc->dev, irq->res,
4166                                             rss_getcpu(q % nbuckets));
4167 #endif
4168                                         irq++;
4169                                         rid++;
4170                                         vi->nintr++;
4171                                 }
4172                         }
4173 #ifdef TCP_OFFLOAD
4174                         if (vi->flags & INTR_OFLD_RXQ) {
4175                                 for_each_ofld_rxq(vi, q, ofld_rxq) {
4176                                         snprintf(s, sizeof(s), "%x%c%x", p,
4177                                             'A' + v, q);
4178                                         rc = t4_alloc_irq(sc, irq, rid,
4179                                             t4_intr, ofld_rxq, s);
4180                                         if (rc != 0)
4181                                                 return (rc);
4182                                         irq++;
4183                                         rid++;
4184                                         vi->nintr++;
4185                                 }
4186                         }
4187 #endif
4188                 }
4189         }
4190         MPASS(irq == &sc->irq[sc->intr_count]);
4191
4192         return (0);
4193 }
4194
4195 int
4196 adapter_full_init(struct adapter *sc)
4197 {
4198         int rc, i;
4199
4200         ASSERT_SYNCHRONIZED_OP(sc);
4201         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
4202         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
4203             ("%s: FULL_INIT_DONE already", __func__));
4204
4205         /*
4206          * queues that belong to the adapter (not any particular port).
4207          */
4208         rc = t4_setup_adapter_queues(sc);
4209         if (rc != 0)
4210                 goto done;
4211
4212         for (i = 0; i < nitems(sc->tq); i++) {
4213                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
4214                     taskqueue_thread_enqueue, &sc->tq[i]);
4215                 if (sc->tq[i] == NULL) {
4216                         device_printf(sc->dev,
4217                             "failed to allocate task queue %d\n", i);
4218                         rc = ENOMEM;
4219                         goto done;
4220                 }
4221                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
4222                     device_get_nameunit(sc->dev), i);
4223         }
4224
4225         if (!(sc->flags & IS_VF))
4226                 t4_intr_enable(sc);
4227         sc->flags |= FULL_INIT_DONE;
4228 done:
4229         if (rc != 0)
4230                 adapter_full_uninit(sc);
4231
4232         return (rc);
4233 }
4234
4235 int
4236 adapter_full_uninit(struct adapter *sc)
4237 {
4238         int i;
4239
4240         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
4241
4242         t4_teardown_adapter_queues(sc);
4243
4244         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
4245                 taskqueue_free(sc->tq[i]);
4246                 sc->tq[i] = NULL;
4247         }
4248
4249         sc->flags &= ~FULL_INIT_DONE;
4250
4251         return (0);
4252 }
4253
4254 #ifdef RSS
4255 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
4256     RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
4257     RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
4258     RSS_HASHTYPE_RSS_UDP_IPV6)
4259
4260 /* Translates kernel hash types to hardware. */
4261 static int
4262 hashconfig_to_hashen(int hashconfig)
4263 {
4264         int hashen = 0;
4265
4266         if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
4267                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
4268         if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
4269                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
4270         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
4271                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
4272                     F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
4273         }
4274         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
4275                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
4276                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
4277         }
4278         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
4279                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
4280         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
4281                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
4282
4283         return (hashen);
4284 }
4285
4286 /* Translates hardware hash types to kernel. */
4287 static int
4288 hashen_to_hashconfig(int hashen)
4289 {
4290         int hashconfig = 0;
4291
4292         if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
4293                 /*
4294                  * If UDP hashing was enabled it must have been enabled for
4295                  * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
4296                  * enabling any 4-tuple hash is nonsense configuration.
4297                  */
4298                 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
4299                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
4300
4301                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
4302                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
4303                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
4304                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
4305         }
4306         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
4307                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
4308         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
4309                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
4310         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
4311                 hashconfig |= RSS_HASHTYPE_RSS_IPV4;
4312         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
4313                 hashconfig |= RSS_HASHTYPE_RSS_IPV6;
4314
4315         return (hashconfig);
4316 }
4317 #endif
4318
4319 int
4320 vi_full_init(struct vi_info *vi)
4321 {
4322         struct adapter *sc = vi->pi->adapter;
4323         struct ifnet *ifp = vi->ifp;
4324         uint16_t *rss;
4325         struct sge_rxq *rxq;
4326         int rc, i, j, hashen;
4327 #ifdef RSS
4328         int nbuckets = rss_getnumbuckets();
4329         int hashconfig = rss_gethashconfig();
4330         int extra;
4331         uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
4332         uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
4333 #endif
4334
4335         ASSERT_SYNCHRONIZED_OP(sc);
4336         KASSERT((vi->flags & VI_INIT_DONE) == 0,
4337             ("%s: VI_INIT_DONE already", __func__));
4338
4339         sysctl_ctx_init(&vi->ctx);
4340         vi->flags |= VI_SYSCTL_CTX;
4341
4342         /*
4343          * Allocate tx/rx/fl queues for this VI.
4344          */
4345         rc = t4_setup_vi_queues(vi);
4346         if (rc != 0)
4347                 goto done;      /* error message displayed already */
4348
4349         /*
4350          * Setup RSS for this VI.  Save a copy of the RSS table for later use.
4351          */
4352         if (vi->nrxq > vi->rss_size) {
4353                 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
4354                     "some queues will never receive traffic.\n", vi->nrxq,
4355                     vi->rss_size);
4356         } else if (vi->rss_size % vi->nrxq) {
4357                 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
4358                     "expect uneven traffic distribution.\n", vi->nrxq,
4359                     vi->rss_size);
4360         }
4361 #ifdef RSS
4362         MPASS(RSS_KEYSIZE == 40);
4363         if (vi->nrxq != nbuckets) {
4364                 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
4365                     "performance will be impacted.\n", vi->nrxq, nbuckets);
4366         }
4367
4368         rss_getkey((void *)&raw_rss_key[0]);
4369         for (i = 0; i < nitems(rss_key); i++) {
4370                 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
4371         }
4372         t4_write_rss_key(sc, &rss_key[0], -1);
4373 #endif
4374         rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
4375         for (i = 0; i < vi->rss_size;) {
4376 #ifdef RSS
4377                 j = rss_get_indirection_to_bucket(i);
4378                 j %= vi->nrxq;
4379                 rxq = &sc->sge.rxq[vi->first_rxq + j];
4380                 rss[i++] = rxq->iq.abs_id;
4381 #else
4382                 for_each_rxq(vi, j, rxq) {
4383                         rss[i++] = rxq->iq.abs_id;
4384                         if (i == vi->rss_size)
4385                                 break;
4386                 }
4387 #endif
4388         }
4389
4390         rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
4391             vi->rss_size);
4392         if (rc != 0) {
4393                 if_printf(ifp, "rss_config failed: %d\n", rc);
4394                 goto done;
4395         }
4396
4397 #ifdef RSS
4398         hashen = hashconfig_to_hashen(hashconfig);
4399
4400         /*
4401          * We may have had to enable some hashes even though the global config
4402          * wants them disabled.  This is a potential problem that must be
4403          * reported to the user.
4404          */
4405         extra = hashen_to_hashconfig(hashen) ^ hashconfig;
4406
4407         /*
4408          * If we consider only the supported hash types, then the enabled hashes
4409          * are a superset of the requested hashes.  In other words, there cannot
4410          * be any supported hash that was requested but not enabled, but there
4411          * can be hashes that were not requested but had to be enabled.
4412          */
4413         extra &= SUPPORTED_RSS_HASHTYPES;
4414         MPASS((extra & hashconfig) == 0);
4415
4416         if (extra) {
4417                 if_printf(ifp,
4418                     "global RSS config (0x%x) cannot be accommodated.\n",
4419                     hashconfig);
4420         }
4421         if (extra & RSS_HASHTYPE_RSS_IPV4)
4422                 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
4423         if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
4424                 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
4425         if (extra & RSS_HASHTYPE_RSS_IPV6)
4426                 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
4427         if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
4428                 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
4429         if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
4430                 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
4431         if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
4432                 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
4433 #else
4434         hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
4435             F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
4436             F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
4437             F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
4438 #endif
4439         rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0], 0, 0);
4440         if (rc != 0) {
4441                 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
4442                 goto done;
4443         }
4444
4445         vi->rss = rss;
4446         vi->flags |= VI_INIT_DONE;
4447 done:
4448         if (rc != 0)
4449                 vi_full_uninit(vi);
4450
4451         return (rc);
4452 }
4453
4454 /*
4455  * Idempotent.
4456  */
4457 int
4458 vi_full_uninit(struct vi_info *vi)
4459 {
4460         struct port_info *pi = vi->pi;
4461         struct adapter *sc = pi->adapter;
4462         int i;
4463         struct sge_rxq *rxq;
4464         struct sge_txq *txq;
4465 #ifdef TCP_OFFLOAD
4466         struct sge_ofld_rxq *ofld_rxq;
4467         struct sge_wrq *ofld_txq;
4468 #endif
4469
4470         if (vi->flags & VI_INIT_DONE) {
4471
4472                 /* Need to quiesce queues.  */
4473
4474                 /* XXX: Only for the first VI? */
4475                 if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF))
4476                         quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
4477
4478                 for_each_txq(vi, i, txq) {
4479                         quiesce_txq(sc, txq);
4480                 }
4481
4482 #ifdef TCP_OFFLOAD
4483                 for_each_ofld_txq(vi, i, ofld_txq) {
4484                         quiesce_wrq(sc, ofld_txq);
4485                 }
4486 #endif
4487
4488                 for_each_rxq(vi, i, rxq) {
4489                         quiesce_iq(sc, &rxq->iq);
4490                         quiesce_fl(sc, &rxq->fl);
4491                 }
4492
4493 #ifdef TCP_OFFLOAD
4494                 for_each_ofld_rxq(vi, i, ofld_rxq) {
4495                         quiesce_iq(sc, &ofld_rxq->iq);
4496                         quiesce_fl(sc, &ofld_rxq->fl);
4497                 }
4498 #endif
4499                 free(vi->rss, M_CXGBE);
4500                 free(vi->nm_rss, M_CXGBE);
4501         }
4502
4503         t4_teardown_vi_queues(vi);
4504         vi->flags &= ~VI_INIT_DONE;
4505
4506         return (0);
4507 }
4508
4509 static void
4510 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
4511 {
4512         struct sge_eq *eq = &txq->eq;
4513         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
4514
4515         (void) sc;      /* unused */
4516
4517 #ifdef INVARIANTS
4518         TXQ_LOCK(txq);
4519         MPASS((eq->flags & EQ_ENABLED) == 0);
4520         TXQ_UNLOCK(txq);
4521 #endif
4522
4523         /* Wait for the mp_ring to empty. */
4524         while (!mp_ring_is_idle(txq->r)) {
4525                 mp_ring_check_drainage(txq->r, 0);
4526                 pause("rquiesce", 1);
4527         }
4528
4529         /* Then wait for the hardware to finish. */
4530         while (spg->cidx != htobe16(eq->pidx))
4531                 pause("equiesce", 1);
4532
4533         /* Finally, wait for the driver to reclaim all descriptors. */
4534         while (eq->cidx != eq->pidx)
4535                 pause("dquiesce", 1);
4536 }
4537
4538 static void
4539 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
4540 {
4541
4542         /* XXXTX */
4543 }
4544
4545 static void
4546 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
4547 {
4548         (void) sc;      /* unused */
4549
4550         /* Synchronize with the interrupt handler */
4551         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
4552                 pause("iqfree", 1);
4553 }
4554
4555 static void
4556 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
4557 {
4558         mtx_lock(&sc->sfl_lock);
4559         FL_LOCK(fl);
4560         fl->flags |= FL_DOOMED;
4561         FL_UNLOCK(fl);
4562         callout_stop(&sc->sfl_callout);
4563         mtx_unlock(&sc->sfl_lock);
4564
4565         KASSERT((fl->flags & FL_STARVING) == 0,
4566             ("%s: still starving", __func__));
4567 }
4568
4569 static int
4570 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
4571     driver_intr_t *handler, void *arg, char *name)
4572 {
4573         int rc;
4574
4575         irq->rid = rid;
4576         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
4577             RF_SHAREABLE | RF_ACTIVE);
4578         if (irq->res == NULL) {
4579                 device_printf(sc->dev,
4580                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
4581                 return (ENOMEM);
4582         }
4583
4584         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
4585             NULL, handler, arg, &irq->tag);
4586         if (rc != 0) {
4587                 device_printf(sc->dev,
4588                     "failed to setup interrupt for rid %d, name %s: %d\n",
4589                     rid, name, rc);
4590         } else if (name)
4591                 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);
4592
4593         return (rc);
4594 }
4595
4596 static int
4597 t4_free_irq(struct adapter *sc, struct irq *irq)
4598 {
4599         if (irq->tag)
4600                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
4601         if (irq->res)
4602                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
4603
4604         bzero(irq, sizeof(*irq));
4605
4606         return (0);
4607 }
4608
4609 static void
4610 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
4611 {
4612
4613         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4614         t4_get_regs(sc, buf, regs->len);
4615 }
4616
4617 #define A_PL_INDIR_CMD  0x1f8
4618
4619 #define S_PL_AUTOINC    31
4620 #define M_PL_AUTOINC    0x1U
4621 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
4622 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
4623
4624 #define S_PL_VFID       20
4625 #define M_PL_VFID       0xffU
4626 #define V_PL_VFID(x)    ((x) << S_PL_VFID)
4627 #define G_PL_VFID(x)    (((x) >> S_PL_VFID) & M_PL_VFID)
4628
4629 #define S_PL_ADDR       0
4630 #define M_PL_ADDR       0xfffffU
4631 #define V_PL_ADDR(x)    ((x) << S_PL_ADDR)
4632 #define G_PL_ADDR(x)    (((x) >> S_PL_ADDR) & M_PL_ADDR)
4633
4634 #define A_PL_INDIR_DATA 0x1fc
4635
4636 static uint64_t
4637 read_vf_stat(struct adapter *sc, unsigned int viid, int reg)
4638 {
4639         u32 stats[2];
4640
4641         mtx_assert(&sc->reg_lock, MA_OWNED);
4642         if (sc->flags & IS_VF) {
4643                 stats[0] = t4_read_reg(sc, VF_MPS_REG(reg));
4644                 stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4));
4645         } else {
4646                 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4647                     V_PL_VFID(G_FW_VIID_VIN(viid)) |
4648                     V_PL_ADDR(VF_MPS_REG(reg)));
4649                 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
4650                 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
4651         }
4652         return (((uint64_t)stats[1]) << 32 | stats[0]);
4653 }
4654
4655 static void
4656 t4_get_vi_stats(struct adapter *sc, unsigned int viid,
4657     struct fw_vi_stats_vf *stats)
4658 {
4659
4660 #define GET_STAT(name) \
4661         read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L)
4662
4663         stats->tx_bcast_bytes    = GET_STAT(TX_VF_BCAST_BYTES);
4664         stats->tx_bcast_frames   = GET_STAT(TX_VF_BCAST_FRAMES);
4665         stats->tx_mcast_bytes    = GET_STAT(TX_VF_MCAST_BYTES);
4666         stats->tx_mcast_frames   = GET_STAT(TX_VF_MCAST_FRAMES);
4667         stats->tx_ucast_bytes    = GET_STAT(TX_VF_UCAST_BYTES);
4668         stats->tx_ucast_frames   = GET_STAT(TX_VF_UCAST_FRAMES);
4669         stats->tx_drop_frames    = GET_STAT(TX_VF_DROP_FRAMES);
4670         stats->tx_offload_bytes  = GET_STAT(TX_VF_OFFLOAD_BYTES);
4671         stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
4672         stats->rx_bcast_bytes    = GET_STAT(RX_VF_BCAST_BYTES);
4673         stats->rx_bcast_frames   = GET_STAT(RX_VF_BCAST_FRAMES);
4674         stats->rx_mcast_bytes    = GET_STAT(RX_VF_MCAST_BYTES);
4675         stats->rx_mcast_frames   = GET_STAT(RX_VF_MCAST_FRAMES);
4676         stats->rx_ucast_bytes    = GET_STAT(RX_VF_UCAST_BYTES);
4677         stats->rx_ucast_frames   = GET_STAT(RX_VF_UCAST_FRAMES);
4678         stats->rx_err_frames     = GET_STAT(RX_VF_ERR_FRAMES);
4679
4680 #undef GET_STAT
4681 }
4682
4683 static void
4684 t4_clr_vi_stats(struct adapter *sc, unsigned int viid)
4685 {
4686         int reg;
4687
4688         t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4689             V_PL_VFID(G_FW_VIID_VIN(viid)) |
4690             V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
4691         for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
4692              reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
4693                 t4_write_reg(sc, A_PL_INDIR_DATA, 0);
4694 }
4695
4696 static void
4697 vi_refresh_stats(struct adapter *sc, struct vi_info *vi)
4698 {
4699         struct timeval tv;
4700         const struct timeval interval = {0, 250000};    /* 250ms */
4701
4702         if (!(vi->flags & VI_INIT_DONE))
4703                 return;
4704
4705         getmicrotime(&tv);
4706         timevalsub(&tv, &interval);
4707         if (timevalcmp(&tv, &vi->last_refreshed, <))
4708                 return;
4709
4710         mtx_lock(&sc->reg_lock);
4711         t4_get_vi_stats(sc, vi->viid, &vi->stats);
4712         getmicrotime(&vi->last_refreshed);
4713         mtx_unlock(&sc->reg_lock);
4714 }
4715
4716 static void
4717 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4718 {
4719         int i;
4720         u_int v, tnl_cong_drops;
4721         struct timeval tv;
4722         const struct timeval interval = {0, 250000};    /* 250ms */
4723
4724         getmicrotime(&tv);
4725         timevalsub(&tv, &interval);
4726         if (timevalcmp(&tv, &pi->last_refreshed, <))
4727                 return;
4728
4729         tnl_cong_drops = 0;
4730         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4731         for (i = 0; i < sc->chip_params->nchan; i++) {
4732                 if (pi->rx_chan_map & (1 << i)) {
4733                         mtx_lock(&sc->reg_lock);
4734                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4735                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4736                         mtx_unlock(&sc->reg_lock);
4737                         tnl_cong_drops += v;
4738                 }
4739         }
4740         pi->tnl_cong_drops = tnl_cong_drops;
4741         getmicrotime(&pi->last_refreshed);
4742 }
4743
4744 static void
4745 cxgbe_tick(void *arg)
4746 {
4747         struct port_info *pi = arg;
4748         struct adapter *sc = pi->adapter;
4749
4750         PORT_LOCK_ASSERT_OWNED(pi);
4751         cxgbe_refresh_stats(sc, pi);
4752
4753         callout_schedule(&pi->tick, hz);
4754 }
4755
4756 void
4757 vi_tick(void *arg)
4758 {
4759         struct vi_info *vi = arg;
4760         struct adapter *sc = vi->pi->adapter;
4761
4762         vi_refresh_stats(sc, vi);
4763
4764         callout_schedule(&vi->tick, hz);
4765 }
4766
4767 static void
4768 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4769 {
4770         struct ifnet *vlan;
4771
4772         if (arg != ifp || ifp->if_type != IFT_ETHER)
4773                 return;
4774
4775         vlan = VLAN_DEVAT(ifp, vid);
4776         VLAN_SETCOOKIE(vlan, ifp);
4777 }
4778
4779 /*
4780  * Should match fw_caps_config_<foo> enums in t4fw_interface.h
4781  */
4782 static char *caps_decoder[] = {
4783         "\20\001IPMI\002NCSI",                          /* 0: NBM */
4784         "\20\001PPP\002QFC\003DCBX",                    /* 1: link */
4785         "\20\001INGRESS\002EGRESS",                     /* 2: switch */
4786         "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL"      /* 3: NIC */
4787             "\006HASHFILTER\007ETHOFLD",
4788         "\20\001TOE",                                   /* 4: TOE */
4789         "\20\001RDDP\002RDMAC",                         /* 5: RDMA */
4790         "\20\001INITIATOR_PDU\002TARGET_PDU"            /* 6: iSCSI */
4791             "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD"
4792             "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD"
4793             "\007T10DIF"
4794             "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD",
4795         "\20\001LOOKASIDE\002TLSKEYS",                  /* 7: Crypto */
4796         "\20\001INITIATOR\002TARGET\003CTRL_OFLD"       /* 8: FCoE */
4797                     "\004PO_INITIATOR\005PO_TARGET",
4798 };
4799
4800 void
4801 t4_sysctls(struct adapter *sc)
4802 {
4803         struct sysctl_ctx_list *ctx;
4804         struct sysctl_oid *oid;
4805         struct sysctl_oid_list *children, *c0;
4806         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4807
4808         ctx = device_get_sysctl_ctx(sc->dev);
4809
4810         /*
4811          * dev.t4nex.X.
4812          */
4813         oid = device_get_sysctl_tree(sc->dev);
4814         c0 = children = SYSCTL_CHILDREN(oid);
4815
4816         sc->sc_do_rxcopy = 1;
4817         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4818             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4819
4820         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4821             sc->params.nports, "# of ports");
4822
4823         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4824             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4825             sysctl_bitfield, "A", "available doorbells");
4826
4827         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4828             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4829
4830         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4831             CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.timer_val,
4832             sizeof(sc->params.sge.timer_val), sysctl_int_array, "A",
4833             "interrupt holdoff timer values (us)");
4834
4835         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4836             CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.counter_val,
4837             sizeof(sc->params.sge.counter_val), sysctl_int_array, "A",
4838             "interrupt holdoff packet counter values");
4839
4840         t4_sge_sysctls(sc, ctx, children);
4841
4842         sc->lro_timeout = 100;
4843         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4844             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4845
4846         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW,
4847             &sc->debug_flags, 0, "flags to enable runtime debugging");
4848
4849         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version",
4850             CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
4851
4852         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4853             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4854
4855         if (sc->flags & IS_VF)
4856                 return;
4857
4858         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4859             NULL, chip_rev(sc), "chip hardware revision");
4860
4861         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn",
4862             CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");
4863
4864         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn",
4865             CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");
4866
4867         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec",
4868             CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");
4869
4870         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na",
4871             CTLFLAG_RD, sc->params.vpd.na, 0, "network address");
4872
4873         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD,
4874             sc->er_version, 0, "expansion ROM version");
4875
4876         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD,
4877             sc->bs_version, 0, "bootstrap firmware version");
4878
4879         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD,
4880             NULL, sc->params.scfg_vers, "serial config version");
4881
4882         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD,
4883             NULL, sc->params.vpd_vers, "VPD version");
4884
4885         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4886             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4887
4888         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4889             sc->cfcsum, "config file checksum");
4890
4891 #define SYSCTL_CAP(name, n, text) \
4892         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \
4893             CTLTYPE_STRING | CTLFLAG_RD, caps_decoder[n], sc->name, \
4894             sysctl_bitfield, "A", "available " text " capabilities")
4895
4896         SYSCTL_CAP(nbmcaps, 0, "NBM");
4897         SYSCTL_CAP(linkcaps, 1, "link");
4898         SYSCTL_CAP(switchcaps, 2, "switch");
4899         SYSCTL_CAP(niccaps, 3, "NIC");
4900         SYSCTL_CAP(toecaps, 4, "TCP offload");
4901         SYSCTL_CAP(rdmacaps, 5, "RDMA");
4902         SYSCTL_CAP(iscsicaps, 6, "iSCSI");
4903         SYSCTL_CAP(cryptocaps, 7, "crypto");
4904         SYSCTL_CAP(fcoecaps, 8, "FCoE");
4905 #undef SYSCTL_CAP
4906
4907         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4908             NULL, sc->tids.nftids, "number of filters");
4909
4910         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4911             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4912             "chip temperature (in Celsius)");
4913
4914 #ifdef SBUF_DRAIN
4915         /*
4916          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4917          */
4918         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4919             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4920             "logs and miscellaneous information");
4921         children = SYSCTL_CHILDREN(oid);
4922
4923         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4924             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4925             sysctl_cctrl, "A", "congestion control");
4926
4927         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4928             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4929             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4930
4931         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4932             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4933             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4934
4935         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4936             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4937             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4938
4939         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4940             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4941             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4942
4943         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4944             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4945             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4946
4947         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4948             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4949             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4950
4951         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4952             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4953             chip_id(sc) <= CHELSIO_T5 ? sysctl_cim_la : sysctl_cim_la_t6,
4954             "A", "CIM logic analyzer");
4955
4956         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4957             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4958             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4959
4960         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4961             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4962             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4963
4964         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4965             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4966             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4967
4968         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4969             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4970             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4971
4972         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4973             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4974             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4975
4976         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4977             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4978             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4979
4980         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4981             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4982             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4983
4984         if (chip_id(sc) > CHELSIO_T4) {
4985                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4986                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4987                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4988
4989                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4990                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4991                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4992         }
4993
4994         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4995             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4996             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4997
4998         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4999             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5000             sysctl_cim_qcfg, "A", "CIM queue configuration");
5001
5002         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
5003             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5004             sysctl_cpl_stats, "A", "CPL statistics");
5005
5006         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
5007             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5008             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
5009
5010         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
5011             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5012             sysctl_devlog, "A", "firmware's device log");
5013
5014         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
5015             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5016             sysctl_fcoe_stats, "A", "FCoE statistics");
5017
5018         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
5019             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5020             sysctl_hw_sched, "A", "hardware scheduler ");
5021
5022         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
5023             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5024             sysctl_l2t, "A", "hardware L2 table");
5025
5026         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
5027             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5028             sysctl_lb_stats, "A", "loopback statistics");
5029
5030         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
5031             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5032             sysctl_meminfo, "A", "memory regions");
5033
5034         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
5035             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5036             chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6,
5037             "A", "MPS TCAM entries");
5038
5039         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
5040             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5041             sysctl_path_mtus, "A", "path MTUs");
5042
5043         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
5044             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5045             sysctl_pm_stats, "A", "PM statistics");
5046
5047         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
5048             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5049             sysctl_rdma_stats, "A", "RDMA statistics");
5050
5051         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
5052             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5053             sysctl_tcp_stats, "A", "TCP statistics");
5054
5055         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
5056             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5057             sysctl_tids, "A", "TID information");
5058
5059         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
5060             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5061             sysctl_tp_err_stats, "A", "TP error statistics");
5062
5063         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask",
5064             CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tp_la_mask, "I",
5065             "TP logic analyzer event capture mask");
5066
5067         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
5068             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5069             sysctl_tp_la, "A", "TP logic analyzer");
5070
5071         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
5072             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5073             sysctl_tx_rate, "A", "Tx rate");
5074
5075         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
5076             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5077             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
5078
5079         if (chip_id(sc) >= CHELSIO_T5) {
5080                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
5081                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5082                     sysctl_wcwr_stats, "A", "write combined work requests");
5083         }
5084 #endif
5085
5086 #ifdef TCP_OFFLOAD
5087         if (is_offload(sc)) {
5088                 /*
5089                  * dev.t4nex.X.toe.
5090                  */
5091                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
5092                     NULL, "TOE parameters");
5093                 children = SYSCTL_CHILDREN(oid);
5094
5095                 sc->tt.sndbuf = 256 * 1024;
5096                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
5097                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
5098
5099                 sc->tt.ddp = 0;
5100                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
5101                     &sc->tt.ddp, 0, "DDP allowed");
5102
5103                 sc->tt.rx_coalesce = 1;
5104                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
5105                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
5106
5107                 sc->tt.tx_align = 1;
5108                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
5109                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
5110
5111                 sc->tt.tx_zcopy = 0;
5112                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy",
5113                     CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
5114                     "Enable zero-copy aio_write(2)");
5115
5116                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick",
5117                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A",
5118                     "TP timer tick (us)");
5119
5120                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick",
5121                     CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_tp_tick, "A",
5122                     "TCP timestamp tick (us)");
5123
5124                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick",
5125                     CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_tp_tick, "A",
5126                     "DACK tick (us)");
5127
5128                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer",
5129                     CTLTYPE_UINT | CTLFLAG_RD, sc, 0, sysctl_tp_dack_timer,
5130                     "IU", "DACK timer (us)");
5131
5132                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min",
5133                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MIN,
5134                     sysctl_tp_timer, "LU", "Retransmit min (us)");
5135
5136                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max",
5137                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MAX,
5138                     sysctl_tp_timer, "LU", "Retransmit max (us)");
5139
5140                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min",
5141                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MIN,
5142                     sysctl_tp_timer, "LU", "Persist timer min (us)");
5143
5144                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max",
5145                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MAX,
5146                     sysctl_tp_timer, "LU", "Persist timer max (us)");
5147
5148                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle",
5149                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_IDLE,
5150                     sysctl_tp_timer, "LU", "Keepidle idle timer (us)");
5151
5152                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_intvl",
5153                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_INTVL,
5154                     sysctl_tp_timer, "LU", "Keepidle interval (us)");
5155
5156                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt",
5157                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_INIT_SRTT,
5158                     sysctl_tp_timer, "LU", "Initial SRTT (us)");
5159
5160                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer",
5161                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_FINWAIT2_TIMER,
5162                     sysctl_tp_timer, "LU", "FINWAIT2 timer (us)");
5163         }
5164 #endif
5165 }
5166
5167 void
5168 vi_sysctls(struct vi_info *vi)
5169 {
5170         struct sysctl_ctx_list *ctx;
5171         struct sysctl_oid *oid;
5172         struct sysctl_oid_list *children;
5173
5174         ctx = device_get_sysctl_ctx(vi->dev);
5175
5176         /*
5177          * dev.v?(cxgbe|cxl).X.
5178          */
5179         oid = device_get_sysctl_tree(vi->dev);
5180         children = SYSCTL_CHILDREN(oid);
5181
5182         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
5183             vi->viid, "VI identifer");
5184         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
5185             &vi->nrxq, 0, "# of rx queues");
5186         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
5187             &vi->ntxq, 0, "# of tx queues");
5188         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
5189             &vi->first_rxq, 0, "index of first rx queue");
5190         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
5191             &vi->first_txq, 0, "index of first tx queue");
5192         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL,
5193             vi->rss_size, "size of RSS indirection table");
5194
5195         if (IS_MAIN_VI(vi)) {
5196                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq",
5197                     CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU",
5198                     "Reserve queue 0 for non-flowid packets");
5199         }
5200
5201 #ifdef TCP_OFFLOAD
5202         if (vi->nofldrxq != 0) {
5203                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
5204                     &vi->nofldrxq, 0,
5205                     "# of rx queues for offloaded TCP connections");
5206                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
5207                     &vi->nofldtxq, 0,
5208                     "# of tx queues for offloaded TCP connections");
5209                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
5210                     CTLFLAG_RD, &vi->first_ofld_rxq, 0,
5211                     "index of first TOE rx queue");
5212                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
5213                     CTLFLAG_RD, &vi->first_ofld_txq, 0,
5214                     "index of first TOE tx queue");
5215         }
5216 #endif
5217 #ifdef DEV_NETMAP
5218         if (vi->nnmrxq != 0) {
5219                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
5220                     &vi->nnmrxq, 0, "# of netmap rx queues");
5221                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
5222                     &vi->nnmtxq, 0, "# of netmap tx queues");
5223                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
5224                     CTLFLAG_RD, &vi->first_nm_rxq, 0,
5225                     "index of first netmap rx queue");
5226                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
5227                     CTLFLAG_RD, &vi->first_nm_txq, 0,
5228                     "index of first netmap tx queue");
5229         }
5230 #endif
5231
5232         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
5233             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I",
5234             "holdoff timer index");
5235         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
5236             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I",
5237             "holdoff packet counter index");
5238
5239         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
5240             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I",
5241             "rx queue size");
5242         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
5243             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I",
5244             "tx queue size");
5245 }
5246
5247 static void
5248 cxgbe_sysctls(struct port_info *pi)
5249 {
5250         struct sysctl_ctx_list *ctx;
5251         struct sysctl_oid *oid;
5252         struct sysctl_oid_list *children, *children2;
5253         struct adapter *sc = pi->adapter;
5254         int i;
5255         char name[16];
5256
5257         ctx = device_get_sysctl_ctx(pi->dev);
5258
5259         /*
5260          * dev.cxgbe.X.
5261          */
5262         oid = device_get_sysctl_tree(pi->dev);
5263         children = SYSCTL_CHILDREN(oid);
5264
5265         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
5266            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
5267         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
5268                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
5269                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
5270                     "PHY temperature (in Celsius)");
5271                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
5272                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
5273                     "PHY firmware version");
5274         }
5275
5276         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
5277             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
5278             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
5279
5280         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL,
5281             port_top_speed(pi), "max speed (in Gbps)");
5282
5283         if (sc->flags & IS_VF)
5284                 return;
5285
5286         /*
5287          * dev.(cxgbe|cxl).X.tc.
5288          */
5289         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL,
5290             "Tx scheduler traffic classes");
5291         for (i = 0; i < sc->chip_params->nsched_cls; i++) {
5292                 struct tx_sched_class *tc = &pi->tc[i];
5293
5294                 snprintf(name, sizeof(name), "%d", i);
5295                 children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx,
5296                     SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL,
5297                     "traffic class"));
5298                 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "flags", CTLFLAG_RD,
5299                     &tc->flags, 0, "flags");
5300                 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount",
5301                     CTLFLAG_RD, &tc->refcount, 0, "references to this class");
5302 #ifdef SBUF_DRAIN
5303                 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params",
5304                     CTLTYPE_STRING | CTLFLAG_RD, sc, (pi->port_id << 16) | i,
5305                     sysctl_tc_params, "A", "traffic class parameters");
5306 #endif
5307         }
5308
5309         /*
5310          * dev.cxgbe.X.stats.
5311          */
5312         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
5313             NULL, "port statistics");
5314         children = SYSCTL_CHILDREN(oid);
5315         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
5316             &pi->tx_parse_error, 0,
5317             "# of tx packets with invalid length or # of segments");
5318
5319 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
5320         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
5321             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
5322             sysctl_handle_t4_reg64, "QU", desc)
5323
5324         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
5325             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
5326         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
5327             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
5328         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
5329             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
5330         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
5331             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
5332         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
5333             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
5334         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
5335             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
5336         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
5337             "# of tx frames in this range",
5338             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
5339         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
5340             "# of tx frames in this range",
5341             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
5342         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
5343             "# of tx frames in this range",
5344             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
5345         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
5346             "# of tx frames in this range",
5347             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
5348         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
5349             "# of tx frames in this range",
5350             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
5351         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
5352             "# of tx frames in this range",
5353             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
5354         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
5355             "# of tx frames in this range",
5356             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
5357         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
5358             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
5359         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
5360             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
5361         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
5362             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
5363         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
5364             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
5365         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
5366             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
5367         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
5368             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
5369         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
5370             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
5371         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
5372             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
5373         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
5374             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
5375         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
5376             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
5377
5378         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
5379             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
5380         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
5381             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
5382         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
5383             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
5384         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
5385             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
5386         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
5387             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
5388         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
5389             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
5390         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
5391             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
5392         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
5393             "# of frames received with bad FCS",
5394             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
5395         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5396             "# of frames received with length error",
5397             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5398         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5399             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5400         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5401             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5402         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5403             "# of rx frames in this range",
5404             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5405         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5406             "# of rx frames in this range",
5407             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5408         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5409             "# of rx frames in this range",
5410             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5411         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5412             "# of rx frames in this range",
5413             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5414         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5415             "# of rx frames in this range",
5416             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5417         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5418             "# of rx frames in this range",
5419             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5420         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5421             "# of rx frames in this range",
5422             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5423         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5424             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5425         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5426             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5427         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5428             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5429         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5430             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5431         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5432             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5433         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5434             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5435         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5436             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5437         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5438             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5439         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5440             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5441
5442 #undef SYSCTL_ADD_T4_REG64
5443
5444 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5445         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5446             &pi->stats.name, desc)
5447
5448         /* We get these from port_stats and they may be stale by up to 1s */
5449         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5450             "# drops due to buffer-group 0 overflows");
5451         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5452             "# drops due to buffer-group 1 overflows");
5453         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5454             "# drops due to buffer-group 2 overflows");
5455         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5456             "# drops due to buffer-group 3 overflows");
5457         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5458             "# of buffer-group 0 truncated packets");
5459         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5460             "# of buffer-group 1 truncated packets");
5461         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5462             "# of buffer-group 2 truncated packets");
5463         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5464             "# of buffer-group 3 truncated packets");
5465
5466 #undef SYSCTL_ADD_T4_PORTSTAT
5467 }
5468
5469 static int
5470 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5471 {
5472         int rc, *i, space = 0;
5473         struct sbuf sb;
5474
5475         sbuf_new_for_sysctl(&sb, NULL, 64, req);
5476         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5477                 if (space)
5478                         sbuf_printf(&sb, " ");
5479                 sbuf_printf(&sb, "%d", *i);
5480                 space = 1;
5481         }
5482         rc = sbuf_finish(&sb);
5483         sbuf_delete(&sb);
5484         return (rc);
5485 }
5486
5487 static int
5488 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5489 {
5490         int rc;
5491         struct sbuf *sb;
5492
5493         rc = sysctl_wire_old_buffer(req, 0);
5494         if (rc != 0)
5495                 return(rc);
5496
5497         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5498         if (sb == NULL)
5499                 return (ENOMEM);
5500
5501         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5502         rc = sbuf_finish(sb);
5503         sbuf_delete(sb);
5504
5505         return (rc);
5506 }
5507
5508 static int
5509 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5510 {
5511         struct port_info *pi = arg1;
5512         int op = arg2;
5513         struct adapter *sc = pi->adapter;
5514         u_int v;
5515         int rc;
5516
5517         rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
5518         if (rc)
5519                 return (rc);
5520         /* XXX: magic numbers */
5521         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5522             &v);
5523         end_synchronized_op(sc, 0);
5524         if (rc)
5525                 return (rc);
5526         if (op == 0)
5527                 v /= 256;
5528
5529         rc = sysctl_handle_int(oidp, &v, 0, req);
5530         return (rc);
5531 }
5532
5533 static int
5534 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5535 {
5536         struct vi_info *vi = arg1;
5537         int rc, val;
5538
5539         val = vi->rsrv_noflowq;
5540         rc = sysctl_handle_int(oidp, &val, 0, req);
5541         if (rc != 0 || req->newptr == NULL)
5542                 return (rc);
5543
5544         if ((val >= 1) && (vi->ntxq > 1))
5545                 vi->rsrv_noflowq = 1;
5546         else
5547                 vi->rsrv_noflowq = 0;
5548
5549         return (rc);
5550 }
5551
5552 static int
5553 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5554 {
5555         struct vi_info *vi = arg1;
5556         struct adapter *sc = vi->pi->adapter;
5557         int idx, rc, i;
5558         struct sge_rxq *rxq;
5559 #ifdef TCP_OFFLOAD
5560         struct sge_ofld_rxq *ofld_rxq;
5561 #endif
5562         uint8_t v;
5563
5564         idx = vi->tmr_idx;
5565
5566         rc = sysctl_handle_int(oidp, &idx, 0, req);
5567         if (rc != 0 || req->newptr == NULL)
5568                 return (rc);
5569
5570         if (idx < 0 || idx >= SGE_NTIMERS)
5571                 return (EINVAL);
5572
5573         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5574             "t4tmr");
5575         if (rc)
5576                 return (rc);
5577
5578         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
5579         for_each_rxq(vi, i, rxq) {
5580 #ifdef atomic_store_rel_8
5581                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5582 #else
5583                 rxq->iq.intr_params = v;
5584 #endif
5585         }
5586 #ifdef TCP_OFFLOAD
5587         for_each_ofld_rxq(vi, i, ofld_rxq) {
5588 #ifdef atomic_store_rel_8
5589                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5590 #else
5591                 ofld_rxq->iq.intr_params = v;
5592 #endif
5593         }
5594 #endif
5595         vi->tmr_idx = idx;
5596
5597         end_synchronized_op(sc, LOCK_HELD);
5598         return (0);
5599 }
5600
5601 static int
5602 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5603 {
5604         struct vi_info *vi = arg1;
5605         struct adapter *sc = vi->pi->adapter;
5606         int idx, rc;
5607
5608         idx = vi->pktc_idx;
5609
5610         rc = sysctl_handle_int(oidp, &idx, 0, req);
5611         if (rc != 0 || req->newptr == NULL)
5612                 return (rc);
5613
5614         if (idx < -1 || idx >= SGE_NCOUNTERS)
5615                 return (EINVAL);
5616
5617         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5618             "t4pktc");
5619         if (rc)
5620                 return (rc);
5621
5622         if (vi->flags & VI_INIT_DONE)
5623                 rc = EBUSY; /* cannot be changed once the queues are created */
5624         else
5625                 vi->pktc_idx = idx;
5626
5627         end_synchronized_op(sc, LOCK_HELD);
5628         return (rc);
5629 }
5630
5631 static int
5632 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5633 {
5634         struct vi_info *vi = arg1;
5635         struct adapter *sc = vi->pi->adapter;
5636         int qsize, rc;
5637
5638         qsize = vi->qsize_rxq;
5639
5640         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5641         if (rc != 0 || req->newptr == NULL)
5642                 return (rc);
5643
5644         if (qsize < 128 || (qsize & 7))
5645                 return (EINVAL);
5646
5647         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5648             "t4rxqs");
5649         if (rc)
5650                 return (rc);
5651
5652         if (vi->flags & VI_INIT_DONE)
5653                 rc = EBUSY; /* cannot be changed once the queues are created */
5654         else
5655                 vi->qsize_rxq = qsize;
5656
5657         end_synchronized_op(sc, LOCK_HELD);
5658         return (rc);
5659 }
5660
5661 static int
5662 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5663 {
5664         struct vi_info *vi = arg1;
5665         struct adapter *sc = vi->pi->adapter;
5666         int qsize, rc;
5667
5668         qsize = vi->qsize_txq;
5669
5670         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5671         if (rc != 0 || req->newptr == NULL)
5672                 return (rc);
5673
5674         if (qsize < 128 || qsize > 65536)
5675                 return (EINVAL);
5676
5677         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5678             "t4txqs");
5679         if (rc)
5680                 return (rc);
5681
5682         if (vi->flags & VI_INIT_DONE)
5683                 rc = EBUSY; /* cannot be changed once the queues are created */
5684         else
5685                 vi->qsize_txq = qsize;
5686
5687         end_synchronized_op(sc, LOCK_HELD);
5688         return (rc);
5689 }
5690
5691 static int
5692 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5693 {
5694         struct port_info *pi = arg1;
5695         struct adapter *sc = pi->adapter;
5696         struct link_config *lc = &pi->link_cfg;
5697         int rc;
5698
5699         if (req->newptr == NULL) {
5700                 struct sbuf *sb;
5701                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5702
5703                 rc = sysctl_wire_old_buffer(req, 0);
5704                 if (rc != 0)
5705                         return(rc);
5706
5707                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5708                 if (sb == NULL)
5709                         return (ENOMEM);
5710
5711                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5712                 rc = sbuf_finish(sb);
5713                 sbuf_delete(sb);
5714         } else {
5715                 char s[2];
5716                 int n;
5717
5718                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5719                 s[1] = 0;
5720
5721                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5722                 if (rc != 0)
5723                         return(rc);
5724
5725                 if (s[1] != 0)
5726                         return (EINVAL);
5727                 if (s[0] < '0' || s[0] > '9')
5728                         return (EINVAL);        /* not a number */
5729                 n = s[0] - '0';
5730                 if (n & ~(PAUSE_TX | PAUSE_RX))
5731                         return (EINVAL);        /* some other bit is set too */
5732
5733                 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
5734                     "t4PAUSE");
5735                 if (rc)
5736                         return (rc);
5737                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5738                         int link_ok = lc->link_ok;
5739
5740                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5741                         lc->requested_fc |= n;
5742                         rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
5743                         lc->link_ok = link_ok;  /* restore */
5744                 }
5745                 end_synchronized_op(sc, 0);
5746         }
5747
5748         return (rc);
5749 }
5750
5751 static int
5752 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5753 {
5754         struct adapter *sc = arg1;
5755         int reg = arg2;
5756         uint64_t val;
5757
5758         val = t4_read_reg64(sc, reg);
5759
5760         return (sysctl_handle_64(oidp, &val, 0, req));
5761 }
5762
5763 static int
5764 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5765 {
5766         struct adapter *sc = arg1;
5767         int rc, t;
5768         uint32_t param, val;
5769
5770         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5771         if (rc)
5772                 return (rc);
5773         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5774             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5775             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5776         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5777         end_synchronized_op(sc, 0);
5778         if (rc)
5779                 return (rc);
5780
5781         /* unknown is returned as 0 but we display -1 in that case */
5782         t = val == 0 ? -1 : val;
5783
5784         rc = sysctl_handle_int(oidp, &t, 0, req);
5785         return (rc);
5786 }
5787
5788 #ifdef SBUF_DRAIN
5789 static int
5790 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5791 {
5792         struct adapter *sc = arg1;
5793         struct sbuf *sb;
5794         int rc, i;
5795         uint16_t incr[NMTUS][NCCTRL_WIN];
5796         static const char *dec_fac[] = {
5797                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5798                 "0.9375"
5799         };
5800
5801         rc = sysctl_wire_old_buffer(req, 0);
5802         if (rc != 0)
5803                 return (rc);
5804
5805         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5806         if (sb == NULL)
5807                 return (ENOMEM);
5808
5809         t4_read_cong_tbl(sc, incr);
5810
5811         for (i = 0; i < NCCTRL_WIN; ++i) {
5812                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5813                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5814                     incr[5][i], incr[6][i], incr[7][i]);
5815                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5816                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5817                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5818                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5819         }
5820
5821         rc = sbuf_finish(sb);
5822         sbuf_delete(sb);
5823
5824         return (rc);
5825 }
5826
5827 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5828         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5829         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5830         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5831 };
5832
5833 static int
5834 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5835 {
5836         struct adapter *sc = arg1;
5837         struct sbuf *sb;
5838         int rc, i, n, qid = arg2;
5839         uint32_t *buf, *p;
5840         char *qtype;
5841         u_int cim_num_obq = sc->chip_params->cim_num_obq;
5842
5843         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5844             ("%s: bad qid %d\n", __func__, qid));
5845
5846         if (qid < CIM_NUM_IBQ) {
5847                 /* inbound queue */
5848                 qtype = "IBQ";
5849                 n = 4 * CIM_IBQ_SIZE;
5850                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5851                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5852         } else {
5853                 /* outbound queue */
5854                 qtype = "OBQ";
5855                 qid -= CIM_NUM_IBQ;
5856                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5857                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5858                 rc = t4_read_cim_obq(sc, qid, buf, n);
5859         }
5860
5861         if (rc < 0) {
5862                 rc = -rc;
5863                 goto done;
5864         }
5865         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5866
5867         rc = sysctl_wire_old_buffer(req, 0);
5868         if (rc != 0)
5869                 goto done;
5870
5871         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5872         if (sb == NULL) {
5873                 rc = ENOMEM;
5874                 goto done;
5875         }
5876
5877         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5878         for (i = 0, p = buf; i < n; i += 16, p += 4)
5879                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5880                     p[2], p[3]);
5881
5882         rc = sbuf_finish(sb);
5883         sbuf_delete(sb);
5884 done:
5885         free(buf, M_CXGBE);
5886         return (rc);
5887 }
5888
5889 static int
5890 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5891 {
5892         struct adapter *sc = arg1;
5893         u_int cfg;
5894         struct sbuf *sb;
5895         uint32_t *buf, *p;
5896         int rc;
5897
5898         MPASS(chip_id(sc) <= CHELSIO_T5);
5899
5900         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5901         if (rc != 0)
5902                 return (rc);
5903
5904         rc = sysctl_wire_old_buffer(req, 0);
5905         if (rc != 0)
5906                 return (rc);
5907
5908         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5909         if (sb == NULL)
5910                 return (ENOMEM);
5911
5912         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5913             M_ZERO | M_WAITOK);
5914
5915         rc = -t4_cim_read_la(sc, buf, NULL);
5916         if (rc != 0)
5917                 goto done;
5918
5919         sbuf_printf(sb, "Status   Data      PC%s",
5920             cfg & F_UPDBGLACAPTPCONLY ? "" :
5921             "     LS0Stat  LS0Addr             LS0Data");
5922
5923         for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
5924                 if (cfg & F_UPDBGLACAPTPCONLY) {
5925                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5926                             p[6], p[7]);
5927                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5928                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5929                             p[4] & 0xff, p[5] >> 8);
5930                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5931                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5932                             p[1] & 0xf, p[2] >> 4);
5933                 } else {
5934                         sbuf_printf(sb,
5935                             "\n  %02x   %x%07x %x%07x %08x %08x "
5936                             "%08x%08x%08x%08x",
5937                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5938                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5939                             p[6], p[7]);
5940                 }
5941         }
5942
5943         rc = sbuf_finish(sb);
5944         sbuf_delete(sb);
5945 done:
5946         free(buf, M_CXGBE);
5947         return (rc);
5948 }
5949
5950 static int
5951 sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS)
5952 {
5953         struct adapter *sc = arg1;
5954         u_int cfg;
5955         struct sbuf *sb;
5956         uint32_t *buf, *p;
5957         int rc;
5958
5959         MPASS(chip_id(sc) > CHELSIO_T5);
5960
5961         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5962         if (rc != 0)
5963                 return (rc);
5964
5965         rc = sysctl_wire_old_buffer(req, 0);
5966         if (rc != 0)
5967                 return (rc);
5968
5969         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5970         if (sb == NULL)
5971                 return (ENOMEM);
5972
5973         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5974             M_ZERO | M_WAITOK);
5975
5976         rc = -t4_cim_read_la(sc, buf, NULL);
5977         if (rc != 0)
5978                 goto done;
5979
5980         sbuf_printf(sb, "Status   Inst    Data      PC%s",
5981             cfg & F_UPDBGLACAPTPCONLY ? "" :
5982             "     LS0Stat  LS0Addr  LS0Data  LS1Stat  LS1Addr  LS1Data");
5983
5984         for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
5985                 if (cfg & F_UPDBGLACAPTPCONLY) {
5986                         sbuf_printf(sb, "\n  %02x   %08x %08x %08x",
5987                             p[3] & 0xff, p[2], p[1], p[0]);
5988                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x %02x%06x",
5989                             (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8,
5990                             p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8);
5991                         sbuf_printf(sb, "\n  %02x   %04x%04x %04x%04x %04x%04x",
5992                             (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16,
5993                             p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff,
5994                             p[6] >> 16);
5995                 } else {
5996                         sbuf_printf(sb, "\n  %02x   %04x%04x %04x%04x %04x%04x "
5997                             "%08x %08x %08x %08x %08x %08x",
5998                             (p[9] >> 16) & 0xff,
5999                             p[9] & 0xffff, p[8] >> 16,
6000                             p[8] & 0xffff, p[7] >> 16,
6001                             p[7] & 0xffff, p[6] >> 16,
6002                             p[2], p[1], p[0], p[5], p[4], p[3]);
6003                 }
6004         }
6005
6006         rc = sbuf_finish(sb);
6007         sbuf_delete(sb);
6008 done:
6009         free(buf, M_CXGBE);
6010         return (rc);
6011 }
6012
6013 static int
6014 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
6015 {
6016         struct adapter *sc = arg1;
6017         u_int i;
6018         struct sbuf *sb;
6019         uint32_t *buf, *p;
6020         int rc;
6021
6022         rc = sysctl_wire_old_buffer(req, 0);
6023         if (rc != 0)
6024                 return (rc);
6025
6026         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6027         if (sb == NULL)
6028                 return (ENOMEM);
6029
6030         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
6031             M_ZERO | M_WAITOK);
6032
6033         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
6034         p = buf;
6035
6036         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6037                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
6038                     p[1], p[0]);
6039         }
6040
6041         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
6042         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6043                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
6044                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
6045                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
6046                     (p[1] >> 2) | ((p[2] & 3) << 30),
6047                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
6048                     p[0] & 1);
6049         }
6050
6051         rc = sbuf_finish(sb);
6052         sbuf_delete(sb);
6053         free(buf, M_CXGBE);
6054         return (rc);
6055 }
6056
6057 static int
6058 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
6059 {
6060         struct adapter *sc = arg1;
6061         u_int i;
6062         struct sbuf *sb;
6063         uint32_t *buf, *p;
6064         int rc;
6065
6066         rc = sysctl_wire_old_buffer(req, 0);
6067         if (rc != 0)
6068                 return (rc);
6069
6070         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6071         if (sb == NULL)
6072                 return (ENOMEM);
6073
6074         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
6075             M_ZERO | M_WAITOK);
6076
6077         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
6078         p = buf;
6079
6080         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
6081         for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
6082                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
6083                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
6084                     p[4], p[3], p[2], p[1], p[0]);
6085         }
6086
6087         sbuf_printf(sb, "\n\nCntl ID               Data");
6088         for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
6089                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
6090                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
6091         }
6092
6093         rc = sbuf_finish(sb);
6094         sbuf_delete(sb);
6095         free(buf, M_CXGBE);
6096         return (rc);
6097 }
6098
6099 static int
6100 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
6101 {
6102         struct adapter *sc = arg1;
6103         struct sbuf *sb;
6104         int rc, i;
6105         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6106         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6107         uint16_t thres[CIM_NUM_IBQ];
6108         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
6109         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
6110         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
6111
6112         cim_num_obq = sc->chip_params->cim_num_obq;
6113         if (is_t4(sc)) {
6114                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
6115                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
6116         } else {
6117                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
6118                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
6119         }
6120         nq = CIM_NUM_IBQ + cim_num_obq;
6121
6122         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
6123         if (rc == 0)
6124                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
6125         if (rc != 0)
6126                 return (rc);
6127
6128         t4_read_cimq_cfg(sc, base, size, thres);
6129
6130         rc = sysctl_wire_old_buffer(req, 0);
6131         if (rc != 0)
6132                 return (rc);
6133
6134         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
6135         if (sb == NULL)
6136                 return (ENOMEM);
6137
6138         sbuf_printf(sb,
6139             "  Queue  Base  Size Thres  RdPtr WrPtr  SOP  EOP Avail");
6140
6141         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
6142                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
6143                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
6144                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6145                     G_QUEREMFLITS(p[2]) * 16);
6146         for ( ; i < nq; i++, p += 4, wr += 2)
6147                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
6148                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
6149                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6150                     G_QUEREMFLITS(p[2]) * 16);
6151
6152         rc = sbuf_finish(sb);
6153         sbuf_delete(sb);
6154
6155         return (rc);
6156 }
6157
6158 static int
6159 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
6160 {
6161         struct adapter *sc = arg1;
6162         struct sbuf *sb;
6163         int rc;
6164         struct tp_cpl_stats stats;
6165
6166         rc = sysctl_wire_old_buffer(req, 0);
6167         if (rc != 0)
6168                 return (rc);
6169
6170         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6171         if (sb == NULL)
6172                 return (ENOMEM);
6173
6174         mtx_lock(&sc->reg_lock);
6175         t4_tp_get_cpl_stats(sc, &stats);
6176         mtx_unlock(&sc->reg_lock);
6177
6178         if (sc->chip_params->nchan > 2) {
6179                 sbuf_printf(sb, "                 channel 0  channel 1"
6180                     "  channel 2  channel 3");
6181                 sbuf_printf(sb, "\nCPL requests:   %10u %10u %10u %10u",
6182                     stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
6183                 sbuf_printf(sb, "\nCPL responses:   %10u %10u %10u %10u",
6184                     stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
6185         } else {
6186                 sbuf_printf(sb, "                 channel 0  channel 1");
6187                 sbuf_printf(sb, "\nCPL requests:   %10u %10u",
6188                     stats.req[0], stats.req[1]);
6189                 sbuf_printf(sb, "\nCPL responses:   %10u %10u",
6190                     stats.rsp[0], stats.rsp[1]);
6191         }
6192
6193         rc = sbuf_finish(sb);
6194         sbuf_delete(sb);
6195
6196         return (rc);
6197 }
6198
6199 static int
6200 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
6201 {
6202         struct adapter *sc = arg1;
6203         struct sbuf *sb;
6204         int rc;
6205         struct tp_usm_stats stats;
6206
6207         rc = sysctl_wire_old_buffer(req, 0);
6208         if (rc != 0)
6209                 return(rc);
6210
6211         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6212         if (sb == NULL)
6213                 return (ENOMEM);
6214
6215         t4_get_usm_stats(sc, &stats);
6216
6217         sbuf_printf(sb, "Frames: %u\n", stats.frames);
6218         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
6219         sbuf_printf(sb, "Drops:  %u", stats.drops);
6220
6221         rc = sbuf_finish(sb);
6222         sbuf_delete(sb);
6223
6224         return (rc);
6225 }
6226
6227 static const char * const devlog_level_strings[] = {
6228         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
6229         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
6230         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
6231         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
6232         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
6233         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
6234 };
6235
6236 static const char * const devlog_facility_strings[] = {
6237         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
6238         [FW_DEVLOG_FACILITY_CF]         = "CF",
6239         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
6240         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
6241         [FW_DEVLOG_FACILITY_RES]        = "RES",
6242         [FW_DEVLOG_FACILITY_HW]         = "HW",
6243         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
6244         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
6245         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
6246         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
6247         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
6248         [FW_DEVLOG_FACILITY_VI]         = "VI",
6249         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
6250         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
6251         [FW_DEVLOG_FACILITY_TM]         = "TM",
6252         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
6253         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
6254         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
6255         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
6256         [FW_DEVLOG_FACILITY_RI]         = "RI",
6257         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
6258         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
6259         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
6260         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE",
6261         [FW_DEVLOG_FACILITY_CHNET]      = "CHNET",
6262 };
6263
6264 static int
6265 sysctl_devlog(SYSCTL_HANDLER_ARGS)
6266 {
6267         struct adapter *sc = arg1;
6268         struct devlog_params *dparams = &sc->params.devlog;
6269         struct fw_devlog_e *buf, *e;
6270         int i, j, rc, nentries, first = 0;
6271         struct sbuf *sb;
6272         uint64_t ftstamp = UINT64_MAX;
6273
6274         if (dparams->addr == 0)
6275                 return (ENXIO);
6276
6277         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
6278         if (buf == NULL)
6279                 return (ENOMEM);
6280
6281         rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size);
6282         if (rc != 0)
6283                 goto done;
6284
6285         nentries = dparams->size / sizeof(struct fw_devlog_e);
6286         for (i = 0; i < nentries; i++) {
6287                 e = &buf[i];
6288
6289                 if (e->timestamp == 0)
6290                         break;  /* end */
6291
6292                 e->timestamp = be64toh(e->timestamp);
6293                 e->seqno = be32toh(e->seqno);
6294                 for (j = 0; j < 8; j++)
6295                         e->params[j] = be32toh(e->params[j]);
6296
6297                 if (e->timestamp < ftstamp) {
6298                         ftstamp = e->timestamp;
6299                         first = i;
6300                 }
6301         }
6302
6303         if (buf[first].timestamp == 0)
6304                 goto done;      /* nothing in the log */
6305
6306         rc = sysctl_wire_old_buffer(req, 0);
6307         if (rc != 0)
6308                 goto done;
6309
6310         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6311         if (sb == NULL) {
6312                 rc = ENOMEM;
6313                 goto done;
6314         }
6315         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
6316             "Seq#", "Tstamp", "Level", "Facility", "Message");
6317
6318         i = first;
6319         do {
6320                 e = &buf[i];
6321                 if (e->timestamp == 0)
6322                         break;  /* end */
6323
6324                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
6325                     e->seqno, e->timestamp,
6326                     (e->level < nitems(devlog_level_strings) ?
6327                         devlog_level_strings[e->level] : "UNKNOWN"),
6328                     (e->facility < nitems(devlog_facility_strings) ?
6329                         devlog_facility_strings[e->facility] : "UNKNOWN"));
6330                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
6331                     e->params[2], e->params[3], e->params[4],
6332                     e->params[5], e->params[6], e->params[7]);
6333
6334                 if (++i == nentries)
6335                         i = 0;
6336         } while (i != first);
6337
6338         rc = sbuf_finish(sb);
6339         sbuf_delete(sb);
6340 done:
6341         free(buf, M_CXGBE);
6342         return (rc);
6343 }
6344
6345 static int
6346 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
6347 {
6348         struct adapter *sc = arg1;
6349         struct sbuf *sb;
6350         int rc;
6351         struct tp_fcoe_stats stats[MAX_NCHAN];
6352         int i, nchan = sc->chip_params->nchan;
6353
6354         rc = sysctl_wire_old_buffer(req, 0);
6355         if (rc != 0)
6356                 return (rc);
6357
6358         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6359         if (sb == NULL)
6360                 return (ENOMEM);
6361
6362         for (i = 0; i < nchan; i++)
6363                 t4_get_fcoe_stats(sc, i, &stats[i]);
6364
6365         if (nchan > 2) {
6366                 sbuf_printf(sb, "                   channel 0        channel 1"
6367                     "        channel 2        channel 3");
6368                 sbuf_printf(sb, "\noctetsDDP:  %16ju %16ju %16ju %16ju",
6369                     stats[0].octets_ddp, stats[1].octets_ddp,
6370                     stats[2].octets_ddp, stats[3].octets_ddp);
6371                 sbuf_printf(sb, "\nframesDDP:  %16u %16u %16u %16u",
6372                     stats[0].frames_ddp, stats[1].frames_ddp,
6373                     stats[2].frames_ddp, stats[3].frames_ddp);
6374                 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u",
6375                     stats[0].frames_drop, stats[1].frames_drop,
6376                     stats[2].frames_drop, stats[3].frames_drop);
6377         } else {
6378                 sbuf_printf(sb, "                   channel 0        channel 1");
6379                 sbuf_printf(sb, "\noctetsDDP:  %16ju %16ju",
6380                     stats[0].octets_ddp, stats[1].octets_ddp);
6381                 sbuf_printf(sb, "\nframesDDP:  %16u %16u",
6382                     stats[0].frames_ddp, stats[1].frames_ddp);
6383                 sbuf_printf(sb, "\nframesDrop: %16u %16u",
6384                     stats[0].frames_drop, stats[1].frames_drop);
6385         }
6386
6387         rc = sbuf_finish(sb);
6388         sbuf_delete(sb);
6389
6390         return (rc);
6391 }
6392
6393 static int
6394 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
6395 {
6396         struct adapter *sc = arg1;
6397         struct sbuf *sb;
6398         int rc, i;
6399         unsigned int map, kbps, ipg, mode;
6400         unsigned int pace_tab[NTX_SCHED];
6401
6402         rc = sysctl_wire_old_buffer(req, 0);
6403         if (rc != 0)
6404                 return (rc);
6405
6406         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6407         if (sb == NULL)
6408                 return (ENOMEM);
6409
6410         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
6411         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
6412         t4_read_pace_tbl(sc, pace_tab);
6413
6414         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
6415             "Class IPG (0.1 ns)   Flow IPG (us)");
6416
6417         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
6418                 t4_get_tx_sched(sc, i, &kbps, &ipg);
6419                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
6420                     (mode & (1 << i)) ? "flow" : "class", map & 3);
6421                 if (kbps)
6422                         sbuf_printf(sb, "%9u     ", kbps);
6423                 else
6424                         sbuf_printf(sb, " disabled     ");
6425
6426                 if (ipg)
6427                         sbuf_printf(sb, "%13u        ", ipg);
6428                 else
6429                         sbuf_printf(sb, "     disabled        ");
6430
6431                 if (pace_tab[i])
6432                         sbuf_printf(sb, "%10u", pace_tab[i]);
6433                 else
6434                         sbuf_printf(sb, "  disabled");
6435         }
6436
6437         rc = sbuf_finish(sb);
6438         sbuf_delete(sb);
6439
6440         return (rc);
6441 }
6442
6443 static int
6444 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
6445 {
6446         struct adapter *sc = arg1;
6447         struct sbuf *sb;
6448         int rc, i, j;
6449         uint64_t *p0, *p1;
6450         struct lb_port_stats s[2];
6451         static const char *stat_name[] = {
6452                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
6453                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
6454                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
6455                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
6456                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
6457                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
6458                 "BG2FramesTrunc:", "BG3FramesTrunc:"
6459         };
6460
6461         rc = sysctl_wire_old_buffer(req, 0);
6462         if (rc != 0)
6463                 return (rc);
6464
6465         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6466         if (sb == NULL)
6467                 return (ENOMEM);
6468
6469         memset(s, 0, sizeof(s));
6470
6471         for (i = 0; i < sc->chip_params->nchan; i += 2) {
6472                 t4_get_lb_stats(sc, i, &s[0]);
6473                 t4_get_lb_stats(sc, i + 1, &s[1]);
6474
6475                 p0 = &s[0].octets;
6476                 p1 = &s[1].octets;
6477                 sbuf_printf(sb, "%s                       Loopback %u"
6478                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6479
6480                 for (j = 0; j < nitems(stat_name); j++)
6481                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6482                                    *p0++, *p1++);
6483         }
6484
6485         rc = sbuf_finish(sb);
6486         sbuf_delete(sb);
6487
6488         return (rc);
6489 }
6490
6491 static int
6492 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6493 {
6494         int rc = 0;
6495         struct port_info *pi = arg1;
6496         struct sbuf *sb;
6497
6498         rc = sysctl_wire_old_buffer(req, 0);
6499         if (rc != 0)
6500                 return(rc);
6501         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6502         if (sb == NULL)
6503                 return (ENOMEM);
6504
6505         if (pi->linkdnrc < 0)
6506                 sbuf_printf(sb, "n/a");
6507         else
6508                 sbuf_printf(sb, "%s", t4_link_down_rc_str(pi->linkdnrc));
6509
6510         rc = sbuf_finish(sb);
6511         sbuf_delete(sb);
6512
6513         return (rc);
6514 }
6515
6516 struct mem_desc {
6517         unsigned int base;
6518         unsigned int limit;
6519         unsigned int idx;
6520 };
6521
6522 static int
6523 mem_desc_cmp(const void *a, const void *b)
6524 {
6525         return ((const struct mem_desc *)a)->base -
6526                ((const struct mem_desc *)b)->base;
6527 }
6528
6529 static void
6530 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6531     unsigned int to)
6532 {
6533         unsigned int size;
6534
6535         if (from == to)
6536                 return;
6537
6538         size = to - from + 1;
6539         if (size == 0)
6540                 return;
6541
6542         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6543         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6544 }
6545
6546 static int
6547 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6548 {
6549         struct adapter *sc = arg1;
6550         struct sbuf *sb;
6551         int rc, i, n;
6552         uint32_t lo, hi, used, alloc;
6553         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6554         static const char *region[] = {
6555                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6556                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6557                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6558                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6559                 "RQUDP region:", "PBL region:", "TXPBL region:",
6560                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6561                 "On-chip queues:"
6562         };
6563         struct mem_desc avail[4];
6564         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6565         struct mem_desc *md = mem;
6566
6567         rc = sysctl_wire_old_buffer(req, 0);
6568         if (rc != 0)
6569                 return (rc);
6570
6571         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6572         if (sb == NULL)
6573                 return (ENOMEM);
6574
6575         for (i = 0; i < nitems(mem); i++) {
6576                 mem[i].limit = 0;
6577                 mem[i].idx = i;
6578         }
6579
6580         /* Find and sort the populated memory ranges */
6581         i = 0;
6582         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6583         if (lo & F_EDRAM0_ENABLE) {
6584                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6585                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6586                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6587                 avail[i].idx = 0;
6588                 i++;
6589         }
6590         if (lo & F_EDRAM1_ENABLE) {
6591                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6592                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6593                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6594                 avail[i].idx = 1;
6595                 i++;
6596         }
6597         if (lo & F_EXT_MEM_ENABLE) {
6598                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6599                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6600                 avail[i].limit = avail[i].base +
6601                     (G_EXT_MEM_SIZE(hi) << 20);
6602                 avail[i].idx = is_t5(sc) ? 3 : 2;       /* Call it MC0 for T5 */
6603                 i++;
6604         }
6605         if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) {
6606                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6607                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6608                 avail[i].limit = avail[i].base +
6609                     (G_EXT_MEM1_SIZE(hi) << 20);
6610                 avail[i].idx = 4;
6611                 i++;
6612         }
6613         if (!i)                                    /* no memory available */
6614                 return 0;
6615         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6616
6617         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6618         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6619         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6620         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6621         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6622         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6623         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6624         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6625         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6626
6627         /* the next few have explicit upper bounds */
6628         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6629         md->limit = md->base - 1 +
6630                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6631                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6632         md++;
6633
6634         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6635         md->limit = md->base - 1 +
6636                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6637                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6638         md++;
6639
6640         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6641                 if (chip_id(sc) <= CHELSIO_T5)
6642                         md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6643                 else
6644                         md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR);
6645                 md->limit = 0;
6646         } else {
6647                 md->base = 0;
6648                 md->idx = nitems(region);  /* hide it */
6649         }
6650         md++;
6651
6652 #define ulp_region(reg) \
6653         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6654         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6655
6656         ulp_region(RX_ISCSI);
6657         ulp_region(RX_TDDP);
6658         ulp_region(TX_TPT);
6659         ulp_region(RX_STAG);
6660         ulp_region(RX_RQ);
6661         ulp_region(RX_RQUDP);
6662         ulp_region(RX_PBL);
6663         ulp_region(TX_PBL);
6664 #undef ulp_region
6665
6666         md->base = 0;
6667         md->idx = nitems(region);
6668         if (!is_t4(sc)) {
6669                 uint32_t size = 0;
6670                 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2);
6671                 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE);
6672
6673                 if (is_t5(sc)) {
6674                         if (sge_ctrl & F_VFIFO_ENABLE)
6675                                 size = G_DBVFIFO_SIZE(fifo_size);
6676                 } else
6677                         size = G_T6_DBVFIFO_SIZE(fifo_size);
6678
6679                 if (size) {
6680                         md->base = G_BASEADDR(t4_read_reg(sc,
6681                             A_SGE_DBVFIFO_BADDR));
6682                         md->limit = md->base + (size << 2) - 1;
6683                 }
6684         }
6685         md++;
6686
6687         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6688         md->limit = 0;
6689         md++;
6690         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6691         md->limit = 0;
6692         md++;
6693
6694         md->base = sc->vres.ocq.start;
6695         if (sc->vres.ocq.size)
6696                 md->limit = md->base + sc->vres.ocq.size - 1;
6697         else
6698                 md->idx = nitems(region);  /* hide it */
6699         md++;
6700
6701         /* add any address-space holes, there can be up to 3 */
6702         for (n = 0; n < i - 1; n++)
6703                 if (avail[n].limit < avail[n + 1].base)
6704                         (md++)->base = avail[n].limit;
6705         if (avail[n].limit)
6706                 (md++)->base = avail[n].limit;
6707
6708         n = md - mem;
6709         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6710
6711         for (lo = 0; lo < i; lo++)
6712                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6713                                 avail[lo].limit - 1);
6714
6715         sbuf_printf(sb, "\n");
6716         for (i = 0; i < n; i++) {
6717                 if (mem[i].idx >= nitems(region))
6718                         continue;                        /* skip holes */
6719                 if (!mem[i].limit)
6720                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6721                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6722                                 mem[i].limit);
6723         }
6724
6725         sbuf_printf(sb, "\n");
6726         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6727         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6728         mem_region_show(sb, "uP RAM:", lo, hi);
6729
6730         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6731         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6732         mem_region_show(sb, "uP Extmem2:", lo, hi);
6733
6734         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6735         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6736                    G_PMRXMAXPAGE(lo),
6737                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6738                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6739
6740         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6741         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6742         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6743                    G_PMTXMAXPAGE(lo),
6744                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6745                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6746         sbuf_printf(sb, "%u p-structs\n",
6747                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6748
6749         for (i = 0; i < 4; i++) {
6750                 if (chip_id(sc) > CHELSIO_T5)
6751                         lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4);
6752                 else
6753                         lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6754                 if (is_t5(sc)) {
6755                         used = G_T5_USED(lo);
6756                         alloc = G_T5_ALLOC(lo);
6757                 } else {
6758                         used = G_USED(lo);
6759                         alloc = G_ALLOC(lo);
6760                 }
6761                 /* For T6 these are MAC buffer groups */
6762                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6763                     i, used, alloc);
6764         }
6765         for (i = 0; i < sc->chip_params->nchan; i++) {
6766                 if (chip_id(sc) > CHELSIO_T5)
6767                         lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4);
6768                 else
6769                         lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6770                 if (is_t5(sc)) {
6771                         used = G_T5_USED(lo);
6772                         alloc = G_T5_ALLOC(lo);
6773                 } else {
6774                         used = G_USED(lo);
6775                         alloc = G_ALLOC(lo);
6776                 }
6777                 /* For T6 these are MAC buffer groups */
6778                 sbuf_printf(sb,
6779                     "\nLoopback %d using %u pages out of %u allocated",
6780                     i, used, alloc);
6781         }
6782
6783         rc = sbuf_finish(sb);
6784         sbuf_delete(sb);
6785
6786         return (rc);
6787 }
6788
6789 static inline void
6790 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6791 {
6792         *mask = x | y;
6793         y = htobe64(y);
6794         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6795 }
6796
6797 static int
6798 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6799 {
6800         struct adapter *sc = arg1;
6801         struct sbuf *sb;
6802         int rc, i;
6803
6804         MPASS(chip_id(sc) <= CHELSIO_T5);
6805
6806         rc = sysctl_wire_old_buffer(req, 0);
6807         if (rc != 0)
6808                 return (rc);
6809
6810         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6811         if (sb == NULL)
6812                 return (ENOMEM);
6813
6814         sbuf_printf(sb,
6815             "Idx  Ethernet address     Mask     Vld Ports PF"
6816             "  VF              Replication             P0 P1 P2 P3  ML");
6817         for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
6818                 uint64_t tcamx, tcamy, mask;
6819                 uint32_t cls_lo, cls_hi;
6820                 uint8_t addr[ETHER_ADDR_LEN];
6821
6822                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6823                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6824                 if (tcamx & tcamy)
6825                         continue;
6826                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6827                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6828                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6829                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6830                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6831                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6832                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6833                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6834                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6835
6836                 if (cls_lo & F_REPLICATE) {
6837                         struct fw_ldst_cmd ldst_cmd;
6838
6839                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6840                         ldst_cmd.op_to_addrspace =
6841                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6842                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6843                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6844                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6845                         ldst_cmd.u.mps.rplc.fid_idx =
6846                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6847                                 V_FW_LDST_CMD_IDX(i));
6848
6849                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6850                             "t4mps");
6851                         if (rc)
6852                                 break;
6853                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6854                             sizeof(ldst_cmd), &ldst_cmd);
6855                         end_synchronized_op(sc, 0);
6856
6857                         if (rc != 0) {
6858                                 sbuf_printf(sb, "%36d", rc);
6859                                 rc = 0;
6860                         } else {
6861                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6862                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6863                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6864                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6865                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6866                         }
6867                 } else
6868                         sbuf_printf(sb, "%36s", "");
6869
6870                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6871                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6872                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6873         }
6874
6875         if (rc)
6876                 (void) sbuf_finish(sb);
6877         else
6878                 rc = sbuf_finish(sb);
6879         sbuf_delete(sb);
6880
6881         return (rc);
6882 }
6883
6884 static int
6885 sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS)
6886 {
6887         struct adapter *sc = arg1;
6888         struct sbuf *sb;
6889         int rc, i;
6890
6891         MPASS(chip_id(sc) > CHELSIO_T5);
6892
6893         rc = sysctl_wire_old_buffer(req, 0);
6894         if (rc != 0)
6895                 return (rc);
6896
6897         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6898         if (sb == NULL)
6899                 return (ENOMEM);
6900
6901         sbuf_printf(sb, "Idx  Ethernet address     Mask       VNI   Mask"
6902             "   IVLAN Vld DIP_Hit   Lookup  Port Vld Ports PF  VF"
6903             "                           Replication"
6904             "                                    P0 P1 P2 P3  ML\n");
6905
6906         for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
6907                 uint8_t dip_hit, vlan_vld, lookup_type, port_num;
6908                 uint16_t ivlan;
6909                 uint64_t tcamx, tcamy, val, mask;
6910                 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy;
6911                 uint8_t addr[ETHER_ADDR_LEN];
6912
6913                 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0);
6914                 if (i < 256)
6915                         ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0);
6916                 else
6917                         ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1);
6918                 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
6919                 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
6920                 tcamy = G_DMACH(val) << 32;
6921                 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
6922                 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
6923                 lookup_type = G_DATALKPTYPE(data2);
6924                 port_num = G_DATAPORTNUM(data2);
6925                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
6926                         /* Inner header VNI */
6927                         vniy = ((data2 & F_DATAVIDH2) << 23) |
6928                                        (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
6929                         dip_hit = data2 & F_DATADIPHIT;
6930                         vlan_vld = 0;
6931                 } else {
6932                         vniy = 0;
6933                         dip_hit = 0;
6934                         vlan_vld = data2 & F_DATAVIDH2;
6935                         ivlan = G_VIDL(val);
6936                 }
6937
6938                 ctl |= V_CTLXYBITSEL(1);
6939                 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
6940                 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
6941                 tcamx = G_DMACH(val) << 32;
6942                 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
6943                 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
6944                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
6945                         /* Inner header VNI mask */
6946                         vnix = ((data2 & F_DATAVIDH2) << 23) |
6947                                (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
6948                 } else
6949                         vnix = 0;
6950
6951                 if (tcamx & tcamy)
6952                         continue;
6953                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6954
6955                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6956                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6957
6958                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
6959                         sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
6960                             "%012jx %06x %06x    -    -   %3c"
6961                             "      'I'  %4x   %3c   %#x%4u%4d", i, addr[0],
6962                             addr[1], addr[2], addr[3], addr[4], addr[5],
6963                             (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N',
6964                             port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
6965                             G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
6966                             cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
6967                 } else {
6968                         sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
6969                             "%012jx    -       -   ", i, addr[0], addr[1],
6970                             addr[2], addr[3], addr[4], addr[5],
6971                             (uintmax_t)mask);
6972
6973                         if (vlan_vld)
6974                                 sbuf_printf(sb, "%4u   Y     ", ivlan);
6975                         else
6976                                 sbuf_printf(sb, "  -    N     ");
6977
6978                         sbuf_printf(sb, "-      %3c  %4x   %3c   %#x%4u%4d",
6979                             lookup_type ? 'I' : 'O', port_num,
6980                             cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
6981                             G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
6982                             cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
6983                 }
6984
6985
6986                 if (cls_lo & F_T6_REPLICATE) {
6987                         struct fw_ldst_cmd ldst_cmd;
6988
6989                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6990                         ldst_cmd.op_to_addrspace =
6991                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6992                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6993                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6994                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6995                         ldst_cmd.u.mps.rplc.fid_idx =
6996                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6997                                 V_FW_LDST_CMD_IDX(i));
6998
6999                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
7000                             "t6mps");
7001                         if (rc)
7002                                 break;
7003                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
7004                             sizeof(ldst_cmd), &ldst_cmd);
7005                         end_synchronized_op(sc, 0);
7006
7007                         if (rc != 0) {
7008                                 sbuf_printf(sb, "%72d", rc);
7009                                 rc = 0;
7010                         } else {
7011                                 sbuf_printf(sb, " %08x %08x %08x %08x"
7012                                     " %08x %08x %08x %08x",
7013                                     be32toh(ldst_cmd.u.mps.rplc.rplc255_224),
7014                                     be32toh(ldst_cmd.u.mps.rplc.rplc223_192),
7015                                     be32toh(ldst_cmd.u.mps.rplc.rplc191_160),
7016                                     be32toh(ldst_cmd.u.mps.rplc.rplc159_128),
7017                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
7018                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
7019                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
7020                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
7021                         }
7022                 } else
7023                         sbuf_printf(sb, "%72s", "");
7024
7025                 sbuf_printf(sb, "%4u%3u%3u%3u %#x",
7026                     G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo),
7027                     G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo),
7028                     (cls_lo >> S_T6_MULTILISTEN0) & 0xf);
7029         }
7030
7031         if (rc)
7032                 (void) sbuf_finish(sb);
7033         else
7034                 rc = sbuf_finish(sb);
7035         sbuf_delete(sb);
7036
7037         return (rc);
7038 }
7039
7040 static int
7041 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
7042 {
7043         struct adapter *sc = arg1;
7044         struct sbuf *sb;
7045         int rc;
7046         uint16_t mtus[NMTUS];
7047
7048         rc = sysctl_wire_old_buffer(req, 0);
7049         if (rc != 0)
7050                 return (rc);
7051
7052         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7053         if (sb == NULL)
7054                 return (ENOMEM);
7055
7056         t4_read_mtu_tbl(sc, mtus, NULL);
7057
7058         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
7059             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
7060             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
7061             mtus[14], mtus[15]);
7062
7063         rc = sbuf_finish(sb);
7064         sbuf_delete(sb);
7065
7066         return (rc);
7067 }
7068
7069 static int
7070 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
7071 {
7072         struct adapter *sc = arg1;
7073         struct sbuf *sb;
7074         int rc, i;
7075         uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS];
7076         uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS];
7077         static const char *tx_stats[MAX_PM_NSTATS] = {
7078                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:",
7079                 "Tx FIFO wait", NULL, "Tx latency"
7080         };
7081         static const char *rx_stats[MAX_PM_NSTATS] = {
7082                 "Read:", "Write bypass:", "Write mem:", "Flush:",
7083                 "Rx FIFO wait", NULL, "Rx latency"
7084         };
7085
7086         rc = sysctl_wire_old_buffer(req, 0);
7087         if (rc != 0)
7088                 return (rc);
7089
7090         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7091         if (sb == NULL)
7092                 return (ENOMEM);
7093
7094         t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
7095         t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
7096
7097         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
7098         for (i = 0; i < 4; i++) {
7099                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
7100                     tx_cyc[i]);
7101         }
7102
7103         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
7104         for (i = 0; i < 4; i++) {
7105                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
7106                     rx_cyc[i]);
7107         }
7108
7109         if (chip_id(sc) > CHELSIO_T5) {
7110                 sbuf_printf(sb,
7111                     "\n              Total wait      Total occupancy");
7112                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
7113                     tx_cyc[i]);
7114                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
7115                     rx_cyc[i]);
7116
7117                 i += 2;
7118                 MPASS(i < nitems(tx_stats));
7119
7120                 sbuf_printf(sb,
7121                     "\n                   Reads           Total wait");
7122                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
7123                     tx_cyc[i]);
7124                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
7125                     rx_cyc[i]);
7126         }
7127
7128         rc = sbuf_finish(sb);
7129         sbuf_delete(sb);
7130
7131         return (rc);
7132 }
7133
7134 static int
7135 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
7136 {
7137         struct adapter *sc = arg1;
7138         struct sbuf *sb;
7139         int rc;
7140         struct tp_rdma_stats stats;
7141
7142         rc = sysctl_wire_old_buffer(req, 0);
7143         if (rc != 0)
7144                 return (rc);
7145
7146         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7147         if (sb == NULL)
7148                 return (ENOMEM);
7149
7150         mtx_lock(&sc->reg_lock);
7151         t4_tp_get_rdma_stats(sc, &stats);
7152         mtx_unlock(&sc->reg_lock);
7153
7154         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
7155         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
7156
7157         rc = sbuf_finish(sb);
7158         sbuf_delete(sb);
7159
7160         return (rc);
7161 }
7162
7163 static int
7164 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
7165 {
7166         struct adapter *sc = arg1;
7167         struct sbuf *sb;
7168         int rc;
7169         struct tp_tcp_stats v4, v6;
7170
7171         rc = sysctl_wire_old_buffer(req, 0);
7172         if (rc != 0)
7173                 return (rc);
7174
7175         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7176         if (sb == NULL)
7177                 return (ENOMEM);
7178
7179         mtx_lock(&sc->reg_lock);
7180         t4_tp_get_tcp_stats(sc, &v4, &v6);
7181         mtx_unlock(&sc->reg_lock);
7182
7183         sbuf_printf(sb,
7184             "                                IP                 IPv6\n");
7185         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
7186             v4.tcp_out_rsts, v6.tcp_out_rsts);
7187         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
7188             v4.tcp_in_segs, v6.tcp_in_segs);
7189         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
7190             v4.tcp_out_segs, v6.tcp_out_segs);
7191         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
7192             v4.tcp_retrans_segs, v6.tcp_retrans_segs);
7193
7194         rc = sbuf_finish(sb);
7195         sbuf_delete(sb);
7196
7197         return (rc);
7198 }
7199
7200 static int
7201 sysctl_tids(SYSCTL_HANDLER_ARGS)
7202 {
7203         struct adapter *sc = arg1;
7204         struct sbuf *sb;
7205         int rc;
7206         struct tid_info *t = &sc->tids;
7207
7208         rc = sysctl_wire_old_buffer(req, 0);
7209         if (rc != 0)
7210                 return (rc);
7211
7212         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7213         if (sb == NULL)
7214                 return (ENOMEM);
7215
7216         if (t->natids) {
7217                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
7218                     t->atids_in_use);
7219         }
7220
7221         if (t->ntids) {
7222                 sbuf_printf(sb, "TID range: ");
7223                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
7224                         uint32_t b, hb;
7225
7226                         if (chip_id(sc) <= CHELSIO_T5) {
7227                                 b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
7228                                 hb = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
7229                         } else {
7230                                 b = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX);
7231                                 hb = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE);
7232                         }
7233
7234                         if (b)
7235                                 sbuf_printf(sb, "0-%u, ", b - 1);
7236                         sbuf_printf(sb, "%u-%u", hb, t->ntids - 1);
7237                 } else
7238                         sbuf_printf(sb, "0-%u", t->ntids - 1);
7239                 sbuf_printf(sb, ", in use: %u\n",
7240                     atomic_load_acq_int(&t->tids_in_use));
7241         }
7242
7243         if (t->nstids) {
7244                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
7245                     t->stid_base + t->nstids - 1, t->stids_in_use);
7246         }
7247
7248         if (t->nftids) {
7249                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
7250                     t->ftid_base + t->nftids - 1);
7251         }
7252
7253         if (t->netids) {
7254                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
7255                     t->etid_base + t->netids - 1);
7256         }
7257
7258         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
7259             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
7260             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
7261
7262         rc = sbuf_finish(sb);
7263         sbuf_delete(sb);
7264
7265         return (rc);
7266 }
7267
7268 static int
7269 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
7270 {
7271         struct adapter *sc = arg1;
7272         struct sbuf *sb;
7273         int rc;
7274         struct tp_err_stats stats;
7275
7276         rc = sysctl_wire_old_buffer(req, 0);
7277         if (rc != 0)
7278                 return (rc);
7279
7280         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7281         if (sb == NULL)
7282                 return (ENOMEM);
7283
7284         mtx_lock(&sc->reg_lock);
7285         t4_tp_get_err_stats(sc, &stats);
7286         mtx_unlock(&sc->reg_lock);
7287
7288         if (sc->chip_params->nchan > 2) {
7289                 sbuf_printf(sb, "                 channel 0  channel 1"
7290                     "  channel 2  channel 3\n");
7291                 sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
7292                     stats.mac_in_errs[0], stats.mac_in_errs[1],
7293                     stats.mac_in_errs[2], stats.mac_in_errs[3]);
7294                 sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
7295                     stats.hdr_in_errs[0], stats.hdr_in_errs[1],
7296                     stats.hdr_in_errs[2], stats.hdr_in_errs[3]);
7297                 sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
7298                     stats.tcp_in_errs[0], stats.tcp_in_errs[1],
7299                     stats.tcp_in_errs[2], stats.tcp_in_errs[3]);
7300                 sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
7301                     stats.tcp6_in_errs[0], stats.tcp6_in_errs[1],
7302                     stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]);
7303                 sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
7304                     stats.tnl_cong_drops[0], stats.tnl_cong_drops[1],
7305                     stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]);
7306                 sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
7307                     stats.tnl_tx_drops[0], stats.tnl_tx_drops[1],
7308                     stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]);
7309                 sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
7310                     stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1],
7311                     stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]);
7312                 sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
7313                     stats.ofld_chan_drops[0], stats.ofld_chan_drops[1],
7314                     stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]);
7315         } else {
7316                 sbuf_printf(sb, "                 channel 0  channel 1\n");
7317                 sbuf_printf(sb, "macInErrs:      %10u %10u\n",
7318                     stats.mac_in_errs[0], stats.mac_in_errs[1]);
7319                 sbuf_printf(sb, "hdrInErrs:      %10u %10u\n",
7320                     stats.hdr_in_errs[0], stats.hdr_in_errs[1]);
7321                 sbuf_printf(sb, "tcpInErrs:      %10u %10u\n",
7322                     stats.tcp_in_errs[0], stats.tcp_in_errs[1]);
7323                 sbuf_printf(sb, "tcp6InErrs:     %10u %10u\n",
7324                     stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]);
7325                 sbuf_printf(sb, "tnlCongDrops:   %10u %10u\n",
7326                     stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]);
7327                 sbuf_printf(sb, "tnlTxDrops:     %10u %10u\n",
7328                     stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]);
7329                 sbuf_printf(sb, "ofldVlanDrops:  %10u %10u\n",
7330                     stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]);
7331                 sbuf_printf(sb, "ofldChanDrops:  %10u %10u\n\n",
7332                     stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]);
7333         }
7334
7335         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
7336             stats.ofld_no_neigh, stats.ofld_cong_defer);
7337
7338         rc = sbuf_finish(sb);
7339         sbuf_delete(sb);
7340
7341         return (rc);
7342 }
7343
7344 static int
7345 sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS)
7346 {
7347         struct adapter *sc = arg1;
7348         struct tp_params *tpp = &sc->params.tp;
7349         u_int mask;
7350         int rc;
7351
7352         mask = tpp->la_mask >> 16;
7353         rc = sysctl_handle_int(oidp, &mask, 0, req);
7354         if (rc != 0 || req->newptr == NULL)
7355                 return (rc);
7356         if (mask > 0xffff)
7357                 return (EINVAL);
7358         tpp->la_mask = mask << 16;
7359         t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask);
7360
7361         return (0);
7362 }
7363
7364 struct field_desc {
7365         const char *name;
7366         u_int start;
7367         u_int width;
7368 };
7369
7370 static void
7371 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
7372 {
7373         char buf[32];
7374         int line_size = 0;
7375
7376         while (f->name) {
7377                 uint64_t mask = (1ULL << f->width) - 1;
7378                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
7379                     ((uintmax_t)v >> f->start) & mask);
7380
7381                 if (line_size + len >= 79) {
7382                         line_size = 8;
7383                         sbuf_printf(sb, "\n        ");
7384                 }
7385                 sbuf_printf(sb, "%s ", buf);
7386                 line_size += len + 1;
7387                 f++;
7388         }
7389         sbuf_printf(sb, "\n");
7390 }
7391
7392 static const struct field_desc tp_la0[] = {
7393         { "RcfOpCodeOut", 60, 4 },
7394         { "State", 56, 4 },
7395         { "WcfState", 52, 4 },
7396         { "RcfOpcSrcOut", 50, 2 },
7397         { "CRxError", 49, 1 },
7398         { "ERxError", 48, 1 },
7399         { "SanityFailed", 47, 1 },
7400         { "SpuriousMsg", 46, 1 },
7401         { "FlushInputMsg", 45, 1 },
7402         { "FlushInputCpl", 44, 1 },
7403         { "RssUpBit", 43, 1 },
7404         { "RssFilterHit", 42, 1 },
7405         { "Tid", 32, 10 },
7406         { "InitTcb", 31, 1 },
7407         { "LineNumber", 24, 7 },
7408         { "Emsg", 23, 1 },
7409         { "EdataOut", 22, 1 },
7410         { "Cmsg", 21, 1 },
7411         { "CdataOut", 20, 1 },
7412         { "EreadPdu", 19, 1 },
7413         { "CreadPdu", 18, 1 },
7414         { "TunnelPkt", 17, 1 },
7415         { "RcfPeerFin", 16, 1 },
7416         { "RcfReasonOut", 12, 4 },
7417         { "TxCchannel", 10, 2 },
7418         { "RcfTxChannel", 8, 2 },
7419         { "RxEchannel", 6, 2 },
7420         { "RcfRxChannel", 5, 1 },
7421         { "RcfDataOutSrdy", 4, 1 },
7422         { "RxDvld", 3, 1 },
7423         { "RxOoDvld", 2, 1 },
7424         { "RxCongestion", 1, 1 },
7425         { "TxCongestion", 0, 1 },
7426         { NULL }
7427 };
7428
7429 static const struct field_desc tp_la1[] = {
7430         { "CplCmdIn", 56, 8 },
7431         { "CplCmdOut", 48, 8 },
7432         { "ESynOut", 47, 1 },
7433         { "EAckOut", 46, 1 },
7434         { "EFinOut", 45, 1 },
7435         { "ERstOut", 44, 1 },
7436         { "SynIn", 43, 1 },
7437         { "AckIn", 42, 1 },
7438         { "FinIn", 41, 1 },
7439         { "RstIn", 40, 1 },
7440         { "DataIn", 39, 1 },
7441         { "DataInVld", 38, 1 },
7442         { "PadIn", 37, 1 },
7443         { "RxBufEmpty", 36, 1 },
7444         { "RxDdp", 35, 1 },
7445         { "RxFbCongestion", 34, 1 },
7446         { "TxFbCongestion", 33, 1 },
7447         { "TxPktSumSrdy", 32, 1 },
7448         { "RcfUlpType", 28, 4 },
7449         { "Eread", 27, 1 },
7450         { "Ebypass", 26, 1 },
7451         { "Esave", 25, 1 },
7452         { "Static0", 24, 1 },
7453         { "Cread", 23, 1 },
7454         { "Cbypass", 22, 1 },
7455         { "Csave", 21, 1 },
7456         { "CPktOut", 20, 1 },
7457         { "RxPagePoolFull", 18, 2 },
7458         { "RxLpbkPkt", 17, 1 },
7459         { "TxLpbkPkt", 16, 1 },
7460         { "RxVfValid", 15, 1 },
7461         { "SynLearned", 14, 1 },
7462         { "SetDelEntry", 13, 1 },
7463         { "SetInvEntry", 12, 1 },
7464         { "CpcmdDvld", 11, 1 },
7465         { "CpcmdSave", 10, 1 },
7466         { "RxPstructsFull", 8, 2 },
7467         { "EpcmdDvld", 7, 1 },
7468         { "EpcmdFlush", 6, 1 },
7469         { "EpcmdTrimPrefix", 5, 1 },
7470         { "EpcmdTrimPostfix", 4, 1 },
7471         { "ERssIp4Pkt", 3, 1 },
7472         { "ERssIp6Pkt", 2, 1 },
7473         { "ERssTcpUdpPkt", 1, 1 },
7474         { "ERssFceFipPkt", 0, 1 },
7475         { NULL }
7476 };
7477
7478 static const struct field_desc tp_la2[] = {
7479         { "CplCmdIn", 56, 8 },
7480         { "MpsVfVld", 55, 1 },
7481         { "MpsPf", 52, 3 },
7482         { "MpsVf", 44, 8 },
7483         { "SynIn", 43, 1 },
7484         { "AckIn", 42, 1 },
7485         { "FinIn", 41, 1 },
7486         { "RstIn", 40, 1 },
7487         { "DataIn", 39, 1 },
7488         { "DataInVld", 38, 1 },
7489         { "PadIn", 37, 1 },
7490         { "RxBufEmpty", 36, 1 },
7491         { "RxDdp", 35, 1 },
7492         { "RxFbCongestion", 34, 1 },
7493         { "TxFbCongestion", 33, 1 },
7494         { "TxPktSumSrdy", 32, 1 },
7495         { "RcfUlpType", 28, 4 },
7496         { "Eread", 27, 1 },
7497         { "Ebypass", 26, 1 },
7498         { "Esave", 25, 1 },
7499         { "Static0", 24, 1 },
7500         { "Cread", 23, 1 },
7501         { "Cbypass", 22, 1 },
7502         { "Csave", 21, 1 },
7503         { "CPktOut", 20, 1 },
7504         { "RxPagePoolFull", 18, 2 },
7505         { "RxLpbkPkt", 17, 1 },
7506         { "TxLpbkPkt", 16, 1 },
7507         { "RxVfValid", 15, 1 },
7508         { "SynLearned", 14, 1 },
7509         { "SetDelEntry", 13, 1 },
7510         { "SetInvEntry", 12, 1 },
7511         { "CpcmdDvld", 11, 1 },
7512         { "CpcmdSave", 10, 1 },
7513         { "RxPstructsFull", 8, 2 },
7514         { "EpcmdDvld", 7, 1 },
7515         { "EpcmdFlush", 6, 1 },
7516         { "EpcmdTrimPrefix", 5, 1 },
7517         { "EpcmdTrimPostfix", 4, 1 },
7518         { "ERssIp4Pkt", 3, 1 },
7519         { "ERssIp6Pkt", 2, 1 },
7520         { "ERssTcpUdpPkt", 1, 1 },
7521         { "ERssFceFipPkt", 0, 1 },
7522         { NULL }
7523 };
7524
7525 static void
7526 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
7527 {
7528
7529         field_desc_show(sb, *p, tp_la0);
7530 }
7531
7532 static void
7533 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
7534 {
7535
7536         if (idx)
7537                 sbuf_printf(sb, "\n");
7538         field_desc_show(sb, p[0], tp_la0);
7539         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7540                 field_desc_show(sb, p[1], tp_la0);
7541 }
7542
7543 static void
7544 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
7545 {
7546
7547         if (idx)
7548                 sbuf_printf(sb, "\n");
7549         field_desc_show(sb, p[0], tp_la0);
7550         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7551                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
7552 }
7553
7554 static int
7555 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
7556 {
7557         struct adapter *sc = arg1;
7558         struct sbuf *sb;
7559         uint64_t *buf, *p;
7560         int rc;
7561         u_int i, inc;
7562         void (*show_func)(struct sbuf *, uint64_t *, int);
7563
7564         rc = sysctl_wire_old_buffer(req, 0);
7565         if (rc != 0)
7566                 return (rc);
7567
7568         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7569         if (sb == NULL)
7570                 return (ENOMEM);
7571
7572         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
7573
7574         t4_tp_read_la(sc, buf, NULL);
7575         p = buf;
7576
7577         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
7578         case 2:
7579                 inc = 2;
7580                 show_func = tp_la_show2;
7581                 break;
7582         case 3:
7583                 inc = 2;
7584                 show_func = tp_la_show3;
7585                 break;
7586         default:
7587                 inc = 1;
7588                 show_func = tp_la_show;
7589         }
7590
7591         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
7592                 (*show_func)(sb, p, i);
7593
7594         rc = sbuf_finish(sb);
7595         sbuf_delete(sb);
7596         free(buf, M_CXGBE);
7597         return (rc);
7598 }
7599
7600 static int
7601 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
7602 {
7603         struct adapter *sc = arg1;
7604         struct sbuf *sb;
7605         int rc;
7606         u64 nrate[MAX_NCHAN], orate[MAX_NCHAN];
7607
7608         rc = sysctl_wire_old_buffer(req, 0);
7609         if (rc != 0)
7610                 return (rc);
7611
7612         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7613         if (sb == NULL)
7614                 return (ENOMEM);
7615
7616         t4_get_chan_txrate(sc, nrate, orate);
7617
7618         if (sc->chip_params->nchan > 2) {
7619                 sbuf_printf(sb, "              channel 0   channel 1"
7620                     "   channel 2   channel 3\n");
7621                 sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
7622                     nrate[0], nrate[1], nrate[2], nrate[3]);
7623                 sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
7624                     orate[0], orate[1], orate[2], orate[3]);
7625         } else {
7626                 sbuf_printf(sb, "              channel 0   channel 1\n");
7627                 sbuf_printf(sb, "NIC B/s:     %10ju  %10ju\n",
7628                     nrate[0], nrate[1]);
7629                 sbuf_printf(sb, "Offload B/s: %10ju  %10ju",
7630                     orate[0], orate[1]);
7631         }
7632
7633         rc = sbuf_finish(sb);
7634         sbuf_delete(sb);
7635
7636         return (rc);
7637 }
7638
7639 static int
7640 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
7641 {
7642         struct adapter *sc = arg1;
7643         struct sbuf *sb;
7644         uint32_t *buf, *p;
7645         int rc, i;
7646
7647         rc = sysctl_wire_old_buffer(req, 0);
7648         if (rc != 0)
7649                 return (rc);
7650
7651         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7652         if (sb == NULL)
7653                 return (ENOMEM);
7654
7655         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
7656             M_ZERO | M_WAITOK);
7657
7658         t4_ulprx_read_la(sc, buf);
7659         p = buf;
7660
7661         sbuf_printf(sb, "      Pcmd        Type   Message"
7662             "                Data");
7663         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
7664                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
7665                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
7666         }
7667
7668         rc = sbuf_finish(sb);
7669         sbuf_delete(sb);
7670         free(buf, M_CXGBE);
7671         return (rc);
7672 }
7673
7674 static int
7675 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
7676 {
7677         struct adapter *sc = arg1;
7678         struct sbuf *sb;
7679         int rc, v;
7680
7681         MPASS(chip_id(sc) >= CHELSIO_T5);
7682
7683         rc = sysctl_wire_old_buffer(req, 0);
7684         if (rc != 0)
7685                 return (rc);
7686
7687         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7688         if (sb == NULL)
7689                 return (ENOMEM);
7690
7691         v = t4_read_reg(sc, A_SGE_STAT_CFG);
7692         if (G_STATSOURCE_T5(v) == 7) {
7693                 int mode;
7694
7695                 mode = is_t5(sc) ? G_STATMODE(v) : G_T6_STATMODE(v);
7696                 if (mode == 0) {
7697                         sbuf_printf(sb, "total %d, incomplete %d",
7698                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7699                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7700                 } else if (mode == 1) {
7701                         sbuf_printf(sb, "total %d, data overflow %d",
7702                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7703                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7704                 } else {
7705                         sbuf_printf(sb, "unknown mode %d", mode);
7706                 }
7707         }
7708         rc = sbuf_finish(sb);
7709         sbuf_delete(sb);
7710
7711         return (rc);
7712 }
7713
7714 static int
7715 sysctl_tc_params(SYSCTL_HANDLER_ARGS)
7716 {
7717         struct adapter *sc = arg1;
7718         struct tx_sched_class *tc;
7719         struct t4_sched_class_params p;
7720         struct sbuf *sb;
7721         int i, rc, port_id, flags, mbps, gbps;
7722
7723         rc = sysctl_wire_old_buffer(req, 0);
7724         if (rc != 0)
7725                 return (rc);
7726
7727         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7728         if (sb == NULL)
7729                 return (ENOMEM);
7730
7731         port_id = arg2 >> 16;
7732         MPASS(port_id < sc->params.nports);
7733         MPASS(sc->port[port_id] != NULL);
7734         i = arg2 & 0xffff;
7735         MPASS(i < sc->chip_params->nsched_cls);
7736         tc = &sc->port[port_id]->tc[i];
7737
7738         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7739             "t4tc_p");
7740         if (rc)
7741                 goto done;
7742         flags = tc->flags;
7743         p = tc->params;
7744         end_synchronized_op(sc, LOCK_HELD);
7745
7746         if ((flags & TX_SC_OK) == 0) {
7747                 sbuf_printf(sb, "none");
7748                 goto done;
7749         }
7750
7751         if (p.level == SCHED_CLASS_LEVEL_CL_WRR) {
7752                 sbuf_printf(sb, "cl-wrr weight %u", p.weight);
7753                 goto done;
7754         } else if (p.level == SCHED_CLASS_LEVEL_CL_RL)
7755                 sbuf_printf(sb, "cl-rl");
7756         else if (p.level == SCHED_CLASS_LEVEL_CH_RL)
7757                 sbuf_printf(sb, "ch-rl");
7758         else {
7759                 rc = ENXIO;
7760                 goto done;
7761         }
7762
7763         if (p.ratemode == SCHED_CLASS_RATEMODE_REL) {
7764                 /* XXX: top speed or actual link speed? */
7765                 gbps = port_top_speed(sc->port[port_id]);
7766                 sbuf_printf(sb, " %u%% of %uGbps", p.maxrate, gbps);
7767         }
7768         else if (p.ratemode == SCHED_CLASS_RATEMODE_ABS) {
7769                 switch (p.rateunit) {
7770                 case SCHED_CLASS_RATEUNIT_BITS:
7771                         mbps = p.maxrate / 1000;
7772                         gbps = p.maxrate / 1000000;
7773                         if (p.maxrate == gbps * 1000000)
7774                                 sbuf_printf(sb, " %uGbps", gbps);
7775                         else if (p.maxrate == mbps * 1000)
7776                                 sbuf_printf(sb, " %uMbps", mbps);
7777                         else
7778                                 sbuf_printf(sb, " %uKbps", p.maxrate);
7779                         break;
7780                 case SCHED_CLASS_RATEUNIT_PKTS:
7781                         sbuf_printf(sb, " %upps", p.maxrate);
7782                         break;
7783                 default:
7784                         rc = ENXIO;
7785                         goto done;
7786                 }
7787         }
7788
7789         switch (p.mode) {
7790         case SCHED_CLASS_MODE_CLASS:
7791                 sbuf_printf(sb, " aggregate");
7792                 break;
7793         case SCHED_CLASS_MODE_FLOW:
7794                 sbuf_printf(sb, " per-flow");
7795                 break;
7796         default:
7797                 rc = ENXIO;
7798                 goto done;
7799         }
7800
7801 done:
7802         if (rc == 0)
7803                 rc = sbuf_finish(sb);
7804         sbuf_delete(sb);
7805
7806         return (rc);
7807 }
7808 #endif
7809
7810 #ifdef TCP_OFFLOAD
7811 static void
7812 unit_conv(char *buf, size_t len, u_int val, u_int factor)
7813 {
7814         u_int rem = val % factor;
7815
7816         if (rem == 0)
7817                 snprintf(buf, len, "%u", val / factor);
7818         else {
7819                 while (rem % 10 == 0)
7820                         rem /= 10;
7821                 snprintf(buf, len, "%u.%u", val / factor, rem);
7822         }
7823 }
7824
7825 static int
7826 sysctl_tp_tick(SYSCTL_HANDLER_ARGS)
7827 {
7828         struct adapter *sc = arg1;
7829         char buf[16];
7830         u_int res, re;
7831         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
7832
7833         res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
7834         switch (arg2) {
7835         case 0:
7836                 /* timer_tick */
7837                 re = G_TIMERRESOLUTION(res);
7838                 break;
7839         case 1:
7840                 /* TCP timestamp tick */
7841                 re = G_TIMESTAMPRESOLUTION(res);
7842                 break;
7843         case 2:
7844                 /* DACK tick */
7845                 re = G_DELAYEDACKRESOLUTION(res);
7846                 break;
7847         default:
7848                 return (EDOOFUS);
7849         }
7850
7851         unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000);
7852
7853         return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
7854 }
7855
7856 static int
7857 sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS)
7858 {
7859         struct adapter *sc = arg1;
7860         u_int res, dack_re, v;
7861         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
7862
7863         res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
7864         dack_re = G_DELAYEDACKRESOLUTION(res);
7865         v = ((cclk_ps << dack_re) / 1000000) * t4_read_reg(sc, A_TP_DACK_TIMER);
7866
7867         return (sysctl_handle_int(oidp, &v, 0, req));
7868 }
7869
7870 static int
7871 sysctl_tp_timer(SYSCTL_HANDLER_ARGS)
7872 {
7873         struct adapter *sc = arg1;
7874         int reg = arg2;
7875         u_int tre;
7876         u_long tp_tick_us, v;
7877         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
7878
7879         MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX ||
7880             reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX ||
7881             reg == A_TP_KEEP_IDLE || A_TP_KEEP_INTVL || reg == A_TP_INIT_SRTT ||
7882             reg == A_TP_FINWAIT2_TIMER);
7883
7884         tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION));
7885         tp_tick_us = (cclk_ps << tre) / 1000000;
7886
7887         if (reg == A_TP_INIT_SRTT)
7888                 v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg));
7889         else
7890                 v = tp_tick_us * t4_read_reg(sc, reg);
7891
7892         return (sysctl_handle_long(oidp, &v, 0, req));
7893 }
7894 #endif
7895
7896 static uint32_t
7897 fconf_iconf_to_mode(uint32_t fconf, uint32_t iconf)
7898 {
7899         uint32_t mode;
7900
7901         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
7902             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
7903
7904         if (fconf & F_FRAGMENTATION)
7905                 mode |= T4_FILTER_IP_FRAGMENT;
7906
7907         if (fconf & F_MPSHITTYPE)
7908                 mode |= T4_FILTER_MPS_HIT_TYPE;
7909
7910         if (fconf & F_MACMATCH)
7911                 mode |= T4_FILTER_MAC_IDX;
7912
7913         if (fconf & F_ETHERTYPE)
7914                 mode |= T4_FILTER_ETH_TYPE;
7915
7916         if (fconf & F_PROTOCOL)
7917                 mode |= T4_FILTER_IP_PROTO;
7918
7919         if (fconf & F_TOS)
7920                 mode |= T4_FILTER_IP_TOS;
7921
7922         if (fconf & F_VLAN)
7923                 mode |= T4_FILTER_VLAN;
7924
7925         if (fconf & F_VNIC_ID) {
7926                 mode |= T4_FILTER_VNIC;
7927                 if (iconf & F_VNIC)
7928                         mode |= T4_FILTER_IC_VNIC;
7929         }
7930
7931         if (fconf & F_PORT)
7932                 mode |= T4_FILTER_PORT;
7933
7934         if (fconf & F_FCOE)
7935                 mode |= T4_FILTER_FCoE;
7936
7937         return (mode);
7938 }
7939
7940 static uint32_t
7941 mode_to_fconf(uint32_t mode)
7942 {
7943         uint32_t fconf = 0;
7944
7945         if (mode & T4_FILTER_IP_FRAGMENT)
7946                 fconf |= F_FRAGMENTATION;
7947
7948         if (mode & T4_FILTER_MPS_HIT_TYPE)
7949                 fconf |= F_MPSHITTYPE;
7950
7951         if (mode & T4_FILTER_MAC_IDX)
7952                 fconf |= F_MACMATCH;
7953
7954         if (mode & T4_FILTER_ETH_TYPE)
7955                 fconf |= F_ETHERTYPE;
7956
7957         if (mode & T4_FILTER_IP_PROTO)
7958                 fconf |= F_PROTOCOL;
7959
7960         if (mode & T4_FILTER_IP_TOS)
7961                 fconf |= F_TOS;
7962
7963         if (mode & T4_FILTER_VLAN)
7964                 fconf |= F_VLAN;
7965
7966         if (mode & T4_FILTER_VNIC)
7967                 fconf |= F_VNIC_ID;
7968
7969         if (mode & T4_FILTER_PORT)
7970                 fconf |= F_PORT;
7971
7972         if (mode & T4_FILTER_FCoE)
7973                 fconf |= F_FCOE;
7974
7975         return (fconf);
7976 }
7977
7978 static uint32_t
7979 mode_to_iconf(uint32_t mode)
7980 {
7981
7982         if (mode & T4_FILTER_IC_VNIC)
7983                 return (F_VNIC);
7984         return (0);
7985 }
7986
7987 static int check_fspec_against_fconf_iconf(struct adapter *sc,
7988     struct t4_filter_specification *fs)
7989 {
7990         struct tp_params *tpp = &sc->params.tp;
7991         uint32_t fconf = 0;
7992
7993         if (fs->val.frag || fs->mask.frag)
7994                 fconf |= F_FRAGMENTATION;
7995
7996         if (fs->val.matchtype || fs->mask.matchtype)
7997                 fconf |= F_MPSHITTYPE;
7998
7999         if (fs->val.macidx || fs->mask.macidx)
8000                 fconf |= F_MACMATCH;
8001
8002         if (fs->val.ethtype || fs->mask.ethtype)
8003                 fconf |= F_ETHERTYPE;
8004
8005         if (fs->val.proto || fs->mask.proto)
8006                 fconf |= F_PROTOCOL;
8007
8008         if (fs->val.tos || fs->mask.tos)
8009                 fconf |= F_TOS;
8010
8011         if (fs->val.vlan_vld || fs->mask.vlan_vld)
8012                 fconf |= F_VLAN;
8013
8014         if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
8015                 fconf |= F_VNIC_ID;
8016                 if (tpp->ingress_config & F_VNIC)
8017                         return (EINVAL);
8018         }
8019
8020         if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
8021                 fconf |= F_VNIC_ID;
8022                 if ((tpp->ingress_config & F_VNIC) == 0)
8023                         return (EINVAL);
8024         }
8025
8026         if (fs->val.iport || fs->mask.iport)
8027                 fconf |= F_PORT;
8028
8029         if (fs->val.fcoe || fs->mask.fcoe)
8030                 fconf |= F_FCOE;
8031
8032         if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map)
8033                 return (E2BIG);
8034
8035         return (0);
8036 }
8037
8038 static int
8039 get_filter_mode(struct adapter *sc, uint32_t *mode)
8040 {
8041         struct tp_params *tpp = &sc->params.tp;
8042
8043         /*
8044          * We trust the cached values of the relevant TP registers.  This means
8045          * things work reliably only if writes to those registers are always via
8046          * t4_set_filter_mode.
8047          */
8048         *mode = fconf_iconf_to_mode(tpp->vlan_pri_map, tpp->ingress_config);
8049
8050         return (0);
8051 }
8052
8053 static int
8054 set_filter_mode(struct adapter *sc, uint32_t mode)
8055 {
8056         struct tp_params *tpp = &sc->params.tp;
8057         uint32_t fconf, iconf;
8058         int rc;
8059
8060         iconf = mode_to_iconf(mode);
8061         if ((iconf ^ tpp->ingress_config) & F_VNIC) {
8062                 /*
8063                  * For now we just complain if A_TP_INGRESS_CONFIG is not
8064                  * already set to the correct value for the requested filter
8065                  * mode.  It's not clear if it's safe to write to this register
8066                  * on the fly.  (And we trust the cached value of the register).
8067                  */
8068                 return (EBUSY);
8069         }
8070
8071         fconf = mode_to_fconf(mode);
8072
8073         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
8074             "t4setfm");
8075         if (rc)
8076                 return (rc);
8077
8078         if (sc->tids.ftids_in_use > 0) {
8079                 rc = EBUSY;
8080                 goto done;
8081         }
8082
8083 #ifdef TCP_OFFLOAD
8084         if (uld_active(sc, ULD_TOM)) {
8085                 rc = EBUSY;
8086                 goto done;
8087         }
8088 #endif
8089
8090         rc = -t4_set_filter_mode(sc, fconf);
8091 done:
8092         end_synchronized_op(sc, LOCK_HELD);
8093         return (rc);
8094 }
8095
8096 static inline uint64_t
8097 get_filter_hits(struct adapter *sc, uint32_t fid)
8098 {
8099         uint32_t tcb_addr;
8100
8101         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) +
8102             (fid + sc->tids.ftid_base) * TCB_SIZE;
8103
8104         if (is_t4(sc)) {
8105                 uint64_t hits;
8106
8107                 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8);
8108                 return (be64toh(hits));
8109         } else {
8110                 uint32_t hits;
8111
8112                 read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4);
8113                 return (be32toh(hits));
8114         }
8115 }
8116
8117 static int
8118 get_filter(struct adapter *sc, struct t4_filter *t)
8119 {
8120         int i, rc, nfilters = sc->tids.nftids;
8121         struct filter_entry *f;
8122
8123         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
8124             "t4getf");
8125         if (rc)
8126                 return (rc);
8127
8128         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
8129             t->idx >= nfilters) {
8130                 t->idx = 0xffffffff;
8131                 goto done;
8132         }
8133
8134         f = &sc->tids.ftid_tab[t->idx];
8135         for (i = t->idx; i < nfilters; i++, f++) {
8136                 if (f->valid) {
8137                         t->idx = i;
8138                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
8139                         t->smtidx = f->smtidx;
8140                         if (f->fs.hitcnts)
8141                                 t->hits = get_filter_hits(sc, t->idx);
8142                         else
8143                                 t->hits = UINT64_MAX;
8144                         t->fs = f->fs;
8145
8146                         goto done;
8147                 }
8148         }
8149
8150         t->idx = 0xffffffff;
8151 done:
8152         end_synchronized_op(sc, LOCK_HELD);
8153         return (0);
8154 }
8155
8156 static int
8157 set_filter(struct adapter *sc, struct t4_filter *t)
8158 {
8159         unsigned int nfilters, nports;
8160         struct filter_entry *f;
8161         int i, rc;
8162
8163         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
8164         if (rc)
8165                 return (rc);
8166
8167         nfilters = sc->tids.nftids;
8168         nports = sc->params.nports;
8169
8170         if (nfilters == 0) {
8171                 rc = ENOTSUP;
8172                 goto done;
8173         }
8174
8175         if (t->idx >= nfilters) {
8176                 rc = EINVAL;
8177                 goto done;
8178         }
8179
8180         /* Validate against the global filter mode and ingress config */
8181         rc = check_fspec_against_fconf_iconf(sc, &t->fs);
8182         if (rc != 0)
8183                 goto done;
8184
8185         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
8186                 rc = EINVAL;
8187                 goto done;
8188         }
8189
8190         if (t->fs.val.iport >= nports) {
8191                 rc = EINVAL;
8192                 goto done;
8193         }
8194
8195         /* Can't specify an iq if not steering to it */
8196         if (!t->fs.dirsteer && t->fs.iq) {
8197                 rc = EINVAL;
8198                 goto done;
8199         }
8200
8201         /* IPv6 filter idx must be 4 aligned */
8202         if (t->fs.type == 1 &&
8203             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
8204                 rc = EINVAL;
8205                 goto done;
8206         }
8207
8208         if (!(sc->flags & FULL_INIT_DONE) &&
8209             ((rc = adapter_full_init(sc)) != 0))
8210                 goto done;
8211
8212         if (sc->tids.ftid_tab == NULL) {
8213                 KASSERT(sc->tids.ftids_in_use == 0,
8214                     ("%s: no memory allocated but filters_in_use > 0",
8215                     __func__));
8216
8217                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
8218                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
8219                 if (sc->tids.ftid_tab == NULL) {
8220                         rc = ENOMEM;
8221                         goto done;
8222                 }
8223                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
8224         }
8225
8226         for (i = 0; i < 4; i++) {
8227                 f = &sc->tids.ftid_tab[t->idx + i];
8228
8229                 if (f->pending || f->valid) {
8230                         rc = EBUSY;
8231                         goto done;
8232                 }
8233                 if (f->locked) {
8234                         rc = EPERM;
8235                         goto done;
8236                 }
8237
8238                 if (t->fs.type == 0)
8239                         break;
8240         }
8241
8242         f = &sc->tids.ftid_tab[t->idx];
8243         f->fs = t->fs;
8244
8245         rc = set_filter_wr(sc, t->idx);
8246 done:
8247         end_synchronized_op(sc, 0);
8248
8249         if (rc == 0) {
8250                 mtx_lock(&sc->tids.ftid_lock);
8251                 for (;;) {
8252                         if (f->pending == 0) {
8253                                 rc = f->valid ? 0 : EIO;
8254                                 break;
8255                         }
8256
8257                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
8258                             PCATCH, "t4setfw", 0)) {
8259                                 rc = EINPROGRESS;
8260                                 break;
8261                         }
8262                 }
8263                 mtx_unlock(&sc->tids.ftid_lock);
8264         }
8265         return (rc);
8266 }
8267
8268 static int
8269 del_filter(struct adapter *sc, struct t4_filter *t)
8270 {
8271         unsigned int nfilters;
8272         struct filter_entry *f;
8273         int rc;
8274
8275         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
8276         if (rc)
8277                 return (rc);
8278
8279         nfilters = sc->tids.nftids;
8280
8281         if (nfilters == 0) {
8282                 rc = ENOTSUP;
8283                 goto done;
8284         }
8285
8286         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
8287             t->idx >= nfilters) {
8288                 rc = EINVAL;
8289                 goto done;
8290         }
8291
8292         if (!(sc->flags & FULL_INIT_DONE)) {
8293                 rc = EAGAIN;
8294                 goto done;
8295         }
8296
8297         f = &sc->tids.ftid_tab[t->idx];
8298
8299         if (f->pending) {
8300                 rc = EBUSY;
8301                 goto done;
8302         }
8303         if (f->locked) {
8304                 rc = EPERM;
8305                 goto done;
8306         }
8307
8308         if (f->valid) {
8309                 t->fs = f->fs;  /* extra info for the caller */
8310                 rc = del_filter_wr(sc, t->idx);
8311         }
8312
8313 done:
8314         end_synchronized_op(sc, 0);
8315
8316         if (rc == 0) {
8317                 mtx_lock(&sc->tids.ftid_lock);
8318                 for (;;) {
8319                         if (f->pending == 0) {
8320                                 rc = f->valid ? EIO : 0;
8321                                 break;
8322                         }
8323
8324                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
8325                             PCATCH, "t4delfw", 0)) {
8326                                 rc = EINPROGRESS;
8327                                 break;
8328                         }
8329                 }
8330                 mtx_unlock(&sc->tids.ftid_lock);
8331         }
8332
8333         return (rc);
8334 }
8335
8336 static void
8337 clear_filter(struct filter_entry *f)
8338 {
8339         if (f->l2t)
8340                 t4_l2t_release(f->l2t);
8341
8342         bzero(f, sizeof (*f));
8343 }
8344
8345 static int
8346 set_filter_wr(struct adapter *sc, int fidx)
8347 {
8348         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
8349         struct fw_filter_wr *fwr;
8350         unsigned int ftid, vnic_vld, vnic_vld_mask;
8351         struct wrq_cookie cookie;
8352
8353         ASSERT_SYNCHRONIZED_OP(sc);
8354
8355         if (f->fs.newdmac || f->fs.newvlan) {
8356                 /* This filter needs an L2T entry; allocate one. */
8357                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
8358                 if (f->l2t == NULL)
8359                         return (EAGAIN);
8360                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
8361                     f->fs.dmac)) {
8362                         t4_l2t_release(f->l2t);
8363                         f->l2t = NULL;
8364                         return (ENOMEM);
8365                 }
8366         }
8367
8368         /* Already validated against fconf, iconf */
8369         MPASS((f->fs.val.pfvf_vld & f->fs.val.ovlan_vld) == 0);
8370         MPASS((f->fs.mask.pfvf_vld & f->fs.mask.ovlan_vld) == 0);
8371         if (f->fs.val.pfvf_vld || f->fs.val.ovlan_vld)
8372                 vnic_vld = 1;
8373         else
8374                 vnic_vld = 0;
8375         if (f->fs.mask.pfvf_vld || f->fs.mask.ovlan_vld)
8376                 vnic_vld_mask = 1;
8377         else
8378                 vnic_vld_mask = 0;
8379
8380         ftid = sc->tids.ftid_base + fidx;
8381
8382         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8383         if (fwr == NULL)
8384                 return (ENOMEM);
8385         bzero(fwr, sizeof(*fwr));
8386
8387         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
8388         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
8389         fwr->tid_to_iq =
8390             htobe32(V_FW_FILTER_WR_TID(ftid) |
8391                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
8392                 V_FW_FILTER_WR_NOREPLY(0) |
8393                 V_FW_FILTER_WR_IQ(f->fs.iq));
8394         fwr->del_filter_to_l2tix =
8395             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
8396                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
8397                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
8398                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
8399                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
8400                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
8401                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
8402                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
8403                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
8404                     f->fs.newvlan == VLAN_REWRITE) |
8405                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
8406                     f->fs.newvlan == VLAN_REWRITE) |
8407                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
8408                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
8409                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
8410                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
8411         fwr->ethtype = htobe16(f->fs.val.ethtype);
8412         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
8413         fwr->frag_to_ovlan_vldm =
8414             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
8415                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
8416                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
8417                 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
8418                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
8419                 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
8420         fwr->smac_sel = 0;
8421         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
8422             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
8423         fwr->maci_to_matchtypem =
8424             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
8425                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
8426                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
8427                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
8428                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
8429                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
8430                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
8431                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
8432         fwr->ptcl = f->fs.val.proto;
8433         fwr->ptclm = f->fs.mask.proto;
8434         fwr->ttyp = f->fs.val.tos;
8435         fwr->ttypm = f->fs.mask.tos;
8436         fwr->ivlan = htobe16(f->fs.val.vlan);
8437         fwr->ivlanm = htobe16(f->fs.mask.vlan);
8438         fwr->ovlan = htobe16(f->fs.val.vnic);
8439         fwr->ovlanm = htobe16(f->fs.mask.vnic);
8440         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
8441         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
8442         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
8443         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
8444         fwr->lp = htobe16(f->fs.val.dport);
8445         fwr->lpm = htobe16(f->fs.mask.dport);
8446         fwr->fp = htobe16(f->fs.val.sport);
8447         fwr->fpm = htobe16(f->fs.mask.sport);
8448         if (f->fs.newsmac)
8449                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
8450
8451         f->pending = 1;
8452         sc->tids.ftids_in_use++;
8453
8454         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8455         return (0);
8456 }
8457
8458 static int
8459 del_filter_wr(struct adapter *sc, int fidx)
8460 {
8461         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
8462         struct fw_filter_wr *fwr;
8463         unsigned int ftid;
8464         struct wrq_cookie cookie;
8465
8466         ftid = sc->tids.ftid_base + fidx;
8467
8468         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8469         if (fwr == NULL)
8470                 return (ENOMEM);
8471         bzero(fwr, sizeof (*fwr));
8472
8473         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
8474
8475         f->pending = 1;
8476         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8477         return (0);
8478 }
8479
8480 int
8481 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8482 {
8483         struct adapter *sc = iq->adapter;
8484         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
8485         unsigned int idx = GET_TID(rpl);
8486         unsigned int rc;
8487         struct filter_entry *f;
8488
8489         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
8490             rss->opcode));
8491         MPASS(iq == &sc->sge.fwq);
8492         MPASS(is_ftid(sc, idx));
8493
8494         idx -= sc->tids.ftid_base;
8495         f = &sc->tids.ftid_tab[idx];
8496         rc = G_COOKIE(rpl->cookie);
8497
8498         mtx_lock(&sc->tids.ftid_lock);
8499         if (rc == FW_FILTER_WR_FLT_ADDED) {
8500                 KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
8501                     __func__, idx));
8502                 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
8503                 f->pending = 0;  /* asynchronous setup completed */
8504                 f->valid = 1;
8505         } else {
8506                 if (rc != FW_FILTER_WR_FLT_DELETED) {
8507                         /* Add or delete failed, display an error */
8508                         log(LOG_ERR,
8509                             "filter %u setup failed with error %u\n",
8510                             idx, rc);
8511                 }
8512
8513                 clear_filter(f);
8514                 sc->tids.ftids_in_use--;
8515         }
8516         wakeup(&sc->tids.ftid_tab);
8517         mtx_unlock(&sc->tids.ftid_lock);
8518
8519         return (0);
8520 }
8521
8522 static int
8523 set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8524 {
8525
8526         MPASS(iq->set_tcb_rpl != NULL);
8527         return (iq->set_tcb_rpl(iq, rss, m));
8528 }
8529
8530 static int
8531 l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8532 {
8533
8534         MPASS(iq->l2t_write_rpl != NULL);
8535         return (iq->l2t_write_rpl(iq, rss, m));
8536 }
8537
8538 static int
8539 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
8540 {
8541         int rc;
8542
8543         if (cntxt->cid > M_CTXTQID)
8544                 return (EINVAL);
8545
8546         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
8547             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
8548                 return (EINVAL);
8549
8550         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
8551         if (rc)
8552                 return (rc);
8553
8554         if (sc->flags & FW_OK) {
8555                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
8556                     &cntxt->data[0]);
8557                 if (rc == 0)
8558                         goto done;
8559         }
8560
8561         /*
8562          * Read via firmware failed or wasn't even attempted.  Read directly via
8563          * the backdoor.
8564          */
8565         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
8566 done:
8567         end_synchronized_op(sc, 0);
8568         return (rc);
8569 }
8570
8571 static int
8572 load_fw(struct adapter *sc, struct t4_data *fw)
8573 {
8574         int rc;
8575         uint8_t *fw_data;
8576
8577         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
8578         if (rc)
8579                 return (rc);
8580
8581         if (sc->flags & FULL_INIT_DONE) {
8582                 rc = EBUSY;
8583                 goto done;
8584         }
8585
8586         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
8587         if (fw_data == NULL) {
8588                 rc = ENOMEM;
8589                 goto done;
8590         }
8591
8592         rc = copyin(fw->data, fw_data, fw->len);
8593         if (rc == 0)
8594                 rc = -t4_load_fw(sc, fw_data, fw->len);
8595
8596         free(fw_data, M_CXGBE);
8597 done:
8598         end_synchronized_op(sc, 0);
8599         return (rc);
8600 }
8601
8602 static int
8603 load_cfg(struct adapter *sc, struct t4_data *cfg)
8604 {
8605         int rc;
8606         uint8_t *cfg_data = NULL;
8607
8608         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf");
8609         if (rc)
8610                 return (rc);
8611
8612         if (cfg->len == 0) {
8613                 /* clear */
8614                 rc = -t4_load_cfg(sc, NULL, 0);
8615                 goto done;
8616         }
8617
8618         cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK);
8619         if (cfg_data == NULL) {
8620                 rc = ENOMEM;
8621                 goto done;
8622         }
8623
8624         rc = copyin(cfg->data, cfg_data, cfg->len);
8625         if (rc == 0)
8626                 rc = -t4_load_cfg(sc, cfg_data, cfg->len);
8627
8628         free(cfg_data, M_CXGBE);
8629 done:
8630         end_synchronized_op(sc, 0);
8631         return (rc);
8632 }
8633
8634 #define MAX_READ_BUF_SIZE (128 * 1024)
8635 static int
8636 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
8637 {
8638         uint32_t addr, remaining, n;
8639         uint32_t *buf;
8640         int rc;
8641         uint8_t *dst;
8642
8643         rc = validate_mem_range(sc, mr->addr, mr->len);
8644         if (rc != 0)
8645                 return (rc);
8646
8647         buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK);
8648         addr = mr->addr;
8649         remaining = mr->len;
8650         dst = (void *)mr->data;
8651
8652         while (remaining) {
8653                 n = min(remaining, MAX_READ_BUF_SIZE);
8654                 read_via_memwin(sc, 2, addr, buf, n);
8655
8656                 rc = copyout(buf, dst, n);
8657                 if (rc != 0)
8658                         break;
8659
8660                 dst += n;
8661                 remaining -= n;
8662                 addr += n;
8663         }
8664
8665         free(buf, M_CXGBE);
8666         return (rc);
8667 }
8668 #undef MAX_READ_BUF_SIZE
8669
8670 static int
8671 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
8672 {
8673         int rc;
8674
8675         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
8676                 return (EINVAL);
8677
8678         if (i2cd->len > sizeof(i2cd->data))
8679                 return (EFBIG);
8680
8681         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
8682         if (rc)
8683                 return (rc);
8684         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
8685             i2cd->offset, i2cd->len, &i2cd->data[0]);
8686         end_synchronized_op(sc, 0);
8687
8688         return (rc);
8689 }
8690
8691 static int
8692 in_range(int val, int lo, int hi)
8693 {
8694
8695         return (val < 0 || (val <= hi && val >= lo));
8696 }
8697
8698 static int
8699 set_sched_class_config(struct adapter *sc, int minmax)
8700 {
8701         int rc;
8702
8703         if (minmax < 0)
8704                 return (EINVAL);
8705
8706         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sscc");
8707         if (rc)
8708                 return (rc);
8709         rc = -t4_sched_config(sc, FW_SCHED_TYPE_PKTSCHED, minmax, 1);
8710         end_synchronized_op(sc, 0);
8711
8712         return (rc);
8713 }
8714
8715 static int
8716 set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p,
8717     int sleep_ok)
8718 {
8719         int rc, top_speed, fw_level, fw_mode, fw_rateunit, fw_ratemode;
8720         struct port_info *pi;
8721         struct tx_sched_class *tc;
8722
8723         if (p->level == SCHED_CLASS_LEVEL_CL_RL)
8724                 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
8725         else if (p->level == SCHED_CLASS_LEVEL_CL_WRR)
8726                 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
8727         else if (p->level == SCHED_CLASS_LEVEL_CH_RL)
8728                 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
8729         else
8730                 return (EINVAL);
8731
8732         if (p->mode == SCHED_CLASS_MODE_CLASS)
8733                 fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
8734         else if (p->mode == SCHED_CLASS_MODE_FLOW)
8735                 fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
8736         else
8737                 return (EINVAL);
8738
8739         if (p->rateunit == SCHED_CLASS_RATEUNIT_BITS)
8740                 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
8741         else if (p->rateunit == SCHED_CLASS_RATEUNIT_PKTS)
8742                 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
8743         else
8744                 return (EINVAL);
8745
8746         if (p->ratemode == SCHED_CLASS_RATEMODE_REL)
8747                 fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
8748         else if (p->ratemode == SCHED_CLASS_RATEMODE_ABS)
8749                 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
8750         else
8751                 return (EINVAL);
8752
8753         /* Vet our parameters ... */
8754         if (!in_range(p->channel, 0, sc->chip_params->nchan - 1))
8755                 return (ERANGE);
8756
8757         pi = sc->port[sc->chan_map[p->channel]];
8758         if (pi == NULL)
8759                 return (ENXIO);
8760         MPASS(pi->tx_chan == p->channel);
8761         top_speed = port_top_speed(pi) * 1000000; /* Gbps -> Kbps */
8762
8763         if (!in_range(p->cl, 0, sc->chip_params->nsched_cls) ||
8764             !in_range(p->minrate, 0, top_speed) ||
8765             !in_range(p->maxrate, 0, top_speed) ||
8766             !in_range(p->weight, 0, 100))
8767                 return (ERANGE);
8768
8769         /*
8770          * Translate any unset parameters into the firmware's
8771          * nomenclature and/or fail the call if the parameters
8772          * are required ...
8773          */
8774         if (p->rateunit < 0 || p->ratemode < 0 || p->channel < 0 || p->cl < 0)
8775                 return (EINVAL);
8776
8777         if (p->minrate < 0)
8778                 p->minrate = 0;
8779         if (p->maxrate < 0) {
8780                 if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
8781                     p->level == SCHED_CLASS_LEVEL_CH_RL)
8782                         return (EINVAL);
8783                 else
8784                         p->maxrate = 0;
8785         }
8786         if (p->weight < 0) {
8787                 if (p->level == SCHED_CLASS_LEVEL_CL_WRR)
8788                         return (EINVAL);
8789                 else
8790                         p->weight = 0;
8791         }
8792         if (p->pktsize < 0) {
8793                 if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
8794                     p->level == SCHED_CLASS_LEVEL_CH_RL)
8795                         return (EINVAL);
8796                 else
8797                         p->pktsize = 0;
8798         }
8799
8800         rc = begin_synchronized_op(sc, NULL,
8801             sleep_ok ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4sscp");
8802         if (rc)
8803                 return (rc);
8804         tc = &pi->tc[p->cl];
8805         tc->params = *p;
8806         rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level, fw_mode,
8807             fw_rateunit, fw_ratemode, p->channel, p->cl, p->minrate, p->maxrate,
8808             p->weight, p->pktsize, sleep_ok);
8809         if (rc == 0)
8810                 tc->flags |= TX_SC_OK;
8811         else {
8812                 /*
8813                  * Unknown state at this point, see tc->params for what was
8814                  * attempted.
8815                  */
8816                 tc->flags &= ~TX_SC_OK;
8817         }
8818         end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD);
8819
8820         return (rc);
8821 }
8822
8823 int
8824 t4_set_sched_class(struct adapter *sc, struct t4_sched_params *p)
8825 {
8826
8827         if (p->type != SCHED_CLASS_TYPE_PACKET)
8828                 return (EINVAL);
8829
8830         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
8831                 return (set_sched_class_config(sc, p->u.config.minmax));
8832
8833         if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
8834                 return (set_sched_class_params(sc, &p->u.params, 1));
8835
8836         return (EINVAL);
8837 }
8838
8839 int
8840 t4_set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
8841 {
8842         struct port_info *pi = NULL;
8843         struct vi_info *vi;
8844         struct sge_txq *txq;
8845         uint32_t fw_mnem, fw_queue, fw_class;
8846         int i, rc;
8847
8848         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
8849         if (rc)
8850                 return (rc);
8851
8852         if (p->port >= sc->params.nports) {
8853                 rc = EINVAL;
8854                 goto done;
8855         }
8856
8857         /* XXX: Only supported for the main VI. */
8858         pi = sc->port[p->port];
8859         vi = &pi->vi[0];
8860         if (!(vi->flags & VI_INIT_DONE)) {
8861                 /* tx queues not set up yet */
8862                 rc = EAGAIN;
8863                 goto done;
8864         }
8865
8866         if (!in_range(p->queue, 0, vi->ntxq - 1) ||
8867             !in_range(p->cl, 0, sc->chip_params->nsched_cls - 1)) {
8868                 rc = EINVAL;
8869                 goto done;
8870         }
8871
8872         /*
8873          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
8874          * Scheduling Class in this case).
8875          */
8876         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
8877             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
8878         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
8879
8880         /*
8881          * If op.queue is non-negative, then we're only changing the scheduling
8882          * on a single specified TX queue.
8883          */
8884         if (p->queue >= 0) {
8885                 txq = &sc->sge.txq[vi->first_txq + p->queue];
8886                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8887                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8888                     &fw_class);
8889                 goto done;
8890         }
8891
8892         /*
8893          * Change the scheduling on all the TX queues for the
8894          * interface.
8895          */
8896         for_each_txq(vi, i, txq) {
8897                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
8898                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
8899                     &fw_class);
8900                 if (rc)
8901                         goto done;
8902         }
8903
8904         rc = 0;
8905 done:
8906         end_synchronized_op(sc, 0);
8907         return (rc);
8908 }
8909
8910 int
8911 t4_os_find_pci_capability(struct adapter *sc, int cap)
8912 {
8913         int i;
8914
8915         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
8916 }
8917
8918 int
8919 t4_os_pci_save_state(struct adapter *sc)
8920 {
8921         device_t dev;
8922         struct pci_devinfo *dinfo;
8923
8924         dev = sc->dev;
8925         dinfo = device_get_ivars(dev);
8926
8927         pci_cfg_save(dev, dinfo, 0);
8928         return (0);
8929 }
8930
8931 int
8932 t4_os_pci_restore_state(struct adapter *sc)
8933 {
8934         device_t dev;
8935         struct pci_devinfo *dinfo;
8936
8937         dev = sc->dev;
8938         dinfo = device_get_ivars(dev);
8939
8940         pci_cfg_restore(dev, dinfo);
8941         return (0);
8942 }
8943
8944 void
8945 t4_os_portmod_changed(const struct adapter *sc, int idx)
8946 {
8947         struct port_info *pi = sc->port[idx];
8948         struct vi_info *vi;
8949         struct ifnet *ifp;
8950         int v;
8951         static const char *mod_str[] = {
8952                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
8953         };
8954
8955         for_each_vi(pi, v, vi) {
8956                 build_medialist(pi, &vi->media);
8957         }
8958
8959         ifp = pi->vi[0].ifp;
8960         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
8961                 if_printf(ifp, "transceiver unplugged.\n");
8962         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
8963                 if_printf(ifp, "unknown transceiver inserted.\n");
8964         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
8965                 if_printf(ifp, "unsupported transceiver inserted.\n");
8966         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
8967                 if_printf(ifp, "%s transceiver inserted.\n",
8968                     mod_str[pi->mod_type]);
8969         } else {
8970                 if_printf(ifp, "transceiver (type %d) inserted.\n",
8971                     pi->mod_type);
8972         }
8973 }
8974
8975 void
8976 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
8977 {
8978         struct port_info *pi = sc->port[idx];
8979         struct vi_info *vi;
8980         struct ifnet *ifp;
8981         int v;
8982
8983         if (link_stat)
8984                 pi->linkdnrc = -1;
8985         else {
8986                 if (reason >= 0)
8987                         pi->linkdnrc = reason;
8988         }
8989         for_each_vi(pi, v, vi) {
8990                 ifp = vi->ifp;
8991                 if (ifp == NULL)
8992                         continue;
8993
8994                 if (link_stat) {
8995                         ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
8996                         if_link_state_change(ifp, LINK_STATE_UP);
8997                 } else {
8998                         if_link_state_change(ifp, LINK_STATE_DOWN);
8999                 }
9000         }
9001 }
9002
9003 void
9004 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
9005 {
9006         struct adapter *sc;
9007
9008         sx_slock(&t4_list_lock);
9009         SLIST_FOREACH(sc, &t4_list, link) {
9010                 /*
9011                  * func should not make any assumptions about what state sc is
9012                  * in - the only guarantee is that sc->sc_lock is a valid lock.
9013                  */
9014                 func(sc, arg);
9015         }
9016         sx_sunlock(&t4_list_lock);
9017 }
9018
9019 static int
9020 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
9021     struct thread *td)
9022 {
9023         int rc;
9024         struct adapter *sc = dev->si_drv1;
9025
9026         rc = priv_check(td, PRIV_DRIVER);
9027         if (rc != 0)
9028                 return (rc);
9029
9030         switch (cmd) {
9031         case CHELSIO_T4_GETREG: {
9032                 struct t4_reg *edata = (struct t4_reg *)data;
9033
9034                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
9035                         return (EFAULT);
9036
9037                 if (edata->size == 4)
9038                         edata->val = t4_read_reg(sc, edata->addr);
9039                 else if (edata->size == 8)
9040                         edata->val = t4_read_reg64(sc, edata->addr);
9041                 else
9042                         return (EINVAL);
9043
9044                 break;
9045         }
9046         case CHELSIO_T4_SETREG: {
9047                 struct t4_reg *edata = (struct t4_reg *)data;
9048
9049                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
9050                         return (EFAULT);
9051
9052                 if (edata->size == 4) {
9053                         if (edata->val & 0xffffffff00000000)
9054                                 return (EINVAL);
9055                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
9056                 } else if (edata->size == 8)
9057                         t4_write_reg64(sc, edata->addr, edata->val);
9058                 else
9059                         return (EINVAL);
9060                 break;
9061         }
9062         case CHELSIO_T4_REGDUMP: {
9063                 struct t4_regdump *regs = (struct t4_regdump *)data;
9064                 int reglen = t4_get_regs_len(sc);
9065                 uint8_t *buf;
9066
9067                 if (regs->len < reglen) {
9068                         regs->len = reglen; /* hint to the caller */
9069                         return (ENOBUFS);
9070                 }
9071
9072                 regs->len = reglen;
9073                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
9074                 get_regs(sc, regs, buf);
9075                 rc = copyout(buf, regs->data, reglen);
9076                 free(buf, M_CXGBE);
9077                 break;
9078         }
9079         case CHELSIO_T4_GET_FILTER_MODE:
9080                 rc = get_filter_mode(sc, (uint32_t *)data);
9081                 break;
9082         case CHELSIO_T4_SET_FILTER_MODE:
9083                 rc = set_filter_mode(sc, *(uint32_t *)data);
9084                 break;
9085         case CHELSIO_T4_GET_FILTER:
9086                 rc = get_filter(sc, (struct t4_filter *)data);
9087                 break;
9088         case CHELSIO_T4_SET_FILTER:
9089                 rc = set_filter(sc, (struct t4_filter *)data);
9090                 break;
9091         case CHELSIO_T4_DEL_FILTER:
9092                 rc = del_filter(sc, (struct t4_filter *)data);
9093                 break;
9094         case CHELSIO_T4_GET_SGE_CONTEXT:
9095                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
9096                 break;
9097         case CHELSIO_T4_LOAD_FW:
9098                 rc = load_fw(sc, (struct t4_data *)data);
9099                 break;
9100         case CHELSIO_T4_GET_MEM:
9101                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
9102                 break;
9103         case CHELSIO_T4_GET_I2C:
9104                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
9105                 break;
9106         case CHELSIO_T4_CLEAR_STATS: {
9107                 int i, v;
9108                 u_int port_id = *(uint32_t *)data;
9109                 struct port_info *pi;
9110                 struct vi_info *vi;
9111
9112                 if (port_id >= sc->params.nports)
9113                         return (EINVAL);
9114                 pi = sc->port[port_id];
9115                 if (pi == NULL)
9116                         return (EIO);
9117
9118                 /* MAC stats */
9119                 t4_clr_port_stats(sc, pi->tx_chan);
9120                 pi->tx_parse_error = 0;
9121                 mtx_lock(&sc->reg_lock);
9122                 for_each_vi(pi, v, vi) {
9123                         if (vi->flags & VI_INIT_DONE)
9124                                 t4_clr_vi_stats(sc, vi->viid);
9125                 }
9126                 mtx_unlock(&sc->reg_lock);
9127
9128                 /*
9129                  * Since this command accepts a port, clear stats for
9130                  * all VIs on this port.
9131                  */
9132                 for_each_vi(pi, v, vi) {
9133                         if (vi->flags & VI_INIT_DONE) {
9134                                 struct sge_rxq *rxq;
9135                                 struct sge_txq *txq;
9136                                 struct sge_wrq *wrq;
9137
9138                                 for_each_rxq(vi, i, rxq) {
9139 #if defined(INET) || defined(INET6)
9140                                         rxq->lro.lro_queued = 0;
9141                                         rxq->lro.lro_flushed = 0;
9142 #endif
9143                                         rxq->rxcsum = 0;
9144                                         rxq->vlan_extraction = 0;
9145                                 }
9146
9147                                 for_each_txq(vi, i, txq) {
9148                                         txq->txcsum = 0;
9149                                         txq->tso_wrs = 0;
9150                                         txq->vlan_insertion = 0;
9151                                         txq->imm_wrs = 0;
9152                                         txq->sgl_wrs = 0;
9153                                         txq->txpkt_wrs = 0;
9154                                         txq->txpkts0_wrs = 0;
9155                                         txq->txpkts1_wrs = 0;
9156                                         txq->txpkts0_pkts = 0;
9157                                         txq->txpkts1_pkts = 0;
9158                                         mp_ring_reset_stats(txq->r);
9159                                 }
9160
9161 #ifdef TCP_OFFLOAD
9162                                 /* nothing to clear for each ofld_rxq */
9163
9164                                 for_each_ofld_txq(vi, i, wrq) {
9165                                         wrq->tx_wrs_direct = 0;
9166                                         wrq->tx_wrs_copied = 0;
9167                                 }
9168 #endif
9169
9170                                 if (IS_MAIN_VI(vi)) {
9171                                         wrq = &sc->sge.ctrlq[pi->port_id];
9172                                         wrq->tx_wrs_direct = 0;
9173                                         wrq->tx_wrs_copied = 0;
9174                                 }
9175                         }
9176                 }
9177                 break;
9178         }
9179         case CHELSIO_T4_SCHED_CLASS:
9180                 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data);
9181                 break;
9182         case CHELSIO_T4_SCHED_QUEUE:
9183                 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data);
9184                 break;
9185         case CHELSIO_T4_GET_TRACER:
9186                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
9187                 break;
9188         case CHELSIO_T4_SET_TRACER:
9189                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
9190                 break;
9191         case CHELSIO_T4_LOAD_CFG:
9192                 rc = load_cfg(sc, (struct t4_data *)data);
9193                 break;
9194         default:
9195                 rc = ENOTTY;
9196         }
9197
9198         return (rc);
9199 }
9200
9201 void
9202 t4_db_full(struct adapter *sc)
9203 {
9204
9205         CXGBE_UNIMPLEMENTED(__func__);
9206 }
9207
9208 void
9209 t4_db_dropped(struct adapter *sc)
9210 {
9211
9212         CXGBE_UNIMPLEMENTED(__func__);
9213 }
9214
9215 #ifdef TCP_OFFLOAD
9216 static int
9217 toe_capability(struct vi_info *vi, int enable)
9218 {
9219         int rc;
9220         struct port_info *pi = vi->pi;
9221         struct adapter *sc = pi->adapter;
9222
9223         ASSERT_SYNCHRONIZED_OP(sc);
9224
9225         if (!is_offload(sc))
9226                 return (ENODEV);
9227
9228         if (enable) {
9229                 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) {
9230                         /* TOE is already enabled. */
9231                         return (0);
9232                 }
9233
9234                 /*
9235                  * We need the port's queues around so that we're able to send
9236                  * and receive CPLs to/from the TOE even if the ifnet for this
9237                  * port has never been UP'd administratively.
9238                  */
9239                 if (!(vi->flags & VI_INIT_DONE)) {
9240                         rc = vi_full_init(vi);
9241                         if (rc)
9242                                 return (rc);
9243                 }
9244                 if (!(pi->vi[0].flags & VI_INIT_DONE)) {
9245                         rc = vi_full_init(&pi->vi[0]);
9246                         if (rc)
9247                                 return (rc);
9248                 }
9249
9250                 if (isset(&sc->offload_map, pi->port_id)) {
9251                         /* TOE is enabled on another VI of this port. */
9252                         pi->uld_vis++;
9253                         return (0);
9254                 }
9255
9256                 if (!uld_active(sc, ULD_TOM)) {
9257                         rc = t4_activate_uld(sc, ULD_TOM);
9258                         if (rc == EAGAIN) {
9259                                 log(LOG_WARNING,
9260                                     "You must kldload t4_tom.ko before trying "
9261                                     "to enable TOE on a cxgbe interface.\n");
9262                         }
9263                         if (rc != 0)
9264                                 return (rc);
9265                         KASSERT(sc->tom_softc != NULL,
9266                             ("%s: TOM activated but softc NULL", __func__));
9267                         KASSERT(uld_active(sc, ULD_TOM),
9268                             ("%s: TOM activated but flag not set", __func__));
9269                 }
9270
9271                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
9272                 if (!uld_active(sc, ULD_IWARP))
9273                         (void) t4_activate_uld(sc, ULD_IWARP);
9274                 if (!uld_active(sc, ULD_ISCSI))
9275                         (void) t4_activate_uld(sc, ULD_ISCSI);
9276
9277                 pi->uld_vis++;
9278                 setbit(&sc->offload_map, pi->port_id);
9279         } else {
9280                 pi->uld_vis--;
9281
9282                 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
9283                         return (0);
9284
9285                 KASSERT(uld_active(sc, ULD_TOM),
9286                     ("%s: TOM never initialized?", __func__));
9287                 clrbit(&sc->offload_map, pi->port_id);
9288         }
9289
9290         return (0);
9291 }
9292
9293 /*
9294  * Add an upper layer driver to the global list.
9295  */
9296 int
9297 t4_register_uld(struct uld_info *ui)
9298 {
9299         int rc = 0;
9300         struct uld_info *u;
9301
9302         sx_xlock(&t4_uld_list_lock);
9303         SLIST_FOREACH(u, &t4_uld_list, link) {
9304             if (u->uld_id == ui->uld_id) {
9305                     rc = EEXIST;
9306                     goto done;
9307             }
9308         }
9309
9310         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
9311         ui->refcount = 0;
9312 done:
9313         sx_xunlock(&t4_uld_list_lock);
9314         return (rc);
9315 }
9316
9317 int
9318 t4_unregister_uld(struct uld_info *ui)
9319 {
9320         int rc = EINVAL;
9321         struct uld_info *u;
9322
9323         sx_xlock(&t4_uld_list_lock);
9324
9325         SLIST_FOREACH(u, &t4_uld_list, link) {
9326             if (u == ui) {
9327                     if (ui->refcount > 0) {
9328                             rc = EBUSY;
9329                             goto done;
9330                     }
9331
9332                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
9333                     rc = 0;
9334                     goto done;
9335             }
9336         }
9337 done:
9338         sx_xunlock(&t4_uld_list_lock);
9339         return (rc);
9340 }
9341
9342 int
9343 t4_activate_uld(struct adapter *sc, int id)
9344 {
9345         int rc;
9346         struct uld_info *ui;
9347
9348         ASSERT_SYNCHRONIZED_OP(sc);
9349
9350         if (id < 0 || id > ULD_MAX)
9351                 return (EINVAL);
9352         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
9353
9354         sx_slock(&t4_uld_list_lock);
9355
9356         SLIST_FOREACH(ui, &t4_uld_list, link) {
9357                 if (ui->uld_id == id) {
9358                         if (!(sc->flags & FULL_INIT_DONE)) {
9359                                 rc = adapter_full_init(sc);
9360                                 if (rc != 0)
9361                                         break;
9362                         }
9363
9364                         rc = ui->activate(sc);
9365                         if (rc == 0) {
9366                                 setbit(&sc->active_ulds, id);
9367                                 ui->refcount++;
9368                         }
9369                         break;
9370                 }
9371         }
9372
9373         sx_sunlock(&t4_uld_list_lock);
9374
9375         return (rc);
9376 }
9377
9378 int
9379 t4_deactivate_uld(struct adapter *sc, int id)
9380 {
9381         int rc;
9382         struct uld_info *ui;
9383
9384         ASSERT_SYNCHRONIZED_OP(sc);
9385
9386         if (id < 0 || id > ULD_MAX)
9387                 return (EINVAL);
9388         rc = ENXIO;
9389
9390         sx_slock(&t4_uld_list_lock);
9391
9392         SLIST_FOREACH(ui, &t4_uld_list, link) {
9393                 if (ui->uld_id == id) {
9394                         rc = ui->deactivate(sc);
9395                         if (rc == 0) {
9396                                 clrbit(&sc->active_ulds, id);
9397                                 ui->refcount--;
9398                         }
9399                         break;
9400                 }
9401         }
9402
9403         sx_sunlock(&t4_uld_list_lock);
9404
9405         return (rc);
9406 }
9407
9408 int
9409 uld_active(struct adapter *sc, int uld_id)
9410 {
9411
9412         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
9413
9414         return (isset(&sc->active_ulds, uld_id));
9415 }
9416 #endif
9417
9418 /*
9419  * Come up with reasonable defaults for some of the tunables, provided they're
9420  * not set by the user (in which case we'll use the values as is).
9421  */
9422 static void
9423 tweak_tunables(void)
9424 {
9425         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
9426
9427         if (t4_ntxq10g < 1) {
9428 #ifdef RSS
9429                 t4_ntxq10g = rss_getnumbuckets();
9430 #else
9431                 t4_ntxq10g = min(nc, NTXQ_10G);
9432 #endif
9433         }
9434
9435         if (t4_ntxq1g < 1) {
9436 #ifdef RSS
9437                 /* XXX: way too many for 1GbE? */
9438                 t4_ntxq1g = rss_getnumbuckets();
9439 #else
9440                 t4_ntxq1g = min(nc, NTXQ_1G);
9441 #endif
9442         }
9443
9444         if (t4_ntxq_vi < 1)
9445                 t4_ntxq_vi = min(nc, NTXQ_VI);
9446
9447         if (t4_nrxq10g < 1) {
9448 #ifdef RSS
9449                 t4_nrxq10g = rss_getnumbuckets();
9450 #else
9451                 t4_nrxq10g = min(nc, NRXQ_10G);
9452 #endif
9453         }
9454
9455         if (t4_nrxq1g < 1) {
9456 #ifdef RSS
9457                 /* XXX: way too many for 1GbE? */
9458                 t4_nrxq1g = rss_getnumbuckets();
9459 #else
9460                 t4_nrxq1g = min(nc, NRXQ_1G);
9461 #endif
9462         }
9463
9464         if (t4_nrxq_vi < 1)
9465                 t4_nrxq_vi = min(nc, NRXQ_VI);
9466
9467 #ifdef TCP_OFFLOAD
9468         if (t4_nofldtxq10g < 1)
9469                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
9470
9471         if (t4_nofldtxq1g < 1)
9472                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
9473
9474         if (t4_nofldtxq_vi < 1)
9475                 t4_nofldtxq_vi = min(nc, NOFLDTXQ_VI);
9476
9477         if (t4_nofldrxq10g < 1)
9478                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
9479
9480         if (t4_nofldrxq1g < 1)
9481                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
9482
9483         if (t4_nofldrxq_vi < 1)
9484                 t4_nofldrxq_vi = min(nc, NOFLDRXQ_VI);
9485
9486         if (t4_toecaps_allowed == -1)
9487                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
9488
9489         if (t4_rdmacaps_allowed == -1) {
9490                 t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP |
9491                     FW_CAPS_CONFIG_RDMA_RDMAC;
9492         }
9493
9494         if (t4_iscsicaps_allowed == -1) {
9495                 t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU |
9496                     FW_CAPS_CONFIG_ISCSI_TARGET_PDU |
9497                     FW_CAPS_CONFIG_ISCSI_T10DIF;
9498         }
9499 #else
9500         if (t4_toecaps_allowed == -1)
9501                 t4_toecaps_allowed = 0;
9502
9503         if (t4_rdmacaps_allowed == -1)
9504                 t4_rdmacaps_allowed = 0;
9505
9506         if (t4_iscsicaps_allowed == -1)
9507                 t4_iscsicaps_allowed = 0;
9508 #endif
9509
9510 #ifdef DEV_NETMAP
9511         if (t4_nnmtxq_vi < 1)
9512                 t4_nnmtxq_vi = min(nc, NNMTXQ_VI);
9513
9514         if (t4_nnmrxq_vi < 1)
9515                 t4_nnmrxq_vi = min(nc, NNMRXQ_VI);
9516 #endif
9517
9518         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
9519                 t4_tmr_idx_10g = TMR_IDX_10G;
9520
9521         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
9522                 t4_pktc_idx_10g = PKTC_IDX_10G;
9523
9524         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
9525                 t4_tmr_idx_1g = TMR_IDX_1G;
9526
9527         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
9528                 t4_pktc_idx_1g = PKTC_IDX_1G;
9529
9530         if (t4_qsize_txq < 128)
9531                 t4_qsize_txq = 128;
9532
9533         if (t4_qsize_rxq < 128)
9534                 t4_qsize_rxq = 128;
9535         while (t4_qsize_rxq & 7)
9536                 t4_qsize_rxq++;
9537
9538         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
9539 }
9540
9541 #ifdef DDB
9542 static void
9543 t4_dump_tcb(struct adapter *sc, int tid)
9544 {
9545         uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos;
9546
9547         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2);
9548         save = t4_read_reg(sc, reg);
9549         base = sc->memwin[2].mw_base;
9550
9551         /* Dump TCB for the tid */
9552         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
9553         tcb_addr += tid * TCB_SIZE;
9554
9555         if (is_t4(sc)) {
9556                 pf = 0;
9557                 win_pos = tcb_addr & ~0xf;      /* start must be 16B aligned */
9558         } else {
9559                 pf = V_PFNUM(sc->pf);
9560                 win_pos = tcb_addr & ~0x7f;     /* start must be 128B aligned */
9561         }
9562         t4_write_reg(sc, reg, win_pos | pf);
9563         t4_read_reg(sc, reg);
9564
9565         off = tcb_addr - win_pos;
9566         for (i = 0; i < 4; i++) {
9567                 uint32_t buf[8];
9568                 for (j = 0; j < 8; j++, off += 4)
9569                         buf[j] = htonl(t4_read_reg(sc, base + off));
9570
9571                 db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n",
9572                     buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
9573                     buf[7]);
9574         }
9575
9576         t4_write_reg(sc, reg, save);
9577         t4_read_reg(sc, reg);
9578 }
9579
9580 static void
9581 t4_dump_devlog(struct adapter *sc)
9582 {
9583         struct devlog_params *dparams = &sc->params.devlog;
9584         struct fw_devlog_e e;
9585         int i, first, j, m, nentries, rc;
9586         uint64_t ftstamp = UINT64_MAX;
9587
9588         if (dparams->start == 0) {
9589                 db_printf("devlog params not valid\n");
9590                 return;
9591         }
9592
9593         nentries = dparams->size / sizeof(struct fw_devlog_e);
9594         m = fwmtype_to_hwmtype(dparams->memtype);
9595
9596         /* Find the first entry. */
9597         first = -1;
9598         for (i = 0; i < nentries && !db_pager_quit; i++) {
9599                 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
9600                     sizeof(e), (void *)&e);
9601                 if (rc != 0)
9602                         break;
9603
9604                 if (e.timestamp == 0)
9605                         break;
9606
9607                 e.timestamp = be64toh(e.timestamp);
9608                 if (e.timestamp < ftstamp) {
9609                         ftstamp = e.timestamp;
9610                         first = i;
9611                 }
9612         }
9613
9614         if (first == -1)
9615                 return;
9616
9617         i = first;
9618         do {
9619                 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
9620                     sizeof(e), (void *)&e);
9621                 if (rc != 0)
9622                         return;
9623
9624                 if (e.timestamp == 0)
9625                         return;
9626
9627                 e.timestamp = be64toh(e.timestamp);
9628                 e.seqno = be32toh(e.seqno);
9629                 for (j = 0; j < 8; j++)
9630                         e.params[j] = be32toh(e.params[j]);
9631
9632                 db_printf("%10d  %15ju  %8s  %8s  ",
9633                     e.seqno, e.timestamp,
9634                     (e.level < nitems(devlog_level_strings) ?
9635                         devlog_level_strings[e.level] : "UNKNOWN"),
9636                     (e.facility < nitems(devlog_facility_strings) ?
9637                         devlog_facility_strings[e.facility] : "UNKNOWN"));
9638                 db_printf(e.fmt, e.params[0], e.params[1], e.params[2],
9639                     e.params[3], e.params[4], e.params[5], e.params[6],
9640                     e.params[7]);
9641
9642                 if (++i == nentries)
9643                         i = 0;
9644         } while (i != first && !db_pager_quit);
9645 }
9646
9647 static struct command_table db_t4_table = LIST_HEAD_INITIALIZER(db_t4_table);
9648 _DB_SET(_show, t4, NULL, db_show_table, 0, &db_t4_table);
9649
9650 DB_FUNC(devlog, db_show_devlog, db_t4_table, CS_OWN, NULL)
9651 {
9652         device_t dev;
9653         int t;
9654         bool valid;
9655
9656         valid = false;
9657         t = db_read_token();
9658         if (t == tIDENT) {
9659                 dev = device_lookup_by_name(db_tok_string);
9660                 valid = true;
9661         }
9662         db_skip_to_eol();
9663         if (!valid) {
9664                 db_printf("usage: show t4 devlog <nexus>\n");
9665                 return;
9666         }
9667
9668         if (dev == NULL) {
9669                 db_printf("device not found\n");
9670                 return;
9671         }
9672
9673         t4_dump_devlog(device_get_softc(dev));
9674 }
9675
9676 DB_FUNC(tcb, db_show_t4tcb, db_t4_table, CS_OWN, NULL)
9677 {
9678         device_t dev;
9679         int radix, tid, t;
9680         bool valid;
9681
9682         valid = false;
9683         radix = db_radix;
9684         db_radix = 10;
9685         t = db_read_token();
9686         if (t == tIDENT) {
9687                 dev = device_lookup_by_name(db_tok_string);
9688                 t = db_read_token();
9689                 if (t == tNUMBER) {
9690                         tid = db_tok_number;
9691                         valid = true;
9692                 }
9693         }
9694         db_radix = radix;
9695         db_skip_to_eol();
9696         if (!valid) {
9697                 db_printf("usage: show t4 tcb <nexus> <tid>\n");
9698                 return;
9699         }
9700
9701         if (dev == NULL) {
9702                 db_printf("device not found\n");
9703                 return;
9704         }
9705         if (tid < 0) {
9706                 db_printf("invalid tid\n");
9707                 return;
9708         }
9709
9710         t4_dump_tcb(device_get_softc(dev), tid);
9711 }
9712 #endif
9713
9714 static struct sx mlu;   /* mod load unload */
9715 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
9716
9717 static int
9718 mod_event(module_t mod, int cmd, void *arg)
9719 {
9720         int rc = 0;
9721         static int loaded = 0;
9722
9723         switch (cmd) {
9724         case MOD_LOAD:
9725                 sx_xlock(&mlu);
9726                 if (loaded++ == 0) {
9727                         t4_sge_modload();
9728                         t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl);
9729                         t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl);
9730                         t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt);
9731                         t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt);
9732                         sx_init(&t4_list_lock, "T4/T5 adapters");
9733                         SLIST_INIT(&t4_list);
9734 #ifdef TCP_OFFLOAD
9735                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
9736                         SLIST_INIT(&t4_uld_list);
9737 #endif
9738                         t4_tracer_modload();
9739                         tweak_tunables();
9740                 }
9741                 sx_xunlock(&mlu);
9742                 break;
9743
9744         case MOD_UNLOAD:
9745                 sx_xlock(&mlu);
9746                 if (--loaded == 0) {
9747                         int tries;
9748
9749                         sx_slock(&t4_list_lock);
9750                         if (!SLIST_EMPTY(&t4_list)) {
9751                                 rc = EBUSY;
9752                                 sx_sunlock(&t4_list_lock);
9753                                 goto done_unload;
9754                         }
9755 #ifdef TCP_OFFLOAD
9756                         sx_slock(&t4_uld_list_lock);
9757                         if (!SLIST_EMPTY(&t4_uld_list)) {
9758                                 rc = EBUSY;
9759                                 sx_sunlock(&t4_uld_list_lock);
9760                                 sx_sunlock(&t4_list_lock);
9761                                 goto done_unload;
9762                         }
9763 #endif
9764                         tries = 0;
9765                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
9766                                 uprintf("%ju clusters with custom free routine "
9767                                     "still is use.\n", t4_sge_extfree_refs());
9768                                 pause("t4unload", 2 * hz);
9769                         }
9770 #ifdef TCP_OFFLOAD
9771                         sx_sunlock(&t4_uld_list_lock);
9772 #endif
9773                         sx_sunlock(&t4_list_lock);
9774
9775                         if (t4_sge_extfree_refs() == 0) {
9776                                 t4_tracer_modunload();
9777 #ifdef TCP_OFFLOAD
9778                                 sx_destroy(&t4_uld_list_lock);
9779 #endif
9780                                 sx_destroy(&t4_list_lock);
9781                                 t4_sge_modunload();
9782                                 loaded = 0;
9783                         } else {
9784                                 rc = EBUSY;
9785                                 loaded++;       /* undo earlier decrement */
9786                         }
9787                 }
9788 done_unload:
9789                 sx_xunlock(&mlu);
9790                 break;
9791         }
9792
9793         return (rc);
9794 }
9795
9796 static devclass_t t4_devclass, t5_devclass, t6_devclass;
9797 static devclass_t cxgbe_devclass, cxl_devclass, cc_devclass;
9798 static devclass_t vcxgbe_devclass, vcxl_devclass, vcc_devclass;
9799
9800 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
9801 MODULE_VERSION(t4nex, 1);
9802 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
9803 #ifdef DEV_NETMAP
9804 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
9805 #endif /* DEV_NETMAP */
9806
9807 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
9808 MODULE_VERSION(t5nex, 1);
9809 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
9810 #ifdef DEV_NETMAP
9811 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
9812 #endif /* DEV_NETMAP */
9813
9814 DRIVER_MODULE(t6nex, pci, t6_driver, t6_devclass, mod_event, 0);
9815 MODULE_VERSION(t6nex, 1);
9816 MODULE_DEPEND(t6nex, firmware, 1, 1, 1);
9817 #ifdef DEV_NETMAP
9818 MODULE_DEPEND(t6nex, netmap, 1, 1, 1);
9819 #endif /* DEV_NETMAP */
9820
9821 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
9822 MODULE_VERSION(cxgbe, 1);
9823
9824 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
9825 MODULE_VERSION(cxl, 1);
9826
9827 DRIVER_MODULE(cc, t6nex, cc_driver, cc_devclass, 0, 0);
9828 MODULE_VERSION(cc, 1);
9829
9830 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0);
9831 MODULE_VERSION(vcxgbe, 1);
9832
9833 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0);
9834 MODULE_VERSION(vcxl, 1);
9835
9836 DRIVER_MODULE(vcc, cc, vcc_driver, vcc_devclass, 0, 0);
9837 MODULE_VERSION(vcc, 1);