sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_ddb.h"
  32 #include "opt_inet.h"
  33 #include "opt_inet6.h"
  34 #include "opt_rss.h"
  35
  36 #include <sys/param.h>
  37 #include <sys/conf.h>
  38 #include <sys/priv.h>
  39 #include <sys/kernel.h>
  40 #include <sys/bus.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #ifdef RSS
  61 #include <net/rss_config.h>
  62 #endif
  63 #if defined(__i386__) || defined(__amd64__)
  64 #include <vm/vm.h>
  65 #include <vm/pmap.h>
  66 #endif
  67 #ifdef DDB
  68 #include <ddb/ddb.h>
  69 #include <ddb/db_lex.h>
  70 #endif
  71
  72 #include "common/common.h"
  73 #include "common/t4_msg.h"
  74 #include "common/t4_regs.h"
  75 #include "common/t4_regs_values.h"
  76 #include "t4_ioctl.h"
  77 #include "t4_l2t.h"
  78 #include "t4_mp_ring.h"
  79 #include "t4_if.h"
  80
  81 /* T4 bus driver interface */
  82 static int t4_probe(device_t);
  83 static int t4_attach(device_t);
  84 static int t4_detach(device_t);
  85 static int t4_ready(device_t);
  86 static int t4_read_port_device(device_t, int, device_t *);
  87 static device_method_t t4_methods[] = {
  88         DEVMETHOD(device_probe,         t4_probe),
  89         DEVMETHOD(device_attach,        t4_attach),
  90         DEVMETHOD(device_detach,        t4_detach),
  91
  92         DEVMETHOD(t4_is_main_ready,     t4_ready),
  93         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
  94
  95         DEVMETHOD_END
  96 };
  97 static driver_t t4_driver = {
  98         "t4nex",
  99         t4_methods,
 100         sizeof(struct adapter)
 101 };
 102
 103
 104 /* T4 port (cxgbe) interface */
 105 static int cxgbe_probe(device_t);
 106 static int cxgbe_attach(device_t);
 107 static int cxgbe_detach(device_t);
 108 device_method_t cxgbe_methods[] = {
 109         DEVMETHOD(device_probe,         cxgbe_probe),
 110         DEVMETHOD(device_attach,        cxgbe_attach),
 111         DEVMETHOD(device_detach,        cxgbe_detach),
 112         { 0, 0 }
 113 };
 114 static driver_t cxgbe_driver = {
 115         "cxgbe",
 116         cxgbe_methods,
 117         sizeof(struct port_info)
 118 };
 119
 120 /* T4 VI (vcxgbe) interface */
 121 static int vcxgbe_probe(device_t);
 122 static int vcxgbe_attach(device_t);
 123 static int vcxgbe_detach(device_t);
 124 static device_method_t vcxgbe_methods[] = {
 125         DEVMETHOD(device_probe,         vcxgbe_probe),
 126         DEVMETHOD(device_attach,        vcxgbe_attach),
 127         DEVMETHOD(device_detach,        vcxgbe_detach),
 128         { 0, 0 }
 129 };
 130 static driver_t vcxgbe_driver = {
 131         "vcxgbe",
 132         vcxgbe_methods,
 133         sizeof(struct vi_info)
 134 };
 135
 136 static d_ioctl_t t4_ioctl;
 137
 138 static struct cdevsw t4_cdevsw = {
 139        .d_version = D_VERSION,
 140        .d_ioctl = t4_ioctl,
 141        .d_name = "t4nex",
 142 };
 143
 144 /* T5 bus driver interface */
 145 static int t5_probe(device_t);
 146 static device_method_t t5_methods[] = {
 147         DEVMETHOD(device_probe,         t5_probe),
 148         DEVMETHOD(device_attach,        t4_attach),
 149         DEVMETHOD(device_detach,        t4_detach),
 150
 151         DEVMETHOD(t4_is_main_ready,     t4_ready),
 152         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
 153
 154         DEVMETHOD_END
 155 };
 156 static driver_t t5_driver = {
 157         "t5nex",
 158         t5_methods,
 159         sizeof(struct adapter)
 160 };
 161
 162
 163 /* T5 port (cxl) interface */
 164 static driver_t cxl_driver = {
 165         "cxl",
 166         cxgbe_methods,
 167         sizeof(struct port_info)
 168 };
 169
 170 /* T5 VI (vcxl) interface */
 171 static driver_t vcxl_driver = {
 172         "vcxl",
 173         vcxgbe_methods,
 174         sizeof(struct vi_info)
 175 };
 176
 177 /* T6 bus driver interface */
 178 static int t6_probe(device_t);
 179 static device_method_t t6_methods[] = {
 180         DEVMETHOD(device_probe,         t6_probe),
 181         DEVMETHOD(device_attach,        t4_attach),
 182         DEVMETHOD(device_detach,        t4_detach),
 183
 184         DEVMETHOD(t4_is_main_ready,     t4_ready),
 185         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
 186
 187         DEVMETHOD_END
 188 };
 189 static driver_t t6_driver = {
 190         "t6nex",
 191         t6_methods,
 192         sizeof(struct adapter)
 193 };
 194
 195
 196 /* T6 port (cc) interface */
 197 static driver_t cc_driver = {
 198         "cc",
 199         cxgbe_methods,
 200         sizeof(struct port_info)
 201 };
 202
 203 /* T6 VI (vcc) interface */
 204 static driver_t vcc_driver = {
 205         "vcc",
 206         vcxgbe_methods,
 207         sizeof(struct vi_info)
 208 };
 209
 210 /* ifnet + media interface */
 211 static void cxgbe_init(void *);
 212 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 213 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 214 static void cxgbe_qflush(struct ifnet *);
 215 static int cxgbe_media_change(struct ifnet *);
 216 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 217
 218 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 219
 220 /*
 221  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 222  * then ADAPTER_LOCK, then t4_uld_list_lock.
 223  */
 224 static struct sx t4_list_lock;
 225 SLIST_HEAD(, adapter) t4_list;
 226 #ifdef TCP_OFFLOAD
 227 static struct sx t4_uld_list_lock;
 228 SLIST_HEAD(, uld_info) t4_uld_list;
 229 #endif
 230
 231 /*
 232  * Tunables.  See tweak_tunables() too.
 233  *
 234  * Each tunable is set to a default value here if it's known at compile-time.
 235  * Otherwise it is set to -n as an indication to tweak_tunables() that it should
 236  * provide a reasonable default (upto n) when the driver is loaded.
 237  *
 238  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 239  * T5 are under hw.cxl.
 240  */
 241
 242 /*
 243  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 244  */
 245 #define NTXQ_10G 16
 246 int t4_ntxq10g = -NTXQ_10G;
 247 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 248
 249 #define NRXQ_10G 8
 250 int t4_nrxq10g = -NRXQ_10G;
 251 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 252
 253 #define NTXQ_1G 4
 254 int t4_ntxq1g = -NTXQ_1G;
 255 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 256
 257 #define NRXQ_1G 2
 258 int t4_nrxq1g = -NRXQ_1G;
 259 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 260
 261 #define NTXQ_VI 1
 262 static int t4_ntxq_vi = -NTXQ_VI;
 263 TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi);
 264
 265 #define NRXQ_VI 1
 266 static int t4_nrxq_vi = -NRXQ_VI;
 267 TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi);
 268
 269 static int t4_rsrv_noflowq = 0;
 270 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 271
 272 #ifdef TCP_OFFLOAD
 273 #define NOFLDTXQ_10G 8
 274 static int t4_nofldtxq10g = -NOFLDTXQ_10G;
 275 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 276
 277 #define NOFLDRXQ_10G 2
 278 static int t4_nofldrxq10g = -NOFLDRXQ_10G;
 279 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 280
 281 #define NOFLDTXQ_1G 2
 282 static int t4_nofldtxq1g = -NOFLDTXQ_1G;
 283 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 284
 285 #define NOFLDRXQ_1G 1
 286 static int t4_nofldrxq1g = -NOFLDRXQ_1G;
 287 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 288
 289 #define NOFLDTXQ_VI 1
 290 static int t4_nofldtxq_vi = -NOFLDTXQ_VI;
 291 TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi);
 292
 293 #define NOFLDRXQ_VI 1
 294 static int t4_nofldrxq_vi = -NOFLDRXQ_VI;
 295 TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi);
 296 #endif
 297
 298 #ifdef DEV_NETMAP
 299 #define NNMTXQ_VI 2
 300 static int t4_nnmtxq_vi = -NNMTXQ_VI;
 301 TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi);
 302
 303 #define NNMRXQ_VI 2
 304 static int t4_nnmrxq_vi = -NNMRXQ_VI;
 305 TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi);
 306 #endif
 307
 308 /*
 309  * Holdoff parameters for 10G and 1G ports.
 310  */
 311 #define TMR_IDX_10G 1
 312 int t4_tmr_idx_10g = TMR_IDX_10G;
 313 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 314
 315 #define PKTC_IDX_10G (-1)
 316 int t4_pktc_idx_10g = PKTC_IDX_10G;
 317 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 318
 319 #define TMR_IDX_1G 1
 320 int t4_tmr_idx_1g = TMR_IDX_1G;
 321 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 322
 323 #define PKTC_IDX_1G (-1)
 324 int t4_pktc_idx_1g = PKTC_IDX_1G;
 325 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 326
 327 /*
 328  * Size (# of entries) of each tx and rx queue.
 329  */
 330 unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 331 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 332
 333 unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 334 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 335
 336 /*
 337  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 338  */
 339 int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 340 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 341
 342 /*
 343  * Configuration file.
 344  */
 345 #define DEFAULT_CF      "default"
 346 #define FLASH_CF        "flash"
 347 #define UWIRE_CF        "uwire"
 348 #define FPGA_CF         "fpga"
 349 static char t4_cfg_file[32] = DEFAULT_CF;
 350 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 351
 352 /*
 353  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 354  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 355  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 356  *            mark or when signalled to do so, 0 to never emit PAUSE.
 357  */
 358 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 359 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 360
 361 /*
 362  * Forward Error Correction settings (bit 0, 1, 2 = FEC_RS, FEC_BASER_RS,
 363  * FEC_RESERVED respectively).
 364  * -1 to run with the firmware default.
 365  *  0 to disable FEC.
 366  */
 367 static int t4_fec = -1;
 368 TUNABLE_INT("hw.cxgbe.fec", &t4_fec);
 369
 370 /*
 371  * Link autonegotiation.
 372  * -1 to run with the firmware default.
 373  *  0 to disable.
 374  *  1 to enable.
 375  */
 376 static int t4_autoneg = -1;
 377 TUNABLE_INT("hw.cxgbe.autoneg", &t4_autoneg);
 378
 379 /*
 380  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 381  * encouraged respectively).
 382  */
 383 static unsigned int t4_fw_install = 1;
 384 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 385
 386 /*
 387  * ASIC features that will be used.  Disable the ones you don't want so that the
 388  * chip resources aren't wasted on features that will not be used.
 389  */
 390 static int t4_nbmcaps_allowed = 0;
 391 TUNABLE_INT("hw.cxgbe.nbmcaps_allowed", &t4_nbmcaps_allowed);
 392
 393 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 394 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 395
 396 static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS |
 397     FW_CAPS_CONFIG_SWITCH_EGRESS;
 398 TUNABLE_INT("hw.cxgbe.switchcaps_allowed", &t4_switchcaps_allowed);
 399
 400 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 401 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 402
 403 static int t4_toecaps_allowed = -1;
 404 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 405
 406 static int t4_rdmacaps_allowed = -1;
 407 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 408
 409 static int t4_cryptocaps_allowed = 0;
 410 TUNABLE_INT("hw.cxgbe.cryptocaps_allowed", &t4_cryptocaps_allowed);
 411
 412 static int t4_iscsicaps_allowed = -1;
 413 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 414
 415 static int t4_fcoecaps_allowed = 0;
 416 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 417
 418 static int t5_write_combine = 0;
 419 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 420
 421 static int t4_num_vis = 1;
 422 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis);
 423
 424 /* Functions used by extra VIs to obtain unique MAC addresses for each VI. */
 425 static int vi_mac_funcs[] = {
 426         FW_VI_FUNC_OFLD,
 427         FW_VI_FUNC_IWARP,
 428         FW_VI_FUNC_OPENISCSI,
 429         FW_VI_FUNC_OPENFCOE,
 430         FW_VI_FUNC_FOISCSI,
 431         FW_VI_FUNC_FOFCOE,
 432 };
 433
 434 struct intrs_and_queues {
 435         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 436         uint16_t nirq;          /* Total # of vectors */
 437         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 438         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 439         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 440         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 441         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 442         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 443         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 444         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 445         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 446         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 447         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 448
 449         /* The vcxgbe/vcxl interfaces use these and not the ones above. */
 450         uint16_t ntxq_vi;       /* # of NIC txq's */
 451         uint16_t nrxq_vi;       /* # of NIC rxq's */
 452         uint16_t nofldtxq_vi;   /* # of TOE txq's */
 453         uint16_t nofldrxq_vi;   /* # of TOE rxq's */
 454         uint16_t nnmtxq_vi;     /* # of netmap txq's */
 455         uint16_t nnmrxq_vi;     /* # of netmap rxq's */
 456 };
 457
 458 struct filter_entry {
 459         uint32_t valid:1;       /* filter allocated and valid */
 460         uint32_t locked:1;      /* filter is administratively locked */
 461         uint32_t pending:1;     /* filter action is pending firmware reply */
 462         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 463         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 464
 465         struct t4_filter_specification fs;
 466 };
 467
 468 static void setup_memwin(struct adapter *);
 469 static void position_memwin(struct adapter *, int, uint32_t);
 470 static int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int);
 471 static inline int read_via_memwin(struct adapter *, int, uint32_t, uint32_t *,
 472     int);
 473 static inline int write_via_memwin(struct adapter *, int, uint32_t,
 474     const uint32_t *, int);
 475 static int validate_mem_range(struct adapter *, uint32_t, int);
 476 static int fwmtype_to_hwmtype(int);
 477 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 478     uint32_t *);
 479 static int fixup_devlog_params(struct adapter *);
 480 static int cfg_itype_and_nqueues(struct adapter *, int, int, int,
 481     struct intrs_and_queues *);
 482 static int prep_firmware(struct adapter *);
 483 static int partition_resources(struct adapter *, const struct firmware *,
 484     const char *);
 485 static int get_params__pre_init(struct adapter *);
 486 static int get_params__post_init(struct adapter *);
 487 static int set_params__post_init(struct adapter *);
 488 static void t4_set_desc(struct adapter *);
 489 static void build_medialist(struct port_info *, struct ifmedia *);
 490 static int cxgbe_init_synchronized(struct vi_info *);
 491 static int cxgbe_uninit_synchronized(struct vi_info *);
 492 static void quiesce_txq(struct adapter *, struct sge_txq *);
 493 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 494 static void quiesce_iq(struct adapter *, struct sge_iq *);
 495 static void quiesce_fl(struct adapter *, struct sge_fl *);
 496 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 497     driver_intr_t *, void *, char *);
 498 static int t4_free_irq(struct adapter *, struct irq *);
 499 static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 500 static void vi_refresh_stats(struct adapter *, struct vi_info *);
 501 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 502 static void cxgbe_tick(void *);
 503 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 504 static void cxgbe_sysctls(struct port_info *);
 505 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 506 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 507 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 508 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 509 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 510 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 511 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 512 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 513 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 514 static int sysctl_fec(SYSCTL_HANDLER_ARGS);
 515 static int sysctl_autoneg(SYSCTL_HANDLER_ARGS);
 516 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 517 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 518 #ifdef SBUF_DRAIN
 519 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 520 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 521 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 522 static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS);
 523 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 524 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 525 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 526 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 527 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 528 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 529 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 530 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 531 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 532 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 533 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 534 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 535 static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS);
 536 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 537 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 538 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 539 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 540 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 541 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 542 static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS);
 543 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 544 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 545 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 546 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 547 static int sysctl_tc_params(SYSCTL_HANDLER_ARGS);
 548 #endif
 549 #ifdef TCP_OFFLOAD
 550 static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
 551 static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
 552 static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
 553 #endif
 554 static uint32_t fconf_iconf_to_mode(uint32_t, uint32_t);
 555 static uint32_t mode_to_fconf(uint32_t);
 556 static uint32_t mode_to_iconf(uint32_t);
 557 static int check_fspec_against_fconf_iconf(struct adapter *,
 558     struct t4_filter_specification *);
 559 static int get_filter_mode(struct adapter *, uint32_t *);
 560 static int set_filter_mode(struct adapter *, uint32_t);
 561 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 562 static int get_filter(struct adapter *, struct t4_filter *);
 563 static int set_filter(struct adapter *, struct t4_filter *);
 564 static int del_filter(struct adapter *, struct t4_filter *);
 565 static void clear_filter(struct filter_entry *);
 566 static int set_filter_wr(struct adapter *, int);
 567 static int del_filter_wr(struct adapter *, int);
 568 static int set_tcb_rpl(struct sge_iq *, const struct rss_header *,
 569     struct mbuf *);
 570 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 571 static int load_fw(struct adapter *, struct t4_data *);
 572 static int load_cfg(struct adapter *, struct t4_data *);
 573 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 574 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 575 #ifdef TCP_OFFLOAD
 576 static int toe_capability(struct vi_info *, int);
 577 #endif
 578 static int mod_event(module_t, int, void *);
 579 static int notify_siblings(device_t, int);
 580
 581 struct {
 582         uint16_t device;
 583         char *desc;
 584 } t4_pciids[] = {
 585         {0xa000, "Chelsio Terminator 4 FPGA"},
 586         {0x4400, "Chelsio T440-dbg"},
 587         {0x4401, "Chelsio T420-CR"},
 588         {0x4402, "Chelsio T422-CR"},
 589         {0x4403, "Chelsio T440-CR"},
 590         {0x4404, "Chelsio T420-BCH"},
 591         {0x4405, "Chelsio T440-BCH"},
 592         {0x4406, "Chelsio T440-CH"},
 593         {0x4407, "Chelsio T420-SO"},
 594         {0x4408, "Chelsio T420-CX"},
 595         {0x4409, "Chelsio T420-BT"},
 596         {0x440a, "Chelsio T404-BT"},
 597         {0x440e, "Chelsio T440-LP-CR"},
 598 }, t5_pciids[] = {
 599         {0xb000, "Chelsio Terminator 5 FPGA"},
 600         {0x5400, "Chelsio T580-dbg"},
 601         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 602         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 603         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 604         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 605         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 606         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 607         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 608         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 609         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 610         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 611         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 612         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 613         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 614 #ifdef notyet
 615         {0x5404,  "Chelsio T520-BCH"},
 616         {0x5405,  "Chelsio T540-BCH"},
 617         {0x5406,  "Chelsio T540-CH"},
 618         {0x5408,  "Chelsio T520-CX"},
 619         {0x540b,  "Chelsio B520-SR"},
 620         {0x540c,  "Chelsio B504-BT"},
 621         {0x540f,  "Chelsio Amsterdam"},
 622         {0x5413,  "Chelsio T580-CHR"},
 623 #endif
 624 }, t6_pciids[] = {
 625         {0xc006, "Chelsio Terminator 6 FPGA"},  /* T6 PE10K6 FPGA (PF0) */
 626         {0x6400, "Chelsio T6225-DBG"},          /* 2 x 10/25G, debug */
 627         {0x6401, "Chelsio T6225-CR"},           /* 2 x 10/25G */
 628         {0x6402, "Chelsio T6225-SO-CR"},        /* 2 x 10/25G, nomem */
 629         {0x6407, "Chelsio T62100-LP-CR"},       /* 2 x 40/50/100G */
 630         {0x6408, "Chelsio T62100-SO-CR"},       /* 2 x 40/50/100G, nomem */
 631         {0x640d, "Chelsio T62100-CR"},          /* 2 x 40/50/100G */
 632         {0x6410, "Chelsio T62100-DBG"},         /* 2 x 40/50/100G, debug */
 633 };
 634
 635 #ifdef TCP_OFFLOAD
 636 /*
 637  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 638  * exactly the same for both rxq and ofld_rxq.
 639  */
 640 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 641 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 642 #endif
 643 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 644
 645 static int
 646 t4_probe(device_t dev)
 647 {
 648         int i;
 649         uint16_t v = pci_get_vendor(dev);
 650         uint16_t d = pci_get_device(dev);
 651         uint8_t f = pci_get_function(dev);
 652
 653         if (v != PCI_VENDOR_ID_CHELSIO)
 654                 return (ENXIO);
 655
 656         /* Attach only to PF0 of the FPGA */
 657         if (d == 0xa000 && f != 0)
 658                 return (ENXIO);
 659
 660         for (i = 0; i < nitems(t4_pciids); i++) {
 661                 if (d == t4_pciids[i].device) {
 662                         device_set_desc(dev, t4_pciids[i].desc);
 663                         return (BUS_PROBE_DEFAULT);
 664                 }
 665         }
 666
 667         return (ENXIO);
 668 }
 669
 670 static int
 671 t5_probe(device_t dev)
 672 {
 673         int i;
 674         uint16_t v = pci_get_vendor(dev);
 675         uint16_t d = pci_get_device(dev);
 676         uint8_t f = pci_get_function(dev);
 677
 678         if (v != PCI_VENDOR_ID_CHELSIO)
 679                 return (ENXIO);
 680
 681         /* Attach only to PF0 of the FPGA */
 682         if (d == 0xb000 && f != 0)
 683                 return (ENXIO);
 684
 685         for (i = 0; i < nitems(t5_pciids); i++) {
 686                 if (d == t5_pciids[i].device) {
 687                         device_set_desc(dev, t5_pciids[i].desc);
 688                         return (BUS_PROBE_DEFAULT);
 689                 }
 690         }
 691
 692         return (ENXIO);
 693 }
 694
 695 static int
 696 t6_probe(device_t dev)
 697 {
 698         int i;
 699         uint16_t v = pci_get_vendor(dev);
 700         uint16_t d = pci_get_device(dev);
 701
 702         if (v != PCI_VENDOR_ID_CHELSIO)
 703                 return (ENXIO);
 704
 705         for (i = 0; i < nitems(t6_pciids); i++) {
 706                 if (d == t6_pciids[i].device) {
 707                         device_set_desc(dev, t6_pciids[i].desc);
 708                         return (BUS_PROBE_DEFAULT);
 709                 }
 710         }
 711
 712         return (ENXIO);
 713 }
 714
 715 static void
 716 t5_attribute_workaround(device_t dev)
 717 {
 718         device_t root_port;
 719         uint32_t v;
 720
 721         /*
 722          * The T5 chips do not properly echo the No Snoop and Relaxed
 723          * Ordering attributes when replying to a TLP from a Root
 724          * Port.  As a workaround, find the parent Root Port and
 725          * disable No Snoop and Relaxed Ordering.  Note that this
 726          * affects all devices under this root port.
 727          */
 728         root_port = pci_find_pcie_root_port(dev);
 729         if (root_port == NULL) {
 730                 device_printf(dev, "Unable to find parent root port\n");
 731                 return;
 732         }
 733
 734         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 735             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 736         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 737             0)
 738                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 739                     device_get_nameunit(root_port));
 740 }
 741
 742 static const struct devnames devnames[] = {
 743         {
 744                 .nexus_name = "t4nex",
 745                 .ifnet_name = "cxgbe",
 746                 .vi_ifnet_name = "vcxgbe",
 747                 .pf03_drv_name = "t4iov",
 748                 .vf_nexus_name = "t4vf",
 749                 .vf_ifnet_name = "cxgbev"
 750         }, {
 751                 .nexus_name = "t5nex",
 752                 .ifnet_name = "cxl",
 753                 .vi_ifnet_name = "vcxl",
 754                 .pf03_drv_name = "t5iov",
 755                 .vf_nexus_name = "t5vf",
 756                 .vf_ifnet_name = "cxlv"
 757         }, {
 758                 .nexus_name = "t6nex",
 759                 .ifnet_name = "cc",
 760                 .vi_ifnet_name = "vcc",
 761                 .pf03_drv_name = "t6iov",
 762                 .vf_nexus_name = "t6vf",
 763                 .vf_ifnet_name = "ccv"
 764         }
 765 };
 766
 767 void
 768 t4_init_devnames(struct adapter *sc)
 769 {
 770         int id;
 771
 772         id = chip_id(sc);
 773         if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames))
 774                 sc->names = &devnames[id - CHELSIO_T4];
 775         else {
 776                 device_printf(sc->dev, "chip id %d is not supported.\n", id);
 777                 sc->names = NULL;
 778         }
 779 }
 780
 781 static int
 782 t4_attach(device_t dev)
 783 {
 784         struct adapter *sc;
 785         int rc = 0, i, j, n10g, n1g, rqidx, tqidx;
 786         struct make_dev_args mda;
 787         struct intrs_and_queues iaq;
 788         struct sge *s;
 789         uint8_t *buf;
 790 #ifdef TCP_OFFLOAD
 791         int ofld_rqidx, ofld_tqidx;
 792 #endif
 793 #ifdef DEV_NETMAP
 794         int nm_rqidx, nm_tqidx;
 795 #endif
 796         int num_vis;
 797
 798         sc = device_get_softc(dev);
 799         sc->dev = dev;
 800         TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
 801
 802         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 803                 t5_attribute_workaround(dev);
 804         pci_enable_busmaster(dev);
 805         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 806                 uint32_t v;
 807
 808                 pci_set_max_read_req(dev, 4096);
 809                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 810                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 811                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 812
 813                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 814         }
 815
 816         sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
 817         sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
 818         sc->traceq = -1;
 819         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 820         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 821             device_get_nameunit(dev));
 822
 823         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 824             device_get_nameunit(dev));
 825         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 826         t4_add_adapter(sc);
 827
 828         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 829         TAILQ_INIT(&sc->sfl);
 830         callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
 831
 832         mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
 833
 834         rc = t4_map_bars_0_and_4(sc);
 835         if (rc != 0)
 836                 goto done; /* error message displayed already */
 837
 838         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 839
 840         /* Prepare the adapter for operation. */
 841         buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK);
 842         rc = -t4_prep_adapter(sc, buf);
 843         free(buf, M_CXGBE);
 844         if (rc != 0) {
 845                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 846                 goto done;
 847         }
 848
 849         /*
 850          * This is the real PF# to which we're attaching.  Works from within PCI
 851          * passthrough environments too, where pci_get_function() could return a
 852          * different PF# depending on the passthrough configuration.  We need to
 853          * use the real PF# in all our communication with the firmware.
 854          */
 855         j = t4_read_reg(sc, A_PL_WHOAMI);
 856         sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
 857         sc->mbox = sc->pf;
 858
 859         t4_init_devnames(sc);
 860         if (sc->names == NULL) {
 861                 rc = ENOTSUP;
 862                 goto done; /* error message displayed already */
 863         }
 864
 865         /*
 866          * Do this really early, with the memory windows set up even before the
 867          * character device.  The userland tool's register i/o and mem read
 868          * will work even in "recovery mode".
 869          */
 870         setup_memwin(sc);
 871         if (t4_init_devlog_params(sc, 0) == 0)
 872                 fixup_devlog_params(sc);
 873         make_dev_args_init(&mda);
 874         mda.mda_devsw = &t4_cdevsw;
 875         mda.mda_uid = UID_ROOT;
 876         mda.mda_gid = GID_WHEEL;
 877         mda.mda_mode = 0600;
 878         mda.mda_si_drv1 = sc;
 879         rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
 880         if (rc != 0)
 881                 device_printf(dev, "failed to create nexus char device: %d.\n",
 882                     rc);
 883
 884         /* Go no further if recovery mode has been requested. */
 885         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 886                 device_printf(dev, "recovery mode.\n");
 887                 goto done;
 888         }
 889
 890 #if defined(__i386__)
 891         if ((cpu_feature & CPUID_CX8) == 0) {
 892                 device_printf(dev, "64 bit atomics not available.\n");
 893                 rc = ENOTSUP;
 894                 goto done;
 895         }
 896 #endif
 897
 898         /* Prepare the firmware for operation */
 899         rc = prep_firmware(sc);
 900         if (rc != 0)
 901                 goto done; /* error message displayed already */
 902
 903         rc = get_params__post_init(sc);
 904         if (rc != 0)
 905                 goto done; /* error message displayed already */
 906
 907         rc = set_params__post_init(sc);
 908         if (rc != 0)
 909                 goto done; /* error message displayed already */
 910
 911         rc = t4_map_bar_2(sc);
 912         if (rc != 0)
 913                 goto done; /* error message displayed already */
 914
 915         rc = t4_create_dma_tag(sc);
 916         if (rc != 0)
 917                 goto done; /* error message displayed already */
 918
 919         /*
 920          * Number of VIs to create per-port.  The first VI is the "main" regular
 921          * VI for the port.  The rest are additional virtual interfaces on the
 922          * same physical port.  Note that the main VI does not have native
 923          * netmap support but the extra VIs do.
 924          *
 925          * Limit the number of VIs per port to the number of available
 926          * MAC addresses per port.
 927          */
 928         if (t4_num_vis >= 1)
 929                 num_vis = t4_num_vis;
 930         else
 931                 num_vis = 1;
 932         if (num_vis > nitems(vi_mac_funcs)) {
 933                 num_vis = nitems(vi_mac_funcs);
 934                 device_printf(dev, "Number of VIs limited to %d\n", num_vis);
 935         }
 936
 937         /*
 938          * First pass over all the ports - allocate VIs and initialize some
 939          * basic parameters like mac address, port type, etc.  We also figure
 940          * out whether a port is 10G or 1G and use that information when
 941          * calculating how many interrupts to attempt to allocate.
 942          */
 943         n10g = n1g = 0;
 944         for_each_port(sc, i) {
 945                 struct port_info *pi;
 946                 struct link_config *lc;
 947
 948                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 949                 sc->port[i] = pi;
 950
 951                 /* These must be set before t4_port_init */
 952                 pi->adapter = sc;
 953                 pi->port_id = i;
 954                 /*
 955                  * XXX: vi[0] is special so we can't delay this allocation until
 956                  * pi->nvi's final value is known.
 957                  */
 958                 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE,
 959                     M_ZERO | M_WAITOK);
 960
 961                 /*
 962                  * Allocate the "main" VI and initialize parameters
 963                  * like mac addr.
 964                  */
 965                 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
 966                 if (rc != 0) {
 967                         device_printf(dev, "unable to initialize port %d: %d\n",
 968                             i, rc);
 969                         free(pi->vi, M_CXGBE);
 970                         free(pi, M_CXGBE);
 971                         sc->port[i] = NULL;
 972                         goto done;
 973                 }
 974
 975                 lc = &pi->link_cfg;
 976                 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 977                 lc->requested_fc |= t4_pause_settings;
 978                 if (t4_fec != -1) {
 979                         lc->requested_fec = t4_fec &
 980                             G_FW_PORT_CAP_FEC(lc->supported);
 981                 }
 982                 if (lc->supported & FW_PORT_CAP_ANEG && t4_autoneg != -1) {
 983                         lc->autoneg = t4_autoneg ? AUTONEG_ENABLE :
 984                             AUTONEG_DISABLE;
 985                 }
 986
 987                 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
 988                 if (rc != 0) {
 989                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 990                         free(pi->vi, M_CXGBE);
 991                         free(pi, M_CXGBE);
 992                         sc->port[i] = NULL;
 993                         goto done;
 994                 }
 995
 996                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 997                     device_get_nameunit(dev), i);
 998                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 999                 sc->chan_map[pi->tx_chan] = i;
1000
1001                 pi->tc = malloc(sizeof(struct tx_sched_class) *
1002                     sc->chip_params->nsched_cls, M_CXGBE, M_ZERO | M_WAITOK);
1003
1004                 if (port_top_speed(pi) >= 10) {
1005                         n10g++;
1006                 } else {
1007                         n1g++;
1008                 }
1009
1010                 pi->dev = device_add_child(dev, sc->names->ifnet_name, -1);
1011                 if (pi->dev == NULL) {
1012                         device_printf(dev,
1013                             "failed to add device for port %d.\n", i);
1014                         rc = ENXIO;
1015                         goto done;
1016                 }
1017                 pi->vi[0].dev = pi->dev;
1018                 device_set_softc(pi->dev, pi);
1019         }
1020
1021         /*
1022          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
1023          */
1024         rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq);
1025         if (rc != 0)
1026                 goto done; /* error message displayed already */
1027         if (iaq.nrxq_vi + iaq.nofldrxq_vi + iaq.nnmrxq_vi == 0)
1028                 num_vis = 1;
1029
1030         sc->intr_type = iaq.intr_type;
1031         sc->intr_count = iaq.nirq;
1032
1033         s = &sc->sge;
1034         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
1035         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
1036         if (num_vis > 1) {
1037                 s->nrxq += (n10g + n1g) * (num_vis - 1) * iaq.nrxq_vi;
1038                 s->ntxq += (n10g + n1g) * (num_vis - 1) * iaq.ntxq_vi;
1039         }
1040         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
1041         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
1042         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
1043 #ifdef TCP_OFFLOAD
1044         if (is_offload(sc)) {
1045                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
1046                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
1047                 if (num_vis > 1) {
1048                         s->nofldrxq += (n10g + n1g) * (num_vis - 1) *
1049                             iaq.nofldrxq_vi;
1050                         s->nofldtxq += (n10g + n1g) * (num_vis - 1) *
1051                             iaq.nofldtxq_vi;
1052                 }
1053                 s->neq += s->nofldtxq + s->nofldrxq;
1054                 s->niq += s->nofldrxq;
1055
1056                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
1057                     M_CXGBE, M_ZERO | M_WAITOK);
1058                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
1059                     M_CXGBE, M_ZERO | M_WAITOK);
1060         }
1061 #endif
1062 #ifdef DEV_NETMAP
1063         if (num_vis > 1) {
1064                 s->nnmrxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmrxq_vi;
1065                 s->nnmtxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmtxq_vi;
1066         }
1067         s->neq += s->nnmtxq + s->nnmrxq;
1068         s->niq += s->nnmrxq;
1069
1070         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
1071             M_CXGBE, M_ZERO | M_WAITOK);
1072         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
1073             M_CXGBE, M_ZERO | M_WAITOK);
1074 #endif
1075
1076         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
1077             M_ZERO | M_WAITOK);
1078         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
1079             M_ZERO | M_WAITOK);
1080         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
1081             M_ZERO | M_WAITOK);
1082         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
1083             M_ZERO | M_WAITOK);
1084         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
1085             M_ZERO | M_WAITOK);
1086
1087         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
1088             M_ZERO | M_WAITOK);
1089
1090         t4_init_l2t(sc, M_WAITOK);
1091
1092         /*
1093          * Second pass over the ports.  This time we know the number of rx and
1094          * tx queues that each port should get.
1095          */
1096         rqidx = tqidx = 0;
1097 #ifdef TCP_OFFLOAD
1098         ofld_rqidx = ofld_tqidx = 0;
1099 #endif
1100 #ifdef DEV_NETMAP
1101         nm_rqidx = nm_tqidx = 0;
1102 #endif
1103         for_each_port(sc, i) {
1104                 struct port_info *pi = sc->port[i];
1105                 struct vi_info *vi;
1106
1107                 if (pi == NULL)
1108                         continue;
1109
1110                 pi->nvi = num_vis;
1111                 for_each_vi(pi, j, vi) {
1112                         vi->pi = pi;
1113                         vi->qsize_rxq = t4_qsize_rxq;
1114                         vi->qsize_txq = t4_qsize_txq;
1115
1116                         vi->first_rxq = rqidx;
1117                         vi->first_txq = tqidx;
1118                         if (port_top_speed(pi) >= 10) {
1119                                 vi->tmr_idx = t4_tmr_idx_10g;
1120                                 vi->pktc_idx = t4_pktc_idx_10g;
1121                                 vi->flags |= iaq.intr_flags_10g & INTR_RXQ;
1122                                 vi->nrxq = j == 0 ? iaq.nrxq10g : iaq.nrxq_vi;
1123                                 vi->ntxq = j == 0 ? iaq.ntxq10g : iaq.ntxq_vi;
1124                         } else {
1125                                 vi->tmr_idx = t4_tmr_idx_1g;
1126                                 vi->pktc_idx = t4_pktc_idx_1g;
1127                                 vi->flags |= iaq.intr_flags_1g & INTR_RXQ;
1128                                 vi->nrxq = j == 0 ? iaq.nrxq1g : iaq.nrxq_vi;
1129                                 vi->ntxq = j == 0 ? iaq.ntxq1g : iaq.ntxq_vi;
1130                         }
1131                         rqidx += vi->nrxq;
1132                         tqidx += vi->ntxq;
1133
1134                         if (j == 0 && vi->ntxq > 1)
1135                                 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
1136                         else
1137                                 vi->rsrv_noflowq = 0;
1138
1139 #ifdef TCP_OFFLOAD
1140                         vi->first_ofld_rxq = ofld_rqidx;
1141                         vi->first_ofld_txq = ofld_tqidx;
1142                         if (port_top_speed(pi) >= 10) {
1143                                 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ;
1144                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g :
1145                                     iaq.nofldrxq_vi;
1146                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g :
1147                                     iaq.nofldtxq_vi;
1148                         } else {
1149                                 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ;
1150                                 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g :
1151                                     iaq.nofldrxq_vi;
1152                                 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g :
1153                                     iaq.nofldtxq_vi;
1154                         }
1155                         ofld_rqidx += vi->nofldrxq;
1156                         ofld_tqidx += vi->nofldtxq;
1157 #endif
1158 #ifdef DEV_NETMAP
1159                         if (j > 0) {
1160                                 vi->first_nm_rxq = nm_rqidx;
1161                                 vi->first_nm_txq = nm_tqidx;
1162                                 vi->nnmrxq = iaq.nnmrxq_vi;
1163                                 vi->nnmtxq = iaq.nnmtxq_vi;
1164                                 nm_rqidx += vi->nnmrxq;
1165                                 nm_tqidx += vi->nnmtxq;
1166                         }
1167 #endif
1168                 }
1169         }
1170
1171         rc = t4_setup_intr_handlers(sc);
1172         if (rc != 0) {
1173                 device_printf(dev,
1174                     "failed to setup interrupt handlers: %d\n", rc);
1175                 goto done;
1176         }
1177
1178         rc = bus_generic_attach(dev);
1179         if (rc != 0) {
1180                 device_printf(dev,
1181                     "failed to attach all child ports: %d\n", rc);
1182                 goto done;
1183         }
1184
1185         device_printf(dev,
1186             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
1187             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1188             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1189             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1190             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1191
1192         t4_set_desc(sc);
1193
1194         notify_siblings(dev, 0);
1195
1196 done:
1197         if (rc != 0 && sc->cdev) {
1198                 /* cdev was created and so cxgbetool works; recover that way. */
1199                 device_printf(dev,
1200                     "error during attach, adapter is now in recovery mode.\n");
1201                 rc = 0;
1202         }
1203
1204         if (rc != 0)
1205                 t4_detach_common(dev);
1206         else
1207                 t4_sysctls(sc);
1208
1209         return (rc);
1210 }
1211
1212 static int
1213 t4_ready(device_t dev)
1214 {
1215         struct adapter *sc;
1216
1217         sc = device_get_softc(dev);
1218         if (sc->flags & FW_OK)
1219                 return (0);
1220         return (ENXIO);
1221 }
1222
1223 static int
1224 t4_read_port_device(device_t dev, int port, device_t *child)
1225 {
1226         struct adapter *sc;
1227         struct port_info *pi;
1228
1229         sc = device_get_softc(dev);
1230         if (port < 0 || port >= MAX_NPORTS)
1231                 return (EINVAL);
1232         pi = sc->port[port];
1233         if (pi == NULL || pi->dev == NULL)
1234                 return (ENXIO);
1235         *child = pi->dev;
1236         return (0);
1237 }
1238
1239 static int
1240 notify_siblings(device_t dev, int detaching)
1241 {
1242         device_t sibling;
1243         int error, i;
1244
1245         error = 0;
1246         for (i = 0; i < PCI_FUNCMAX; i++) {
1247                 if (i == pci_get_function(dev))
1248                         continue;
1249                 sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
1250                     pci_get_slot(dev), i);
1251                 if (sibling == NULL || !device_is_attached(sibling))
1252                         continue;
1253                 if (detaching)
1254                         error = T4_DETACH_CHILD(sibling);
1255                 else
1256                         (void)T4_ATTACH_CHILD(sibling);
1257                 if (error)
1258                         break;
1259         }
1260         return (error);
1261 }
1262
1263 /*
1264  * Idempotent
1265  */
1266 static int
1267 t4_detach(device_t dev)
1268 {
1269         struct adapter *sc;
1270         int rc;
1271
1272         sc = device_get_softc(dev);
1273
1274         rc = notify_siblings(dev, 1);
1275         if (rc) {
1276                 device_printf(dev,
1277                     "failed to detach sibling devices: %d\n", rc);
1278                 return (rc);
1279         }
1280
1281         return (t4_detach_common(dev));
1282 }
1283
1284 int
1285 t4_detach_common(device_t dev)
1286 {
1287         struct adapter *sc;
1288         struct port_info *pi;
1289         int i, rc;
1290
1291         sc = device_get_softc(dev);
1292
1293         if (sc->flags & FULL_INIT_DONE) {
1294                 if (!(sc->flags & IS_VF))
1295                         t4_intr_disable(sc);
1296         }
1297
1298         if (sc->cdev) {
1299                 destroy_dev(sc->cdev);
1300                 sc->cdev = NULL;
1301         }
1302
1303         if (device_is_attached(dev)) {
1304                 rc = bus_generic_detach(dev);
1305                 if (rc) {
1306                         device_printf(dev,
1307                             "failed to detach child devices: %d\n", rc);
1308                         return (rc);
1309                 }
1310         }
1311
1312         for (i = 0; i < sc->intr_count; i++)
1313                 t4_free_irq(sc, &sc->irq[i]);
1314
1315         for (i = 0; i < MAX_NPORTS; i++) {
1316                 pi = sc->port[i];
1317                 if (pi) {
1318                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1319                         if (pi->dev)
1320                                 device_delete_child(dev, pi->dev);
1321
1322                         mtx_destroy(&pi->pi_lock);
1323                         free(pi->vi, M_CXGBE);
1324                         free(pi->tc, M_CXGBE);
1325                         free(pi, M_CXGBE);
1326                 }
1327         }
1328
1329         if (sc->flags & FULL_INIT_DONE)
1330                 adapter_full_uninit(sc);
1331
1332         if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1333                 t4_fw_bye(sc, sc->mbox);
1334
1335         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1336                 pci_release_msi(dev);
1337
1338         if (sc->regs_res)
1339                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1340                     sc->regs_res);
1341
1342         if (sc->udbs_res)
1343                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1344                     sc->udbs_res);
1345
1346         if (sc->msix_res)
1347                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1348                     sc->msix_res);
1349
1350         if (sc->l2t)
1351                 t4_free_l2t(sc->l2t);
1352
1353 #ifdef TCP_OFFLOAD
1354         free(sc->sge.ofld_rxq, M_CXGBE);
1355         free(sc->sge.ofld_txq, M_CXGBE);
1356 #endif
1357 #ifdef DEV_NETMAP
1358         free(sc->sge.nm_rxq, M_CXGBE);
1359         free(sc->sge.nm_txq, M_CXGBE);
1360 #endif
1361         free(sc->irq, M_CXGBE);
1362         free(sc->sge.rxq, M_CXGBE);
1363         free(sc->sge.txq, M_CXGBE);
1364         free(sc->sge.ctrlq, M_CXGBE);
1365         free(sc->sge.iqmap, M_CXGBE);
1366         free(sc->sge.eqmap, M_CXGBE);
1367         free(sc->tids.ftid_tab, M_CXGBE);
1368         t4_destroy_dma_tag(sc);
1369         if (mtx_initialized(&sc->sc_lock)) {
1370                 sx_xlock(&t4_list_lock);
1371                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1372                 sx_xunlock(&t4_list_lock);
1373                 mtx_destroy(&sc->sc_lock);
1374         }
1375
1376         callout_drain(&sc->sfl_callout);
1377         if (mtx_initialized(&sc->tids.ftid_lock))
1378                 mtx_destroy(&sc->tids.ftid_lock);
1379         if (mtx_initialized(&sc->sfl_lock))
1380                 mtx_destroy(&sc->sfl_lock);
1381         if (mtx_initialized(&sc->ifp_lock))
1382                 mtx_destroy(&sc->ifp_lock);
1383         if (mtx_initialized(&sc->reg_lock))
1384                 mtx_destroy(&sc->reg_lock);
1385
1386         for (i = 0; i < NUM_MEMWIN; i++) {
1387                 struct memwin *mw = &sc->memwin[i];
1388
1389                 if (rw_initialized(&mw->mw_lock))
1390                         rw_destroy(&mw->mw_lock);
1391         }
1392
1393         bzero(sc, sizeof(*sc));
1394
1395         return (0);
1396 }
1397
1398 static int
1399 cxgbe_probe(device_t dev)
1400 {
1401         char buf[128];
1402         struct port_info *pi = device_get_softc(dev);
1403
1404         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1405         device_set_desc_copy(dev, buf);
1406
1407         return (BUS_PROBE_DEFAULT);
1408 }
1409
1410 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1411     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1412     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1413 #define T4_CAP_ENABLE (T4_CAP)
1414
1415 static int
1416 cxgbe_vi_attach(device_t dev, struct vi_info *vi)
1417 {
1418         struct ifnet *ifp;
1419         struct sbuf *sb;
1420
1421         vi->xact_addr_filt = -1;
1422         callout_init(&vi->tick, 1);
1423
1424         /* Allocate an ifnet and set it up */
1425         ifp = if_alloc(IFT_ETHER);
1426         if (ifp == NULL) {
1427                 device_printf(dev, "Cannot allocate ifnet\n");
1428                 return (ENOMEM);
1429         }
1430         vi->ifp = ifp;
1431         ifp->if_softc = vi;
1432
1433         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1434         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1435
1436         ifp->if_init = cxgbe_init;
1437         ifp->if_ioctl = cxgbe_ioctl;
1438         ifp->if_transmit = cxgbe_transmit;
1439         ifp->if_qflush = cxgbe_qflush;
1440         ifp->if_get_counter = cxgbe_get_counter;
1441
1442         ifp->if_capabilities = T4_CAP;
1443 #ifdef TCP_OFFLOAD
1444         if (vi->nofldrxq != 0)
1445                 ifp->if_capabilities |= IFCAP_TOE;
1446 #endif
1447 #ifdef DEV_NETMAP
1448         if (vi->nnmrxq != 0)
1449                 ifp->if_capabilities |= IFCAP_NETMAP;
1450 #endif
1451         ifp->if_capenable = T4_CAP_ENABLE;
1452         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1453             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1454
1455         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1456         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1457         ifp->if_hw_tsomaxsegsize = 65536;
1458
1459         /* Initialize ifmedia for this VI */
1460         ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change,
1461             cxgbe_media_status);
1462         build_medialist(vi->pi, &vi->media);
1463
1464         vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1465             EVENTHANDLER_PRI_ANY);
1466
1467         ether_ifattach(ifp, vi->hw_addr);
1468 #ifdef DEV_NETMAP
1469         if (ifp->if_capabilities & IFCAP_NETMAP)
1470                 cxgbe_nm_attach(vi);
1471 #endif
1472         sb = sbuf_new_auto();
1473         sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
1474 #ifdef TCP_OFFLOAD
1475         if (ifp->if_capabilities & IFCAP_TOE)
1476                 sbuf_printf(sb, "; %d txq, %d rxq (TOE)",
1477                     vi->nofldtxq, vi->nofldrxq);
1478 #endif
1479 #ifdef DEV_NETMAP
1480         if (ifp->if_capabilities & IFCAP_NETMAP)
1481                 sbuf_printf(sb, "; %d txq, %d rxq (netmap)",
1482                     vi->nnmtxq, vi->nnmrxq);
1483 #endif
1484         sbuf_finish(sb);
1485         device_printf(dev, "%s\n", sbuf_data(sb));
1486         sbuf_delete(sb);
1487
1488         vi_sysctls(vi);
1489
1490         return (0);
1491 }
1492
1493 static int
1494 cxgbe_attach(device_t dev)
1495 {
1496         struct port_info *pi = device_get_softc(dev);
1497         struct adapter *sc = pi->adapter;
1498         struct vi_info *vi;
1499         int i, rc;
1500
1501         callout_init_mtx(&pi->tick, &pi->pi_lock, 0);
1502
1503         rc = cxgbe_vi_attach(dev, &pi->vi[0]);
1504         if (rc)
1505                 return (rc);
1506
1507         for_each_vi(pi, i, vi) {
1508                 if (i == 0)
1509                         continue;
1510                 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1);
1511                 if (vi->dev == NULL) {
1512                         device_printf(dev, "failed to add VI %d\n", i);
1513                         continue;
1514                 }
1515                 device_set_softc(vi->dev, vi);
1516         }
1517
1518         cxgbe_sysctls(pi);
1519
1520         bus_generic_attach(dev);
1521
1522         return (0);
1523 }
1524
1525 static void
1526 cxgbe_vi_detach(struct vi_info *vi)
1527 {
1528         struct ifnet *ifp = vi->ifp;
1529
1530         ether_ifdetach(ifp);
1531
1532         if (vi->vlan_c)
1533                 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c);
1534
1535         /* Let detach proceed even if these fail. */
1536 #ifdef DEV_NETMAP
1537         if (ifp->if_capabilities & IFCAP_NETMAP)
1538                 cxgbe_nm_detach(vi);
1539 #endif
1540         cxgbe_uninit_synchronized(vi);
1541         callout_drain(&vi->tick);
1542         vi_full_uninit(vi);
1543
1544         ifmedia_removeall(&vi->media);
1545         if_free(vi->ifp);
1546         vi->ifp = NULL;
1547 }
1548
1549 static int
1550 cxgbe_detach(device_t dev)
1551 {
1552         struct port_info *pi = device_get_softc(dev);
1553         struct adapter *sc = pi->adapter;
1554         int rc;
1555
1556         /* Detach the extra VIs first. */
1557         rc = bus_generic_detach(dev);
1558         if (rc)
1559                 return (rc);
1560         device_delete_children(dev);
1561
1562         doom_vi(sc, &pi->vi[0]);
1563
1564         if (pi->flags & HAS_TRACEQ) {
1565                 sc->traceq = -1;        /* cloner should not create ifnet */
1566                 t4_tracer_port_detach(sc);
1567         }
1568
1569         cxgbe_vi_detach(&pi->vi[0]);
1570         callout_drain(&pi->tick);
1571
1572         end_synchronized_op(sc, 0);
1573
1574         return (0);
1575 }
1576
1577 static void
1578 cxgbe_init(void *arg)
1579 {
1580         struct vi_info *vi = arg;
1581         struct adapter *sc = vi->pi->adapter;
1582
1583         if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
1584                 return;
1585         cxgbe_init_synchronized(vi);
1586         end_synchronized_op(sc, 0);
1587 }
1588
1589 static int
1590 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1591 {
1592         int rc = 0, mtu, flags, can_sleep;
1593         struct vi_info *vi = ifp->if_softc;
1594         struct adapter *sc = vi->pi->adapter;
1595         struct ifreq *ifr = (struct ifreq *)data;
1596         uint32_t mask;
1597
1598         switch (cmd) {
1599         case SIOCSIFMTU:
1600                 mtu = ifr->ifr_mtu;
1601                 if (mtu < ETHERMIN || mtu > MAX_MTU)
1602                         return (EINVAL);
1603
1604                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
1605                 if (rc)
1606                         return (rc);
1607                 ifp->if_mtu = mtu;
1608                 if (vi->flags & VI_INIT_DONE) {
1609                         t4_update_fl_bufsize(ifp);
1610                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1611                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1612                 }
1613                 end_synchronized_op(sc, 0);
1614                 break;
1615
1616         case SIOCSIFFLAGS:
1617                 can_sleep = 0;
1618 redo_sifflags:
1619                 rc = begin_synchronized_op(sc, vi,
1620                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1621                 if (rc)
1622                         return (rc);
1623
1624                 if (ifp->if_flags & IFF_UP) {
1625                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1626                                 flags = vi->if_flags;
1627                                 if ((ifp->if_flags ^ flags) &
1628                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1629                                         if (can_sleep == 1) {
1630                                                 end_synchronized_op(sc, 0);
1631                                                 can_sleep = 0;
1632                                                 goto redo_sifflags;
1633                                         }
1634                                         rc = update_mac_settings(ifp,
1635                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1636                                 }
1637                         } else {
1638                                 if (can_sleep == 0) {
1639                                         end_synchronized_op(sc, LOCK_HELD);
1640                                         can_sleep = 1;
1641                                         goto redo_sifflags;
1642                                 }
1643                                 rc = cxgbe_init_synchronized(vi);
1644                         }
1645                         vi->if_flags = ifp->if_flags;
1646                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1647                         if (can_sleep == 0) {
1648                                 end_synchronized_op(sc, LOCK_HELD);
1649                                 can_sleep = 1;
1650                                 goto redo_sifflags;
1651                         }
1652                         rc = cxgbe_uninit_synchronized(vi);
1653                 }
1654                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1655                 break;
1656
1657         case SIOCADDMULTI:
1658         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1659                 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi");
1660                 if (rc)
1661                         return (rc);
1662                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1663                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1664                 end_synchronized_op(sc, LOCK_HELD);
1665                 break;
1666
1667         case SIOCSIFCAP:
1668                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
1669                 if (rc)
1670                         return (rc);
1671
1672                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1673                 if (mask & IFCAP_TXCSUM) {
1674                         ifp->if_capenable ^= IFCAP_TXCSUM;
1675                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1676
1677                         if (IFCAP_TSO4 & ifp->if_capenable &&
1678                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1679                                 ifp->if_capenable &= ~IFCAP_TSO4;
1680                                 if_printf(ifp,
1681                                     "tso4 disabled due to -txcsum.\n");
1682                         }
1683                 }
1684                 if (mask & IFCAP_TXCSUM_IPV6) {
1685                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1686                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1687
1688                         if (IFCAP_TSO6 & ifp->if_capenable &&
1689                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1690                                 ifp->if_capenable &= ~IFCAP_TSO6;
1691                                 if_printf(ifp,
1692                                     "tso6 disabled due to -txcsum6.\n");
1693                         }
1694                 }
1695                 if (mask & IFCAP_RXCSUM)
1696                         ifp->if_capenable ^= IFCAP_RXCSUM;
1697                 if (mask & IFCAP_RXCSUM_IPV6)
1698                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1699
1700                 /*
1701                  * Note that we leave CSUM_TSO alone (it is always set).  The
1702                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1703                  * sending a TSO request our way, so it's sufficient to toggle
1704                  * IFCAP_TSOx only.
1705                  */
1706                 if (mask & IFCAP_TSO4) {
1707                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1708                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1709                                 if_printf(ifp, "enable txcsum first.\n");
1710                                 rc = EAGAIN;
1711                                 goto fail;
1712                         }
1713                         ifp->if_capenable ^= IFCAP_TSO4;
1714                 }
1715                 if (mask & IFCAP_TSO6) {
1716                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1717                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1718                                 if_printf(ifp, "enable txcsum6 first.\n");
1719                                 rc = EAGAIN;
1720                                 goto fail;
1721                         }
1722                         ifp->if_capenable ^= IFCAP_TSO6;
1723                 }
1724                 if (mask & IFCAP_LRO) {
1725 #if defined(INET) || defined(INET6)
1726                         int i;
1727                         struct sge_rxq *rxq;
1728
1729                         ifp->if_capenable ^= IFCAP_LRO;
1730                         for_each_rxq(vi, i, rxq) {
1731                                 if (ifp->if_capenable & IFCAP_LRO)
1732                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1733                                 else
1734                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1735                         }
1736 #endif
1737                 }
1738 #ifdef TCP_OFFLOAD
1739                 if (mask & IFCAP_TOE) {
1740                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1741
1742                         rc = toe_capability(vi, enable);
1743                         if (rc != 0)
1744                                 goto fail;
1745
1746                         ifp->if_capenable ^= mask;
1747                 }
1748 #endif
1749                 if (mask & IFCAP_VLAN_HWTAGGING) {
1750                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1751                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1752                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1753                 }
1754                 if (mask & IFCAP_VLAN_MTU) {
1755                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1756
1757                         /* Need to find out how to disable auto-mtu-inflation */
1758                 }
1759                 if (mask & IFCAP_VLAN_HWTSO)
1760                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1761                 if (mask & IFCAP_VLAN_HWCSUM)
1762                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1763
1764 #ifdef VLAN_CAPABILITIES
1765                 VLAN_CAPABILITIES(ifp);
1766 #endif
1767 fail:
1768                 end_synchronized_op(sc, 0);
1769                 break;
1770
1771         case SIOCSIFMEDIA:
1772         case SIOCGIFMEDIA:
1773         case SIOCGIFXMEDIA:
1774                 ifmedia_ioctl(ifp, ifr, &vi->media, cmd);
1775                 break;
1776
1777         case SIOCGI2C: {
1778                 struct ifi2creq i2c;
1779
1780                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1781                 if (rc != 0)
1782                         break;
1783                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1784                         rc = EPERM;
1785                         break;
1786                 }
1787                 if (i2c.len > sizeof(i2c.data)) {
1788                         rc = EINVAL;
1789                         break;
1790                 }
1791                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
1792                 if (rc)
1793                         return (rc);
1794                 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr,
1795                     i2c.offset, i2c.len, &i2c.data[0]);
1796                 end_synchronized_op(sc, 0);
1797                 if (rc == 0)
1798                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1799                 break;
1800         }
1801
1802         default:
1803                 rc = ether_ioctl(ifp, cmd, data);
1804         }
1805
1806         return (rc);
1807 }
1808
1809 static int
1810 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1811 {
1812         struct vi_info *vi = ifp->if_softc;
1813         struct port_info *pi = vi->pi;
1814         struct adapter *sc = pi->adapter;
1815         struct sge_txq *txq;
1816         void *items[1];
1817         int rc;
1818
1819         M_ASSERTPKTHDR(m);
1820         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1821
1822         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1823                 m_freem(m);
1824                 return (ENETDOWN);
1825         }
1826
1827         rc = parse_pkt(sc, &m);
1828         if (__predict_false(rc != 0)) {
1829                 MPASS(m == NULL);                       /* was freed already */
1830                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1831                 return (rc);
1832         }
1833
1834         /* Select a txq. */
1835         txq = &sc->sge.txq[vi->first_txq];
1836         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1837                 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
1838                     vi->rsrv_noflowq);
1839
1840         items[0] = m;
1841         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1842         if (__predict_false(rc != 0))
1843                 m_freem(m);
1844
1845         return (rc);
1846 }
1847
1848 static void
1849 cxgbe_qflush(struct ifnet *ifp)
1850 {
1851         struct vi_info *vi = ifp->if_softc;
1852         struct sge_txq *txq;
1853         int i;
1854
1855         /* queues do not exist if !VI_INIT_DONE. */
1856         if (vi->flags & VI_INIT_DONE) {
1857                 for_each_txq(vi, i, txq) {
1858                         TXQ_LOCK(txq);
1859                         txq->eq.flags &= ~EQ_ENABLED;
1860                         TXQ_UNLOCK(txq);
1861                         while (!mp_ring_is_idle(txq->r)) {
1862                                 mp_ring_check_drainage(txq->r, 0);
1863                                 pause("qflush", 1);
1864                         }
1865                 }
1866         }
1867         if_qflush(ifp);
1868 }
1869
1870 static uint64_t
1871 vi_get_counter(struct ifnet *ifp, ift_counter c)
1872 {
1873         struct vi_info *vi = ifp->if_softc;
1874         struct fw_vi_stats_vf *s = &vi->stats;
1875
1876         vi_refresh_stats(vi->pi->adapter, vi);
1877
1878         switch (c) {
1879         case IFCOUNTER_IPACKETS:
1880                 return (s->rx_bcast_frames + s->rx_mcast_frames +
1881                     s->rx_ucast_frames);
1882         case IFCOUNTER_IERRORS:
1883                 return (s->rx_err_frames);
1884         case IFCOUNTER_OPACKETS:
1885                 return (s->tx_bcast_frames + s->tx_mcast_frames +
1886                     s->tx_ucast_frames + s->tx_offload_frames);
1887         case IFCOUNTER_OERRORS:
1888                 return (s->tx_drop_frames);
1889         case IFCOUNTER_IBYTES:
1890                 return (s->rx_bcast_bytes + s->rx_mcast_bytes +
1891                     s->rx_ucast_bytes);
1892         case IFCOUNTER_OBYTES:
1893                 return (s->tx_bcast_bytes + s->tx_mcast_bytes +
1894                     s->tx_ucast_bytes + s->tx_offload_bytes);
1895         case IFCOUNTER_IMCASTS:
1896                 return (s->rx_mcast_frames);
1897         case IFCOUNTER_OMCASTS:
1898                 return (s->tx_mcast_frames);
1899         case IFCOUNTER_OQDROPS: {
1900                 uint64_t drops;
1901
1902                 drops = 0;
1903                 if (vi->flags & VI_INIT_DONE) {
1904                         int i;
1905                         struct sge_txq *txq;
1906
1907                         for_each_txq(vi, i, txq)
1908                                 drops += counter_u64_fetch(txq->r->drops);
1909                 }
1910
1911                 return (drops);
1912
1913         }
1914
1915         default:
1916                 return (if_get_counter_default(ifp, c));
1917         }
1918 }
1919
1920 uint64_t
1921 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1922 {
1923         struct vi_info *vi = ifp->if_softc;
1924         struct port_info *pi = vi->pi;
1925         struct adapter *sc = pi->adapter;
1926         struct port_stats *s = &pi->stats;
1927
1928         if (pi->nvi > 1 || sc->flags & IS_VF)
1929                 return (vi_get_counter(ifp, c));
1930
1931         cxgbe_refresh_stats(sc, pi);
1932
1933         switch (c) {
1934         case IFCOUNTER_IPACKETS:
1935                 return (s->rx_frames);
1936
1937         case IFCOUNTER_IERRORS:
1938                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1939                     s->rx_fcs_err + s->rx_len_err);
1940
1941         case IFCOUNTER_OPACKETS:
1942                 return (s->tx_frames);
1943
1944         case IFCOUNTER_OERRORS:
1945                 return (s->tx_error_frames);
1946
1947         case IFCOUNTER_IBYTES:
1948                 return (s->rx_octets);
1949
1950         case IFCOUNTER_OBYTES:
1951                 return (s->tx_octets);
1952
1953         case IFCOUNTER_IMCASTS:
1954                 return (s->rx_mcast_frames);
1955
1956         case IFCOUNTER_OMCASTS:
1957                 return (s->tx_mcast_frames);
1958
1959         case IFCOUNTER_IQDROPS:
1960                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1961                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1962                     s->rx_trunc3 + pi->tnl_cong_drops);
1963
1964         case IFCOUNTER_OQDROPS: {
1965                 uint64_t drops;
1966
1967                 drops = s->tx_drop;
1968                 if (vi->flags & VI_INIT_DONE) {
1969                         int i;
1970                         struct sge_txq *txq;
1971
1972                         for_each_txq(vi, i, txq)
1973                                 drops += counter_u64_fetch(txq->r->drops);
1974                 }
1975
1976                 return (drops);
1977
1978         }
1979
1980         default:
1981                 return (if_get_counter_default(ifp, c));
1982         }
1983 }
1984
1985 static int
1986 cxgbe_media_change(struct ifnet *ifp)
1987 {
1988         struct vi_info *vi = ifp->if_softc;
1989
1990         device_printf(vi->dev, "%s unimplemented.\n", __func__);
1991
1992         return (EOPNOTSUPP);
1993 }
1994
1995 static void
1996 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1997 {
1998         struct vi_info *vi = ifp->if_softc;
1999         struct port_info *pi = vi->pi;
2000         struct ifmedia_entry *cur;
2001         int speed = pi->link_cfg.speed;
2002
2003         cur = vi->media.ifm_cur;
2004
2005         ifmr->ifm_status = IFM_AVALID;
2006         if (!pi->link_cfg.link_ok)
2007                 return;
2008
2009         ifmr->ifm_status |= IFM_ACTIVE;
2010
2011         /* active and current will differ iff current media is autoselect. */
2012         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2013                 return;
2014
2015         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2016         if (speed == 10000)
2017                 ifmr->ifm_active |= IFM_10G_T;
2018         else if (speed == 1000)
2019                 ifmr->ifm_active |= IFM_1000_T;
2020         else if (speed == 100)
2021                 ifmr->ifm_active |= IFM_100_TX;
2022         else if (speed == 10)
2023                 ifmr->ifm_active |= IFM_10_T;
2024         else
2025                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2026                             speed));
2027 }
2028
2029 static int
2030 vcxgbe_probe(device_t dev)
2031 {
2032         char buf[128];
2033         struct vi_info *vi = device_get_softc(dev);
2034
2035         snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
2036             vi - vi->pi->vi);
2037         device_set_desc_copy(dev, buf);
2038
2039         return (BUS_PROBE_DEFAULT);
2040 }
2041
2042 static int
2043 vcxgbe_attach(device_t dev)
2044 {
2045         struct vi_info *vi;
2046         struct port_info *pi;
2047         struct adapter *sc;
2048         int func, index, rc;
2049         u32 param, val;
2050
2051         vi = device_get_softc(dev);
2052         pi = vi->pi;
2053         sc = pi->adapter;
2054
2055         index = vi - pi->vi;
2056         KASSERT(index < nitems(vi_mac_funcs),
2057             ("%s: VI %s doesn't have a MAC func", __func__,
2058             device_get_nameunit(dev)));
2059         func = vi_mac_funcs[index];
2060         rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
2061             vi->hw_addr, &vi->rss_size, func, 0);
2062         if (rc < 0) {
2063                 device_printf(dev, "Failed to allocate virtual interface "
2064                     "for port %d: %d\n", pi->port_id, -rc);
2065                 return (-rc);
2066         }
2067         vi->viid = rc;
2068         if (chip_id(sc) <= CHELSIO_T5)
2069                 vi->smt_idx = (rc & 0x7f) << 1;
2070         else
2071                 vi->smt_idx = (rc & 0x7f);
2072
2073         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
2074             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
2075             V_FW_PARAMS_PARAM_YZ(vi->viid);
2076         rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2077         if (rc)
2078                 vi->rss_base = 0xffff;
2079         else {
2080                 /* MPASS((val >> 16) == rss_size); */
2081                 vi->rss_base = val & 0xffff;
2082         }
2083
2084         rc = cxgbe_vi_attach(dev, vi);
2085         if (rc) {
2086                 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
2087                 return (rc);
2088         }
2089         return (0);
2090 }
2091
2092 static int
2093 vcxgbe_detach(device_t dev)
2094 {
2095         struct vi_info *vi;
2096         struct adapter *sc;
2097
2098         vi = device_get_softc(dev);
2099         sc = vi->pi->adapter;
2100
2101         doom_vi(sc, vi);
2102
2103         cxgbe_vi_detach(vi);
2104         t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
2105
2106         end_synchronized_op(sc, 0);
2107
2108         return (0);
2109 }
2110
2111 void
2112 t4_fatal_err(struct adapter *sc)
2113 {
2114         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
2115         t4_intr_disable(sc);
2116         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
2117             device_get_nameunit(sc->dev));
2118 }
2119
2120 void
2121 t4_add_adapter(struct adapter *sc)
2122 {
2123         sx_xlock(&t4_list_lock);
2124         SLIST_INSERT_HEAD(&t4_list, sc, link);
2125         sx_xunlock(&t4_list_lock);
2126 }
2127
2128 int
2129 t4_map_bars_0_and_4(struct adapter *sc)
2130 {
2131         sc->regs_rid = PCIR_BAR(0);
2132         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2133             &sc->regs_rid, RF_ACTIVE);
2134         if (sc->regs_res == NULL) {
2135                 device_printf(sc->dev, "cannot map registers.\n");
2136                 return (ENXIO);
2137         }
2138         sc->bt = rman_get_bustag(sc->regs_res);
2139         sc->bh = rman_get_bushandle(sc->regs_res);
2140         sc->mmio_len = rman_get_size(sc->regs_res);
2141         setbit(&sc->doorbells, DOORBELL_KDB);
2142
2143         sc->msix_rid = PCIR_BAR(4);
2144         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2145             &sc->msix_rid, RF_ACTIVE);
2146         if (sc->msix_res == NULL) {
2147                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
2148                 return (ENXIO);
2149         }
2150
2151         return (0);
2152 }
2153
2154 int
2155 t4_map_bar_2(struct adapter *sc)
2156 {
2157
2158         /*
2159          * T4: only iWARP driver uses the userspace doorbells.  There is no need
2160          * to map it if RDMA is disabled.
2161          */
2162         if (is_t4(sc) && sc->rdmacaps == 0)
2163                 return (0);
2164
2165         sc->udbs_rid = PCIR_BAR(2);
2166         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2167             &sc->udbs_rid, RF_ACTIVE);
2168         if (sc->udbs_res == NULL) {
2169                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
2170                 return (ENXIO);
2171         }
2172         sc->udbs_base = rman_get_virtual(sc->udbs_res);
2173
2174         if (chip_id(sc) >= CHELSIO_T5) {
2175                 setbit(&sc->doorbells, DOORBELL_UDB);
2176 #if defined(__i386__) || defined(__amd64__)
2177                 if (t5_write_combine) {
2178                         int rc, mode;
2179
2180                         /*
2181                          * Enable write combining on BAR2.  This is the
2182                          * userspace doorbell BAR and is split into 128B
2183                          * (UDBS_SEG_SIZE) doorbell regions, each associated
2184                          * with an egress queue.  The first 64B has the doorbell
2185                          * and the second 64B can be used to submit a tx work
2186                          * request with an implicit doorbell.
2187                          */
2188
2189                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
2190                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
2191                         if (rc == 0) {
2192                                 clrbit(&sc->doorbells, DOORBELL_UDB);
2193                                 setbit(&sc->doorbells, DOORBELL_WCWR);
2194                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
2195                         } else {
2196                                 device_printf(sc->dev,
2197                                     "couldn't enable write combining: %d\n",
2198                                     rc);
2199                         }
2200
2201                         mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0);
2202                         t4_write_reg(sc, A_SGE_STAT_CFG,
2203                             V_STATSOURCE_T5(7) | mode);
2204                 }
2205 #endif
2206         }
2207
2208         return (0);
2209 }
2210
2211 struct memwin_init {
2212         uint32_t base;
2213         uint32_t aperture;
2214 };
2215
2216 static const struct memwin_init t4_memwin[NUM_MEMWIN] = {
2217         { MEMWIN0_BASE, MEMWIN0_APERTURE },
2218         { MEMWIN1_BASE, MEMWIN1_APERTURE },
2219         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
2220 };
2221
2222 static const struct memwin_init t5_memwin[NUM_MEMWIN] = {
2223         { MEMWIN0_BASE, MEMWIN0_APERTURE },
2224         { MEMWIN1_BASE, MEMWIN1_APERTURE },
2225         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
2226 };
2227
2228 static void
2229 setup_memwin(struct adapter *sc)
2230 {
2231         const struct memwin_init *mw_init;
2232         struct memwin *mw;
2233         int i;
2234         uint32_t bar0;
2235
2236         if (is_t4(sc)) {
2237                 /*
2238                  * Read low 32b of bar0 indirectly via the hardware backdoor
2239                  * mechanism.  Works from within PCI passthrough environments
2240                  * too, where rman_get_start() can return a different value.  We
2241                  * need to program the T4 memory window decoders with the actual
2242                  * addresses that will be coming across the PCIe link.
2243                  */
2244                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
2245                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
2246
2247                 mw_init = &t4_memwin[0];
2248         } else {
2249                 /* T5+ use the relative offset inside the PCIe BAR */
2250                 bar0 = 0;
2251
2252                 mw_init = &t5_memwin[0];
2253         }
2254
2255         for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
2256                 rw_init(&mw->mw_lock, "memory window access");
2257                 mw->mw_base = mw_init->base;
2258                 mw->mw_aperture = mw_init->aperture;
2259                 mw->mw_curpos = 0;
2260                 t4_write_reg(sc,
2261                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
2262                     (mw->mw_base + bar0) | V_BIR(0) |
2263                     V_WINDOW(ilog2(mw->mw_aperture) - 10));
2264                 rw_wlock(&mw->mw_lock);
2265                 position_memwin(sc, i, 0);
2266                 rw_wunlock(&mw->mw_lock);
2267         }
2268
2269         /* flush */
2270         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
2271 }
2272
2273 /*
2274  * Positions the memory window at the given address in the card's address space.
2275  * There are some alignment requirements and the actual position may be at an
2276  * address prior to the requested address.  mw->mw_curpos always has the actual
2277  * position of the window.
2278  */
2279 static void
2280 position_memwin(struct adapter *sc, int idx, uint32_t addr)
2281 {
2282         struct memwin *mw;
2283         uint32_t pf;
2284         uint32_t reg;
2285
2286         MPASS(idx >= 0 && idx < NUM_MEMWIN);
2287         mw = &sc->memwin[idx];
2288         rw_assert(&mw->mw_lock, RA_WLOCKED);
2289
2290         if (is_t4(sc)) {
2291                 pf = 0;
2292                 mw->mw_curpos = addr & ~0xf;    /* start must be 16B aligned */
2293         } else {
2294                 pf = V_PFNUM(sc->pf);
2295                 mw->mw_curpos = addr & ~0x7f;   /* start must be 128B aligned */
2296         }
2297         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx);
2298         t4_write_reg(sc, reg, mw->mw_curpos | pf);
2299         t4_read_reg(sc, reg);   /* flush */
2300 }
2301
2302 static int
2303 rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
2304     int len, int rw)
2305 {
2306         struct memwin *mw;
2307         uint32_t mw_end, v;
2308
2309         MPASS(idx >= 0 && idx < NUM_MEMWIN);
2310
2311         /* Memory can only be accessed in naturally aligned 4 byte units */
2312         if (addr & 3 || len & 3 || len <= 0)
2313                 return (EINVAL);
2314
2315         mw = &sc->memwin[idx];
2316         while (len > 0) {
2317                 rw_rlock(&mw->mw_lock);
2318                 mw_end = mw->mw_curpos + mw->mw_aperture;
2319                 if (addr >= mw_end || addr < mw->mw_curpos) {
2320                         /* Will need to reposition the window */
2321                         if (!rw_try_upgrade(&mw->mw_lock)) {
2322                                 rw_runlock(&mw->mw_lock);
2323                                 rw_wlock(&mw->mw_lock);
2324                         }
2325                         rw_assert(&mw->mw_lock, RA_WLOCKED);
2326                         position_memwin(sc, idx, addr);
2327                         rw_downgrade(&mw->mw_lock);
2328                         mw_end = mw->mw_curpos + mw->mw_aperture;
2329                 }
2330                 rw_assert(&mw->mw_lock, RA_RLOCKED);
2331                 while (addr < mw_end && len > 0) {
2332                         if (rw == 0) {
2333                                 v = t4_read_reg(sc, mw->mw_base + addr -
2334                                     mw->mw_curpos);
2335                                 *val++ = le32toh(v);
2336                         } else {
2337                                 v = *val++;
2338                                 t4_write_reg(sc, mw->mw_base + addr -
2339                                     mw->mw_curpos, htole32(v));
2340                         }
2341                         addr += 4;
2342                         len -= 4;
2343                 }
2344                 rw_runlock(&mw->mw_lock);
2345         }
2346
2347         return (0);
2348 }
2349
2350 static inline int
2351 read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
2352     int len)
2353 {
2354
2355         return (rw_via_memwin(sc, idx, addr, val, len, 0));
2356 }
2357
2358 static inline int
2359 write_via_memwin(struct adapter *sc, int idx, uint32_t addr,
2360     const uint32_t *val, int len)
2361 {
2362
2363         return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1));
2364 }
2365
2366 static int
2367 t4_range_cmp(const void *a, const void *b)
2368 {
2369         return ((const struct t4_range *)a)->start -
2370                ((const struct t4_range *)b)->start;
2371 }
2372
2373 /*
2374  * Verify that the memory range specified by the addr/len pair is valid within
2375  * the card's address space.
2376  */
2377 static int
2378 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
2379 {
2380         struct t4_range mem_ranges[4], *r, *next;
2381         uint32_t em, addr_len;
2382         int i, n, remaining;
2383
2384         /* Memory can only be accessed in naturally aligned 4 byte units */
2385         if (addr & 3 || len & 3 || len <= 0)
2386                 return (EINVAL);
2387
2388         /* Enabled memories */
2389         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2390
2391         r = &mem_ranges[0];
2392         n = 0;
2393         bzero(r, sizeof(mem_ranges));
2394         if (em & F_EDRAM0_ENABLE) {
2395                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2396                 r->size = G_EDRAM0_SIZE(addr_len) << 20;
2397                 if (r->size > 0) {
2398                         r->start = G_EDRAM0_BASE(addr_len) << 20;
2399                         if (addr >= r->start &&
2400                             addr + len <= r->start + r->size)
2401                                 return (0);
2402                         r++;
2403                         n++;
2404                 }
2405         }
2406         if (em & F_EDRAM1_ENABLE) {
2407                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2408                 r->size = G_EDRAM1_SIZE(addr_len) << 20;
2409                 if (r->size > 0) {
2410                         r->start = G_EDRAM1_BASE(addr_len) << 20;
2411                         if (addr >= r->start &&
2412                             addr + len <= r->start + r->size)
2413                                 return (0);
2414                         r++;
2415                         n++;
2416                 }
2417         }
2418         if (em & F_EXT_MEM_ENABLE) {
2419                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2420                 r->size = G_EXT_MEM_SIZE(addr_len) << 20;
2421                 if (r->size > 0) {
2422                         r->start = G_EXT_MEM_BASE(addr_len) << 20;
2423                         if (addr >= r->start &&
2424                             addr + len <= r->start + r->size)
2425                                 return (0);
2426                         r++;
2427                         n++;
2428                 }
2429         }
2430         if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) {
2431                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2432                 r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
2433                 if (r->size > 0) {
2434                         r->start = G_EXT_MEM1_BASE(addr_len) << 20;
2435                         if (addr >= r->start &&
2436                             addr + len <= r->start + r->size)
2437                                 return (0);
2438                         r++;
2439                         n++;
2440                 }
2441         }
2442         MPASS(n <= nitems(mem_ranges));
2443
2444         if (n > 1) {
2445                 /* Sort and merge the ranges. */
2446                 qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp);
2447
2448                 /* Start from index 0 and examine the next n - 1 entries. */
2449                 r = &mem_ranges[0];
2450                 for (remaining = n - 1; remaining > 0; remaining--, r++) {
2451
2452                         MPASS(r->size > 0);     /* r is a valid entry. */
2453                         next = r + 1;
2454                         MPASS(next->size > 0);  /* and so is the next one. */
2455
2456                         while (r->start + r->size >= next->start) {
2457                                 /* Merge the next one into the current entry. */
2458                                 r->size = max(r->start + r->size,
2459                                     next->start + next->size) - r->start;
2460                                 n--;    /* One fewer entry in total. */
2461                                 if (--remaining == 0)
2462                                         goto done;      /* short circuit */
2463                                 next++;
2464                         }
2465                         if (next != r + 1) {
2466                                 /*
2467                                  * Some entries were merged into r and next
2468                                  * points to the first valid entry that couldn't
2469                                  * be merged.
2470                                  */
2471                                 MPASS(next->size > 0);  /* must be valid */
2472                                 memcpy(r + 1, next, remaining * sizeof(*r));
2473 #ifdef INVARIANTS
2474                                 /*
2475                                  * This so that the foo->size assertion in the
2476                                  * next iteration of the loop do the right
2477                                  * thing for entries that were pulled up and are
2478                                  * no longer valid.
2479                                  */
2480                                 MPASS(n < nitems(mem_ranges));
2481                                 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
2482                                     sizeof(struct t4_range));
2483 #endif
2484                         }
2485                 }
2486 done:
2487                 /* Done merging the ranges. */
2488                 MPASS(n > 0);
2489                 r = &mem_ranges[0];
2490                 for (i = 0; i < n; i++, r++) {
2491                         if (addr >= r->start &&
2492                             addr + len <= r->start + r->size)
2493                                 return (0);
2494                 }
2495         }
2496
2497         return (EFAULT);
2498 }
2499
2500 static int
2501 fwmtype_to_hwmtype(int mtype)
2502 {
2503
2504         switch (mtype) {
2505         case FW_MEMTYPE_EDC0:
2506                 return (MEM_EDC0);
2507         case FW_MEMTYPE_EDC1:
2508                 return (MEM_EDC1);
2509         case FW_MEMTYPE_EXTMEM:
2510                 return (MEM_MC0);
2511         case FW_MEMTYPE_EXTMEM1:
2512                 return (MEM_MC1);
2513         default:
2514                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
2515         }
2516 }
2517
2518 /*
2519  * Verify that the memory range specified by the memtype/offset/len pair is
2520  * valid and lies entirely within the memtype specified.  The global address of
2521  * the start of the range is returned in addr.
2522  */
2523 static int
2524 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
2525     uint32_t *addr)
2526 {
2527         uint32_t em, addr_len, maddr;
2528
2529         /* Memory can only be accessed in naturally aligned 4 byte units */
2530         if (off & 3 || len & 3 || len == 0)
2531                 return (EINVAL);
2532
2533         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2534         switch (fwmtype_to_hwmtype(mtype)) {
2535         case MEM_EDC0:
2536                 if (!(em & F_EDRAM0_ENABLE))
2537                         return (EINVAL);
2538                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2539                 maddr = G_EDRAM0_BASE(addr_len) << 20;
2540                 break;
2541         case MEM_EDC1:
2542                 if (!(em & F_EDRAM1_ENABLE))
2543                         return (EINVAL);
2544                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2545                 maddr = G_EDRAM1_BASE(addr_len) << 20;
2546                 break;
2547         case MEM_MC:
2548                 if (!(em & F_EXT_MEM_ENABLE))
2549                         return (EINVAL);
2550                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2551                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
2552                 break;
2553         case MEM_MC1:
2554                 if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE))
2555                         return (EINVAL);
2556                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2557                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
2558                 break;
2559         default:
2560                 return (EINVAL);
2561         }
2562
2563         *addr = maddr + off;    /* global address */
2564         return (validate_mem_range(sc, *addr, len));
2565 }
2566
2567 static int
2568 fixup_devlog_params(struct adapter *sc)
2569 {
2570         struct devlog_params *dparams = &sc->params.devlog;
2571         int rc;
2572
2573         rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
2574             dparams->size, &dparams->addr);
2575
2576         return (rc);
2577 }
2578
2579 static int
2580 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis,
2581     struct intrs_and_queues *iaq)
2582 {
2583         int rc, itype, navail, nrxq10g, nrxq1g, n;
2584         int nofldrxq10g = 0, nofldrxq1g = 0;
2585
2586         bzero(iaq, sizeof(*iaq));
2587
2588         iaq->ntxq10g = t4_ntxq10g;
2589         iaq->ntxq1g = t4_ntxq1g;
2590         iaq->ntxq_vi = t4_ntxq_vi;
2591         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
2592         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
2593         iaq->nrxq_vi = t4_nrxq_vi;
2594         iaq->rsrv_noflowq = t4_rsrv_noflowq;
2595 #ifdef TCP_OFFLOAD
2596         if (is_offload(sc)) {
2597                 iaq->nofldtxq10g = t4_nofldtxq10g;
2598                 iaq->nofldtxq1g = t4_nofldtxq1g;
2599                 iaq->nofldtxq_vi = t4_nofldtxq_vi;
2600                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
2601                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
2602                 iaq->nofldrxq_vi = t4_nofldrxq_vi;
2603         }
2604 #endif
2605 #ifdef DEV_NETMAP
2606         iaq->nnmtxq_vi = t4_nnmtxq_vi;
2607         iaq->nnmrxq_vi = t4_nnmrxq_vi;
2608 #endif
2609
2610         for (itype = INTR_MSIX; itype; itype >>= 1) {
2611
2612                 if ((itype & t4_intr_types) == 0)
2613                         continue;       /* not allowed */
2614
2615                 if (itype == INTR_MSIX)
2616                         navail = pci_msix_count(sc->dev);
2617                 else if (itype == INTR_MSI)
2618                         navail = pci_msi_count(sc->dev);
2619                 else
2620                         navail = 1;
2621 restart:
2622                 if (navail == 0)
2623                         continue;
2624
2625                 iaq->intr_type = itype;
2626                 iaq->intr_flags_10g = 0;
2627                 iaq->intr_flags_1g = 0;
2628
2629                 /*
2630                  * Best option: an interrupt vector for errors, one for the
2631                  * firmware event queue, and one for every rxq (NIC and TOE) of
2632                  * every VI.  The VIs that support netmap use the same
2633                  * interrupts for the NIC rx queues and the netmap rx queues
2634                  * because only one set of queues is active at a time.
2635                  */
2636                 iaq->nirq = T4_EXTRA_INTR;
2637                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
2638                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
2639                 iaq->nirq += (n10g + n1g) * (num_vis - 1) *
2640                     max(iaq->nrxq_vi, iaq->nnmrxq_vi);  /* See comment above. */
2641                 iaq->nirq += (n10g + n1g) * (num_vis - 1) * iaq->nofldrxq_vi;
2642                 if (iaq->nirq <= navail &&
2643                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
2644                         iaq->intr_flags_10g = INTR_ALL;
2645                         iaq->intr_flags_1g = INTR_ALL;
2646                         goto allocate;
2647                 }
2648
2649                 /* Disable the VIs (and netmap) if there aren't enough intrs */
2650                 if (num_vis > 1) {
2651                         device_printf(sc->dev, "virtual interfaces disabled "
2652                             "because num_vis=%u with current settings "
2653                             "(nrxq10g=%u, nrxq1g=%u, nofldrxq10g=%u, "
2654                             "nofldrxq1g=%u, nrxq_vi=%u nofldrxq_vi=%u, "
2655                             "nnmrxq_vi=%u) would need %u interrupts but "
2656                             "only %u are available.\n", num_vis, nrxq10g,
2657                             nrxq1g, nofldrxq10g, nofldrxq1g, iaq->nrxq_vi,
2658                             iaq->nofldrxq_vi, iaq->nnmrxq_vi, iaq->nirq,
2659                             navail);
2660                         num_vis = 1;
2661                         iaq->ntxq_vi = iaq->nrxq_vi = 0;
2662                         iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
2663                         iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
2664                         goto restart;
2665                 }
2666
2667                 /*
2668                  * Second best option: a vector for errors, one for the firmware
2669                  * event queue, and vectors for either all the NIC rx queues or
2670                  * all the TOE rx queues.  The queues that don't get vectors
2671                  * will forward their interrupts to those that do.
2672                  */
2673                 iaq->nirq = T4_EXTRA_INTR;
2674                 if (nrxq10g >= nofldrxq10g) {
2675                         iaq->intr_flags_10g = INTR_RXQ;
2676                         iaq->nirq += n10g * nrxq10g;
2677                 } else {
2678                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2679                         iaq->nirq += n10g * nofldrxq10g;
2680                 }
2681                 if (nrxq1g >= nofldrxq1g) {
2682                         iaq->intr_flags_1g = INTR_RXQ;
2683                         iaq->nirq += n1g * nrxq1g;
2684                 } else {
2685                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2686                         iaq->nirq += n1g * nofldrxq1g;
2687                 }
2688                 if (iaq->nirq <= navail &&
2689                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2690                         goto allocate;
2691
2692                 /*
2693                  * Next best option: an interrupt vector for errors, one for the
2694                  * firmware event queue, and at least one per main-VI.  At this
2695                  * point we know we'll have to downsize nrxq and/or nofldrxq to
2696                  * fit what's available to us.
2697                  */
2698                 iaq->nirq = T4_EXTRA_INTR;
2699                 iaq->nirq += n10g + n1g;
2700                 if (iaq->nirq <= navail) {
2701                         int leftover = navail - iaq->nirq;
2702
2703                         if (n10g > 0) {
2704                                 int target = max(nrxq10g, nofldrxq10g);
2705
2706                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2707                                     INTR_RXQ : INTR_OFLD_RXQ;
2708
2709                                 n = 1;
2710                                 while (n < target && leftover >= n10g) {
2711                                         leftover -= n10g;
2712                                         iaq->nirq += n10g;
2713                                         n++;
2714                                 }
2715                                 iaq->nrxq10g = min(n, nrxq10g);
2716 #ifdef TCP_OFFLOAD
2717                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2718 #endif
2719                         }
2720
2721                         if (n1g > 0) {
2722                                 int target = max(nrxq1g, nofldrxq1g);
2723
2724                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2725                                     INTR_RXQ : INTR_OFLD_RXQ;
2726
2727                                 n = 1;
2728                                 while (n < target && leftover >= n1g) {
2729                                         leftover -= n1g;
2730                                         iaq->nirq += n1g;
2731                                         n++;
2732                                 }
2733                                 iaq->nrxq1g = min(n, nrxq1g);
2734 #ifdef TCP_OFFLOAD
2735                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2736 #endif
2737                         }
2738
2739                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2740                                 goto allocate;
2741                 }
2742
2743                 /*
2744                  * Least desirable option: one interrupt vector for everything.
2745                  */
2746                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2747                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2748 #ifdef TCP_OFFLOAD
2749                 if (is_offload(sc))
2750                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2751 #endif
2752 allocate:
2753                 navail = iaq->nirq;
2754                 rc = 0;
2755                 if (itype == INTR_MSIX)
2756                         rc = pci_alloc_msix(sc->dev, &navail);
2757                 else if (itype == INTR_MSI)
2758                         rc = pci_alloc_msi(sc->dev, &navail);
2759
2760                 if (rc == 0) {
2761                         if (navail == iaq->nirq)
2762                                 return (0);
2763
2764                         /*
2765                          * Didn't get the number requested.  Use whatever number
2766                          * the kernel is willing to allocate (it's in navail).
2767                          */
2768                         device_printf(sc->dev, "fewer vectors than requested, "
2769                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2770                             itype, iaq->nirq, navail);
2771                         pci_release_msi(sc->dev);
2772                         goto restart;
2773                 }
2774
2775                 device_printf(sc->dev,
2776                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2777                     itype, rc, iaq->nirq, navail);
2778         }
2779
2780         device_printf(sc->dev,
2781             "failed to find a usable interrupt type.  "
2782             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2783             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2784
2785         return (ENXIO);
2786 }
2787
2788 #define FW_VERSION(chip) ( \
2789     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2790     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2791     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2792     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2793 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2794
2795 struct fw_info {
2796         uint8_t chip;
2797         char *kld_name;
2798         char *fw_mod_name;
2799         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2800 } fw_info[] = {
2801         {
2802                 .chip = CHELSIO_T4,
2803                 .kld_name = "t4fw_cfg",
2804                 .fw_mod_name = "t4fw",
2805                 .fw_hdr = {
2806                         .chip = FW_HDR_CHIP_T4,
2807                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2808                         .intfver_nic = FW_INTFVER(T4, NIC),
2809                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2810                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2811                         .intfver_ri = FW_INTFVER(T4, RI),
2812                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2813                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2814                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2815                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2816                 },
2817         }, {
2818                 .chip = CHELSIO_T5,
2819                 .kld_name = "t5fw_cfg",
2820                 .fw_mod_name = "t5fw",
2821                 .fw_hdr = {
2822                         .chip = FW_HDR_CHIP_T5,
2823                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2824                         .intfver_nic = FW_INTFVER(T5, NIC),
2825                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2826                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2827                         .intfver_ri = FW_INTFVER(T5, RI),
2828                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2829                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2830                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2831                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2832                 },
2833         }, {
2834                 .chip = CHELSIO_T6,
2835                 .kld_name = "t6fw_cfg",
2836                 .fw_mod_name = "t6fw",
2837                 .fw_hdr = {
2838                         .chip = FW_HDR_CHIP_T6,
2839                         .fw_ver = htobe32_const(FW_VERSION(T6)),
2840                         .intfver_nic = FW_INTFVER(T6, NIC),
2841                         .intfver_vnic = FW_INTFVER(T6, VNIC),
2842                         .intfver_ofld = FW_INTFVER(T6, OFLD),
2843                         .intfver_ri = FW_INTFVER(T6, RI),
2844                         .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
2845                         .intfver_iscsi = FW_INTFVER(T6, ISCSI),
2846                         .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
2847                         .intfver_fcoe = FW_INTFVER(T6, FCOE),
2848                 },
2849         }
2850 };
2851
2852 static struct fw_info *
2853 find_fw_info(int chip)
2854 {
2855         int i;
2856
2857         for (i = 0; i < nitems(fw_info); i++) {
2858                 if (fw_info[i].chip == chip)
2859                         return (&fw_info[i]);
2860         }
2861         return (NULL);
2862 }
2863
2864 /*
2865  * Is the given firmware API compatible with the one the driver was compiled
2866  * with?
2867  */
2868 static int
2869 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2870 {
2871
2872         /* short circuit if it's the exact same firmware version */
2873         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2874                 return (1);
2875
2876         /*
2877          * XXX: Is this too conservative?  Perhaps I should limit this to the
2878          * features that are supported in the driver.
2879          */
2880 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2881         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2882             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2883             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2884                 return (1);
2885 #undef SAME_INTF
2886
2887         return (0);
2888 }
2889
2890 /*
2891  * The firmware in the KLD is usable, but should it be installed?  This routine
2892  * explains itself in detail if it indicates the KLD firmware should be
2893  * installed.
2894  */
2895 static int
2896 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2897 {
2898         const char *reason;
2899
2900         if (!card_fw_usable) {
2901                 reason = "incompatible or unusable";
2902                 goto install;
2903         }
2904
2905         if (k > c) {
2906                 reason = "older than the version bundled with this driver";
2907                 goto install;
2908         }
2909
2910         if (t4_fw_install == 2 && k != c) {
2911                 reason = "different than the version bundled with this driver";
2912                 goto install;
2913         }
2914
2915         return (0);
2916
2917 install:
2918         if (t4_fw_install == 0) {
2919                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2920                     "but the driver is prohibited from installing a different "
2921                     "firmware on the card.\n",
2922                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2923                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2924
2925                 return (0);
2926         }
2927
2928         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2929             "installing firmware %u.%u.%u.%u on card.\n",
2930             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2931             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2932             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2933             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2934
2935         return (1);
2936 }
2937 /*
2938  * Establish contact with the firmware and determine if we are the master driver
2939  * or not, and whether we are responsible for chip initialization.
2940  */
2941 static int
2942 prep_firmware(struct adapter *sc)
2943 {
2944         const struct firmware *fw = NULL, *default_cfg;
2945         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2946         enum dev_state state;
2947         struct fw_info *fw_info;
2948         struct fw_hdr *card_fw;         /* fw on the card */
2949         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2950         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2951                                            against */
2952
2953         /* Contact firmware. */
2954         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2955         if (rc < 0 || state == DEV_STATE_ERR) {
2956                 rc = -rc;
2957                 device_printf(sc->dev,
2958                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2959                 return (rc);
2960         }
2961         pf = rc;
2962         if (pf == sc->mbox)
2963                 sc->flags |= MASTER_PF;
2964         else if (state == DEV_STATE_UNINIT) {
2965                 /*
2966                  * We didn't get to be the master so we definitely won't be
2967                  * configuring the chip.  It's a bug if someone else hasn't
2968                  * configured it already.
2969                  */
2970                 device_printf(sc->dev, "couldn't be master(%d), "
2971                     "device not already initialized either(%d).\n", rc, state);
2972                 return (EDOOFUS);
2973         }
2974
2975         /* This is the firmware whose headers the driver was compiled against */
2976         fw_info = find_fw_info(chip_id(sc));
2977         if (fw_info == NULL) {
2978                 device_printf(sc->dev,
2979                     "unable to look up firmware information for chip %d.\n",
2980                     chip_id(sc));
2981                 return (EINVAL);
2982         }
2983         drv_fw = &fw_info->fw_hdr;
2984
2985         /*
2986          * The firmware KLD contains many modules.  The KLD name is also the
2987          * name of the module that contains the default config file.
2988          */
2989         default_cfg = firmware_get(fw_info->kld_name);
2990
2991         /* Read the header of the firmware on the card */
2992         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2993         rc = -t4_read_flash(sc, FLASH_FW_START,
2994             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2995         if (rc == 0)
2996                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2997         else {
2998                 device_printf(sc->dev,
2999                     "Unable to read card's firmware header: %d\n", rc);
3000                 card_fw_usable = 0;
3001         }
3002
3003         /* This is the firmware in the KLD */
3004         fw = firmware_get(fw_info->fw_mod_name);
3005         if (fw != NULL) {
3006                 kld_fw = (const void *)fw->data;
3007                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
3008         } else {
3009                 kld_fw = NULL;
3010                 kld_fw_usable = 0;
3011         }
3012
3013         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
3014             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
3015                 /*
3016                  * Common case: the firmware on the card is an exact match and
3017                  * the KLD is an exact match too, or the KLD is
3018                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
3019                  * here -- use cxgbetool loadfw if you want to reinstall the
3020                  * same firmware as the one on the card.
3021                  */
3022         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
3023             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
3024             be32toh(card_fw->fw_ver))) {
3025
3026                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
3027                 if (rc != 0) {
3028                         device_printf(sc->dev,
3029                             "failed to install firmware: %d\n", rc);
3030                         goto done;
3031                 }
3032
3033                 /* Installed successfully, update the cached header too. */
3034                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
3035                 card_fw_usable = 1;
3036                 need_fw_reset = 0;      /* already reset as part of load_fw */
3037         }
3038
3039         if (!card_fw_usable) {
3040                 uint32_t d, c, k;
3041
3042                 d = ntohl(drv_fw->fw_ver);
3043                 c = ntohl(card_fw->fw_ver);
3044                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
3045
3046                 device_printf(sc->dev, "Cannot find a usable firmware: "
3047                     "fw_install %d, chip state %d, "
3048                     "driver compiled with %d.%d.%d.%d, "
3049                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
3050                     t4_fw_install, state,
3051                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
3052                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
3053                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
3054                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
3055                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
3056                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
3057                 rc = EINVAL;
3058                 goto done;
3059         }
3060
3061         /* Reset device */
3062         if (need_fw_reset &&
3063             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
3064                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
3065                 if (rc != ETIMEDOUT && rc != EIO)
3066                         t4_fw_bye(sc, sc->mbox);
3067                 goto done;
3068         }
3069         sc->flags |= FW_OK;
3070
3071         rc = get_params__pre_init(sc);
3072         if (rc != 0)
3073                 goto done; /* error message displayed already */
3074
3075         /* Partition adapter resources as specified in the config file. */
3076         if (state == DEV_STATE_UNINIT) {
3077
3078                 KASSERT(sc->flags & MASTER_PF,
3079                     ("%s: trying to change chip settings when not master.",
3080                     __func__));
3081
3082                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
3083                 if (rc != 0)
3084                         goto done;      /* error message displayed already */
3085
3086                 t4_tweak_chip_settings(sc);
3087
3088                 /* get basic stuff going */
3089                 rc = -t4_fw_initialize(sc, sc->mbox);
3090                 if (rc != 0) {
3091                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
3092                         goto done;
3093                 }
3094         } else {
3095                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
3096                 sc->cfcsum = 0;
3097         }
3098
3099 done:
3100         free(card_fw, M_CXGBE);
3101         if (fw != NULL)
3102                 firmware_put(fw, FIRMWARE_UNLOAD);
3103         if (default_cfg != NULL)
3104                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
3105
3106         return (rc);
3107 }
3108
3109 #define FW_PARAM_DEV(param) \
3110         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3111          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3112 #define FW_PARAM_PFVF(param) \
3113         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3114          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
3115
3116 /*
3117  * Partition chip resources for use between various PFs, VFs, etc.
3118  */
3119 static int
3120 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
3121     const char *name_prefix)
3122 {
3123         const struct firmware *cfg = NULL;
3124         int rc = 0;
3125         struct fw_caps_config_cmd caps;
3126         uint32_t mtype, moff, finicsum, cfcsum;
3127
3128         /*
3129          * Figure out what configuration file to use.  Pick the default config
3130          * file for the card if the user hasn't specified one explicitly.
3131          */
3132         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
3133         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
3134                 /* Card specific overrides go here. */
3135                 if (pci_get_device(sc->dev) == 0x440a)
3136                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
3137                 if (is_fpga(sc))
3138                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
3139         }
3140
3141         /*
3142          * We need to load another module if the profile is anything except
3143          * "default" or "flash".
3144          */
3145         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
3146             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
3147                 char s[32];
3148
3149                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
3150                 cfg = firmware_get(s);
3151                 if (cfg == NULL) {
3152                         if (default_cfg != NULL) {
3153                                 device_printf(sc->dev,
3154                                     "unable to load module \"%s\" for "
3155                                     "configuration profile \"%s\", will use "
3156                                     "the default config file instead.\n",
3157                                     s, sc->cfg_file);
3158                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
3159                                     "%s", DEFAULT_CF);
3160                         } else {
3161                                 device_printf(sc->dev,
3162                                     "unable to load module \"%s\" for "
3163                                     "configuration profile \"%s\", will use "
3164                                     "the config file on the card's flash "
3165                                     "instead.\n", s, sc->cfg_file);
3166                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
3167                                     "%s", FLASH_CF);
3168                         }
3169                 }
3170         }
3171
3172         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
3173             default_cfg == NULL) {
3174                 device_printf(sc->dev,
3175                     "default config file not available, will use the config "
3176                     "file on the card's flash instead.\n");
3177                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
3178         }
3179
3180         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
3181                 u_int cflen;
3182                 const uint32_t *cfdata;
3183                 uint32_t param, val, addr;
3184
3185                 KASSERT(cfg != NULL || default_cfg != NULL,
3186                     ("%s: no config to upload", __func__));
3187
3188                 /*
3189                  * Ask the firmware where it wants us to upload the config file.
3190                  */
3191                 param = FW_PARAM_DEV(CF);
3192                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3193                 if (rc != 0) {
3194                         /* No support for config file?  Shouldn't happen. */
3195                         device_printf(sc->dev,
3196                             "failed to query config file location: %d.\n", rc);
3197                         goto done;
3198                 }
3199                 mtype = G_FW_PARAMS_PARAM_Y(val);
3200                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
3201
3202                 /*
3203                  * XXX: sheer laziness.  We deliberately added 4 bytes of
3204                  * useless stuffing/comments at the end of the config file so
3205                  * it's ok to simply throw away the last remaining bytes when
3206                  * the config file is not an exact multiple of 4.  This also
3207                  * helps with the validate_mt_off_len check.
3208                  */
3209                 if (cfg != NULL) {
3210                         cflen = cfg->datasize & ~3;
3211                         cfdata = cfg->data;
3212                 } else {
3213                         cflen = default_cfg->datasize & ~3;
3214                         cfdata = default_cfg->data;
3215                 }
3216
3217                 if (cflen > FLASH_CFG_MAX_SIZE) {
3218                         device_printf(sc->dev,
3219                             "config file too long (%d, max allowed is %d).  "
3220                             "Will try to use the config on the card, if any.\n",
3221                             cflen, FLASH_CFG_MAX_SIZE);
3222                         goto use_config_on_flash;
3223                 }
3224
3225                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
3226                 if (rc != 0) {
3227                         device_printf(sc->dev,
3228                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
3229                             "Will try to use the config on the card, if any.\n",
3230                             __func__, mtype, moff, cflen, rc);
3231                         goto use_config_on_flash;
3232                 }
3233                 write_via_memwin(sc, 2, addr, cfdata, cflen);
3234         } else {
3235 use_config_on_flash:
3236                 mtype = FW_MEMTYPE_FLASH;
3237                 moff = t4_flash_cfg_addr(sc);
3238         }
3239
3240         bzero(&caps, sizeof(caps));
3241         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3242             F_FW_CMD_REQUEST | F_FW_CMD_READ);
3243         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
3244             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
3245             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
3246         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3247         if (rc != 0) {
3248                 device_printf(sc->dev,
3249                     "failed to pre-process config file: %d "
3250                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
3251                 goto done;
3252         }
3253
3254         finicsum = be32toh(caps.finicsum);
3255         cfcsum = be32toh(caps.cfcsum);
3256         if (finicsum != cfcsum) {
3257                 device_printf(sc->dev,
3258                     "WARNING: config file checksum mismatch: %08x %08x\n",
3259                     finicsum, cfcsum);
3260         }
3261         sc->cfcsum = cfcsum;
3262
3263 #define LIMIT_CAPS(x) do { \
3264         caps.x &= htobe16(t4_##x##_allowed); \
3265 } while (0)
3266
3267         /*
3268          * Let the firmware know what features will (not) be used so it can tune
3269          * things accordingly.
3270          */
3271         LIMIT_CAPS(nbmcaps);
3272         LIMIT_CAPS(linkcaps);
3273         LIMIT_CAPS(switchcaps);
3274         LIMIT_CAPS(niccaps);
3275         LIMIT_CAPS(toecaps);
3276         LIMIT_CAPS(rdmacaps);
3277         LIMIT_CAPS(cryptocaps);
3278         LIMIT_CAPS(iscsicaps);
3279         LIMIT_CAPS(fcoecaps);
3280 #undef LIMIT_CAPS
3281
3282         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3283             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
3284         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
3285         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
3286         if (rc != 0) {
3287                 device_printf(sc->dev,
3288                     "failed to process config file: %d.\n", rc);
3289         }
3290 done:
3291         if (cfg != NULL)
3292                 firmware_put(cfg, FIRMWARE_UNLOAD);
3293         return (rc);
3294 }
3295
3296 /*
3297  * Retrieve parameters that are needed (or nice to have) very early.
3298  */
3299 static int
3300 get_params__pre_init(struct adapter *sc)
3301 {
3302         int rc;
3303         uint32_t param[2], val[2];
3304
3305         t4_get_version_info(sc);
3306
3307         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
3308             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
3309             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
3310             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
3311             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
3312
3313         snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
3314             G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
3315             G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
3316             G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
3317             G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));
3318
3319         snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
3320             G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
3321             G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
3322             G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
3323             G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
3324
3325         snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
3326             G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
3327             G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
3328             G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
3329             G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));
3330
3331         param[0] = FW_PARAM_DEV(PORTVEC);
3332         param[1] = FW_PARAM_DEV(CCLK);
3333         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3334         if (rc != 0) {
3335                 device_printf(sc->dev,
3336                     "failed to query parameters (pre_init): %d.\n", rc);
3337                 return (rc);
3338         }
3339
3340         sc->params.portvec = val[0];
3341         sc->params.nports = bitcount32(val[0]);
3342         sc->params.vpd.cclk = val[1];
3343
3344         /* Read device log parameters. */
3345         rc = -t4_init_devlog_params(sc, 1);
3346         if (rc == 0)
3347                 fixup_devlog_params(sc);
3348         else {
3349                 device_printf(sc->dev,
3350                     "failed to get devlog parameters: %d.\n", rc);
3351                 rc = 0; /* devlog isn't critical for device operation */
3352         }
3353
3354         return (rc);
3355 }
3356
3357 /*
3358  * Retrieve various parameters that are of interest to the driver.  The device
3359  * has been initialized by the firmware at this point.
3360  */
3361 static int
3362 get_params__post_init(struct adapter *sc)
3363 {
3364         int rc;
3365         uint32_t param[7], val[7];
3366         struct fw_caps_config_cmd caps;
3367
3368         param[0] = FW_PARAM_PFVF(IQFLINT_START);
3369         param[1] = FW_PARAM_PFVF(EQ_START);
3370         param[2] = FW_PARAM_PFVF(FILTER_START);
3371         param[3] = FW_PARAM_PFVF(FILTER_END);
3372         param[4] = FW_PARAM_PFVF(L2T_START);
3373         param[5] = FW_PARAM_PFVF(L2T_END);
3374         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3375         if (rc != 0) {
3376                 device_printf(sc->dev,
3377                     "failed to query parameters (post_init): %d.\n", rc);
3378                 return (rc);
3379         }
3380
3381         sc->sge.iq_start = val[0];
3382         sc->sge.eq_start = val[1];
3383         sc->tids.ftid_base = val[2];
3384         sc->tids.nftids = val[3] - val[2] + 1;
3385         sc->params.ftid_min = val[2];
3386         sc->params.ftid_max = val[3];
3387         sc->vres.l2t.start = val[4];
3388         sc->vres.l2t.size = val[5] - val[4] + 1;
3389         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
3390             ("%s: L2 table size (%u) larger than expected (%u)",
3391             __func__, sc->vres.l2t.size, L2T_SIZE));
3392
3393         /* get capabilites */
3394         bzero(&caps, sizeof(caps));
3395         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3396             F_FW_CMD_REQUEST | F_FW_CMD_READ);
3397         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
3398         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3399         if (rc != 0) {
3400                 device_printf(sc->dev,
3401                     "failed to get card capabilities: %d.\n", rc);
3402                 return (rc);
3403         }
3404
3405 #define READ_CAPS(x) do { \
3406         sc->x = htobe16(caps.x); \
3407 } while (0)
3408         READ_CAPS(nbmcaps);
3409         READ_CAPS(linkcaps);
3410         READ_CAPS(switchcaps);
3411         READ_CAPS(niccaps);
3412         READ_CAPS(toecaps);
3413         READ_CAPS(rdmacaps);
3414         READ_CAPS(cryptocaps);
3415         READ_CAPS(iscsicaps);
3416         READ_CAPS(fcoecaps);
3417
3418         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
3419                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
3420                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
3421                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3422                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
3423                 if (rc != 0) {
3424                         device_printf(sc->dev,
3425                             "failed to query NIC parameters: %d.\n", rc);
3426                         return (rc);
3427                 }
3428                 sc->tids.etid_base = val[0];
3429                 sc->params.etid_min = val[0];
3430                 sc->tids.netids = val[1] - val[0] + 1;
3431                 sc->params.netids = sc->tids.netids;
3432                 sc->params.eo_wr_cred = val[2];
3433                 sc->params.ethoffload = 1;
3434         }
3435
3436         if (sc->toecaps) {
3437                 /* query offload-related parameters */
3438                 param[0] = FW_PARAM_DEV(NTID);
3439                 param[1] = FW_PARAM_PFVF(SERVER_START);
3440                 param[2] = FW_PARAM_PFVF(SERVER_END);
3441                 param[3] = FW_PARAM_PFVF(TDDP_START);
3442                 param[4] = FW_PARAM_PFVF(TDDP_END);
3443                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3444                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3445                 if (rc != 0) {
3446                         device_printf(sc->dev,
3447                             "failed to query TOE parameters: %d.\n", rc);
3448                         return (rc);
3449                 }
3450                 sc->tids.ntids = val[0];
3451                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
3452                 sc->tids.stid_base = val[1];
3453                 sc->tids.nstids = val[2] - val[1] + 1;
3454                 sc->vres.ddp.start = val[3];
3455                 sc->vres.ddp.size = val[4] - val[3] + 1;
3456                 sc->params.ofldq_wr_cred = val[5];
3457                 sc->params.offload = 1;
3458         }
3459         if (sc->rdmacaps) {
3460                 param[0] = FW_PARAM_PFVF(STAG_START);
3461                 param[1] = FW_PARAM_PFVF(STAG_END);
3462                 param[2] = FW_PARAM_PFVF(RQ_START);
3463                 param[3] = FW_PARAM_PFVF(RQ_END);
3464                 param[4] = FW_PARAM_PFVF(PBL_START);
3465                 param[5] = FW_PARAM_PFVF(PBL_END);
3466                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3467                 if (rc != 0) {
3468                         device_printf(sc->dev,
3469                             "failed to query RDMA parameters(1): %d.\n", rc);
3470                         return (rc);
3471                 }
3472                 sc->vres.stag.start = val[0];
3473                 sc->vres.stag.size = val[1] - val[0] + 1;
3474                 sc->vres.rq.start = val[2];
3475                 sc->vres.rq.size = val[3] - val[2] + 1;
3476                 sc->vres.pbl.start = val[4];
3477                 sc->vres.pbl.size = val[5] - val[4] + 1;
3478
3479                 param[0] = FW_PARAM_PFVF(SQRQ_START);
3480                 param[1] = FW_PARAM_PFVF(SQRQ_END);
3481                 param[2] = FW_PARAM_PFVF(CQ_START);
3482                 param[3] = FW_PARAM_PFVF(CQ_END);
3483                 param[4] = FW_PARAM_PFVF(OCQ_START);
3484                 param[5] = FW_PARAM_PFVF(OCQ_END);
3485                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
3486                 if (rc != 0) {
3487                         device_printf(sc->dev,
3488                             "failed to query RDMA parameters(2): %d.\n", rc);
3489                         return (rc);
3490                 }
3491                 sc->vres.qp.start = val[0];
3492                 sc->vres.qp.size = val[1] - val[0] + 1;
3493                 sc->vres.cq.start = val[2];
3494                 sc->vres.cq.size = val[3] - val[2] + 1;
3495                 sc->vres.ocq.start = val[4];
3496                 sc->vres.ocq.size = val[5] - val[4] + 1;
3497         }
3498         if (sc->iscsicaps) {
3499                 param[0] = FW_PARAM_PFVF(ISCSI_START);
3500                 param[1] = FW_PARAM_PFVF(ISCSI_END);
3501                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3502                 if (rc != 0) {
3503                         device_printf(sc->dev,
3504                             "failed to query iSCSI parameters: %d.\n", rc);
3505                         return (rc);
3506                 }
3507                 sc->vres.iscsi.start = val[0];
3508                 sc->vres.iscsi.size = val[1] - val[0] + 1;
3509         }
3510
3511         t4_init_sge_params(sc);
3512
3513         /*
3514          * We've got the params we wanted to query via the firmware.  Now grab
3515          * some others directly from the chip.
3516          */
3517         rc = t4_read_chip_settings(sc);
3518
3519         return (rc);
3520 }
3521
3522 static int
3523 set_params__post_init(struct adapter *sc)
3524 {
3525         uint32_t param, val;
3526
3527         /* ask for encapsulated CPLs */
3528         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
3529         val = 1;
3530         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3531
3532         return (0);
3533 }
3534
3535 #undef FW_PARAM_PFVF
3536 #undef FW_PARAM_DEV
3537
3538 static void
3539 t4_set_desc(struct adapter *sc)
3540 {
3541         char buf[128];
3542         struct adapter_params *p = &sc->params;
3543
3544         snprintf(buf, sizeof(buf), "Chelsio %s", p->vpd.id);
3545
3546         device_set_desc_copy(sc->dev, buf);
3547 }
3548
3549 static void
3550 build_medialist(struct port_info *pi, struct ifmedia *media)
3551 {
3552         int m;
3553
3554         PORT_LOCK(pi);
3555
3556         ifmedia_removeall(media);
3557
3558         m = IFM_ETHER | IFM_FDX;
3559
3560         switch(pi->port_type) {
3561         case FW_PORT_TYPE_BT_XFI:
3562         case FW_PORT_TYPE_BT_XAUI:
3563                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
3564                 /* fall through */
3565
3566         case FW_PORT_TYPE_BT_SGMII:
3567                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
3568                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
3569                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
3570                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
3571                 break;
3572
3573         case FW_PORT_TYPE_CX4:
3574                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
3575                 ifmedia_set(media, m | IFM_10G_CX4);
3576                 break;
3577
3578         case FW_PORT_TYPE_QSFP_10G:
3579         case FW_PORT_TYPE_SFP:
3580         case FW_PORT_TYPE_FIBER_XFI:
3581         case FW_PORT_TYPE_FIBER_XAUI:
3582                 switch (pi->mod_type) {
3583
3584                 case FW_PORT_MOD_TYPE_LR:
3585                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
3586                         ifmedia_set(media, m | IFM_10G_LR);
3587                         break;
3588
3589                 case FW_PORT_MOD_TYPE_SR:
3590                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
3591                         ifmedia_set(media, m | IFM_10G_SR);
3592                         break;
3593
3594                 case FW_PORT_MOD_TYPE_LRM:
3595                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
3596                         ifmedia_set(media, m | IFM_10G_LRM);
3597                         break;
3598
3599                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3600                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3601                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
3602                         ifmedia_set(media, m | IFM_10G_TWINAX);
3603                         break;
3604
3605                 case FW_PORT_MOD_TYPE_NONE:
3606                         m &= ~IFM_FDX;
3607                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3608                         ifmedia_set(media, m | IFM_NONE);
3609                         break;
3610
3611                 case FW_PORT_MOD_TYPE_NA:
3612                 case FW_PORT_MOD_TYPE_ER:
3613                 default:
3614                         device_printf(pi->dev,
3615                             "unknown port_type (%d), mod_type (%d)\n",
3616                             pi->port_type, pi->mod_type);
3617                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3618                         ifmedia_set(media, m | IFM_UNKNOWN);
3619                         break;
3620                 }
3621                 break;
3622
3623         case FW_PORT_TYPE_CR_QSFP:
3624         case FW_PORT_TYPE_SFP28:
3625         case FW_PORT_TYPE_KR_SFP28:
3626                 switch (pi->mod_type) {
3627
3628                 case FW_PORT_MOD_TYPE_SR:
3629                         ifmedia_add(media, m | IFM_25G_SR, 0, NULL);
3630                         ifmedia_set(media, m | IFM_25G_SR);
3631                         break;
3632
3633                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3634                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3635                         ifmedia_add(media, m | IFM_25G_CR, 0, NULL);
3636                         ifmedia_set(media, m | IFM_25G_CR);
3637                         break;
3638
3639                 case FW_PORT_MOD_TYPE_NONE:
3640                         m &= ~IFM_FDX;
3641                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3642                         ifmedia_set(media, m | IFM_NONE);
3643                         break;
3644
3645                 default:
3646                         device_printf(pi->dev,
3647                             "unknown port_type (%d), mod_type (%d)\n",
3648                             pi->port_type, pi->mod_type);
3649                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3650                         ifmedia_set(media, m | IFM_UNKNOWN);
3651                         break;
3652                 }
3653                 break;
3654
3655         case FW_PORT_TYPE_QSFP:
3656                 switch (pi->mod_type) {
3657
3658                 case FW_PORT_MOD_TYPE_LR:
3659                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
3660                         ifmedia_set(media, m | IFM_40G_LR4);
3661                         break;
3662
3663                 case FW_PORT_MOD_TYPE_SR:
3664                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
3665                         ifmedia_set(media, m | IFM_40G_SR4);
3666                         break;
3667
3668                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3669                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3670                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
3671                         ifmedia_set(media, m | IFM_40G_CR4);
3672                         break;
3673
3674                 case FW_PORT_MOD_TYPE_NONE:
3675                         m &= ~IFM_FDX;
3676                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3677                         ifmedia_set(media, m | IFM_NONE);
3678                         break;
3679
3680                 default:
3681                         device_printf(pi->dev,
3682                             "unknown port_type (%d), mod_type (%d)\n",
3683                             pi->port_type, pi->mod_type);
3684                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3685                         ifmedia_set(media, m | IFM_UNKNOWN);
3686                         break;
3687                 }
3688                 break;
3689
3690         case FW_PORT_TYPE_KR4_100G:
3691         case FW_PORT_TYPE_CR4_QSFP:
3692                 switch (pi->mod_type) {
3693
3694                 case FW_PORT_MOD_TYPE_LR:
3695                         ifmedia_add(media, m | IFM_100G_LR4, 0, NULL);
3696                         ifmedia_set(media, m | IFM_100G_LR4);
3697                         break;
3698
3699                 case FW_PORT_MOD_TYPE_SR:
3700                         ifmedia_add(media, m | IFM_100G_SR4, 0, NULL);
3701                         ifmedia_set(media, m | IFM_100G_SR4);
3702                         break;
3703
3704                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3705                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3706                         ifmedia_add(media, m | IFM_100G_CR4, 0, NULL);
3707                         ifmedia_set(media, m | IFM_100G_CR4);
3708                         break;
3709
3710                 case FW_PORT_MOD_TYPE_NONE:
3711                         m &= ~IFM_FDX;
3712                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
3713                         ifmedia_set(media, m | IFM_NONE);
3714                         break;
3715
3716                 default:
3717                         device_printf(pi->dev,
3718                             "unknown port_type (%d), mod_type (%d)\n",
3719                             pi->port_type, pi->mod_type);
3720                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3721                         ifmedia_set(media, m | IFM_UNKNOWN);
3722                         break;
3723                 }
3724                 break;
3725
3726         default:
3727                 device_printf(pi->dev,
3728                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3729                     pi->mod_type);
3730                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
3731                 ifmedia_set(media, m | IFM_UNKNOWN);
3732                 break;
3733         }
3734
3735         PORT_UNLOCK(pi);
3736 }
3737
3738 #define FW_MAC_EXACT_CHUNK      7
3739
3740 /*
3741  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3742  * indicates which parameters should be programmed (the rest are left alone).
3743  */
3744 int
3745 update_mac_settings(struct ifnet *ifp, int flags)
3746 {
3747         int rc = 0;
3748         struct vi_info *vi = ifp->if_softc;
3749         struct port_info *pi = vi->pi;
3750         struct adapter *sc = pi->adapter;
3751         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3752
3753         ASSERT_SYNCHRONIZED_OP(sc);
3754         KASSERT(flags, ("%s: not told what to update.", __func__));
3755
3756         if (flags & XGMAC_MTU)
3757                 mtu = ifp->if_mtu;
3758
3759         if (flags & XGMAC_PROMISC)
3760                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3761
3762         if (flags & XGMAC_ALLMULTI)
3763                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3764
3765         if (flags & XGMAC_VLANEX)
3766                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3767
3768         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3769                 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
3770                     allmulti, 1, vlanex, false);
3771                 if (rc) {
3772                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3773                             rc);
3774                         return (rc);
3775                 }
3776         }
3777
3778         if (flags & XGMAC_UCADDR) {
3779                 uint8_t ucaddr[ETHER_ADDR_LEN];
3780
3781                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3782                 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
3783                     ucaddr, true, true);
3784                 if (rc < 0) {
3785                         rc = -rc;
3786                         if_printf(ifp, "change_mac failed: %d\n", rc);
3787                         return (rc);
3788                 } else {
3789                         vi->xact_addr_filt = rc;
3790                         rc = 0;
3791                 }
3792         }
3793
3794         if (flags & XGMAC_MCADDRS) {
3795                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3796                 int del = 1;
3797                 uint64_t hash = 0;
3798                 struct ifmultiaddr *ifma;
3799                 int i = 0, j;
3800
3801                 if_maddr_rlock(ifp);
3802                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3803                         if (ifma->ifma_addr->sa_family != AF_LINK)
3804                                 continue;
3805                         mcaddr[i] =
3806                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3807                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3808                         i++;
3809
3810                         if (i == FW_MAC_EXACT_CHUNK) {
3811                                 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
3812                                     del, i, mcaddr, NULL, &hash, 0);
3813                                 if (rc < 0) {
3814                                         rc = -rc;
3815                                         for (j = 0; j < i; j++) {
3816                                                 if_printf(ifp,
3817                                                     "failed to add mc address"
3818                                                     " %02x:%02x:%02x:"
3819                                                     "%02x:%02x:%02x rc=%d\n",
3820                                                     mcaddr[j][0], mcaddr[j][1],
3821                                                     mcaddr[j][2], mcaddr[j][3],
3822                                                     mcaddr[j][4], mcaddr[j][5],
3823                                                     rc);
3824                                         }
3825                                         goto mcfail;
3826                                 }
3827                                 del = 0;
3828                                 i = 0;
3829                         }
3830                 }
3831                 if (i > 0) {
3832                         rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i,
3833                             mcaddr, NULL, &hash, 0);
3834                         if (rc < 0) {
3835                                 rc = -rc;
3836                                 for (j = 0; j < i; j++) {
3837                                         if_printf(ifp,
3838                                             "failed to add mc address"
3839                                             " %02x:%02x:%02x:"
3840                                             "%02x:%02x:%02x rc=%d\n",
3841                                             mcaddr[j][0], mcaddr[j][1],
3842                                             mcaddr[j][2], mcaddr[j][3],
3843                                             mcaddr[j][4], mcaddr[j][5],
3844                                             rc);
3845                                 }
3846                                 goto mcfail;
3847                         }
3848                 }
3849
3850                 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0);
3851                 if (rc != 0)
3852                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3853 mcfail:
3854                 if_maddr_runlock(ifp);
3855         }
3856
3857         return (rc);
3858 }
3859
3860 /*
3861  * {begin|end}_synchronized_op must be called from the same thread.
3862  */
3863 int
3864 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
3865     char *wmesg)
3866 {
3867         int rc, pri;
3868
3869 #ifdef WITNESS
3870         /* the caller thinks it's ok to sleep, but is it really? */
3871         if (flags & SLEEP_OK)
3872                 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
3873                     "begin_synchronized_op");
3874 #endif
3875
3876         if (INTR_OK)
3877                 pri = PCATCH;
3878         else
3879                 pri = 0;
3880
3881         ADAPTER_LOCK(sc);
3882         for (;;) {
3883
3884                 if (vi && IS_DOOMED(vi)) {
3885                         rc = ENXIO;
3886                         goto done;
3887                 }
3888
3889                 if (!IS_BUSY(sc)) {
3890                         rc = 0;
3891                         break;
3892                 }
3893
3894                 if (!(flags & SLEEP_OK)) {
3895                         rc = EBUSY;
3896                         goto done;
3897                 }
3898
3899                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3900                         rc = EINTR;
3901                         goto done;
3902                 }
3903         }
3904
3905         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3906         SET_BUSY(sc);
3907 #ifdef INVARIANTS
3908         sc->last_op = wmesg;
3909         sc->last_op_thr = curthread;
3910         sc->last_op_flags = flags;
3911 #endif
3912
3913 done:
3914         if (!(flags & HOLD_LOCK) || rc)
3915                 ADAPTER_UNLOCK(sc);
3916
3917         return (rc);
3918 }
3919
3920 /*
3921  * Tell if_ioctl and if_init that the VI is going away.  This is
3922  * special variant of begin_synchronized_op and must be paired with a
3923  * call to end_synchronized_op.
3924  */
3925 void
3926 doom_vi(struct adapter *sc, struct vi_info *vi)
3927 {
3928
3929         ADAPTER_LOCK(sc);
3930         SET_DOOMED(vi);
3931         wakeup(&sc->flags);
3932         while (IS_BUSY(sc))
3933                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
3934         SET_BUSY(sc);
3935 #ifdef INVARIANTS
3936         sc->last_op = "t4detach";
3937         sc->last_op_thr = curthread;
3938         sc->last_op_flags = 0;
3939 #endif
3940         ADAPTER_UNLOCK(sc);
3941 }
3942
3943 /*
3944  * {begin|end}_synchronized_op must be called from the same thread.
3945  */
3946 void
3947 end_synchronized_op(struct adapter *sc, int flags)
3948 {
3949
3950         if (flags & LOCK_HELD)
3951                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3952         else
3953                 ADAPTER_LOCK(sc);
3954
3955         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3956         CLR_BUSY(sc);
3957         wakeup(&sc->flags);
3958         ADAPTER_UNLOCK(sc);
3959 }
3960
3961 static int
3962 cxgbe_init_synchronized(struct vi_info *vi)
3963 {
3964         struct port_info *pi = vi->pi;
3965         struct adapter *sc = pi->adapter;
3966         struct ifnet *ifp = vi->ifp;
3967         int rc = 0, i;
3968         struct sge_txq *txq;
3969
3970         ASSERT_SYNCHRONIZED_OP(sc);
3971
3972         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
3973                 return (0);     /* already running */
3974
3975         if (!(sc->flags & FULL_INIT_DONE) &&
3976             ((rc = adapter_full_init(sc)) != 0))
3977                 return (rc);    /* error message displayed already */
3978
3979         if (!(vi->flags & VI_INIT_DONE) &&
3980             ((rc = vi_full_init(vi)) != 0))
3981                 return (rc); /* error message displayed already */
3982
3983         rc = update_mac_settings(ifp, XGMAC_ALL);
3984         if (rc)
3985                 goto done;      /* error message displayed already */
3986
3987         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
3988         if (rc != 0) {
3989                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3990                 goto done;
3991         }
3992
3993         /*
3994          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3995          * if this changes.
3996          */
3997
3998         for_each_txq(vi, i, txq) {
3999                 TXQ_LOCK(txq);
4000                 txq->eq.flags |= EQ_ENABLED;
4001                 TXQ_UNLOCK(txq);
4002         }
4003
4004         /*
4005          * The first iq of the first port to come up is used for tracing.
4006          */
4007         if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
4008                 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
4009                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
4010                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
4011                     V_QUEUENUMBER(sc->traceq));
4012                 pi->flags |= HAS_TRACEQ;
4013         }
4014
4015         /* all ok */
4016         PORT_LOCK(pi);
4017         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4018         pi->up_vis++;
4019
4020         if (pi->nvi > 1 || sc->flags & IS_VF)
4021                 callout_reset(&vi->tick, hz, vi_tick, vi);
4022         else
4023                 callout_reset(&pi->tick, hz, cxgbe_tick, pi);
4024         PORT_UNLOCK(pi);
4025 done:
4026         if (rc != 0)
4027                 cxgbe_uninit_synchronized(vi);
4028
4029         return (rc);
4030 }
4031
4032 /*
4033  * Idempotent.
4034  */
4035 static int
4036 cxgbe_uninit_synchronized(struct vi_info *vi)
4037 {
4038         struct port_info *pi = vi->pi;
4039         struct adapter *sc = pi->adapter;
4040         struct ifnet *ifp = vi->ifp;
4041         int rc, i;
4042         struct sge_txq *txq;
4043
4044         ASSERT_SYNCHRONIZED_OP(sc);
4045
4046         if (!(vi->flags & VI_INIT_DONE)) {
4047                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
4048                     ("uninited VI is running"));
4049                 return (0);
4050         }
4051
4052         /*
4053          * Disable the VI so that all its data in either direction is discarded
4054          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
4055          * tick) intact as the TP can deliver negative advice or data that it's
4056          * holding in its RAM (for an offloaded connection) even after the VI is
4057          * disabled.
4058          */
4059         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
4060         if (rc) {
4061                 if_printf(ifp, "disable_vi failed: %d\n", rc);
4062                 return (rc);
4063         }
4064
4065         for_each_txq(vi, i, txq) {
4066                 TXQ_LOCK(txq);
4067                 txq->eq.flags &= ~EQ_ENABLED;
4068                 TXQ_UNLOCK(txq);
4069         }
4070
4071         PORT_LOCK(pi);
4072         if (pi->nvi > 1 || sc->flags & IS_VF)
4073                 callout_stop(&vi->tick);
4074         else
4075                 callout_stop(&pi->tick);
4076         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4077                 PORT_UNLOCK(pi);
4078                 return (0);
4079         }
4080         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
4081         pi->up_vis--;
4082         if (pi->up_vis > 0) {
4083                 PORT_UNLOCK(pi);
4084                 return (0);
4085         }
4086         PORT_UNLOCK(pi);
4087
4088         pi->link_cfg.link_ok = 0;
4089         pi->link_cfg.speed = 0;
4090         pi->link_cfg.link_down_rc = 255;
4091         t4_os_link_changed(sc, pi->port_id, 0);
4092
4093         return (0);
4094 }
4095
4096 /*
4097  * It is ok for this function to fail midway and return right away.  t4_detach
4098  * will walk the entire sc->irq list and clean up whatever is valid.
4099  */
4100 int
4101 t4_setup_intr_handlers(struct adapter *sc)
4102 {
4103         int rc, rid, p, q, v;
4104         char s[8];
4105         struct irq *irq;
4106         struct port_info *pi;
4107         struct vi_info *vi;
4108         struct sge *sge = &sc->sge;
4109         struct sge_rxq *rxq;
4110 #ifdef TCP_OFFLOAD
4111         struct sge_ofld_rxq *ofld_rxq;
4112 #endif
4113 #ifdef DEV_NETMAP
4114         struct sge_nm_rxq *nm_rxq;
4115 #endif
4116 #ifdef RSS
4117         int nbuckets = rss_getnumbuckets();
4118 #endif
4119
4120         /*
4121          * Setup interrupts.
4122          */
4123         irq = &sc->irq[0];
4124         rid = sc->intr_type == INTR_INTX ? 0 : 1;
4125         if (sc->intr_count == 1)
4126                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
4127
4128         /* Multiple interrupts. */
4129         if (sc->flags & IS_VF)
4130                 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
4131                     ("%s: too few intr.", __func__));
4132         else
4133                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
4134                     ("%s: too few intr.", __func__));
4135
4136         /* The first one is always error intr on PFs */
4137         if (!(sc->flags & IS_VF)) {
4138                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
4139                 if (rc != 0)
4140                         return (rc);
4141                 irq++;
4142                 rid++;
4143         }
4144
4145         /* The second one is always the firmware event queue (first on VFs) */
4146         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
4147         if (rc != 0)
4148                 return (rc);
4149         irq++;
4150         rid++;
4151
4152         for_each_port(sc, p) {
4153                 pi = sc->port[p];
4154                 for_each_vi(pi, v, vi) {
4155                         vi->first_intr = rid - 1;
4156
4157                         if (vi->nnmrxq > 0) {
4158                                 int n = max(vi->nrxq, vi->nnmrxq);
4159
4160                                 MPASS(vi->flags & INTR_RXQ);
4161
4162                                 rxq = &sge->rxq[vi->first_rxq];
4163 #ifdef DEV_NETMAP
4164                                 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
4165 #endif
4166                                 for (q = 0; q < n; q++) {
4167                                         snprintf(s, sizeof(s), "%x%c%x", p,
4168                                             'a' + v, q);
4169                                         if (q < vi->nrxq)
4170                                                 irq->rxq = rxq++;
4171 #ifdef DEV_NETMAP
4172                                         if (q < vi->nnmrxq)
4173                                                 irq->nm_rxq = nm_rxq++;
4174 #endif
4175                                         rc = t4_alloc_irq(sc, irq, rid,
4176                                             t4_vi_intr, irq, s);
4177                                         if (rc != 0)
4178                                                 return (rc);
4179                                         irq++;
4180                                         rid++;
4181                                         vi->nintr++;
4182                                 }
4183                         } else if (vi->flags & INTR_RXQ) {
4184                                 for_each_rxq(vi, q, rxq) {
4185                                         snprintf(s, sizeof(s), "%x%c%x", p,
4186                                             'a' + v, q);
4187                                         rc = t4_alloc_irq(sc, irq, rid,
4188                                             t4_intr, rxq, s);
4189                                         if (rc != 0)
4190                                                 return (rc);
4191 #ifdef RSS
4192                                         bus_bind_intr(sc->dev, irq->res,
4193                                             rss_getcpu(q % nbuckets));
4194 #endif
4195                                         irq++;
4196                                         rid++;
4197                                         vi->nintr++;
4198                                 }
4199                         }
4200 #ifdef TCP_OFFLOAD
4201                         if (vi->flags & INTR_OFLD_RXQ) {
4202                                 for_each_ofld_rxq(vi, q, ofld_rxq) {
4203                                         snprintf(s, sizeof(s), "%x%c%x", p,
4204                                             'A' + v, q);
4205                                         rc = t4_alloc_irq(sc, irq, rid,
4206                                             t4_intr, ofld_rxq, s);
4207                                         if (rc != 0)
4208                                                 return (rc);
4209                                         irq++;
4210                                         rid++;
4211                                         vi->nintr++;
4212                                 }
4213                         }
4214 #endif
4215                 }
4216         }
4217         MPASS(irq == &sc->irq[sc->intr_count]);
4218
4219         return (0);
4220 }
4221
4222 int
4223 adapter_full_init(struct adapter *sc)
4224 {
4225         int rc, i;
4226
4227         ASSERT_SYNCHRONIZED_OP(sc);
4228         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
4229         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
4230             ("%s: FULL_INIT_DONE already", __func__));
4231
4232         /*
4233          * queues that belong to the adapter (not any particular port).
4234          */
4235         rc = t4_setup_adapter_queues(sc);
4236         if (rc != 0)
4237                 goto done;
4238
4239         for (i = 0; i < nitems(sc->tq); i++) {
4240                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
4241                     taskqueue_thread_enqueue, &sc->tq[i]);
4242                 if (sc->tq[i] == NULL) {
4243                         device_printf(sc->dev,
4244                             "failed to allocate task queue %d\n", i);
4245                         rc = ENOMEM;
4246                         goto done;
4247                 }
4248                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
4249                     device_get_nameunit(sc->dev), i);
4250         }
4251
4252         if (!(sc->flags & IS_VF))
4253                 t4_intr_enable(sc);
4254         sc->flags |= FULL_INIT_DONE;
4255 done:
4256         if (rc != 0)
4257                 adapter_full_uninit(sc);
4258
4259         return (rc);
4260 }
4261
4262 int
4263 adapter_full_uninit(struct adapter *sc)
4264 {
4265         int i;
4266
4267         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
4268
4269         t4_teardown_adapter_queues(sc);
4270
4271         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
4272                 taskqueue_free(sc->tq[i]);
4273                 sc->tq[i] = NULL;
4274         }
4275
4276         sc->flags &= ~FULL_INIT_DONE;
4277
4278         return (0);
4279 }
4280
4281 #ifdef RSS
4282 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
4283     RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
4284     RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
4285     RSS_HASHTYPE_RSS_UDP_IPV6)
4286
4287 /* Translates kernel hash types to hardware. */
4288 static int
4289 hashconfig_to_hashen(int hashconfig)
4290 {
4291         int hashen = 0;
4292
4293         if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
4294                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
4295         if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
4296                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
4297         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
4298                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
4299                     F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
4300         }
4301         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
4302                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
4303                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
4304         }
4305         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
4306                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
4307         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
4308                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
4309
4310         return (hashen);
4311 }
4312
4313 /* Translates hardware hash types to kernel. */
4314 static int
4315 hashen_to_hashconfig(int hashen)
4316 {
4317         int hashconfig = 0;
4318
4319         if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
4320                 /*
4321                  * If UDP hashing was enabled it must have been enabled for
4322                  * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
4323                  * enabling any 4-tuple hash is nonsense configuration.
4324                  */
4325                 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
4326                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
4327
4328                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
4329                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
4330                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
4331                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
4332         }
4333         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
4334                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
4335         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
4336                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
4337         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
4338                 hashconfig |= RSS_HASHTYPE_RSS_IPV4;
4339         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
4340                 hashconfig |= RSS_HASHTYPE_RSS_IPV6;
4341
4342         return (hashconfig);
4343 }
4344 #endif
4345
4346 int
4347 vi_full_init(struct vi_info *vi)
4348 {
4349         struct adapter *sc = vi->pi->adapter;
4350         struct ifnet *ifp = vi->ifp;
4351         uint16_t *rss;
4352         struct sge_rxq *rxq;
4353         int rc, i, j, hashen;
4354 #ifdef RSS
4355         int nbuckets = rss_getnumbuckets();
4356         int hashconfig = rss_gethashconfig();
4357         int extra;
4358         uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
4359         uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
4360 #endif
4361
4362         ASSERT_SYNCHRONIZED_OP(sc);
4363         KASSERT((vi->flags & VI_INIT_DONE) == 0,
4364             ("%s: VI_INIT_DONE already", __func__));
4365
4366         sysctl_ctx_init(&vi->ctx);
4367         vi->flags |= VI_SYSCTL_CTX;
4368
4369         /*
4370          * Allocate tx/rx/fl queues for this VI.
4371          */
4372         rc = t4_setup_vi_queues(vi);
4373         if (rc != 0)
4374                 goto done;      /* error message displayed already */
4375
4376         /*
4377          * Setup RSS for this VI.  Save a copy of the RSS table for later use.
4378          */
4379         if (vi->nrxq > vi->rss_size) {
4380                 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
4381                     "some queues will never receive traffic.\n", vi->nrxq,
4382                     vi->rss_size);
4383         } else if (vi->rss_size % vi->nrxq) {
4384                 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
4385                     "expect uneven traffic distribution.\n", vi->nrxq,
4386                     vi->rss_size);
4387         }
4388 #ifdef RSS
4389         MPASS(RSS_KEYSIZE == 40);
4390         if (vi->nrxq != nbuckets) {
4391                 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
4392                     "performance will be impacted.\n", vi->nrxq, nbuckets);
4393         }
4394
4395         rss_getkey((void *)&raw_rss_key[0]);
4396         for (i = 0; i < nitems(rss_key); i++) {
4397                 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
4398         }
4399         t4_write_rss_key(sc, &rss_key[0], -1);
4400 #endif
4401         rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
4402         for (i = 0; i < vi->rss_size;) {
4403 #ifdef RSS
4404                 j = rss_get_indirection_to_bucket(i);
4405                 j %= vi->nrxq;
4406                 rxq = &sc->sge.rxq[vi->first_rxq + j];
4407                 rss[i++] = rxq->iq.abs_id;
4408 #else
4409                 for_each_rxq(vi, j, rxq) {
4410                         rss[i++] = rxq->iq.abs_id;
4411                         if (i == vi->rss_size)
4412                                 break;
4413                 }
4414 #endif
4415         }
4416
4417         rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
4418             vi->rss_size);
4419         if (rc != 0) {
4420                 if_printf(ifp, "rss_config failed: %d\n", rc);
4421                 goto done;
4422         }
4423
4424 #ifdef RSS
4425         hashen = hashconfig_to_hashen(hashconfig);
4426
4427         /*
4428          * We may have had to enable some hashes even though the global config
4429          * wants them disabled.  This is a potential problem that must be
4430          * reported to the user.
4431          */
4432         extra = hashen_to_hashconfig(hashen) ^ hashconfig;
4433
4434         /*
4435          * If we consider only the supported hash types, then the enabled hashes
4436          * are a superset of the requested hashes.  In other words, there cannot
4437          * be any supported hash that was requested but not enabled, but there
4438          * can be hashes that were not requested but had to be enabled.
4439          */
4440         extra &= SUPPORTED_RSS_HASHTYPES;
4441         MPASS((extra & hashconfig) == 0);
4442
4443         if (extra) {
4444                 if_printf(ifp,
4445                     "global RSS config (0x%x) cannot be accommodated.\n",
4446                     hashconfig);
4447         }
4448         if (extra & RSS_HASHTYPE_RSS_IPV4)
4449                 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
4450         if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
4451                 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
4452         if (extra & RSS_HASHTYPE_RSS_IPV6)
4453                 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
4454         if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
4455                 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
4456         if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
4457                 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
4458         if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
4459                 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
4460 #else
4461         hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
4462             F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
4463             F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
4464             F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
4465 #endif
4466         rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0], 0, 0);
4467         if (rc != 0) {
4468                 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
4469                 goto done;
4470         }
4471
4472         vi->rss = rss;
4473         vi->flags |= VI_INIT_DONE;
4474 done:
4475         if (rc != 0)
4476                 vi_full_uninit(vi);
4477
4478         return (rc);
4479 }
4480
4481 /*
4482  * Idempotent.
4483  */
4484 int
4485 vi_full_uninit(struct vi_info *vi)
4486 {
4487         struct port_info *pi = vi->pi;
4488         struct adapter *sc = pi->adapter;
4489         int i;
4490         struct sge_rxq *rxq;
4491         struct sge_txq *txq;
4492 #ifdef TCP_OFFLOAD
4493         struct sge_ofld_rxq *ofld_rxq;
4494         struct sge_wrq *ofld_txq;
4495 #endif
4496
4497         if (vi->flags & VI_INIT_DONE) {
4498
4499                 /* Need to quiesce queues.  */
4500
4501                 /* XXX: Only for the first VI? */
4502                 if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF))
4503                         quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
4504
4505                 for_each_txq(vi, i, txq) {
4506                         quiesce_txq(sc, txq);
4507                 }
4508
4509 #ifdef TCP_OFFLOAD
4510                 for_each_ofld_txq(vi, i, ofld_txq) {
4511                         quiesce_wrq(sc, ofld_txq);
4512                 }
4513 #endif
4514
4515                 for_each_rxq(vi, i, rxq) {
4516                         quiesce_iq(sc, &rxq->iq);
4517                         quiesce_fl(sc, &rxq->fl);
4518                 }
4519
4520 #ifdef TCP_OFFLOAD
4521                 for_each_ofld_rxq(vi, i, ofld_rxq) {
4522                         quiesce_iq(sc, &ofld_rxq->iq);
4523                         quiesce_fl(sc, &ofld_rxq->fl);
4524                 }
4525 #endif
4526                 free(vi->rss, M_CXGBE);
4527                 free(vi->nm_rss, M_CXGBE);
4528         }
4529
4530         t4_teardown_vi_queues(vi);
4531         vi->flags &= ~VI_INIT_DONE;
4532
4533         return (0);
4534 }
4535
4536 static void
4537 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
4538 {
4539         struct sge_eq *eq = &txq->eq;
4540         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
4541
4542         (void) sc;      /* unused */
4543
4544 #ifdef INVARIANTS
4545         TXQ_LOCK(txq);
4546         MPASS((eq->flags & EQ_ENABLED) == 0);
4547         TXQ_UNLOCK(txq);
4548 #endif
4549
4550         /* Wait for the mp_ring to empty. */
4551         while (!mp_ring_is_idle(txq->r)) {
4552                 mp_ring_check_drainage(txq->r, 0);
4553                 pause("rquiesce", 1);
4554         }
4555
4556         /* Then wait for the hardware to finish. */
4557         while (spg->cidx != htobe16(eq->pidx))
4558                 pause("equiesce", 1);
4559
4560         /* Finally, wait for the driver to reclaim all descriptors. */
4561         while (eq->cidx != eq->pidx)
4562                 pause("dquiesce", 1);
4563 }
4564
4565 static void
4566 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
4567 {
4568
4569         /* XXXTX */
4570 }
4571
4572 static void
4573 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
4574 {
4575         (void) sc;      /* unused */
4576
4577         /* Synchronize with the interrupt handler */
4578         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
4579                 pause("iqfree", 1);
4580 }
4581
4582 static void
4583 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
4584 {
4585         mtx_lock(&sc->sfl_lock);
4586         FL_LOCK(fl);
4587         fl->flags |= FL_DOOMED;
4588         FL_UNLOCK(fl);
4589         callout_stop(&sc->sfl_callout);
4590         mtx_unlock(&sc->sfl_lock);
4591
4592         KASSERT((fl->flags & FL_STARVING) == 0,
4593             ("%s: still starving", __func__));
4594 }
4595
4596 static int
4597 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
4598     driver_intr_t *handler, void *arg, char *name)
4599 {
4600         int rc;
4601
4602         irq->rid = rid;
4603         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
4604             RF_SHAREABLE | RF_ACTIVE);
4605         if (irq->res == NULL) {
4606                 device_printf(sc->dev,
4607                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
4608                 return (ENOMEM);
4609         }
4610
4611         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
4612             NULL, handler, arg, &irq->tag);
4613         if (rc != 0) {
4614                 device_printf(sc->dev,
4615                     "failed to setup interrupt for rid %d, name %s: %d\n",
4616                     rid, name, rc);
4617         } else if (name)
4618                 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);
4619
4620         return (rc);
4621 }
4622
4623 static int
4624 t4_free_irq(struct adapter *sc, struct irq *irq)
4625 {
4626         if (irq->tag)
4627                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
4628         if (irq->res)
4629                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
4630
4631         bzero(irq, sizeof(*irq));
4632
4633         return (0);
4634 }
4635
4636 static void
4637 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
4638 {
4639
4640         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4641         t4_get_regs(sc, buf, regs->len);
4642 }
4643
4644 #define A_PL_INDIR_CMD  0x1f8
4645
4646 #define S_PL_AUTOINC    31
4647 #define M_PL_AUTOINC    0x1U
4648 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
4649 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
4650
4651 #define S_PL_VFID       20
4652 #define M_PL_VFID       0xffU
4653 #define V_PL_VFID(x)    ((x) << S_PL_VFID)
4654 #define G_PL_VFID(x)    (((x) >> S_PL_VFID) & M_PL_VFID)
4655
4656 #define S_PL_ADDR       0
4657 #define M_PL_ADDR       0xfffffU
4658 #define V_PL_ADDR(x)    ((x) << S_PL_ADDR)
4659 #define G_PL_ADDR(x)    (((x) >> S_PL_ADDR) & M_PL_ADDR)
4660
4661 #define A_PL_INDIR_DATA 0x1fc
4662
4663 static uint64_t
4664 read_vf_stat(struct adapter *sc, unsigned int viid, int reg)
4665 {
4666         u32 stats[2];
4667
4668         mtx_assert(&sc->reg_lock, MA_OWNED);
4669         if (sc->flags & IS_VF) {
4670                 stats[0] = t4_read_reg(sc, VF_MPS_REG(reg));
4671                 stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4));
4672         } else {
4673                 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4674                     V_PL_VFID(G_FW_VIID_VIN(viid)) |
4675                     V_PL_ADDR(VF_MPS_REG(reg)));
4676                 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
4677                 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
4678         }
4679         return (((uint64_t)stats[1]) << 32 | stats[0]);
4680 }
4681
4682 static void
4683 t4_get_vi_stats(struct adapter *sc, unsigned int viid,
4684     struct fw_vi_stats_vf *stats)
4685 {
4686
4687 #define GET_STAT(name) \
4688         read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L)
4689
4690         stats->tx_bcast_bytes    = GET_STAT(TX_VF_BCAST_BYTES);
4691         stats->tx_bcast_frames   = GET_STAT(TX_VF_BCAST_FRAMES);
4692         stats->tx_mcast_bytes    = GET_STAT(TX_VF_MCAST_BYTES);
4693         stats->tx_mcast_frames   = GET_STAT(TX_VF_MCAST_FRAMES);
4694         stats->tx_ucast_bytes    = GET_STAT(TX_VF_UCAST_BYTES);
4695         stats->tx_ucast_frames   = GET_STAT(TX_VF_UCAST_FRAMES);
4696         stats->tx_drop_frames    = GET_STAT(TX_VF_DROP_FRAMES);
4697         stats->tx_offload_bytes  = GET_STAT(TX_VF_OFFLOAD_BYTES);
4698         stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
4699         stats->rx_bcast_bytes    = GET_STAT(RX_VF_BCAST_BYTES);
4700         stats->rx_bcast_frames   = GET_STAT(RX_VF_BCAST_FRAMES);
4701         stats->rx_mcast_bytes    = GET_STAT(RX_VF_MCAST_BYTES);
4702         stats->rx_mcast_frames   = GET_STAT(RX_VF_MCAST_FRAMES);
4703         stats->rx_ucast_bytes    = GET_STAT(RX_VF_UCAST_BYTES);
4704         stats->rx_ucast_frames   = GET_STAT(RX_VF_UCAST_FRAMES);
4705         stats->rx_err_frames     = GET_STAT(RX_VF_ERR_FRAMES);
4706
4707 #undef GET_STAT
4708 }
4709
4710 static void
4711 t4_clr_vi_stats(struct adapter *sc, unsigned int viid)
4712 {
4713         int reg;
4714
4715         t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
4716             V_PL_VFID(G_FW_VIID_VIN(viid)) |
4717             V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
4718         for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
4719              reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
4720                 t4_write_reg(sc, A_PL_INDIR_DATA, 0);
4721 }
4722
4723 static void
4724 vi_refresh_stats(struct adapter *sc, struct vi_info *vi)
4725 {
4726         struct timeval tv;
4727         const struct timeval interval = {0, 250000};    /* 250ms */
4728
4729         if (!(vi->flags & VI_INIT_DONE))
4730                 return;
4731
4732         getmicrotime(&tv);
4733         timevalsub(&tv, &interval);
4734         if (timevalcmp(&tv, &vi->last_refreshed, <))
4735                 return;
4736
4737         mtx_lock(&sc->reg_lock);
4738         t4_get_vi_stats(sc, vi->viid, &vi->stats);
4739         getmicrotime(&vi->last_refreshed);
4740         mtx_unlock(&sc->reg_lock);
4741 }
4742
4743 static void
4744 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4745 {
4746         int i;
4747         u_int v, tnl_cong_drops;
4748         struct timeval tv;
4749         const struct timeval interval = {0, 250000};    /* 250ms */
4750
4751         getmicrotime(&tv);
4752         timevalsub(&tv, &interval);
4753         if (timevalcmp(&tv, &pi->last_refreshed, <))
4754                 return;
4755
4756         tnl_cong_drops = 0;
4757         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4758         for (i = 0; i < sc->chip_params->nchan; i++) {
4759                 if (pi->rx_chan_map & (1 << i)) {
4760                         mtx_lock(&sc->reg_lock);
4761                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4762                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4763                         mtx_unlock(&sc->reg_lock);
4764                         tnl_cong_drops += v;
4765                 }
4766         }
4767         pi->tnl_cong_drops = tnl_cong_drops;
4768         getmicrotime(&pi->last_refreshed);
4769 }
4770
4771 static void
4772 cxgbe_tick(void *arg)
4773 {
4774         struct port_info *pi = arg;
4775         struct adapter *sc = pi->adapter;
4776
4777         PORT_LOCK_ASSERT_OWNED(pi);
4778         cxgbe_refresh_stats(sc, pi);
4779
4780         callout_schedule(&pi->tick, hz);
4781 }
4782
4783 void
4784 vi_tick(void *arg)
4785 {
4786         struct vi_info *vi = arg;
4787         struct adapter *sc = vi->pi->adapter;
4788
4789         vi_refresh_stats(sc, vi);
4790
4791         callout_schedule(&vi->tick, hz);
4792 }
4793
4794 static void
4795 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4796 {
4797         struct ifnet *vlan;
4798
4799         if (arg != ifp || ifp->if_type != IFT_ETHER)
4800                 return;
4801
4802         vlan = VLAN_DEVAT(ifp, vid);
4803         VLAN_SETCOOKIE(vlan, ifp);
4804 }
4805
4806 /*
4807  * Should match fw_caps_config_<foo> enums in t4fw_interface.h
4808  */
4809 static char *caps_decoder[] = {
4810         "\20\001IPMI\002NCSI",                          /* 0: NBM */
4811         "\20\001PPP\002QFC\003DCBX",                    /* 1: link */
4812         "\20\001INGRESS\002EGRESS",                     /* 2: switch */
4813         "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL"      /* 3: NIC */
4814             "\006HASHFILTER\007ETHOFLD",
4815         "\20\001TOE",                                   /* 4: TOE */
4816         "\20\001RDDP\002RDMAC",                         /* 5: RDMA */
4817         "\20\001INITIATOR_PDU\002TARGET_PDU"            /* 6: iSCSI */
4818             "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD"
4819             "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD"
4820             "\007T10DIF"
4821             "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD",
4822         "\20\001LOOKASIDE\002TLSKEYS",                  /* 7: Crypto */
4823         "\20\001INITIATOR\002TARGET\003CTRL_OFLD"       /* 8: FCoE */
4824                     "\004PO_INITIATOR\005PO_TARGET",
4825 };
4826
4827 void
4828 t4_sysctls(struct adapter *sc)
4829 {
4830         struct sysctl_ctx_list *ctx;
4831         struct sysctl_oid *oid;
4832         struct sysctl_oid_list *children, *c0;
4833         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4834
4835         ctx = device_get_sysctl_ctx(sc->dev);
4836
4837         /*
4838          * dev.t4nex.X.
4839          */
4840         oid = device_get_sysctl_tree(sc->dev);
4841         c0 = children = SYSCTL_CHILDREN(oid);
4842
4843         sc->sc_do_rxcopy = 1;
4844         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4845             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4846
4847         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4848             sc->params.nports, "# of ports");
4849
4850         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4851             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4852             sysctl_bitfield, "A", "available doorbells");
4853
4854         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4855             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4856
4857         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4858             CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.timer_val,
4859             sizeof(sc->params.sge.timer_val), sysctl_int_array, "A",
4860             "interrupt holdoff timer values (us)");
4861
4862         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4863             CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.counter_val,
4864             sizeof(sc->params.sge.counter_val), sysctl_int_array, "A",
4865             "interrupt holdoff packet counter values");
4866
4867         t4_sge_sysctls(sc, ctx, children);
4868
4869         sc->lro_timeout = 100;
4870         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4871             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4872
4873         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW,
4874             &sc->debug_flags, 0, "flags to enable runtime debugging");
4875
4876         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version",
4877             CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
4878
4879         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4880             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4881
4882         if (sc->flags & IS_VF)
4883                 return;
4884
4885         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4886             NULL, chip_rev(sc), "chip hardware revision");
4887
4888         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn",
4889             CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");
4890
4891         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn",
4892             CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");
4893
4894         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec",
4895             CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");
4896
4897         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na",
4898             CTLFLAG_RD, sc->params.vpd.na, 0, "network address");
4899
4900         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD,
4901             sc->er_version, 0, "expansion ROM version");
4902
4903         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD,
4904             sc->bs_version, 0, "bootstrap firmware version");
4905
4906         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD,
4907             NULL, sc->params.scfg_vers, "serial config version");
4908
4909         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD,
4910             NULL, sc->params.vpd_vers, "VPD version");
4911
4912         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4913             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4914
4915         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4916             sc->cfcsum, "config file checksum");
4917
4918 #define SYSCTL_CAP(name, n, text) \
4919         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \
4920             CTLTYPE_STRING | CTLFLAG_RD, caps_decoder[n], sc->name, \
4921             sysctl_bitfield, "A", "available " text " capabilities")
4922
4923         SYSCTL_CAP(nbmcaps, 0, "NBM");
4924         SYSCTL_CAP(linkcaps, 1, "link");
4925         SYSCTL_CAP(switchcaps, 2, "switch");
4926         SYSCTL_CAP(niccaps, 3, "NIC");
4927         SYSCTL_CAP(toecaps, 4, "TCP offload");
4928         SYSCTL_CAP(rdmacaps, 5, "RDMA");
4929         SYSCTL_CAP(iscsicaps, 6, "iSCSI");
4930         SYSCTL_CAP(cryptocaps, 7, "crypto");
4931         SYSCTL_CAP(fcoecaps, 8, "FCoE");
4932 #undef SYSCTL_CAP
4933
4934         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4935             NULL, sc->tids.nftids, "number of filters");
4936
4937         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4938             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4939             "chip temperature (in Celsius)");
4940
4941 #ifdef SBUF_DRAIN
4942         /*
4943          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4944          */
4945         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4946             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4947             "logs and miscellaneous information");
4948         children = SYSCTL_CHILDREN(oid);
4949
4950         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4951             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4952             sysctl_cctrl, "A", "congestion control");
4953
4954         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4955             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4956             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4957
4958         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4959             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4960             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4961
4962         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4963             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4964             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4965
4966         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4967             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4968             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4969
4970         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4971             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4972             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4973
4974         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4975             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4976             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4977
4978         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4979             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4980             chip_id(sc) <= CHELSIO_T5 ? sysctl_cim_la : sysctl_cim_la_t6,
4981             "A", "CIM logic analyzer");
4982
4983         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4984             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4985             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4986
4987         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4988             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4989             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4990
4991         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4992             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4993             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4994
4995         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4996             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4997             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4998
4999         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
5000             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
5001             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
5002
5003         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
5004             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
5005             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
5006
5007         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
5008             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
5009             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
5010
5011         if (chip_id(sc) > CHELSIO_T4) {
5012                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
5013                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
5014                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
5015
5016                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
5017                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
5018                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
5019         }
5020
5021         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
5022             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5023             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
5024
5025         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
5026             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5027             sysctl_cim_qcfg, "A", "CIM queue configuration");
5028
5029         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
5030             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5031             sysctl_cpl_stats, "A", "CPL statistics");
5032
5033         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
5034             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5035             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
5036
5037         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
5038             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5039             sysctl_devlog, "A", "firmware's device log");
5040
5041         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
5042             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5043             sysctl_fcoe_stats, "A", "FCoE statistics");
5044
5045         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
5046             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5047             sysctl_hw_sched, "A", "hardware scheduler ");
5048
5049         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
5050             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5051             sysctl_l2t, "A", "hardware L2 table");
5052
5053         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
5054             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5055             sysctl_lb_stats, "A", "loopback statistics");
5056
5057         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
5058             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5059             sysctl_meminfo, "A", "memory regions");
5060
5061         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
5062             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5063             chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6,
5064             "A", "MPS TCAM entries");
5065
5066         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
5067             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5068             sysctl_path_mtus, "A", "path MTUs");
5069
5070         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
5071             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5072             sysctl_pm_stats, "A", "PM statistics");
5073
5074         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
5075             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5076             sysctl_rdma_stats, "A", "RDMA statistics");
5077
5078         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
5079             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5080             sysctl_tcp_stats, "A", "TCP statistics");
5081
5082         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
5083             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5084             sysctl_tids, "A", "TID information");
5085
5086         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
5087             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5088             sysctl_tp_err_stats, "A", "TP error statistics");
5089
5090         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask",
5091             CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tp_la_mask, "I",
5092             "TP logic analyzer event capture mask");
5093
5094         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
5095             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5096             sysctl_tp_la, "A", "TP logic analyzer");
5097
5098         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
5099             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5100             sysctl_tx_rate, "A", "Tx rate");
5101
5102         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
5103             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5104             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
5105
5106         if (chip_id(sc) >= CHELSIO_T5) {
5107                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
5108                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5109                     sysctl_wcwr_stats, "A", "write combined work requests");
5110         }
5111 #endif
5112
5113 #ifdef TCP_OFFLOAD
5114         if (is_offload(sc)) {
5115                 /*
5116                  * dev.t4nex.X.toe.
5117                  */
5118                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
5119                     NULL, "TOE parameters");
5120                 children = SYSCTL_CHILDREN(oid);
5121
5122                 sc->tt.sndbuf = 256 * 1024;
5123                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
5124                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
5125
5126                 sc->tt.ddp = 0;
5127                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
5128                     &sc->tt.ddp, 0, "DDP allowed");
5129
5130                 sc->tt.rx_coalesce = 1;
5131                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
5132                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
5133
5134                 sc->tt.tx_align = 1;
5135                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
5136                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
5137
5138                 sc->tt.tx_zcopy = 0;
5139                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy",
5140                     CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
5141                     "Enable zero-copy aio_write(2)");
5142
5143                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick",
5144                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A",
5145                     "TP timer tick (us)");
5146
5147                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick",
5148                     CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_tp_tick, "A",
5149                     "TCP timestamp tick (us)");
5150
5151                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick",
5152                     CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_tp_tick, "A",
5153                     "DACK tick (us)");
5154
5155                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer",
5156                     CTLTYPE_UINT | CTLFLAG_RD, sc, 0, sysctl_tp_dack_timer,
5157                     "IU", "DACK timer (us)");
5158
5159                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min",
5160                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MIN,
5161                     sysctl_tp_timer, "LU", "Retransmit min (us)");
5162
5163                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max",
5164                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MAX,
5165                     sysctl_tp_timer, "LU", "Retransmit max (us)");
5166
5167                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min",
5168                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MIN,
5169                     sysctl_tp_timer, "LU", "Persist timer min (us)");
5170
5171                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max",
5172                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MAX,
5173                     sysctl_tp_timer, "LU", "Persist timer max (us)");
5174
5175                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle",
5176                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_IDLE,
5177                     sysctl_tp_timer, "LU", "Keepidle idle timer (us)");
5178
5179                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_intvl",
5180                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_INTVL,
5181                     sysctl_tp_timer, "LU", "Keepidle interval (us)");
5182
5183                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt",
5184                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_INIT_SRTT,
5185                     sysctl_tp_timer, "LU", "Initial SRTT (us)");
5186
5187                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer",
5188                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_FINWAIT2_TIMER,
5189                     sysctl_tp_timer, "LU", "FINWAIT2 timer (us)");
5190         }
5191 #endif
5192 }
5193
5194 void
5195 vi_sysctls(struct vi_info *vi)
5196 {
5197         struct sysctl_ctx_list *ctx;
5198         struct sysctl_oid *oid;
5199         struct sysctl_oid_list *children;
5200
5201         ctx = device_get_sysctl_ctx(vi->dev);
5202
5203         /*
5204          * dev.v?(cxgbe|cxl).X.
5205          */
5206         oid = device_get_sysctl_tree(vi->dev);
5207         children = SYSCTL_CHILDREN(oid);
5208
5209         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
5210             vi->viid, "VI identifer");
5211         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
5212             &vi->nrxq, 0, "# of rx queues");
5213         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
5214             &vi->ntxq, 0, "# of tx queues");
5215         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
5216             &vi->first_rxq, 0, "index of first rx queue");
5217         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
5218             &vi->first_txq, 0, "index of first tx queue");
5219         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL,
5220             vi->rss_size, "size of RSS indirection table");
5221
5222         if (IS_MAIN_VI(vi)) {
5223                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq",
5224                     CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU",
5225                     "Reserve queue 0 for non-flowid packets");
5226         }
5227
5228 #ifdef TCP_OFFLOAD
5229         if (vi->nofldrxq != 0) {
5230                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
5231                     &vi->nofldrxq, 0,
5232                     "# of rx queues for offloaded TCP connections");
5233                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
5234                     &vi->nofldtxq, 0,
5235                     "# of tx queues for offloaded TCP connections");
5236                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
5237                     CTLFLAG_RD, &vi->first_ofld_rxq, 0,
5238                     "index of first TOE rx queue");
5239                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
5240                     CTLFLAG_RD, &vi->first_ofld_txq, 0,
5241                     "index of first TOE tx queue");
5242         }
5243 #endif
5244 #ifdef DEV_NETMAP
5245         if (vi->nnmrxq != 0) {
5246                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
5247                     &vi->nnmrxq, 0, "# of netmap rx queues");
5248                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
5249                     &vi->nnmtxq, 0, "# of netmap tx queues");
5250                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
5251                     CTLFLAG_RD, &vi->first_nm_rxq, 0,
5252                     "index of first netmap rx queue");
5253                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
5254                     CTLFLAG_RD, &vi->first_nm_txq, 0,
5255                     "index of first netmap tx queue");
5256         }
5257 #endif
5258
5259         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
5260             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I",
5261             "holdoff timer index");
5262         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
5263             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I",
5264             "holdoff packet counter index");
5265
5266         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
5267             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I",
5268             "rx queue size");
5269         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
5270             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I",
5271             "tx queue size");
5272 }
5273
5274 static void
5275 cxgbe_sysctls(struct port_info *pi)
5276 {
5277         struct sysctl_ctx_list *ctx;
5278         struct sysctl_oid *oid;
5279         struct sysctl_oid_list *children, *children2;
5280         struct adapter *sc = pi->adapter;
5281         int i;
5282         char name[16];
5283
5284         ctx = device_get_sysctl_ctx(pi->dev);
5285
5286         /*
5287          * dev.cxgbe.X.
5288          */
5289         oid = device_get_sysctl_tree(pi->dev);
5290         children = SYSCTL_CHILDREN(oid);
5291
5292         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
5293            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
5294         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
5295                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
5296                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
5297                     "PHY temperature (in Celsius)");
5298                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
5299                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
5300                     "PHY firmware version");
5301         }
5302
5303         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
5304             CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_pause_settings, "A",
5305             "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
5306         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fec",
5307             CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_fec, "A",
5308             "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)");
5309         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg",
5310             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_autoneg, "I",
5311             "autonegotiation (-1 = not supported)");
5312
5313         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL,
5314             port_top_speed(pi), "max speed (in Gbps)");
5315
5316         if (sc->flags & IS_VF)
5317                 return;
5318
5319         /*
5320          * dev.(cxgbe|cxl).X.tc.
5321          */
5322         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL,
5323             "Tx scheduler traffic classes");
5324         for (i = 0; i < sc->chip_params->nsched_cls; i++) {
5325                 struct tx_sched_class *tc = &pi->tc[i];
5326
5327                 snprintf(name, sizeof(name), "%d", i);
5328                 children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx,
5329                     SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL,
5330                     "traffic class"));
5331                 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "flags", CTLFLAG_RD,
5332                     &tc->flags, 0, "flags");
5333                 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount",
5334                     CTLFLAG_RD, &tc->refcount, 0, "references to this class");
5335 #ifdef SBUF_DRAIN
5336                 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params",
5337                     CTLTYPE_STRING | CTLFLAG_RD, sc, (pi->port_id << 16) | i,
5338                     sysctl_tc_params, "A", "traffic class parameters");
5339 #endif
5340         }
5341
5342         /*
5343          * dev.cxgbe.X.stats.
5344          */
5345         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
5346             NULL, "port statistics");
5347         children = SYSCTL_CHILDREN(oid);
5348         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
5349             &pi->tx_parse_error, 0,
5350             "# of tx packets with invalid length or # of segments");
5351
5352 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
5353         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
5354             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
5355             sysctl_handle_t4_reg64, "QU", desc)
5356
5357         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
5358             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
5359         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
5360             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
5361         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
5362             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
5363         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
5364             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
5365         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
5366             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
5367         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
5368             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
5369         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
5370             "# of tx frames in this range",
5371             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
5372         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
5373             "# of tx frames in this range",
5374             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
5375         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
5376             "# of tx frames in this range",
5377             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
5378         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
5379             "# of tx frames in this range",
5380             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
5381         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
5382             "# of tx frames in this range",
5383             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
5384         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
5385             "# of tx frames in this range",
5386             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
5387         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
5388             "# of tx frames in this range",
5389             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
5390         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
5391             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
5392         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
5393             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
5394         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
5395             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
5396         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
5397             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
5398         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
5399             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
5400         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
5401             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
5402         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
5403             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
5404         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
5405             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
5406         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
5407             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
5408         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
5409             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
5410
5411         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
5412             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
5413         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
5414             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
5415         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
5416             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
5417         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
5418             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
5419         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
5420             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
5421         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
5422             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
5423         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
5424             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
5425         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
5426             "# of frames received with bad FCS",
5427             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
5428         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
5429             "# of frames received with length error",
5430             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
5431         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
5432             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
5433         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
5434             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
5435         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
5436             "# of rx frames in this range",
5437             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
5438         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
5439             "# of rx frames in this range",
5440             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
5441         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5442             "# of rx frames in this range",
5443             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5444         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5445             "# of rx frames in this range",
5446             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5447         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5448             "# of rx frames in this range",
5449             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5450         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5451             "# of rx frames in this range",
5452             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5453         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5454             "# of rx frames in this range",
5455             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5456         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5457             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5458         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5459             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5460         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5461             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5462         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5463             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5464         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5465             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5466         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5467             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5468         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5469             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5470         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5471             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5472         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5473             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5474
5475 #undef SYSCTL_ADD_T4_REG64
5476
5477 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5478         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5479             &pi->stats.name, desc)
5480
5481         /* We get these from port_stats and they may be stale by up to 1s */
5482         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5483             "# drops due to buffer-group 0 overflows");
5484         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5485             "# drops due to buffer-group 1 overflows");
5486         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5487             "# drops due to buffer-group 2 overflows");
5488         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5489             "# drops due to buffer-group 3 overflows");
5490         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5491             "# of buffer-group 0 truncated packets");
5492         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5493             "# of buffer-group 1 truncated packets");
5494         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5495             "# of buffer-group 2 truncated packets");
5496         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5497             "# of buffer-group 3 truncated packets");
5498
5499 #undef SYSCTL_ADD_T4_PORTSTAT
5500 }
5501
5502 static int
5503 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5504 {
5505         int rc, *i, space = 0;
5506         struct sbuf sb;
5507
5508         sbuf_new_for_sysctl(&sb, NULL, 64, req);
5509         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5510                 if (space)
5511                         sbuf_printf(&sb, " ");
5512                 sbuf_printf(&sb, "%d", *i);
5513                 space = 1;
5514         }
5515         rc = sbuf_finish(&sb);
5516         sbuf_delete(&sb);
5517         return (rc);
5518 }
5519
5520 static int
5521 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5522 {
5523         int rc;
5524         struct sbuf *sb;
5525
5526         rc = sysctl_wire_old_buffer(req, 0);
5527         if (rc != 0)
5528                 return(rc);
5529
5530         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5531         if (sb == NULL)
5532                 return (ENOMEM);
5533
5534         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5535         rc = sbuf_finish(sb);
5536         sbuf_delete(sb);
5537
5538         return (rc);
5539 }
5540
5541 static int
5542 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5543 {
5544         struct port_info *pi = arg1;
5545         int op = arg2;
5546         struct adapter *sc = pi->adapter;
5547         u_int v;
5548         int rc;
5549
5550         rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
5551         if (rc)
5552                 return (rc);
5553         /* XXX: magic numbers */
5554         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5555             &v);
5556         end_synchronized_op(sc, 0);
5557         if (rc)
5558                 return (rc);
5559         if (op == 0)
5560                 v /= 256;
5561
5562         rc = sysctl_handle_int(oidp, &v, 0, req);
5563         return (rc);
5564 }
5565
5566 static int
5567 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5568 {
5569         struct vi_info *vi = arg1;
5570         int rc, val;
5571
5572         val = vi->rsrv_noflowq;
5573         rc = sysctl_handle_int(oidp, &val, 0, req);
5574         if (rc != 0 || req->newptr == NULL)
5575                 return (rc);
5576
5577         if ((val >= 1) && (vi->ntxq > 1))
5578                 vi->rsrv_noflowq = 1;
5579         else
5580                 vi->rsrv_noflowq = 0;
5581
5582         return (rc);
5583 }
5584
5585 static int
5586 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5587 {
5588         struct vi_info *vi = arg1;
5589         struct adapter *sc = vi->pi->adapter;
5590         int idx, rc, i;
5591         struct sge_rxq *rxq;
5592 #ifdef TCP_OFFLOAD
5593         struct sge_ofld_rxq *ofld_rxq;
5594 #endif
5595         uint8_t v;
5596
5597         idx = vi->tmr_idx;
5598
5599         rc = sysctl_handle_int(oidp, &idx, 0, req);
5600         if (rc != 0 || req->newptr == NULL)
5601                 return (rc);
5602
5603         if (idx < 0 || idx >= SGE_NTIMERS)
5604                 return (EINVAL);
5605
5606         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5607             "t4tmr");
5608         if (rc)
5609                 return (rc);
5610
5611         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
5612         for_each_rxq(vi, i, rxq) {
5613 #ifdef atomic_store_rel_8
5614                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5615 #else
5616                 rxq->iq.intr_params = v;
5617 #endif
5618         }
5619 #ifdef TCP_OFFLOAD
5620         for_each_ofld_rxq(vi, i, ofld_rxq) {
5621 #ifdef atomic_store_rel_8
5622                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5623 #else
5624                 ofld_rxq->iq.intr_params = v;
5625 #endif
5626         }
5627 #endif
5628         vi->tmr_idx = idx;
5629
5630         end_synchronized_op(sc, LOCK_HELD);
5631         return (0);
5632 }
5633
5634 static int
5635 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5636 {
5637         struct vi_info *vi = arg1;
5638         struct adapter *sc = vi->pi->adapter;
5639         int idx, rc;
5640
5641         idx = vi->pktc_idx;
5642
5643         rc = sysctl_handle_int(oidp, &idx, 0, req);
5644         if (rc != 0 || req->newptr == NULL)
5645                 return (rc);
5646
5647         if (idx < -1 || idx >= SGE_NCOUNTERS)
5648                 return (EINVAL);
5649
5650         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5651             "t4pktc");
5652         if (rc)
5653                 return (rc);
5654
5655         if (vi->flags & VI_INIT_DONE)
5656                 rc = EBUSY; /* cannot be changed once the queues are created */
5657         else
5658                 vi->pktc_idx = idx;
5659
5660         end_synchronized_op(sc, LOCK_HELD);
5661         return (rc);
5662 }
5663
5664 static int
5665 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5666 {
5667         struct vi_info *vi = arg1;
5668         struct adapter *sc = vi->pi->adapter;
5669         int qsize, rc;
5670
5671         qsize = vi->qsize_rxq;
5672
5673         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5674         if (rc != 0 || req->newptr == NULL)
5675                 return (rc);
5676
5677         if (qsize < 128 || (qsize & 7))
5678                 return (EINVAL);
5679
5680         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5681             "t4rxqs");
5682         if (rc)
5683                 return (rc);
5684
5685         if (vi->flags & VI_INIT_DONE)
5686                 rc = EBUSY; /* cannot be changed once the queues are created */
5687         else
5688                 vi->qsize_rxq = qsize;
5689
5690         end_synchronized_op(sc, LOCK_HELD);
5691         return (rc);
5692 }
5693
5694 static int
5695 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5696 {
5697         struct vi_info *vi = arg1;
5698         struct adapter *sc = vi->pi->adapter;
5699         int qsize, rc;
5700
5701         qsize = vi->qsize_txq;
5702
5703         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5704         if (rc != 0 || req->newptr == NULL)
5705                 return (rc);
5706
5707         if (qsize < 128 || qsize > 65536)
5708                 return (EINVAL);
5709
5710         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5711             "t4txqs");
5712         if (rc)
5713                 return (rc);
5714
5715         if (vi->flags & VI_INIT_DONE)
5716                 rc = EBUSY; /* cannot be changed once the queues are created */
5717         else
5718                 vi->qsize_txq = qsize;
5719
5720         end_synchronized_op(sc, LOCK_HELD);
5721         return (rc);
5722 }
5723
5724 static int
5725 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5726 {
5727         struct port_info *pi = arg1;
5728         struct adapter *sc = pi->adapter;
5729         struct link_config *lc = &pi->link_cfg;
5730         int rc;
5731
5732         if (req->newptr == NULL) {
5733                 struct sbuf *sb;
5734                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5735
5736                 rc = sysctl_wire_old_buffer(req, 0);
5737                 if (rc != 0)
5738                         return(rc);
5739
5740                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5741                 if (sb == NULL)
5742                         return (ENOMEM);
5743
5744                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5745                 rc = sbuf_finish(sb);
5746                 sbuf_delete(sb);
5747         } else {
5748                 char s[2];
5749                 int n;
5750
5751                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5752                 s[1] = 0;
5753
5754                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5755                 if (rc != 0)
5756                         return(rc);
5757
5758                 if (s[1] != 0)
5759                         return (EINVAL);
5760                 if (s[0] < '0' || s[0] > '9')
5761                         return (EINVAL);        /* not a number */
5762                 n = s[0] - '0';
5763                 if (n & ~(PAUSE_TX | PAUSE_RX))
5764                         return (EINVAL);        /* some other bit is set too */
5765
5766                 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
5767                     "t4PAUSE");
5768                 if (rc)
5769                         return (rc);
5770                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5771                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5772                         lc->requested_fc |= n;
5773                         rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
5774                 }
5775                 end_synchronized_op(sc, 0);
5776         }
5777
5778         return (rc);
5779 }
5780
5781 static int
5782 sysctl_fec(SYSCTL_HANDLER_ARGS)
5783 {
5784         struct port_info *pi = arg1;
5785         struct adapter *sc = pi->adapter;
5786         struct link_config *lc = &pi->link_cfg;
5787         int rc;
5788
5789         if (req->newptr == NULL) {
5790                 struct sbuf *sb;
5791                 static char *bits = "\20\1RS\2BASER_RS\3RESERVED";
5792
5793                 rc = sysctl_wire_old_buffer(req, 0);
5794                 if (rc != 0)
5795                         return(rc);
5796
5797                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5798                 if (sb == NULL)
5799                         return (ENOMEM);
5800
5801                 sbuf_printf(sb, "%b", lc->fec & M_FW_PORT_CAP_FEC, bits);
5802                 rc = sbuf_finish(sb);
5803                 sbuf_delete(sb);
5804         } else {
5805                 char s[2];
5806                 int n;
5807
5808                 s[0] = '0' + (lc->requested_fec & M_FW_PORT_CAP_FEC);
5809                 s[1] = 0;
5810
5811                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5812                 if (rc != 0)
5813                         return(rc);
5814
5815                 if (s[1] != 0)
5816                         return (EINVAL);
5817                 if (s[0] < '0' || s[0] > '9')
5818                         return (EINVAL);        /* not a number */
5819                 n = s[0] - '0';
5820                 if (n & ~M_FW_PORT_CAP_FEC)
5821                         return (EINVAL);        /* some other bit is set too */
5822
5823                 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
5824                     "t4fec");
5825                 if (rc)
5826                         return (rc);
5827                 if ((lc->requested_fec & M_FW_PORT_CAP_FEC) != n) {
5828                         lc->requested_fec = n &
5829                             G_FW_PORT_CAP_FEC(lc->supported);
5830                         rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
5831                 }
5832                 end_synchronized_op(sc, 0);
5833         }
5834
5835         return (rc);
5836 }
5837
5838 static int
5839 sysctl_autoneg(SYSCTL_HANDLER_ARGS)
5840 {
5841         struct port_info *pi = arg1;
5842         struct adapter *sc = pi->adapter;
5843         struct link_config *lc = &pi->link_cfg;
5844         int rc, val, old;
5845
5846         if (lc->supported & FW_PORT_CAP_ANEG)
5847                 val = lc->autoneg == AUTONEG_ENABLE ? 1 : 0;
5848         else
5849                 val = -1;
5850         rc = sysctl_handle_int(oidp, &val, 0, req);
5851         if (rc != 0 || req->newptr == NULL)
5852                 return (rc);
5853         if ((lc->supported & FW_PORT_CAP_ANEG) == 0)
5854                 return (ENOTSUP);
5855
5856         val = val ? AUTONEG_ENABLE : AUTONEG_DISABLE;
5857         if (lc->autoneg == val)
5858                 return (0);     /* no change */
5859
5860         rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
5861             "t4aneg");
5862         if (rc)
5863                 return (rc);
5864         old = lc->autoneg;
5865         lc->autoneg = val;
5866         rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
5867         if (rc != 0)
5868                 lc->autoneg = old;
5869         return (rc);
5870 }
5871
5872 static int
5873 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5874 {
5875         struct adapter *sc = arg1;
5876         int reg = arg2;
5877         uint64_t val;
5878
5879         val = t4_read_reg64(sc, reg);
5880
5881         return (sysctl_handle_64(oidp, &val, 0, req));
5882 }
5883
5884 static int
5885 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5886 {
5887         struct adapter *sc = arg1;
5888         int rc, t;
5889         uint32_t param, val;
5890
5891         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5892         if (rc)
5893                 return (rc);
5894         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5895             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5896             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5897         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5898         end_synchronized_op(sc, 0);
5899         if (rc)
5900                 return (rc);
5901
5902         /* unknown is returned as 0 but we display -1 in that case */
5903         t = val == 0 ? -1 : val;
5904
5905         rc = sysctl_handle_int(oidp, &t, 0, req);
5906         return (rc);
5907 }
5908
5909 #ifdef SBUF_DRAIN
5910 static int
5911 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5912 {
5913         struct adapter *sc = arg1;
5914         struct sbuf *sb;
5915         int rc, i;
5916         uint16_t incr[NMTUS][NCCTRL_WIN];
5917         static const char *dec_fac[] = {
5918                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5919                 "0.9375"
5920         };
5921
5922         rc = sysctl_wire_old_buffer(req, 0);
5923         if (rc != 0)
5924                 return (rc);
5925
5926         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5927         if (sb == NULL)
5928                 return (ENOMEM);
5929
5930         t4_read_cong_tbl(sc, incr);
5931
5932         for (i = 0; i < NCCTRL_WIN; ++i) {
5933                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5934                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5935                     incr[5][i], incr[6][i], incr[7][i]);
5936                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5937                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5938                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5939                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5940         }
5941
5942         rc = sbuf_finish(sb);
5943         sbuf_delete(sb);
5944
5945         return (rc);
5946 }
5947
5948 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5949         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5950         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5951         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5952 };
5953
5954 static int
5955 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5956 {
5957         struct adapter *sc = arg1;
5958         struct sbuf *sb;
5959         int rc, i, n, qid = arg2;
5960         uint32_t *buf, *p;
5961         char *qtype;
5962         u_int cim_num_obq = sc->chip_params->cim_num_obq;
5963
5964         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5965             ("%s: bad qid %d\n", __func__, qid));
5966
5967         if (qid < CIM_NUM_IBQ) {
5968                 /* inbound queue */
5969                 qtype = "IBQ";
5970                 n = 4 * CIM_IBQ_SIZE;
5971                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5972                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5973         } else {
5974                 /* outbound queue */
5975                 qtype = "OBQ";
5976                 qid -= CIM_NUM_IBQ;
5977                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5978                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5979                 rc = t4_read_cim_obq(sc, qid, buf, n);
5980         }
5981
5982         if (rc < 0) {
5983                 rc = -rc;
5984                 goto done;
5985         }
5986         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5987
5988         rc = sysctl_wire_old_buffer(req, 0);
5989         if (rc != 0)
5990                 goto done;
5991
5992         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5993         if (sb == NULL) {
5994                 rc = ENOMEM;
5995                 goto done;
5996         }
5997
5998         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5999         for (i = 0, p = buf; i < n; i += 16, p += 4)
6000                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
6001                     p[2], p[3]);
6002
6003         rc = sbuf_finish(sb);
6004         sbuf_delete(sb);
6005 done:
6006         free(buf, M_CXGBE);
6007         return (rc);
6008 }
6009
6010 static int
6011 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
6012 {
6013         struct adapter *sc = arg1;
6014         u_int cfg;
6015         struct sbuf *sb;
6016         uint32_t *buf, *p;
6017         int rc;
6018
6019         MPASS(chip_id(sc) <= CHELSIO_T5);
6020
6021         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
6022         if (rc != 0)
6023                 return (rc);
6024
6025         rc = sysctl_wire_old_buffer(req, 0);
6026         if (rc != 0)
6027                 return (rc);
6028
6029         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6030         if (sb == NULL)
6031                 return (ENOMEM);
6032
6033         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
6034             M_ZERO | M_WAITOK);
6035
6036         rc = -t4_cim_read_la(sc, buf, NULL);
6037         if (rc != 0)
6038                 goto done;
6039
6040         sbuf_printf(sb, "Status   Data      PC%s",
6041             cfg & F_UPDBGLACAPTPCONLY ? "" :
6042             "     LS0Stat  LS0Addr             LS0Data");
6043
6044         for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
6045                 if (cfg & F_UPDBGLACAPTPCONLY) {
6046                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
6047                             p[6], p[7]);
6048                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
6049                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
6050                             p[4] & 0xff, p[5] >> 8);
6051                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
6052                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6053                             p[1] & 0xf, p[2] >> 4);
6054                 } else {
6055                         sbuf_printf(sb,
6056                             "\n  %02x   %x%07x %x%07x %08x %08x "
6057                             "%08x%08x%08x%08x",
6058                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
6059                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
6060                             p[6], p[7]);
6061                 }
6062         }
6063
6064         rc = sbuf_finish(sb);
6065         sbuf_delete(sb);
6066 done:
6067         free(buf, M_CXGBE);
6068         return (rc);
6069 }
6070
6071 static int
6072 sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS)
6073 {
6074         struct adapter *sc = arg1;
6075         u_int cfg;
6076         struct sbuf *sb;
6077         uint32_t *buf, *p;
6078         int rc;
6079
6080         MPASS(chip_id(sc) > CHELSIO_T5);
6081
6082         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
6083         if (rc != 0)
6084                 return (rc);
6085
6086         rc = sysctl_wire_old_buffer(req, 0);
6087         if (rc != 0)
6088                 return (rc);
6089
6090         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6091         if (sb == NULL)
6092                 return (ENOMEM);
6093
6094         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
6095             M_ZERO | M_WAITOK);
6096
6097         rc = -t4_cim_read_la(sc, buf, NULL);
6098         if (rc != 0)
6099                 goto done;
6100
6101         sbuf_printf(sb, "Status   Inst    Data      PC%s",
6102             cfg & F_UPDBGLACAPTPCONLY ? "" :
6103             "     LS0Stat  LS0Addr  LS0Data  LS1Stat  LS1Addr  LS1Data");
6104
6105         for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
6106                 if (cfg & F_UPDBGLACAPTPCONLY) {
6107                         sbuf_printf(sb, "\n  %02x   %08x %08x %08x",
6108                             p[3] & 0xff, p[2], p[1], p[0]);
6109                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x %02x%06x",
6110                             (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8,
6111                             p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8);
6112                         sbuf_printf(sb, "\n  %02x   %04x%04x %04x%04x %04x%04x",
6113                             (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16,
6114                             p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff,
6115                             p[6] >> 16);
6116                 } else {
6117                         sbuf_printf(sb, "\n  %02x   %04x%04x %04x%04x %04x%04x "
6118                             "%08x %08x %08x %08x %08x %08x",
6119                             (p[9] >> 16) & 0xff,
6120                             p[9] & 0xffff, p[8] >> 16,
6121                             p[8] & 0xffff, p[7] >> 16,
6122                             p[7] & 0xffff, p[6] >> 16,
6123                             p[2], p[1], p[0], p[5], p[4], p[3]);
6124                 }
6125         }
6126
6127         rc = sbuf_finish(sb);
6128         sbuf_delete(sb);
6129 done:
6130         free(buf, M_CXGBE);
6131         return (rc);
6132 }
6133
6134 static int
6135 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
6136 {
6137         struct adapter *sc = arg1;
6138         u_int i;
6139         struct sbuf *sb;
6140         uint32_t *buf, *p;
6141         int rc;
6142
6143         rc = sysctl_wire_old_buffer(req, 0);
6144         if (rc != 0)
6145                 return (rc);
6146
6147         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6148         if (sb == NULL)
6149                 return (ENOMEM);
6150
6151         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
6152             M_ZERO | M_WAITOK);
6153
6154         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
6155         p = buf;
6156
6157         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6158                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
6159                     p[1], p[0]);
6160         }
6161
6162         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
6163         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
6164                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
6165                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
6166                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
6167                     (p[1] >> 2) | ((p[2] & 3) << 30),
6168                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
6169                     p[0] & 1);
6170         }
6171
6172         rc = sbuf_finish(sb);
6173         sbuf_delete(sb);
6174         free(buf, M_CXGBE);
6175         return (rc);
6176 }
6177
6178 static int
6179 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
6180 {
6181         struct adapter *sc = arg1;
6182         u_int i;
6183         struct sbuf *sb;
6184         uint32_t *buf, *p;
6185         int rc;
6186
6187         rc = sysctl_wire_old_buffer(req, 0);
6188         if (rc != 0)
6189                 return (rc);
6190
6191         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6192         if (sb == NULL)
6193                 return (ENOMEM);
6194
6195         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
6196             M_ZERO | M_WAITOK);
6197
6198         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
6199         p = buf;
6200
6201         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
6202         for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
6203                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
6204                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
6205                     p[4], p[3], p[2], p[1], p[0]);
6206         }
6207
6208         sbuf_printf(sb, "\n\nCntl ID               Data");
6209         for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
6210                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
6211                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
6212         }
6213
6214         rc = sbuf_finish(sb);
6215         sbuf_delete(sb);
6216         free(buf, M_CXGBE);
6217         return (rc);
6218 }
6219
6220 static int
6221 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
6222 {
6223         struct adapter *sc = arg1;
6224         struct sbuf *sb;
6225         int rc, i;
6226         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6227         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
6228         uint16_t thres[CIM_NUM_IBQ];
6229         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
6230         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
6231         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
6232
6233         cim_num_obq = sc->chip_params->cim_num_obq;
6234         if (is_t4(sc)) {
6235                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
6236                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
6237         } else {
6238                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
6239                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
6240         }
6241         nq = CIM_NUM_IBQ + cim_num_obq;
6242
6243         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
6244         if (rc == 0)
6245                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
6246         if (rc != 0)
6247                 return (rc);
6248
6249         t4_read_cimq_cfg(sc, base, size, thres);
6250
6251         rc = sysctl_wire_old_buffer(req, 0);
6252         if (rc != 0)
6253                 return (rc);
6254
6255         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
6256         if (sb == NULL)
6257                 return (ENOMEM);
6258
6259         sbuf_printf(sb,
6260             "  Queue  Base  Size Thres  RdPtr WrPtr  SOP  EOP Avail");
6261
6262         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
6263                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
6264                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
6265                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6266                     G_QUEREMFLITS(p[2]) * 16);
6267         for ( ; i < nq; i++, p += 4, wr += 2)
6268                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
6269                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
6270                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
6271                     G_QUEREMFLITS(p[2]) * 16);
6272
6273         rc = sbuf_finish(sb);
6274         sbuf_delete(sb);
6275
6276         return (rc);
6277 }
6278
6279 static int
6280 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
6281 {
6282         struct adapter *sc = arg1;
6283         struct sbuf *sb;
6284         int rc;
6285         struct tp_cpl_stats stats;
6286
6287         rc = sysctl_wire_old_buffer(req, 0);
6288         if (rc != 0)
6289                 return (rc);
6290
6291         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6292         if (sb == NULL)
6293                 return (ENOMEM);
6294
6295         mtx_lock(&sc->reg_lock);
6296         t4_tp_get_cpl_stats(sc, &stats);
6297         mtx_unlock(&sc->reg_lock);
6298
6299         if (sc->chip_params->nchan > 2) {
6300                 sbuf_printf(sb, "                 channel 0  channel 1"
6301                     "  channel 2  channel 3");
6302                 sbuf_printf(sb, "\nCPL requests:   %10u %10u %10u %10u",
6303                     stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
6304                 sbuf_printf(sb, "\nCPL responses:   %10u %10u %10u %10u",
6305                     stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
6306         } else {
6307                 sbuf_printf(sb, "                 channel 0  channel 1");
6308                 sbuf_printf(sb, "\nCPL requests:   %10u %10u",
6309                     stats.req[0], stats.req[1]);
6310                 sbuf_printf(sb, "\nCPL responses:   %10u %10u",
6311                     stats.rsp[0], stats.rsp[1]);
6312         }
6313
6314         rc = sbuf_finish(sb);
6315         sbuf_delete(sb);
6316
6317         return (rc);
6318 }
6319
6320 static int
6321 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
6322 {
6323         struct adapter *sc = arg1;
6324         struct sbuf *sb;
6325         int rc;
6326         struct tp_usm_stats stats;
6327
6328         rc = sysctl_wire_old_buffer(req, 0);
6329         if (rc != 0)
6330                 return(rc);
6331
6332         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6333         if (sb == NULL)
6334                 return (ENOMEM);
6335
6336         t4_get_usm_stats(sc, &stats);
6337
6338         sbuf_printf(sb, "Frames: %u\n", stats.frames);
6339         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
6340         sbuf_printf(sb, "Drops:  %u", stats.drops);
6341
6342         rc = sbuf_finish(sb);
6343         sbuf_delete(sb);
6344
6345         return (rc);
6346 }
6347
6348 static const char * const devlog_level_strings[] = {
6349         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
6350         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
6351         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
6352         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
6353         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
6354         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
6355 };
6356
6357 static const char * const devlog_facility_strings[] = {
6358         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
6359         [FW_DEVLOG_FACILITY_CF]         = "CF",
6360         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
6361         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
6362         [FW_DEVLOG_FACILITY_RES]        = "RES",
6363         [FW_DEVLOG_FACILITY_HW]         = "HW",
6364         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
6365         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
6366         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
6367         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
6368         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
6369         [FW_DEVLOG_FACILITY_VI]         = "VI",
6370         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
6371         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
6372         [FW_DEVLOG_FACILITY_TM]         = "TM",
6373         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
6374         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
6375         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
6376         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
6377         [FW_DEVLOG_FACILITY_RI]         = "RI",
6378         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
6379         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
6380         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
6381         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE",
6382         [FW_DEVLOG_FACILITY_CHNET]      = "CHNET",
6383 };
6384
6385 static int
6386 sysctl_devlog(SYSCTL_HANDLER_ARGS)
6387 {
6388         struct adapter *sc = arg1;
6389         struct devlog_params *dparams = &sc->params.devlog;
6390         struct fw_devlog_e *buf, *e;
6391         int i, j, rc, nentries, first = 0;
6392         struct sbuf *sb;
6393         uint64_t ftstamp = UINT64_MAX;
6394
6395         if (dparams->addr == 0)
6396                 return (ENXIO);
6397
6398         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
6399         if (buf == NULL)
6400                 return (ENOMEM);
6401
6402         rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size);
6403         if (rc != 0)
6404                 goto done;
6405
6406         nentries = dparams->size / sizeof(struct fw_devlog_e);
6407         for (i = 0; i < nentries; i++) {
6408                 e = &buf[i];
6409
6410                 if (e->timestamp == 0)
6411                         break;  /* end */
6412
6413                 e->timestamp = be64toh(e->timestamp);
6414                 e->seqno = be32toh(e->seqno);
6415                 for (j = 0; j < 8; j++)
6416                         e->params[j] = be32toh(e->params[j]);
6417
6418                 if (e->timestamp < ftstamp) {
6419                         ftstamp = e->timestamp;
6420                         first = i;
6421                 }
6422         }
6423
6424         if (buf[first].timestamp == 0)
6425                 goto done;      /* nothing in the log */
6426
6427         rc = sysctl_wire_old_buffer(req, 0);
6428         if (rc != 0)
6429                 goto done;
6430
6431         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6432         if (sb == NULL) {
6433                 rc = ENOMEM;
6434                 goto done;
6435         }
6436         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
6437             "Seq#", "Tstamp", "Level", "Facility", "Message");
6438
6439         i = first;
6440         do {
6441                 e = &buf[i];
6442                 if (e->timestamp == 0)
6443                         break;  /* end */
6444
6445                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
6446                     e->seqno, e->timestamp,
6447                     (e->level < nitems(devlog_level_strings) ?
6448                         devlog_level_strings[e->level] : "UNKNOWN"),
6449                     (e->facility < nitems(devlog_facility_strings) ?
6450                         devlog_facility_strings[e->facility] : "UNKNOWN"));
6451                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
6452                     e->params[2], e->params[3], e->params[4],
6453                     e->params[5], e->params[6], e->params[7]);
6454
6455                 if (++i == nentries)
6456                         i = 0;
6457         } while (i != first);
6458
6459         rc = sbuf_finish(sb);
6460         sbuf_delete(sb);
6461 done:
6462         free(buf, M_CXGBE);
6463         return (rc);
6464 }
6465
6466 static int
6467 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
6468 {
6469         struct adapter *sc = arg1;
6470         struct sbuf *sb;
6471         int rc;
6472         struct tp_fcoe_stats stats[MAX_NCHAN];
6473         int i, nchan = sc->chip_params->nchan;
6474
6475         rc = sysctl_wire_old_buffer(req, 0);
6476         if (rc != 0)
6477                 return (rc);
6478
6479         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6480         if (sb == NULL)
6481                 return (ENOMEM);
6482
6483         for (i = 0; i < nchan; i++)
6484                 t4_get_fcoe_stats(sc, i, &stats[i]);
6485
6486         if (nchan > 2) {
6487                 sbuf_printf(sb, "                   channel 0        channel 1"
6488                     "        channel 2        channel 3");
6489                 sbuf_printf(sb, "\noctetsDDP:  %16ju %16ju %16ju %16ju",
6490                     stats[0].octets_ddp, stats[1].octets_ddp,
6491                     stats[2].octets_ddp, stats[3].octets_ddp);
6492                 sbuf_printf(sb, "\nframesDDP:  %16u %16u %16u %16u",
6493                     stats[0].frames_ddp, stats[1].frames_ddp,
6494                     stats[2].frames_ddp, stats[3].frames_ddp);
6495                 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u",
6496                     stats[0].frames_drop, stats[1].frames_drop,
6497                     stats[2].frames_drop, stats[3].frames_drop);
6498         } else {
6499                 sbuf_printf(sb, "                   channel 0        channel 1");
6500                 sbuf_printf(sb, "\noctetsDDP:  %16ju %16ju",
6501                     stats[0].octets_ddp, stats[1].octets_ddp);
6502                 sbuf_printf(sb, "\nframesDDP:  %16u %16u",
6503                     stats[0].frames_ddp, stats[1].frames_ddp);
6504                 sbuf_printf(sb, "\nframesDrop: %16u %16u",
6505                     stats[0].frames_drop, stats[1].frames_drop);
6506         }
6507
6508         rc = sbuf_finish(sb);
6509         sbuf_delete(sb);
6510
6511         return (rc);
6512 }
6513
6514 static int
6515 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
6516 {
6517         struct adapter *sc = arg1;
6518         struct sbuf *sb;
6519         int rc, i;
6520         unsigned int map, kbps, ipg, mode;
6521         unsigned int pace_tab[NTX_SCHED];
6522
6523         rc = sysctl_wire_old_buffer(req, 0);
6524         if (rc != 0)
6525                 return (rc);
6526
6527         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6528         if (sb == NULL)
6529                 return (ENOMEM);
6530
6531         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
6532         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
6533         t4_read_pace_tbl(sc, pace_tab);
6534
6535         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
6536             "Class IPG (0.1 ns)   Flow IPG (us)");
6537
6538         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
6539                 t4_get_tx_sched(sc, i, &kbps, &ipg);
6540                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
6541                     (mode & (1 << i)) ? "flow" : "class", map & 3);
6542                 if (kbps)
6543                         sbuf_printf(sb, "%9u     ", kbps);
6544                 else
6545                         sbuf_printf(sb, " disabled     ");
6546
6547                 if (ipg)
6548                         sbuf_printf(sb, "%13u        ", ipg);
6549                 else
6550                         sbuf_printf(sb, "     disabled        ");
6551
6552                 if (pace_tab[i])
6553                         sbuf_printf(sb, "%10u", pace_tab[i]);
6554                 else
6555                         sbuf_printf(sb, "  disabled");
6556         }
6557
6558         rc = sbuf_finish(sb);
6559         sbuf_delete(sb);
6560
6561         return (rc);
6562 }
6563
6564 static int
6565 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
6566 {
6567         struct adapter *sc = arg1;
6568         struct sbuf *sb;
6569         int rc, i, j;
6570         uint64_t *p0, *p1;
6571         struct lb_port_stats s[2];
6572         static const char *stat_name[] = {
6573                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
6574                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
6575                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
6576                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
6577                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
6578                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
6579                 "BG2FramesTrunc:", "BG3FramesTrunc:"
6580         };
6581
6582         rc = sysctl_wire_old_buffer(req, 0);
6583         if (rc != 0)
6584                 return (rc);
6585
6586         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6587         if (sb == NULL)
6588                 return (ENOMEM);
6589
6590         memset(s, 0, sizeof(s));
6591
6592         for (i = 0; i < sc->chip_params->nchan; i += 2) {
6593                 t4_get_lb_stats(sc, i, &s[0]);
6594                 t4_get_lb_stats(sc, i + 1, &s[1]);
6595
6596                 p0 = &s[0].octets;
6597                 p1 = &s[1].octets;
6598                 sbuf_printf(sb, "%s                       Loopback %u"
6599                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
6600
6601                 for (j = 0; j < nitems(stat_name); j++)
6602                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
6603                                    *p0++, *p1++);
6604         }
6605
6606         rc = sbuf_finish(sb);
6607         sbuf_delete(sb);
6608
6609         return (rc);
6610 }
6611
6612 static int
6613 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6614 {
6615         int rc = 0;
6616         struct port_info *pi = arg1;
6617         struct link_config *lc = &pi->link_cfg;
6618         struct sbuf *sb;
6619
6620         rc = sysctl_wire_old_buffer(req, 0);
6621         if (rc != 0)
6622                 return(rc);
6623         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6624         if (sb == NULL)
6625                 return (ENOMEM);
6626
6627         if (lc->link_ok || lc->link_down_rc == 255)
6628                 sbuf_printf(sb, "n/a");
6629         else
6630                 sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc));
6631
6632         rc = sbuf_finish(sb);
6633         sbuf_delete(sb);
6634
6635         return (rc);
6636 }
6637
6638 struct mem_desc {
6639         unsigned int base;
6640         unsigned int limit;
6641         unsigned int idx;
6642 };
6643
6644 static int
6645 mem_desc_cmp(const void *a, const void *b)
6646 {
6647         return ((const struct mem_desc *)a)->base -
6648                ((const struct mem_desc *)b)->base;
6649 }
6650
6651 static void
6652 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6653     unsigned int to)
6654 {
6655         unsigned int size;
6656
6657         if (from == to)
6658                 return;
6659
6660         size = to - from + 1;
6661         if (size == 0)
6662                 return;
6663
6664         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6665         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6666 }
6667
6668 static int
6669 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6670 {
6671         struct adapter *sc = arg1;
6672         struct sbuf *sb;
6673         int rc, i, n;
6674         uint32_t lo, hi, used, alloc;
6675         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6676         static const char *region[] = {
6677                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6678                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6679                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6680                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6681                 "RQUDP region:", "PBL region:", "TXPBL region:",
6682                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6683                 "On-chip queues:"
6684         };
6685         struct mem_desc avail[4];
6686         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6687         struct mem_desc *md = mem;
6688
6689         rc = sysctl_wire_old_buffer(req, 0);
6690         if (rc != 0)
6691                 return (rc);
6692
6693         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6694         if (sb == NULL)
6695                 return (ENOMEM);
6696
6697         for (i = 0; i < nitems(mem); i++) {
6698                 mem[i].limit = 0;
6699                 mem[i].idx = i;
6700         }
6701
6702         /* Find and sort the populated memory ranges */
6703         i = 0;
6704         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6705         if (lo & F_EDRAM0_ENABLE) {
6706                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6707                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6708                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6709                 avail[i].idx = 0;
6710                 i++;
6711         }
6712         if (lo & F_EDRAM1_ENABLE) {
6713                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6714                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6715                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6716                 avail[i].idx = 1;
6717                 i++;
6718         }
6719         if (lo & F_EXT_MEM_ENABLE) {
6720                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6721                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6722                 avail[i].limit = avail[i].base +
6723                     (G_EXT_MEM_SIZE(hi) << 20);
6724                 avail[i].idx = is_t5(sc) ? 3 : 2;       /* Call it MC0 for T5 */
6725                 i++;
6726         }
6727         if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) {
6728                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6729                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6730                 avail[i].limit = avail[i].base +
6731                     (G_EXT_MEM1_SIZE(hi) << 20);
6732                 avail[i].idx = 4;
6733                 i++;
6734         }
6735         if (!i)                                    /* no memory available */
6736                 return 0;
6737         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6738
6739         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6740         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6741         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6742         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6743         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6744         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6745         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6746         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6747         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6748
6749         /* the next few have explicit upper bounds */
6750         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6751         md->limit = md->base - 1 +
6752                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6753                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6754         md++;
6755
6756         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6757         md->limit = md->base - 1 +
6758                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6759                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6760         md++;
6761
6762         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6763                 if (chip_id(sc) <= CHELSIO_T5)
6764                         md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6765                 else
6766                         md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR);
6767                 md->limit = 0;
6768         } else {
6769                 md->base = 0;
6770                 md->idx = nitems(region);  /* hide it */
6771         }
6772         md++;
6773
6774 #define ulp_region(reg) \
6775         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6776         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6777
6778         ulp_region(RX_ISCSI);
6779         ulp_region(RX_TDDP);
6780         ulp_region(TX_TPT);
6781         ulp_region(RX_STAG);
6782         ulp_region(RX_RQ);
6783         ulp_region(RX_RQUDP);
6784         ulp_region(RX_PBL);
6785         ulp_region(TX_PBL);
6786 #undef ulp_region
6787
6788         md->base = 0;
6789         md->idx = nitems(region);
6790         if (!is_t4(sc)) {
6791                 uint32_t size = 0;
6792                 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2);
6793                 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE);
6794
6795                 if (is_t5(sc)) {
6796                         if (sge_ctrl & F_VFIFO_ENABLE)
6797                                 size = G_DBVFIFO_SIZE(fifo_size);
6798                 } else
6799                         size = G_T6_DBVFIFO_SIZE(fifo_size);
6800
6801                 if (size) {
6802                         md->base = G_BASEADDR(t4_read_reg(sc,
6803                             A_SGE_DBVFIFO_BADDR));
6804                         md->limit = md->base + (size << 2) - 1;
6805                 }
6806         }
6807         md++;
6808
6809         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6810         md->limit = 0;
6811         md++;
6812         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6813         md->limit = 0;
6814         md++;
6815
6816         md->base = sc->vres.ocq.start;
6817         if (sc->vres.ocq.size)
6818                 md->limit = md->base + sc->vres.ocq.size - 1;
6819         else
6820                 md->idx = nitems(region);  /* hide it */
6821         md++;
6822
6823         /* add any address-space holes, there can be up to 3 */
6824         for (n = 0; n < i - 1; n++)
6825                 if (avail[n].limit < avail[n + 1].base)
6826                         (md++)->base = avail[n].limit;
6827         if (avail[n].limit)
6828                 (md++)->base = avail[n].limit;
6829
6830         n = md - mem;
6831         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6832
6833         for (lo = 0; lo < i; lo++)
6834                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6835                                 avail[lo].limit - 1);
6836
6837         sbuf_printf(sb, "\n");
6838         for (i = 0; i < n; i++) {
6839                 if (mem[i].idx >= nitems(region))
6840                         continue;                        /* skip holes */
6841                 if (!mem[i].limit)
6842                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6843                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6844                                 mem[i].limit);
6845         }
6846
6847         sbuf_printf(sb, "\n");
6848         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6849         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6850         mem_region_show(sb, "uP RAM:", lo, hi);
6851
6852         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6853         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6854         mem_region_show(sb, "uP Extmem2:", lo, hi);
6855
6856         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6857         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6858                    G_PMRXMAXPAGE(lo),
6859                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6860                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6861
6862         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6863         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6864         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6865                    G_PMTXMAXPAGE(lo),
6866                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6867                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6868         sbuf_printf(sb, "%u p-structs\n",
6869                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6870
6871         for (i = 0; i < 4; i++) {
6872                 if (chip_id(sc) > CHELSIO_T5)
6873                         lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4);
6874                 else
6875                         lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6876                 if (is_t5(sc)) {
6877                         used = G_T5_USED(lo);
6878                         alloc = G_T5_ALLOC(lo);
6879                 } else {
6880                         used = G_USED(lo);
6881                         alloc = G_ALLOC(lo);
6882                 }
6883                 /* For T6 these are MAC buffer groups */
6884                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6885                     i, used, alloc);
6886         }
6887         for (i = 0; i < sc->chip_params->nchan; i++) {
6888                 if (chip_id(sc) > CHELSIO_T5)
6889                         lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4);
6890                 else
6891                         lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6892                 if (is_t5(sc)) {
6893                         used = G_T5_USED(lo);
6894                         alloc = G_T5_ALLOC(lo);
6895                 } else {
6896                         used = G_USED(lo);
6897                         alloc = G_ALLOC(lo);
6898                 }
6899                 /* For T6 these are MAC buffer groups */
6900                 sbuf_printf(sb,
6901                     "\nLoopback %d using %u pages out of %u allocated",
6902                     i, used, alloc);
6903         }
6904
6905         rc = sbuf_finish(sb);
6906         sbuf_delete(sb);
6907
6908         return (rc);
6909 }
6910
6911 static inline void
6912 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6913 {
6914         *mask = x | y;
6915         y = htobe64(y);
6916         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6917 }
6918
6919 static int
6920 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6921 {
6922         struct adapter *sc = arg1;
6923         struct sbuf *sb;
6924         int rc, i;
6925
6926         MPASS(chip_id(sc) <= CHELSIO_T5);
6927
6928         rc = sysctl_wire_old_buffer(req, 0);
6929         if (rc != 0)
6930                 return (rc);
6931
6932         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6933         if (sb == NULL)
6934                 return (ENOMEM);
6935
6936         sbuf_printf(sb,
6937             "Idx  Ethernet address     Mask     Vld Ports PF"
6938             "  VF              Replication             P0 P1 P2 P3  ML");
6939         for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
6940                 uint64_t tcamx, tcamy, mask;
6941                 uint32_t cls_lo, cls_hi;
6942                 uint8_t addr[ETHER_ADDR_LEN];
6943
6944                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6945                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6946                 if (tcamx & tcamy)
6947                         continue;
6948                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6949                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6950                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6951                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6952                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6953                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6954                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6955                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6956                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6957
6958                 if (cls_lo & F_REPLICATE) {
6959                         struct fw_ldst_cmd ldst_cmd;
6960
6961                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6962                         ldst_cmd.op_to_addrspace =
6963                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6964                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6965                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6966                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6967                         ldst_cmd.u.mps.rplc.fid_idx =
6968                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6969                                 V_FW_LDST_CMD_IDX(i));
6970
6971                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6972                             "t4mps");
6973                         if (rc)
6974                                 break;
6975                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6976                             sizeof(ldst_cmd), &ldst_cmd);
6977                         end_synchronized_op(sc, 0);
6978
6979                         if (rc != 0) {
6980                                 sbuf_printf(sb, "%36d", rc);
6981                                 rc = 0;
6982                         } else {
6983                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6984                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6985                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6986                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6987                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6988                         }
6989                 } else
6990                         sbuf_printf(sb, "%36s", "");
6991
6992                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6993                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6994                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6995         }
6996
6997         if (rc)
6998                 (void) sbuf_finish(sb);
6999         else
7000                 rc = sbuf_finish(sb);
7001         sbuf_delete(sb);
7002
7003         return (rc);
7004 }
7005
7006 static int
7007 sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS)
7008 {
7009         struct adapter *sc = arg1;
7010         struct sbuf *sb;
7011         int rc, i;
7012
7013         MPASS(chip_id(sc) > CHELSIO_T5);
7014
7015         rc = sysctl_wire_old_buffer(req, 0);
7016         if (rc != 0)
7017                 return (rc);
7018
7019         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7020         if (sb == NULL)
7021                 return (ENOMEM);
7022
7023         sbuf_printf(sb, "Idx  Ethernet address     Mask       VNI   Mask"
7024             "   IVLAN Vld DIP_Hit   Lookup  Port Vld Ports PF  VF"
7025             "                           Replication"
7026             "                                    P0 P1 P2 P3  ML\n");
7027
7028         for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
7029                 uint8_t dip_hit, vlan_vld, lookup_type, port_num;
7030                 uint16_t ivlan;
7031                 uint64_t tcamx, tcamy, val, mask;
7032                 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy;
7033                 uint8_t addr[ETHER_ADDR_LEN];
7034
7035                 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0);
7036                 if (i < 256)
7037                         ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0);
7038                 else
7039                         ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1);
7040                 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
7041                 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
7042                 tcamy = G_DMACH(val) << 32;
7043                 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
7044                 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
7045                 lookup_type = G_DATALKPTYPE(data2);
7046                 port_num = G_DATAPORTNUM(data2);
7047                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
7048                         /* Inner header VNI */
7049                         vniy = ((data2 & F_DATAVIDH2) << 23) |
7050                                        (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
7051                         dip_hit = data2 & F_DATADIPHIT;
7052                         vlan_vld = 0;
7053                 } else {
7054                         vniy = 0;
7055                         dip_hit = 0;
7056                         vlan_vld = data2 & F_DATAVIDH2;
7057                         ivlan = G_VIDL(val);
7058                 }
7059
7060                 ctl |= V_CTLXYBITSEL(1);
7061                 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
7062                 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
7063                 tcamx = G_DMACH(val) << 32;
7064                 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
7065                 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
7066                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
7067                         /* Inner header VNI mask */
7068                         vnix = ((data2 & F_DATAVIDH2) << 23) |
7069                                (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
7070                 } else
7071                         vnix = 0;
7072
7073                 if (tcamx & tcamy)
7074                         continue;
7075                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
7076
7077                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
7078                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
7079
7080                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
7081                         sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
7082                             "%012jx %06x %06x    -    -   %3c"
7083                             "      'I'  %4x   %3c   %#x%4u%4d", i, addr[0],
7084                             addr[1], addr[2], addr[3], addr[4], addr[5],
7085                             (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N',
7086                             port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
7087                             G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
7088                             cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
7089                 } else {
7090                         sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
7091                             "%012jx    -       -   ", i, addr[0], addr[1],
7092                             addr[2], addr[3], addr[4], addr[5],
7093                             (uintmax_t)mask);
7094
7095                         if (vlan_vld)
7096                                 sbuf_printf(sb, "%4u   Y     ", ivlan);
7097                         else
7098                                 sbuf_printf(sb, "  -    N     ");
7099
7100                         sbuf_printf(sb, "-      %3c  %4x   %3c   %#x%4u%4d",
7101                             lookup_type ? 'I' : 'O', port_num,
7102                             cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
7103                             G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
7104                             cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
7105                 }
7106
7107
7108                 if (cls_lo & F_T6_REPLICATE) {
7109                         struct fw_ldst_cmd ldst_cmd;
7110
7111                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
7112                         ldst_cmd.op_to_addrspace =
7113                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
7114                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
7115                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
7116                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
7117                         ldst_cmd.u.mps.rplc.fid_idx =
7118                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
7119                                 V_FW_LDST_CMD_IDX(i));
7120
7121                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
7122                             "t6mps");
7123                         if (rc)
7124                                 break;
7125                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
7126                             sizeof(ldst_cmd), &ldst_cmd);
7127                         end_synchronized_op(sc, 0);
7128
7129                         if (rc != 0) {
7130                                 sbuf_printf(sb, "%72d", rc);
7131                                 rc = 0;
7132                         } else {
7133                                 sbuf_printf(sb, " %08x %08x %08x %08x"
7134                                     " %08x %08x %08x %08x",
7135                                     be32toh(ldst_cmd.u.mps.rplc.rplc255_224),
7136                                     be32toh(ldst_cmd.u.mps.rplc.rplc223_192),
7137                                     be32toh(ldst_cmd.u.mps.rplc.rplc191_160),
7138                                     be32toh(ldst_cmd.u.mps.rplc.rplc159_128),
7139                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
7140                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
7141                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
7142                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
7143                         }
7144                 } else
7145                         sbuf_printf(sb, "%72s", "");
7146
7147                 sbuf_printf(sb, "%4u%3u%3u%3u %#x",
7148                     G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo),
7149                     G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo),
7150                     (cls_lo >> S_T6_MULTILISTEN0) & 0xf);
7151         }
7152
7153         if (rc)
7154                 (void) sbuf_finish(sb);
7155         else
7156                 rc = sbuf_finish(sb);
7157         sbuf_delete(sb);
7158
7159         return (rc);
7160 }
7161
7162 static int
7163 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
7164 {
7165         struct adapter *sc = arg1;
7166         struct sbuf *sb;
7167         int rc;
7168         uint16_t mtus[NMTUS];
7169
7170         rc = sysctl_wire_old_buffer(req, 0);
7171         if (rc != 0)
7172                 return (rc);
7173
7174         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7175         if (sb == NULL)
7176                 return (ENOMEM);
7177
7178         t4_read_mtu_tbl(sc, mtus, NULL);
7179
7180         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
7181             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
7182             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
7183             mtus[14], mtus[15]);
7184
7185         rc = sbuf_finish(sb);
7186         sbuf_delete(sb);
7187
7188         return (rc);
7189 }
7190
7191 static int
7192 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
7193 {
7194         struct adapter *sc = arg1;
7195         struct sbuf *sb;
7196         int rc, i;
7197         uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS];
7198         uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS];
7199         static const char *tx_stats[MAX_PM_NSTATS] = {
7200                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:",
7201                 "Tx FIFO wait", NULL, "Tx latency"
7202         };
7203         static const char *rx_stats[MAX_PM_NSTATS] = {
7204                 "Read:", "Write bypass:", "Write mem:", "Flush:",
7205                 "Rx FIFO wait", NULL, "Rx latency"
7206         };
7207
7208         rc = sysctl_wire_old_buffer(req, 0);
7209         if (rc != 0)
7210                 return (rc);
7211
7212         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7213         if (sb == NULL)
7214                 return (ENOMEM);
7215
7216         t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
7217         t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
7218
7219         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
7220         for (i = 0; i < 4; i++) {
7221                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
7222                     tx_cyc[i]);
7223         }
7224
7225         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
7226         for (i = 0; i < 4; i++) {
7227                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
7228                     rx_cyc[i]);
7229         }
7230
7231         if (chip_id(sc) > CHELSIO_T5) {
7232                 sbuf_printf(sb,
7233                     "\n              Total wait      Total occupancy");
7234                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
7235                     tx_cyc[i]);
7236                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
7237                     rx_cyc[i]);
7238
7239                 i += 2;
7240                 MPASS(i < nitems(tx_stats));
7241
7242                 sbuf_printf(sb,
7243                     "\n                   Reads           Total wait");
7244                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
7245                     tx_cyc[i]);
7246                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
7247                     rx_cyc[i]);
7248         }
7249
7250         rc = sbuf_finish(sb);
7251         sbuf_delete(sb);
7252
7253         return (rc);
7254 }
7255
7256 static int
7257 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
7258 {
7259         struct adapter *sc = arg1;
7260         struct sbuf *sb;
7261         int rc;
7262         struct tp_rdma_stats stats;
7263
7264         rc = sysctl_wire_old_buffer(req, 0);
7265         if (rc != 0)
7266                 return (rc);
7267
7268         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7269         if (sb == NULL)
7270                 return (ENOMEM);
7271
7272         mtx_lock(&sc->reg_lock);
7273         t4_tp_get_rdma_stats(sc, &stats);
7274         mtx_unlock(&sc->reg_lock);
7275
7276         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
7277         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
7278
7279         rc = sbuf_finish(sb);
7280         sbuf_delete(sb);
7281
7282         return (rc);
7283 }
7284
7285 static int
7286 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
7287 {
7288         struct adapter *sc = arg1;
7289         struct sbuf *sb;
7290         int rc;
7291         struct tp_tcp_stats v4, v6;
7292
7293         rc = sysctl_wire_old_buffer(req, 0);
7294         if (rc != 0)
7295                 return (rc);
7296
7297         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7298         if (sb == NULL)
7299                 return (ENOMEM);
7300
7301         mtx_lock(&sc->reg_lock);
7302         t4_tp_get_tcp_stats(sc, &v4, &v6);
7303         mtx_unlock(&sc->reg_lock);
7304
7305         sbuf_printf(sb,
7306             "                                IP                 IPv6\n");
7307         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
7308             v4.tcp_out_rsts, v6.tcp_out_rsts);
7309         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
7310             v4.tcp_in_segs, v6.tcp_in_segs);
7311         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
7312             v4.tcp_out_segs, v6.tcp_out_segs);
7313         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
7314             v4.tcp_retrans_segs, v6.tcp_retrans_segs);
7315
7316         rc = sbuf_finish(sb);
7317         sbuf_delete(sb);
7318
7319         return (rc);
7320 }
7321
7322 static int
7323 sysctl_tids(SYSCTL_HANDLER_ARGS)
7324 {
7325         struct adapter *sc = arg1;
7326         struct sbuf *sb;
7327         int rc;
7328         struct tid_info *t = &sc->tids;
7329
7330         rc = sysctl_wire_old_buffer(req, 0);
7331         if (rc != 0)
7332                 return (rc);
7333
7334         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7335         if (sb == NULL)
7336                 return (ENOMEM);
7337
7338         if (t->natids) {
7339                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
7340                     t->atids_in_use);
7341         }
7342
7343         if (t->ntids) {
7344                 sbuf_printf(sb, "TID range: ");
7345                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
7346                         uint32_t b, hb;
7347
7348                         if (chip_id(sc) <= CHELSIO_T5) {
7349                                 b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
7350                                 hb = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
7351                         } else {
7352                                 b = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX);
7353                                 hb = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE);
7354                         }
7355
7356                         if (b)
7357                                 sbuf_printf(sb, "0-%u, ", b - 1);
7358                         sbuf_printf(sb, "%u-%u", hb, t->ntids - 1);
7359                 } else
7360                         sbuf_printf(sb, "0-%u", t->ntids - 1);
7361                 sbuf_printf(sb, ", in use: %u\n",
7362                     atomic_load_acq_int(&t->tids_in_use));
7363         }
7364
7365         if (t->nstids) {
7366                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
7367                     t->stid_base + t->nstids - 1, t->stids_in_use);
7368         }
7369
7370         if (t->nftids) {
7371                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
7372                     t->ftid_base + t->nftids - 1);
7373         }
7374
7375         if (t->netids) {
7376                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
7377                     t->etid_base + t->netids - 1);
7378         }
7379
7380         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
7381             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
7382             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
7383
7384         rc = sbuf_finish(sb);
7385         sbuf_delete(sb);
7386
7387         return (rc);
7388 }
7389
7390 static int
7391 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
7392 {
7393         struct adapter *sc = arg1;
7394         struct sbuf *sb;
7395         int rc;
7396         struct tp_err_stats stats;
7397
7398         rc = sysctl_wire_old_buffer(req, 0);
7399         if (rc != 0)
7400                 return (rc);
7401
7402         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7403         if (sb == NULL)
7404                 return (ENOMEM);
7405
7406         mtx_lock(&sc->reg_lock);
7407         t4_tp_get_err_stats(sc, &stats);
7408         mtx_unlock(&sc->reg_lock);
7409
7410         if (sc->chip_params->nchan > 2) {
7411                 sbuf_printf(sb, "                 channel 0  channel 1"
7412                     "  channel 2  channel 3\n");
7413                 sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
7414                     stats.mac_in_errs[0], stats.mac_in_errs[1],
7415                     stats.mac_in_errs[2], stats.mac_in_errs[3]);
7416                 sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
7417                     stats.hdr_in_errs[0], stats.hdr_in_errs[1],
7418                     stats.hdr_in_errs[2], stats.hdr_in_errs[3]);
7419                 sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
7420                     stats.tcp_in_errs[0], stats.tcp_in_errs[1],
7421                     stats.tcp_in_errs[2], stats.tcp_in_errs[3]);
7422                 sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
7423                     stats.tcp6_in_errs[0], stats.tcp6_in_errs[1],
7424                     stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]);
7425                 sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
7426                     stats.tnl_cong_drops[0], stats.tnl_cong_drops[1],
7427                     stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]);
7428                 sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
7429                     stats.tnl_tx_drops[0], stats.tnl_tx_drops[1],
7430                     stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]);
7431                 sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
7432                     stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1],
7433                     stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]);
7434                 sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
7435                     stats.ofld_chan_drops[0], stats.ofld_chan_drops[1],
7436                     stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]);
7437         } else {
7438                 sbuf_printf(sb, "                 channel 0  channel 1\n");
7439                 sbuf_printf(sb, "macInErrs:      %10u %10u\n",
7440                     stats.mac_in_errs[0], stats.mac_in_errs[1]);
7441                 sbuf_printf(sb, "hdrInErrs:      %10u %10u\n",
7442                     stats.hdr_in_errs[0], stats.hdr_in_errs[1]);
7443                 sbuf_printf(sb, "tcpInErrs:      %10u %10u\n",
7444                     stats.tcp_in_errs[0], stats.tcp_in_errs[1]);
7445                 sbuf_printf(sb, "tcp6InErrs:     %10u %10u\n",
7446                     stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]);
7447                 sbuf_printf(sb, "tnlCongDrops:   %10u %10u\n",
7448                     stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]);
7449                 sbuf_printf(sb, "tnlTxDrops:     %10u %10u\n",
7450                     stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]);
7451                 sbuf_printf(sb, "ofldVlanDrops:  %10u %10u\n",
7452                     stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]);
7453                 sbuf_printf(sb, "ofldChanDrops:  %10u %10u\n\n",
7454                     stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]);
7455         }
7456
7457         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
7458             stats.ofld_no_neigh, stats.ofld_cong_defer);
7459
7460         rc = sbuf_finish(sb);
7461         sbuf_delete(sb);
7462
7463         return (rc);
7464 }
7465
7466 static int
7467 sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS)
7468 {
7469         struct adapter *sc = arg1;
7470         struct tp_params *tpp = &sc->params.tp;
7471         u_int mask;
7472         int rc;
7473
7474         mask = tpp->la_mask >> 16;
7475         rc = sysctl_handle_int(oidp, &mask, 0, req);
7476         if (rc != 0 || req->newptr == NULL)
7477                 return (rc);
7478         if (mask > 0xffff)
7479                 return (EINVAL);
7480         tpp->la_mask = mask << 16;
7481         t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask);
7482
7483         return (0);
7484 }
7485
7486 struct field_desc {
7487         const char *name;
7488         u_int start;
7489         u_int width;
7490 };
7491
7492 static void
7493 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
7494 {
7495         char buf[32];
7496         int line_size = 0;
7497
7498         while (f->name) {
7499                 uint64_t mask = (1ULL << f->width) - 1;
7500                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
7501                     ((uintmax_t)v >> f->start) & mask);
7502
7503                 if (line_size + len >= 79) {
7504                         line_size = 8;
7505                         sbuf_printf(sb, "\n        ");
7506                 }
7507                 sbuf_printf(sb, "%s ", buf);
7508                 line_size += len + 1;
7509                 f++;
7510         }
7511         sbuf_printf(sb, "\n");
7512 }
7513
7514 static const struct field_desc tp_la0[] = {
7515         { "RcfOpCodeOut", 60, 4 },
7516         { "State", 56, 4 },
7517         { "WcfState", 52, 4 },
7518         { "RcfOpcSrcOut", 50, 2 },
7519         { "CRxError", 49, 1 },
7520         { "ERxError", 48, 1 },
7521         { "SanityFailed", 47, 1 },
7522         { "SpuriousMsg", 46, 1 },
7523         { "FlushInputMsg", 45, 1 },
7524         { "FlushInputCpl", 44, 1 },
7525         { "RssUpBit", 43, 1 },
7526         { "RssFilterHit", 42, 1 },
7527         { "Tid", 32, 10 },
7528         { "InitTcb", 31, 1 },
7529         { "LineNumber", 24, 7 },
7530         { "Emsg", 23, 1 },
7531         { "EdataOut", 22, 1 },
7532         { "Cmsg", 21, 1 },
7533         { "CdataOut", 20, 1 },
7534         { "EreadPdu", 19, 1 },
7535         { "CreadPdu", 18, 1 },
7536         { "TunnelPkt", 17, 1 },
7537         { "RcfPeerFin", 16, 1 },
7538         { "RcfReasonOut", 12, 4 },
7539         { "TxCchannel", 10, 2 },
7540         { "RcfTxChannel", 8, 2 },
7541         { "RxEchannel", 6, 2 },
7542         { "RcfRxChannel", 5, 1 },
7543         { "RcfDataOutSrdy", 4, 1 },
7544         { "RxDvld", 3, 1 },
7545         { "RxOoDvld", 2, 1 },
7546         { "RxCongestion", 1, 1 },
7547         { "TxCongestion", 0, 1 },
7548         { NULL }
7549 };
7550
7551 static const struct field_desc tp_la1[] = {
7552         { "CplCmdIn", 56, 8 },
7553         { "CplCmdOut", 48, 8 },
7554         { "ESynOut", 47, 1 },
7555         { "EAckOut", 46, 1 },
7556         { "EFinOut", 45, 1 },
7557         { "ERstOut", 44, 1 },
7558         { "SynIn", 43, 1 },
7559         { "AckIn", 42, 1 },
7560         { "FinIn", 41, 1 },
7561         { "RstIn", 40, 1 },
7562         { "DataIn", 39, 1 },
7563         { "DataInVld", 38, 1 },
7564         { "PadIn", 37, 1 },
7565         { "RxBufEmpty", 36, 1 },
7566         { "RxDdp", 35, 1 },
7567         { "RxFbCongestion", 34, 1 },
7568         { "TxFbCongestion", 33, 1 },
7569         { "TxPktSumSrdy", 32, 1 },
7570         { "RcfUlpType", 28, 4 },
7571         { "Eread", 27, 1 },
7572         { "Ebypass", 26, 1 },
7573         { "Esave", 25, 1 },
7574         { "Static0", 24, 1 },
7575         { "Cread", 23, 1 },
7576         { "Cbypass", 22, 1 },
7577         { "Csave", 21, 1 },
7578         { "CPktOut", 20, 1 },
7579         { "RxPagePoolFull", 18, 2 },
7580         { "RxLpbkPkt", 17, 1 },
7581         { "TxLpbkPkt", 16, 1 },
7582         { "RxVfValid", 15, 1 },
7583         { "SynLearned", 14, 1 },
7584         { "SetDelEntry", 13, 1 },
7585         { "SetInvEntry", 12, 1 },
7586         { "CpcmdDvld", 11, 1 },
7587         { "CpcmdSave", 10, 1 },
7588         { "RxPstructsFull", 8, 2 },
7589         { "EpcmdDvld", 7, 1 },
7590         { "EpcmdFlush", 6, 1 },
7591         { "EpcmdTrimPrefix", 5, 1 },
7592         { "EpcmdTrimPostfix", 4, 1 },
7593         { "ERssIp4Pkt", 3, 1 },
7594         { "ERssIp6Pkt", 2, 1 },
7595         { "ERssTcpUdpPkt", 1, 1 },
7596         { "ERssFceFipPkt", 0, 1 },
7597         { NULL }
7598 };
7599
7600 static const struct field_desc tp_la2[] = {
7601         { "CplCmdIn", 56, 8 },
7602         { "MpsVfVld", 55, 1 },
7603         { "MpsPf", 52, 3 },
7604         { "MpsVf", 44, 8 },
7605         { "SynIn", 43, 1 },
7606         { "AckIn", 42, 1 },
7607         { "FinIn", 41, 1 },
7608         { "RstIn", 40, 1 },
7609         { "DataIn", 39, 1 },
7610         { "DataInVld", 38, 1 },
7611         { "PadIn", 37, 1 },
7612         { "RxBufEmpty", 36, 1 },
7613         { "RxDdp", 35, 1 },
7614         { "RxFbCongestion", 34, 1 },
7615         { "TxFbCongestion", 33, 1 },
7616         { "TxPktSumSrdy", 32, 1 },
7617         { "RcfUlpType", 28, 4 },
7618         { "Eread", 27, 1 },
7619         { "Ebypass", 26, 1 },
7620         { "Esave", 25, 1 },
7621         { "Static0", 24, 1 },
7622         { "Cread", 23, 1 },
7623         { "Cbypass", 22, 1 },
7624         { "Csave", 21, 1 },
7625         { "CPktOut", 20, 1 },
7626         { "RxPagePoolFull", 18, 2 },
7627         { "RxLpbkPkt", 17, 1 },
7628         { "TxLpbkPkt", 16, 1 },
7629         { "RxVfValid", 15, 1 },
7630         { "SynLearned", 14, 1 },
7631         { "SetDelEntry", 13, 1 },
7632         { "SetInvEntry", 12, 1 },
7633         { "CpcmdDvld", 11, 1 },
7634         { "CpcmdSave", 10, 1 },
7635         { "RxPstructsFull", 8, 2 },
7636         { "EpcmdDvld", 7, 1 },
7637         { "EpcmdFlush", 6, 1 },
7638         { "EpcmdTrimPrefix", 5, 1 },
7639         { "EpcmdTrimPostfix", 4, 1 },
7640         { "ERssIp4Pkt", 3, 1 },
7641         { "ERssIp6Pkt", 2, 1 },
7642         { "ERssTcpUdpPkt", 1, 1 },
7643         { "ERssFceFipPkt", 0, 1 },
7644         { NULL }
7645 };
7646
7647 static void
7648 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
7649 {
7650
7651         field_desc_show(sb, *p, tp_la0);
7652 }
7653
7654 static void
7655 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
7656 {
7657
7658         if (idx)
7659                 sbuf_printf(sb, "\n");
7660         field_desc_show(sb, p[0], tp_la0);
7661         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7662                 field_desc_show(sb, p[1], tp_la0);
7663 }
7664
7665 static void
7666 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
7667 {
7668
7669         if (idx)
7670                 sbuf_printf(sb, "\n");
7671         field_desc_show(sb, p[0], tp_la0);
7672         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
7673                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
7674 }
7675
7676 static int
7677 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
7678 {
7679         struct adapter *sc = arg1;
7680         struct sbuf *sb;
7681         uint64_t *buf, *p;
7682         int rc;
7683         u_int i, inc;
7684         void (*show_func)(struct sbuf *, uint64_t *, int);
7685
7686         rc = sysctl_wire_old_buffer(req, 0);
7687         if (rc != 0)
7688                 return (rc);
7689
7690         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7691         if (sb == NULL)
7692                 return (ENOMEM);
7693
7694         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
7695
7696         t4_tp_read_la(sc, buf, NULL);
7697         p = buf;
7698
7699         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
7700         case 2:
7701                 inc = 2;
7702                 show_func = tp_la_show2;
7703                 break;
7704         case 3:
7705                 inc = 2;
7706                 show_func = tp_la_show3;
7707                 break;
7708         default:
7709                 inc = 1;
7710                 show_func = tp_la_show;
7711         }
7712
7713         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
7714                 (*show_func)(sb, p, i);
7715
7716         rc = sbuf_finish(sb);
7717         sbuf_delete(sb);
7718         free(buf, M_CXGBE);
7719         return (rc);
7720 }
7721
7722 static int
7723 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
7724 {
7725         struct adapter *sc = arg1;
7726         struct sbuf *sb;
7727         int rc;
7728         u64 nrate[MAX_NCHAN], orate[MAX_NCHAN];
7729
7730         rc = sysctl_wire_old_buffer(req, 0);
7731         if (rc != 0)
7732                 return (rc);
7733
7734         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7735         if (sb == NULL)
7736                 return (ENOMEM);
7737
7738         t4_get_chan_txrate(sc, nrate, orate);
7739
7740         if (sc->chip_params->nchan > 2) {
7741                 sbuf_printf(sb, "              channel 0   channel 1"
7742                     "   channel 2   channel 3\n");
7743                 sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
7744                     nrate[0], nrate[1], nrate[2], nrate[3]);
7745                 sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
7746                     orate[0], orate[1], orate[2], orate[3]);
7747         } else {
7748                 sbuf_printf(sb, "              channel 0   channel 1\n");
7749                 sbuf_printf(sb, "NIC B/s:     %10ju  %10ju\n",
7750                     nrate[0], nrate[1]);
7751                 sbuf_printf(sb, "Offload B/s: %10ju  %10ju",
7752                     orate[0], orate[1]);
7753         }
7754
7755         rc = sbuf_finish(sb);
7756         sbuf_delete(sb);
7757
7758         return (rc);
7759 }
7760
7761 static int
7762 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
7763 {
7764         struct adapter *sc = arg1;
7765         struct sbuf *sb;
7766         uint32_t *buf, *p;
7767         int rc, i;
7768
7769         rc = sysctl_wire_old_buffer(req, 0);
7770         if (rc != 0)
7771                 return (rc);
7772
7773         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7774         if (sb == NULL)
7775                 return (ENOMEM);
7776
7777         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
7778             M_ZERO | M_WAITOK);
7779
7780         t4_ulprx_read_la(sc, buf);
7781         p = buf;
7782
7783         sbuf_printf(sb, "      Pcmd        Type   Message"
7784             "                Data");
7785         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
7786                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
7787                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
7788         }
7789
7790         rc = sbuf_finish(sb);
7791         sbuf_delete(sb);
7792         free(buf, M_CXGBE);
7793         return (rc);
7794 }
7795
7796 static int
7797 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
7798 {
7799         struct adapter *sc = arg1;
7800         struct sbuf *sb;
7801         int rc, v;
7802
7803         MPASS(chip_id(sc) >= CHELSIO_T5);
7804
7805         rc = sysctl_wire_old_buffer(req, 0);
7806         if (rc != 0)
7807                 return (rc);
7808
7809         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7810         if (sb == NULL)
7811                 return (ENOMEM);
7812
7813         v = t4_read_reg(sc, A_SGE_STAT_CFG);
7814         if (G_STATSOURCE_T5(v) == 7) {
7815                 int mode;
7816
7817                 mode = is_t5(sc) ? G_STATMODE(v) : G_T6_STATMODE(v);
7818                 if (mode == 0) {
7819                         sbuf_printf(sb, "total %d, incomplete %d",
7820                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7821                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7822                 } else if (mode == 1) {
7823                         sbuf_printf(sb, "total %d, data overflow %d",
7824                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
7825                             t4_read_reg(sc, A_SGE_STAT_MATCH));
7826                 } else {
7827                         sbuf_printf(sb, "unknown mode %d", mode);
7828                 }
7829         }
7830         rc = sbuf_finish(sb);
7831         sbuf_delete(sb);
7832
7833         return (rc);
7834 }
7835
7836 static int
7837 sysctl_tc_params(SYSCTL_HANDLER_ARGS)
7838 {
7839         struct adapter *sc = arg1;
7840         struct tx_sched_class *tc;
7841         struct t4_sched_class_params p;
7842         struct sbuf *sb;
7843         int i, rc, port_id, flags, mbps, gbps;
7844
7845         rc = sysctl_wire_old_buffer(req, 0);
7846         if (rc != 0)
7847                 return (rc);
7848
7849         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7850         if (sb == NULL)
7851                 return (ENOMEM);
7852
7853         port_id = arg2 >> 16;
7854         MPASS(port_id < sc->params.nports);
7855         MPASS(sc->port[port_id] != NULL);
7856         i = arg2 & 0xffff;
7857         MPASS(i < sc->chip_params->nsched_cls);
7858         tc = &sc->port[port_id]->tc[i];
7859
7860         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7861             "t4tc_p");
7862         if (rc)
7863                 goto done;
7864         flags = tc->flags;
7865         p = tc->params;
7866         end_synchronized_op(sc, LOCK_HELD);
7867
7868         if ((flags & TX_SC_OK) == 0) {
7869                 sbuf_printf(sb, "none");
7870                 goto done;
7871         }
7872
7873         if (p.level == SCHED_CLASS_LEVEL_CL_WRR) {
7874                 sbuf_printf(sb, "cl-wrr weight %u", p.weight);
7875                 goto done;
7876         } else if (p.level == SCHED_CLASS_LEVEL_CL_RL)
7877                 sbuf_printf(sb, "cl-rl");
7878         else if (p.level == SCHED_CLASS_LEVEL_CH_RL)
7879                 sbuf_printf(sb, "ch-rl");
7880         else {
7881                 rc = ENXIO;
7882                 goto done;
7883         }
7884
7885         if (p.ratemode == SCHED_CLASS_RATEMODE_REL) {
7886                 /* XXX: top speed or actual link speed? */
7887                 gbps = port_top_speed(sc->port[port_id]);
7888                 sbuf_printf(sb, " %u%% of %uGbps", p.maxrate, gbps);
7889         }
7890         else if (p.ratemode == SCHED_CLASS_RATEMODE_ABS) {
7891                 switch (p.rateunit) {
7892                 case SCHED_CLASS_RATEUNIT_BITS:
7893                         mbps = p.maxrate / 1000;
7894                         gbps = p.maxrate / 1000000;
7895                         if (p.maxrate == gbps * 1000000)
7896                                 sbuf_printf(sb, " %uGbps", gbps);
7897                         else if (p.maxrate == mbps * 1000)
7898                                 sbuf_printf(sb, " %uMbps", mbps);
7899                         else
7900                                 sbuf_printf(sb, " %uKbps", p.maxrate);
7901                         break;
7902                 case SCHED_CLASS_RATEUNIT_PKTS:
7903                         sbuf_printf(sb, " %upps", p.maxrate);
7904                         break;
7905                 default:
7906                         rc = ENXIO;
7907                         goto done;
7908                 }
7909         }
7910
7911         switch (p.mode) {
7912         case SCHED_CLASS_MODE_CLASS:
7913                 sbuf_printf(sb, " aggregate");
7914                 break;
7915         case SCHED_CLASS_MODE_FLOW:
7916                 sbuf_printf(sb, " per-flow");
7917                 break;
7918         default:
7919                 rc = ENXIO;
7920                 goto done;
7921         }
7922
7923 done:
7924         if (rc == 0)
7925                 rc = sbuf_finish(sb);
7926         sbuf_delete(sb);
7927
7928         return (rc);
7929 }
7930 #endif
7931
7932 #ifdef TCP_OFFLOAD
7933 static void
7934 unit_conv(char *buf, size_t len, u_int val, u_int factor)
7935 {
7936         u_int rem = val % factor;
7937
7938         if (rem == 0)
7939                 snprintf(buf, len, "%u", val / factor);
7940         else {
7941                 while (rem % 10 == 0)
7942                         rem /= 10;
7943                 snprintf(buf, len, "%u.%u", val / factor, rem);
7944         }
7945 }
7946
7947 static int
7948 sysctl_tp_tick(SYSCTL_HANDLER_ARGS)
7949 {
7950         struct adapter *sc = arg1;
7951         char buf[16];
7952         u_int res, re;
7953         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
7954
7955         res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
7956         switch (arg2) {
7957         case 0:
7958                 /* timer_tick */
7959                 re = G_TIMERRESOLUTION(res);
7960                 break;
7961         case 1:
7962                 /* TCP timestamp tick */
7963                 re = G_TIMESTAMPRESOLUTION(res);
7964                 break;
7965         case 2:
7966                 /* DACK tick */
7967                 re = G_DELAYEDACKRESOLUTION(res);
7968                 break;
7969         default:
7970                 return (EDOOFUS);
7971         }
7972
7973         unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000);
7974
7975         return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
7976 }
7977
7978 static int
7979 sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS)
7980 {
7981         struct adapter *sc = arg1;
7982         u_int res, dack_re, v;
7983         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
7984
7985         res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
7986         dack_re = G_DELAYEDACKRESOLUTION(res);
7987         v = ((cclk_ps << dack_re) / 1000000) * t4_read_reg(sc, A_TP_DACK_TIMER);
7988
7989         return (sysctl_handle_int(oidp, &v, 0, req));
7990 }
7991
7992 static int
7993 sysctl_tp_timer(SYSCTL_HANDLER_ARGS)
7994 {
7995         struct adapter *sc = arg1;
7996         int reg = arg2;
7997         u_int tre;
7998         u_long tp_tick_us, v;
7999         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
8000
8001         MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX ||
8002             reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX ||
8003             reg == A_TP_KEEP_IDLE || A_TP_KEEP_INTVL || reg == A_TP_INIT_SRTT ||
8004             reg == A_TP_FINWAIT2_TIMER);
8005
8006         tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION));
8007         tp_tick_us = (cclk_ps << tre) / 1000000;
8008
8009         if (reg == A_TP_INIT_SRTT)
8010                 v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg));
8011         else
8012                 v = tp_tick_us * t4_read_reg(sc, reg);
8013
8014         return (sysctl_handle_long(oidp, &v, 0, req));
8015 }
8016 #endif
8017
8018 static uint32_t
8019 fconf_iconf_to_mode(uint32_t fconf, uint32_t iconf)
8020 {
8021         uint32_t mode;
8022
8023         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
8024             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
8025
8026         if (fconf & F_FRAGMENTATION)
8027                 mode |= T4_FILTER_IP_FRAGMENT;
8028
8029         if (fconf & F_MPSHITTYPE)
8030                 mode |= T4_FILTER_MPS_HIT_TYPE;
8031
8032         if (fconf & F_MACMATCH)
8033                 mode |= T4_FILTER_MAC_IDX;
8034
8035         if (fconf & F_ETHERTYPE)
8036                 mode |= T4_FILTER_ETH_TYPE;
8037
8038         if (fconf & F_PROTOCOL)
8039                 mode |= T4_FILTER_IP_PROTO;
8040
8041         if (fconf & F_TOS)
8042                 mode |= T4_FILTER_IP_TOS;
8043
8044         if (fconf & F_VLAN)
8045                 mode |= T4_FILTER_VLAN;
8046
8047         if (fconf & F_VNIC_ID) {
8048                 mode |= T4_FILTER_VNIC;
8049                 if (iconf & F_VNIC)
8050                         mode |= T4_FILTER_IC_VNIC;
8051         }
8052
8053         if (fconf & F_PORT)
8054                 mode |= T4_FILTER_PORT;
8055
8056         if (fconf & F_FCOE)
8057                 mode |= T4_FILTER_FCoE;
8058
8059         return (mode);
8060 }
8061
8062 static uint32_t
8063 mode_to_fconf(uint32_t mode)
8064 {
8065         uint32_t fconf = 0;
8066
8067         if (mode & T4_FILTER_IP_FRAGMENT)
8068                 fconf |= F_FRAGMENTATION;
8069
8070         if (mode & T4_FILTER_MPS_HIT_TYPE)
8071                 fconf |= F_MPSHITTYPE;
8072
8073         if (mode & T4_FILTER_MAC_IDX)
8074                 fconf |= F_MACMATCH;
8075
8076         if (mode & T4_FILTER_ETH_TYPE)
8077                 fconf |= F_ETHERTYPE;
8078
8079         if (mode & T4_FILTER_IP_PROTO)
8080                 fconf |= F_PROTOCOL;
8081
8082         if (mode & T4_FILTER_IP_TOS)
8083                 fconf |= F_TOS;
8084
8085         if (mode & T4_FILTER_VLAN)
8086                 fconf |= F_VLAN;
8087
8088         if (mode & T4_FILTER_VNIC)
8089                 fconf |= F_VNIC_ID;
8090
8091         if (mode & T4_FILTER_PORT)
8092                 fconf |= F_PORT;
8093
8094         if (mode & T4_FILTER_FCoE)
8095                 fconf |= F_FCOE;
8096
8097         return (fconf);
8098 }
8099
8100 static uint32_t
8101 mode_to_iconf(uint32_t mode)
8102 {
8103
8104         if (mode & T4_FILTER_IC_VNIC)
8105                 return (F_VNIC);
8106         return (0);
8107 }
8108
8109 static int check_fspec_against_fconf_iconf(struct adapter *sc,
8110     struct t4_filter_specification *fs)
8111 {
8112         struct tp_params *tpp = &sc->params.tp;
8113         uint32_t fconf = 0;
8114
8115         if (fs->val.frag || fs->mask.frag)
8116                 fconf |= F_FRAGMENTATION;
8117
8118         if (fs->val.matchtype || fs->mask.matchtype)
8119                 fconf |= F_MPSHITTYPE;
8120
8121         if (fs->val.macidx || fs->mask.macidx)
8122                 fconf |= F_MACMATCH;
8123
8124         if (fs->val.ethtype || fs->mask.ethtype)
8125                 fconf |= F_ETHERTYPE;
8126
8127         if (fs->val.proto || fs->mask.proto)
8128                 fconf |= F_PROTOCOL;
8129
8130         if (fs->val.tos || fs->mask.tos)
8131                 fconf |= F_TOS;
8132
8133         if (fs->val.vlan_vld || fs->mask.vlan_vld)
8134                 fconf |= F_VLAN;
8135
8136         if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
8137                 fconf |= F_VNIC_ID;
8138                 if (tpp->ingress_config & F_VNIC)
8139                         return (EINVAL);
8140         }
8141
8142         if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
8143                 fconf |= F_VNIC_ID;
8144                 if ((tpp->ingress_config & F_VNIC) == 0)
8145                         return (EINVAL);
8146         }
8147
8148         if (fs->val.iport || fs->mask.iport)
8149                 fconf |= F_PORT;
8150
8151         if (fs->val.fcoe || fs->mask.fcoe)
8152                 fconf |= F_FCOE;
8153
8154         if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map)
8155                 return (E2BIG);
8156
8157         return (0);
8158 }
8159
8160 static int
8161 get_filter_mode(struct adapter *sc, uint32_t *mode)
8162 {
8163         struct tp_params *tpp = &sc->params.tp;
8164
8165         /*
8166          * We trust the cached values of the relevant TP registers.  This means
8167          * things work reliably only if writes to those registers are always via
8168          * t4_set_filter_mode.
8169          */
8170         *mode = fconf_iconf_to_mode(tpp->vlan_pri_map, tpp->ingress_config);
8171
8172         return (0);
8173 }
8174
8175 static int
8176 set_filter_mode(struct adapter *sc, uint32_t mode)
8177 {
8178         struct tp_params *tpp = &sc->params.tp;
8179         uint32_t fconf, iconf;
8180         int rc;
8181
8182         iconf = mode_to_iconf(mode);
8183         if ((iconf ^ tpp->ingress_config) & F_VNIC) {
8184                 /*
8185                  * For now we just complain if A_TP_INGRESS_CONFIG is not
8186                  * already set to the correct value for the requested filter
8187                  * mode.  It's not clear if it's safe to write to this register
8188                  * on the fly.  (And we trust the cached value of the register).
8189                  */
8190                 return (EBUSY);
8191         }
8192
8193         fconf = mode_to_fconf(mode);
8194
8195         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
8196             "t4setfm");
8197         if (rc)
8198                 return (rc);
8199
8200         if (sc->tids.ftids_in_use > 0) {
8201                 rc = EBUSY;
8202                 goto done;
8203         }
8204
8205 #ifdef TCP_OFFLOAD
8206         if (uld_active(sc, ULD_TOM)) {
8207                 rc = EBUSY;
8208                 goto done;
8209         }
8210 #endif
8211
8212         rc = -t4_set_filter_mode(sc, fconf);
8213 done:
8214         end_synchronized_op(sc, LOCK_HELD);
8215         return (rc);
8216 }
8217
8218 static inline uint64_t
8219 get_filter_hits(struct adapter *sc, uint32_t fid)
8220 {
8221         uint32_t tcb_addr;
8222
8223         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) +
8224             (fid + sc->tids.ftid_base) * TCB_SIZE;
8225
8226         if (is_t4(sc)) {
8227                 uint64_t hits;
8228
8229                 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8);
8230                 return (be64toh(hits));
8231         } else {
8232                 uint32_t hits;
8233
8234                 read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4);
8235                 return (be32toh(hits));
8236         }
8237 }
8238
8239 static int
8240 get_filter(struct adapter *sc, struct t4_filter *t)
8241 {
8242         int i, rc, nfilters = sc->tids.nftids;
8243         struct filter_entry *f;
8244
8245         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
8246             "t4getf");
8247         if (rc)
8248                 return (rc);
8249
8250         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
8251             t->idx >= nfilters) {
8252                 t->idx = 0xffffffff;
8253                 goto done;
8254         }
8255
8256         f = &sc->tids.ftid_tab[t->idx];
8257         for (i = t->idx; i < nfilters; i++, f++) {
8258                 if (f->valid) {
8259                         t->idx = i;
8260                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
8261                         t->smtidx = f->smtidx;
8262                         if (f->fs.hitcnts)
8263                                 t->hits = get_filter_hits(sc, t->idx);
8264                         else
8265                                 t->hits = UINT64_MAX;
8266                         t->fs = f->fs;
8267
8268                         goto done;
8269                 }
8270         }
8271
8272         t->idx = 0xffffffff;
8273 done:
8274         end_synchronized_op(sc, LOCK_HELD);
8275         return (0);
8276 }
8277
8278 static int
8279 set_filter(struct adapter *sc, struct t4_filter *t)
8280 {
8281         unsigned int nfilters, nports;
8282         struct filter_entry *f;
8283         int i, rc;
8284
8285         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
8286         if (rc)
8287                 return (rc);
8288
8289         nfilters = sc->tids.nftids;
8290         nports = sc->params.nports;
8291
8292         if (nfilters == 0) {
8293                 rc = ENOTSUP;
8294                 goto done;
8295         }
8296
8297         if (t->idx >= nfilters) {
8298                 rc = EINVAL;
8299                 goto done;
8300         }
8301
8302         /* Validate against the global filter mode and ingress config */
8303         rc = check_fspec_against_fconf_iconf(sc, &t->fs);
8304         if (rc != 0)
8305                 goto done;
8306
8307         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
8308                 rc = EINVAL;
8309                 goto done;
8310         }
8311
8312         if (t->fs.val.iport >= nports) {
8313                 rc = EINVAL;
8314                 goto done;
8315         }
8316
8317         /* Can't specify an iq if not steering to it */
8318         if (!t->fs.dirsteer && t->fs.iq) {
8319                 rc = EINVAL;
8320                 goto done;
8321         }
8322
8323         /* IPv6 filter idx must be 4 aligned */
8324         if (t->fs.type == 1 &&
8325             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
8326                 rc = EINVAL;
8327                 goto done;
8328         }
8329
8330         if (!(sc->flags & FULL_INIT_DONE) &&
8331             ((rc = adapter_full_init(sc)) != 0))
8332                 goto done;
8333
8334         if (sc->tids.ftid_tab == NULL) {
8335                 KASSERT(sc->tids.ftids_in_use == 0,
8336                     ("%s: no memory allocated but filters_in_use > 0",
8337                     __func__));
8338
8339                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
8340                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
8341                 if (sc->tids.ftid_tab == NULL) {
8342                         rc = ENOMEM;
8343                         goto done;
8344                 }
8345                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
8346         }
8347
8348         for (i = 0; i < 4; i++) {
8349                 f = &sc->tids.ftid_tab[t->idx + i];
8350
8351                 if (f->pending || f->valid) {
8352                         rc = EBUSY;
8353                         goto done;
8354                 }
8355                 if (f->locked) {
8356                         rc = EPERM;
8357                         goto done;
8358                 }
8359
8360                 if (t->fs.type == 0)
8361                         break;
8362         }
8363
8364         f = &sc->tids.ftid_tab[t->idx];
8365         f->fs = t->fs;
8366
8367         rc = set_filter_wr(sc, t->idx);
8368 done:
8369         end_synchronized_op(sc, 0);
8370
8371         if (rc == 0) {
8372                 mtx_lock(&sc->tids.ftid_lock);
8373                 for (;;) {
8374                         if (f->pending == 0) {
8375                                 rc = f->valid ? 0 : EIO;
8376                                 break;
8377                         }
8378
8379                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
8380                             PCATCH, "t4setfw", 0)) {
8381                                 rc = EINPROGRESS;
8382                                 break;
8383                         }
8384                 }
8385                 mtx_unlock(&sc->tids.ftid_lock);
8386         }
8387         return (rc);
8388 }
8389
8390 static int
8391 del_filter(struct adapter *sc, struct t4_filter *t)
8392 {
8393         unsigned int nfilters;
8394         struct filter_entry *f;
8395         int rc;
8396
8397         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
8398         if (rc)
8399                 return (rc);
8400
8401         nfilters = sc->tids.nftids;
8402
8403         if (nfilters == 0) {
8404                 rc = ENOTSUP;
8405                 goto done;
8406         }
8407
8408         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
8409             t->idx >= nfilters) {
8410                 rc = EINVAL;
8411                 goto done;
8412         }
8413
8414         if (!(sc->flags & FULL_INIT_DONE)) {
8415                 rc = EAGAIN;
8416                 goto done;
8417         }
8418
8419         f = &sc->tids.ftid_tab[t->idx];
8420
8421         if (f->pending) {
8422                 rc = EBUSY;
8423                 goto done;
8424         }
8425         if (f->locked) {
8426                 rc = EPERM;
8427                 goto done;
8428         }
8429
8430         if (f->valid) {
8431                 t->fs = f->fs;  /* extra info for the caller */
8432                 rc = del_filter_wr(sc, t->idx);
8433         }
8434
8435 done:
8436         end_synchronized_op(sc, 0);
8437
8438         if (rc == 0) {
8439                 mtx_lock(&sc->tids.ftid_lock);
8440                 for (;;) {
8441                         if (f->pending == 0) {
8442                                 rc = f->valid ? EIO : 0;
8443                                 break;
8444                         }
8445
8446                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
8447                             PCATCH, "t4delfw", 0)) {
8448                                 rc = EINPROGRESS;
8449                                 break;
8450                         }
8451                 }
8452                 mtx_unlock(&sc->tids.ftid_lock);
8453         }
8454
8455         return (rc);
8456 }
8457
8458 static void
8459 clear_filter(struct filter_entry *f)
8460 {
8461         if (f->l2t)
8462                 t4_l2t_release(f->l2t);
8463
8464         bzero(f, sizeof (*f));
8465 }
8466
8467 static int
8468 set_filter_wr(struct adapter *sc, int fidx)
8469 {
8470         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
8471         struct fw_filter_wr *fwr;
8472         unsigned int ftid, vnic_vld, vnic_vld_mask;
8473         struct wrq_cookie cookie;
8474
8475         ASSERT_SYNCHRONIZED_OP(sc);
8476
8477         if (f->fs.newdmac || f->fs.newvlan) {
8478                 /* This filter needs an L2T entry; allocate one. */
8479                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
8480                 if (f->l2t == NULL)
8481                         return (EAGAIN);
8482                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
8483                     f->fs.dmac)) {
8484                         t4_l2t_release(f->l2t);
8485                         f->l2t = NULL;
8486                         return (ENOMEM);
8487                 }
8488         }
8489
8490         /* Already validated against fconf, iconf */
8491         MPASS((f->fs.val.pfvf_vld & f->fs.val.ovlan_vld) == 0);
8492         MPASS((f->fs.mask.pfvf_vld & f->fs.mask.ovlan_vld) == 0);
8493         if (f->fs.val.pfvf_vld || f->fs.val.ovlan_vld)
8494                 vnic_vld = 1;
8495         else
8496                 vnic_vld = 0;
8497         if (f->fs.mask.pfvf_vld || f->fs.mask.ovlan_vld)
8498                 vnic_vld_mask = 1;
8499         else
8500                 vnic_vld_mask = 0;
8501
8502         ftid = sc->tids.ftid_base + fidx;
8503
8504         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8505         if (fwr == NULL)
8506                 return (ENOMEM);
8507         bzero(fwr, sizeof(*fwr));
8508
8509         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
8510         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
8511         fwr->tid_to_iq =
8512             htobe32(V_FW_FILTER_WR_TID(ftid) |
8513                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
8514                 V_FW_FILTER_WR_NOREPLY(0) |
8515                 V_FW_FILTER_WR_IQ(f->fs.iq));
8516         fwr->del_filter_to_l2tix =
8517             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
8518                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
8519                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
8520                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
8521                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
8522                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
8523                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
8524                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
8525                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
8526                     f->fs.newvlan == VLAN_REWRITE) |
8527                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
8528                     f->fs.newvlan == VLAN_REWRITE) |
8529                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
8530                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
8531                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
8532                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
8533         fwr->ethtype = htobe16(f->fs.val.ethtype);
8534         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
8535         fwr->frag_to_ovlan_vldm =
8536             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
8537                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
8538                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
8539                 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
8540                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
8541                 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
8542         fwr->smac_sel = 0;
8543         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
8544             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
8545         fwr->maci_to_matchtypem =
8546             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
8547                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
8548                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
8549                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
8550                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
8551                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
8552                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
8553                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
8554         fwr->ptcl = f->fs.val.proto;
8555         fwr->ptclm = f->fs.mask.proto;
8556         fwr->ttyp = f->fs.val.tos;
8557         fwr->ttypm = f->fs.mask.tos;
8558         fwr->ivlan = htobe16(f->fs.val.vlan);
8559         fwr->ivlanm = htobe16(f->fs.mask.vlan);
8560         fwr->ovlan = htobe16(f->fs.val.vnic);
8561         fwr->ovlanm = htobe16(f->fs.mask.vnic);
8562         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
8563         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
8564         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
8565         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
8566         fwr->lp = htobe16(f->fs.val.dport);
8567         fwr->lpm = htobe16(f->fs.mask.dport);
8568         fwr->fp = htobe16(f->fs.val.sport);
8569         fwr->fpm = htobe16(f->fs.mask.sport);
8570         if (f->fs.newsmac)
8571                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
8572
8573         f->pending = 1;
8574         sc->tids.ftids_in_use++;
8575
8576         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8577         return (0);
8578 }
8579
8580 static int
8581 del_filter_wr(struct adapter *sc, int fidx)
8582 {
8583         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
8584         struct fw_filter_wr *fwr;
8585         unsigned int ftid;
8586         struct wrq_cookie cookie;
8587
8588         ftid = sc->tids.ftid_base + fidx;
8589
8590         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
8591         if (fwr == NULL)
8592                 return (ENOMEM);
8593         bzero(fwr, sizeof (*fwr));
8594
8595         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
8596
8597         f->pending = 1;
8598         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
8599         return (0);
8600 }
8601
8602 int
8603 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8604 {
8605         struct adapter *sc = iq->adapter;
8606         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
8607         unsigned int idx = GET_TID(rpl);
8608         unsigned int rc;
8609         struct filter_entry *f;
8610
8611         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
8612             rss->opcode));
8613         MPASS(iq == &sc->sge.fwq);
8614         MPASS(is_ftid(sc, idx));
8615
8616         idx -= sc->tids.ftid_base;
8617         f = &sc->tids.ftid_tab[idx];
8618         rc = G_COOKIE(rpl->cookie);
8619
8620         mtx_lock(&sc->tids.ftid_lock);
8621         if (rc == FW_FILTER_WR_FLT_ADDED) {
8622                 KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
8623                     __func__, idx));
8624                 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
8625                 f->pending = 0;  /* asynchronous setup completed */
8626                 f->valid = 1;
8627         } else {
8628                 if (rc != FW_FILTER_WR_FLT_DELETED) {
8629                         /* Add or delete failed, display an error */
8630                         log(LOG_ERR,
8631                             "filter %u setup failed with error %u\n",
8632                             idx, rc);
8633                 }
8634
8635                 clear_filter(f);
8636                 sc->tids.ftids_in_use--;
8637         }
8638         wakeup(&sc->tids.ftid_tab);
8639         mtx_unlock(&sc->tids.ftid_lock);
8640
8641         return (0);
8642 }
8643
8644 static int
8645 set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8646 {
8647
8648         MPASS(iq->set_tcb_rpl != NULL);
8649         return (iq->set_tcb_rpl(iq, rss, m));
8650 }
8651
8652 static int
8653 l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
8654 {
8655
8656         MPASS(iq->l2t_write_rpl != NULL);
8657         return (iq->l2t_write_rpl(iq, rss, m));
8658 }
8659
8660 static int
8661 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
8662 {
8663         int rc;
8664
8665         if (cntxt->cid > M_CTXTQID)
8666                 return (EINVAL);
8667
8668         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
8669             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
8670                 return (EINVAL);
8671
8672         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
8673         if (rc)
8674                 return (rc);
8675
8676         if (sc->flags & FW_OK) {
8677                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
8678                     &cntxt->data[0]);
8679                 if (rc == 0)
8680                         goto done;
8681         }
8682
8683         /*
8684          * Read via firmware failed or wasn't even attempted.  Read directly via
8685          * the backdoor.
8686          */
8687         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
8688 done:
8689         end_synchronized_op(sc, 0);
8690         return (rc);
8691 }
8692
8693 static int
8694 load_fw(struct adapter *sc, struct t4_data *fw)
8695 {
8696         int rc;
8697         uint8_t *fw_data;
8698
8699         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
8700         if (rc)
8701                 return (rc);
8702
8703         if (sc->flags & FULL_INIT_DONE) {
8704                 rc = EBUSY;
8705                 goto done;
8706         }
8707
8708         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
8709         if (fw_data == NULL) {
8710                 rc = ENOMEM;
8711                 goto done;
8712         }
8713
8714         rc = copyin(fw->data, fw_data, fw->len);
8715         if (rc == 0)
8716                 rc = -t4_load_fw(sc, fw_data, fw->len);
8717
8718         free(fw_data, M_CXGBE);
8719 done:
8720         end_synchronized_op(sc, 0);
8721         return (rc);
8722 }
8723
8724 static int
8725 load_cfg(struct adapter *sc, struct t4_data *cfg)
8726 {
8727         int rc;
8728         uint8_t *cfg_data = NULL;
8729
8730         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf");
8731         if (rc)
8732                 return (rc);
8733
8734         if (cfg->len == 0) {
8735                 /* clear */
8736                 rc = -t4_load_cfg(sc, NULL, 0);
8737                 goto done;
8738         }
8739
8740         cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK);
8741         if (cfg_data == NULL) {
8742                 rc = ENOMEM;
8743                 goto done;
8744         }
8745
8746         rc = copyin(cfg->data, cfg_data, cfg->len);
8747         if (rc == 0)
8748                 rc = -t4_load_cfg(sc, cfg_data, cfg->len);
8749
8750         free(cfg_data, M_CXGBE);
8751 done:
8752         end_synchronized_op(sc, 0);
8753         return (rc);
8754 }
8755
8756 #define MAX_READ_BUF_SIZE (128 * 1024)
8757 static int
8758 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
8759 {
8760         uint32_t addr, remaining, n;
8761         uint32_t *buf;
8762         int rc;
8763         uint8_t *dst;
8764
8765         rc = validate_mem_range(sc, mr->addr, mr->len);
8766         if (rc != 0)
8767                 return (rc);
8768
8769         buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK);
8770         addr = mr->addr;
8771         remaining = mr->len;
8772         dst = (void *)mr->data;
8773
8774         while (remaining) {
8775                 n = min(remaining, MAX_READ_BUF_SIZE);
8776                 read_via_memwin(sc, 2, addr, buf, n);
8777
8778                 rc = copyout(buf, dst, n);
8779                 if (rc != 0)
8780                         break;
8781
8782                 dst += n;
8783                 remaining -= n;
8784                 addr += n;
8785         }
8786
8787         free(buf, M_CXGBE);
8788         return (rc);
8789 }
8790 #undef MAX_READ_BUF_SIZE
8791
8792 static int
8793 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
8794 {
8795         int rc;
8796
8797         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
8798                 return (EINVAL);
8799
8800         if (i2cd->len > sizeof(i2cd->data))
8801                 return (EFBIG);
8802
8803         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
8804         if (rc)
8805                 return (rc);
8806         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
8807             i2cd->offset, i2cd->len, &i2cd->data[0]);
8808         end_synchronized_op(sc, 0);
8809
8810         return (rc);
8811 }
8812
8813 static int
8814 in_range(int val, int lo, int hi)
8815 {
8816
8817         return (val < 0 || (val <= hi && val >= lo));
8818 }
8819
8820 static int
8821 set_sched_class_config(struct adapter *sc, int minmax)
8822 {
8823         int rc;
8824
8825         if (minmax < 0)
8826                 return (EINVAL);
8827
8828         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sscc");
8829         if (rc)
8830                 return (rc);
8831         rc = -t4_sched_config(sc, FW_SCHED_TYPE_PKTSCHED, minmax, 1);
8832         end_synchronized_op(sc, 0);
8833
8834         return (rc);
8835 }
8836
8837 static int
8838 set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p,
8839     int sleep_ok)
8840 {
8841         int rc, top_speed, fw_level, fw_mode, fw_rateunit, fw_ratemode;
8842         struct port_info *pi;
8843         struct tx_sched_class *tc;
8844
8845         if (p->level == SCHED_CLASS_LEVEL_CL_RL)
8846                 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
8847         else if (p->level == SCHED_CLASS_LEVEL_CL_WRR)
8848                 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
8849         else if (p->level == SCHED_CLASS_LEVEL_CH_RL)
8850                 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
8851         else
8852                 return (EINVAL);
8853
8854         if (p->mode == SCHED_CLASS_MODE_CLASS)
8855                 fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
8856         else if (p->mode == SCHED_CLASS_MODE_FLOW)
8857                 fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
8858         else
8859                 return (EINVAL);
8860
8861         if (p->rateunit == SCHED_CLASS_RATEUNIT_BITS)
8862                 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
8863         else if (p->rateunit == SCHED_CLASS_RATEUNIT_PKTS)
8864                 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
8865         else
8866                 return (EINVAL);
8867
8868         if (p->ratemode == SCHED_CLASS_RATEMODE_REL)
8869                 fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
8870         else if (p->ratemode == SCHED_CLASS_RATEMODE_ABS)
8871                 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
8872         else
8873                 return (EINVAL);
8874
8875         /* Vet our parameters ... */
8876         if (!in_range(p->channel, 0, sc->chip_params->nchan - 1))
8877                 return (ERANGE);
8878
8879         pi = sc->port[sc->chan_map[p->channel]];
8880         if (pi == NULL)
8881                 return (ENXIO);
8882         MPASS(pi->tx_chan == p->channel);
8883         top_speed = port_top_speed(pi) * 1000000; /* Gbps -> Kbps */
8884
8885         if (!in_range(p->cl, 0, sc->chip_params->nsched_cls) ||
8886             !in_range(p->minrate, 0, top_speed) ||
8887             !in_range(p->maxrate, 0, top_speed) ||
8888             !in_range(p->weight, 0, 100))
8889                 return (ERANGE);
8890
8891         /*
8892          * Translate any unset parameters into the firmware's
8893          * nomenclature and/or fail the call if the parameters
8894          * are required ...
8895          */
8896         if (p->rateunit < 0 || p->ratemode < 0 || p->channel < 0 || p->cl < 0)
8897                 return (EINVAL);
8898
8899         if (p->minrate < 0)
8900                 p->minrate = 0;
8901         if (p->maxrate < 0) {
8902                 if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
8903                     p->level == SCHED_CLASS_LEVEL_CH_RL)
8904                         return (EINVAL);
8905                 else
8906                         p->maxrate = 0;
8907         }
8908         if (p->weight < 0) {
8909                 if (p->level == SCHED_CLASS_LEVEL_CL_WRR)
8910                         return (EINVAL);
8911                 else
8912                         p->weight = 0;
8913         }
8914         if (p->pktsize < 0) {
8915                 if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
8916                     p->level == SCHED_CLASS_LEVEL_CH_RL)
8917                         return (EINVAL);
8918                 else
8919                         p->pktsize = 0;
8920         }
8921
8922         rc = begin_synchronized_op(sc, NULL,
8923             sleep_ok ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4sscp");
8924         if (rc)
8925                 return (rc);
8926         tc = &pi->tc[p->cl];
8927         tc->params = *p;
8928         rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level, fw_mode,
8929             fw_rateunit, fw_ratemode, p->channel, p->cl, p->minrate, p->maxrate,
8930             p->weight, p->pktsize, sleep_ok);
8931         if (rc == 0)
8932                 tc->flags |= TX_SC_OK;
8933         else {
8934                 /*
8935                  * Unknown state at this point, see tc->params for what was
8936                  * attempted.
8937                  */
8938                 tc->flags &= ~TX_SC_OK;
8939         }
8940         end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD);
8941
8942         return (rc);
8943 }
8944
8945 int
8946 t4_set_sched_class(struct adapter *sc, struct t4_sched_params *p)
8947 {
8948
8949         if (p->type != SCHED_CLASS_TYPE_PACKET)
8950                 return (EINVAL);
8951
8952         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
8953                 return (set_sched_class_config(sc, p->u.config.minmax));
8954
8955         if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
8956                 return (set_sched_class_params(sc, &p->u.params, 1));
8957
8958         return (EINVAL);
8959 }
8960
8961 int
8962 t4_set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
8963 {
8964         struct port_info *pi = NULL;
8965         struct vi_info *vi;
8966         struct sge_txq *txq;
8967         uint32_t fw_mnem, fw_queue, fw_class;
8968         int i, rc;
8969
8970         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
8971         if (rc)
8972                 return (rc);
8973
8974         if (p->port >= sc->params.nports) {
8975                 rc = EINVAL;
8976                 goto done;
8977         }
8978
8979         /* XXX: Only supported for the main VI. */
8980         pi = sc->port[p->port];
8981         vi = &pi->vi[0];
8982         if (!(vi->flags & VI_INIT_DONE)) {
8983                 /* tx queues not set up yet */
8984                 rc = EAGAIN;
8985                 goto done;
8986         }
8987
8988         if (!in_range(p->queue, 0, vi->ntxq - 1) ||
8989             !in_range(p->cl, 0, sc->chip_params->nsched_cls - 1)) {
8990                 rc = EINVAL;
8991                 goto done;
8992         }
8993
8994         /*
8995          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
8996          * Scheduling Class in this case).
8997          */
8998         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
8999             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
9000         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
9001
9002         /*
9003          * If op.queue is non-negative, then we're only changing the scheduling
9004          * on a single specified TX queue.
9005          */
9006         if (p->queue >= 0) {
9007                 txq = &sc->sge.txq[vi->first_txq + p->queue];
9008                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
9009                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
9010                     &fw_class);
9011                 goto done;
9012         }
9013
9014         /*
9015          * Change the scheduling on all the TX queues for the
9016          * interface.
9017          */
9018         for_each_txq(vi, i, txq) {
9019                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
9020                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
9021                     &fw_class);
9022                 if (rc)
9023                         goto done;
9024         }
9025
9026         rc = 0;
9027 done:
9028         end_synchronized_op(sc, 0);
9029         return (rc);
9030 }
9031
9032 int
9033 t4_os_find_pci_capability(struct adapter *sc, int cap)
9034 {
9035         int i;
9036
9037         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
9038 }
9039
9040 int
9041 t4_os_pci_save_state(struct adapter *sc)
9042 {
9043         device_t dev;
9044         struct pci_devinfo *dinfo;
9045
9046         dev = sc->dev;
9047         dinfo = device_get_ivars(dev);
9048
9049         pci_cfg_save(dev, dinfo, 0);
9050         return (0);
9051 }
9052
9053 int
9054 t4_os_pci_restore_state(struct adapter *sc)
9055 {
9056         device_t dev;
9057         struct pci_devinfo *dinfo;
9058
9059         dev = sc->dev;
9060         dinfo = device_get_ivars(dev);
9061
9062         pci_cfg_restore(dev, dinfo);
9063         return (0);
9064 }
9065
9066 void
9067 t4_os_portmod_changed(const struct adapter *sc, int idx)
9068 {
9069         struct port_info *pi = sc->port[idx];
9070         struct vi_info *vi;
9071         struct ifnet *ifp;
9072         int v;
9073         static const char *mod_str[] = {
9074                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
9075         };
9076
9077         for_each_vi(pi, v, vi) {
9078                 build_medialist(pi, &vi->media);
9079         }
9080
9081         ifp = pi->vi[0].ifp;
9082         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
9083                 if_printf(ifp, "transceiver unplugged.\n");
9084         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
9085                 if_printf(ifp, "unknown transceiver inserted.\n");
9086         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
9087                 if_printf(ifp, "unsupported transceiver inserted.\n");
9088         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
9089                 if_printf(ifp, "%s transceiver inserted.\n",
9090                     mod_str[pi->mod_type]);
9091         } else {
9092                 if_printf(ifp, "transceiver (type %d) inserted.\n",
9093                     pi->mod_type);
9094         }
9095 }
9096
9097 void
9098 t4_os_link_changed(struct adapter *sc, int idx, int link_stat)
9099 {
9100         struct port_info *pi = sc->port[idx];
9101         struct vi_info *vi;
9102         struct ifnet *ifp;
9103         int v;
9104
9105         for_each_vi(pi, v, vi) {
9106                 ifp = vi->ifp;
9107                 if (ifp == NULL)
9108                         continue;
9109
9110                 if (link_stat) {
9111                         ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
9112                         if_link_state_change(ifp, LINK_STATE_UP);
9113                 } else {
9114                         if_link_state_change(ifp, LINK_STATE_DOWN);
9115                 }
9116         }
9117 }
9118
9119 void
9120 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
9121 {
9122         struct adapter *sc;
9123
9124         sx_slock(&t4_list_lock);
9125         SLIST_FOREACH(sc, &t4_list, link) {
9126                 /*
9127                  * func should not make any assumptions about what state sc is
9128                  * in - the only guarantee is that sc->sc_lock is a valid lock.
9129                  */
9130                 func(sc, arg);
9131         }
9132         sx_sunlock(&t4_list_lock);
9133 }
9134
9135 static int
9136 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
9137     struct thread *td)
9138 {
9139         int rc;
9140         struct adapter *sc = dev->si_drv1;
9141
9142         rc = priv_check(td, PRIV_DRIVER);
9143         if (rc != 0)
9144                 return (rc);
9145
9146         switch (cmd) {
9147         case CHELSIO_T4_GETREG: {
9148                 struct t4_reg *edata = (struct t4_reg *)data;
9149
9150                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
9151                         return (EFAULT);
9152
9153                 if (edata->size == 4)
9154                         edata->val = t4_read_reg(sc, edata->addr);
9155                 else if (edata->size == 8)
9156                         edata->val = t4_read_reg64(sc, edata->addr);
9157                 else
9158                         return (EINVAL);
9159
9160                 break;
9161         }
9162         case CHELSIO_T4_SETREG: {
9163                 struct t4_reg *edata = (struct t4_reg *)data;
9164
9165                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
9166                         return (EFAULT);
9167
9168                 if (edata->size == 4) {
9169                         if (edata->val & 0xffffffff00000000)
9170                                 return (EINVAL);
9171                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
9172                 } else if (edata->size == 8)
9173                         t4_write_reg64(sc, edata->addr, edata->val);
9174                 else
9175                         return (EINVAL);
9176                 break;
9177         }
9178         case CHELSIO_T4_REGDUMP: {
9179                 struct t4_regdump *regs = (struct t4_regdump *)data;
9180                 int reglen = t4_get_regs_len(sc);
9181                 uint8_t *buf;
9182
9183                 if (regs->len < reglen) {
9184                         regs->len = reglen; /* hint to the caller */
9185                         return (ENOBUFS);
9186                 }
9187
9188                 regs->len = reglen;
9189                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
9190                 get_regs(sc, regs, buf);
9191                 rc = copyout(buf, regs->data, reglen);
9192                 free(buf, M_CXGBE);
9193                 break;
9194         }
9195         case CHELSIO_T4_GET_FILTER_MODE:
9196                 rc = get_filter_mode(sc, (uint32_t *)data);
9197                 break;
9198         case CHELSIO_T4_SET_FILTER_MODE:
9199                 rc = set_filter_mode(sc, *(uint32_t *)data);
9200                 break;
9201         case CHELSIO_T4_GET_FILTER:
9202                 rc = get_filter(sc, (struct t4_filter *)data);
9203                 break;
9204         case CHELSIO_T4_SET_FILTER:
9205                 rc = set_filter(sc, (struct t4_filter *)data);
9206                 break;
9207         case CHELSIO_T4_DEL_FILTER:
9208                 rc = del_filter(sc, (struct t4_filter *)data);
9209                 break;
9210         case CHELSIO_T4_GET_SGE_CONTEXT:
9211                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
9212                 break;
9213         case CHELSIO_T4_LOAD_FW:
9214                 rc = load_fw(sc, (struct t4_data *)data);
9215                 break;
9216         case CHELSIO_T4_GET_MEM:
9217                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
9218                 break;
9219         case CHELSIO_T4_GET_I2C:
9220                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
9221                 break;
9222         case CHELSIO_T4_CLEAR_STATS: {
9223                 int i, v;
9224                 u_int port_id = *(uint32_t *)data;
9225                 struct port_info *pi;
9226                 struct vi_info *vi;
9227
9228                 if (port_id >= sc->params.nports)
9229                         return (EINVAL);
9230                 pi = sc->port[port_id];
9231                 if (pi == NULL)
9232                         return (EIO);
9233
9234                 /* MAC stats */
9235                 t4_clr_port_stats(sc, pi->tx_chan);
9236                 pi->tx_parse_error = 0;
9237                 mtx_lock(&sc->reg_lock);
9238                 for_each_vi(pi, v, vi) {
9239                         if (vi->flags & VI_INIT_DONE)
9240                                 t4_clr_vi_stats(sc, vi->viid);
9241                 }
9242                 mtx_unlock(&sc->reg_lock);
9243
9244                 /*
9245                  * Since this command accepts a port, clear stats for
9246                  * all VIs on this port.
9247                  */
9248                 for_each_vi(pi, v, vi) {
9249                         if (vi->flags & VI_INIT_DONE) {
9250                                 struct sge_rxq *rxq;
9251                                 struct sge_txq *txq;
9252                                 struct sge_wrq *wrq;
9253
9254                                 for_each_rxq(vi, i, rxq) {
9255 #if defined(INET) || defined(INET6)
9256                                         rxq->lro.lro_queued = 0;
9257                                         rxq->lro.lro_flushed = 0;
9258 #endif
9259                                         rxq->rxcsum = 0;
9260                                         rxq->vlan_extraction = 0;
9261                                 }
9262
9263                                 for_each_txq(vi, i, txq) {
9264                                         txq->txcsum = 0;
9265                                         txq->tso_wrs = 0;
9266                                         txq->vlan_insertion = 0;
9267                                         txq->imm_wrs = 0;
9268                                         txq->sgl_wrs = 0;
9269                                         txq->txpkt_wrs = 0;
9270                                         txq->txpkts0_wrs = 0;
9271                                         txq->txpkts1_wrs = 0;
9272                                         txq->txpkts0_pkts = 0;
9273                                         txq->txpkts1_pkts = 0;
9274                                         mp_ring_reset_stats(txq->r);
9275                                 }
9276
9277 #ifdef TCP_OFFLOAD
9278                                 /* nothing to clear for each ofld_rxq */
9279
9280                                 for_each_ofld_txq(vi, i, wrq) {
9281                                         wrq->tx_wrs_direct = 0;
9282                                         wrq->tx_wrs_copied = 0;
9283                                 }
9284 #endif
9285
9286                                 if (IS_MAIN_VI(vi)) {
9287                                         wrq = &sc->sge.ctrlq[pi->port_id];
9288                                         wrq->tx_wrs_direct = 0;
9289                                         wrq->tx_wrs_copied = 0;
9290                                 }
9291                         }
9292                 }
9293                 break;
9294         }
9295         case CHELSIO_T4_SCHED_CLASS:
9296                 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data);
9297                 break;
9298         case CHELSIO_T4_SCHED_QUEUE:
9299                 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data);
9300                 break;
9301         case CHELSIO_T4_GET_TRACER:
9302                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
9303                 break;
9304         case CHELSIO_T4_SET_TRACER:
9305                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
9306                 break;
9307         case CHELSIO_T4_LOAD_CFG:
9308                 rc = load_cfg(sc, (struct t4_data *)data);
9309                 break;
9310         default:
9311                 rc = ENOTTY;
9312         }
9313
9314         return (rc);
9315 }
9316
9317 void
9318 t4_db_full(struct adapter *sc)
9319 {
9320
9321         CXGBE_UNIMPLEMENTED(__func__);
9322 }
9323
9324 void
9325 t4_db_dropped(struct adapter *sc)
9326 {
9327
9328         CXGBE_UNIMPLEMENTED(__func__);
9329 }
9330
9331 #ifdef TCP_OFFLOAD
9332 static int
9333 toe_capability(struct vi_info *vi, int enable)
9334 {
9335         int rc;
9336         struct port_info *pi = vi->pi;
9337         struct adapter *sc = pi->adapter;
9338
9339         ASSERT_SYNCHRONIZED_OP(sc);
9340
9341         if (!is_offload(sc))
9342                 return (ENODEV);
9343
9344         if (enable) {
9345                 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) {
9346                         /* TOE is already enabled. */
9347                         return (0);
9348                 }
9349
9350                 /*
9351                  * We need the port's queues around so that we're able to send
9352                  * and receive CPLs to/from the TOE even if the ifnet for this
9353                  * port has never been UP'd administratively.
9354                  */
9355                 if (!(vi->flags & VI_INIT_DONE)) {
9356                         rc = vi_full_init(vi);
9357                         if (rc)
9358                                 return (rc);
9359                 }
9360                 if (!(pi->vi[0].flags & VI_INIT_DONE)) {
9361                         rc = vi_full_init(&pi->vi[0]);
9362                         if (rc)
9363                                 return (rc);
9364                 }
9365
9366                 if (isset(&sc->offload_map, pi->port_id)) {
9367                         /* TOE is enabled on another VI of this port. */
9368                         pi->uld_vis++;
9369                         return (0);
9370                 }
9371
9372                 if (!uld_active(sc, ULD_TOM)) {
9373                         rc = t4_activate_uld(sc, ULD_TOM);
9374                         if (rc == EAGAIN) {
9375                                 log(LOG_WARNING,
9376                                     "You must kldload t4_tom.ko before trying "
9377                                     "to enable TOE on a cxgbe interface.\n");
9378                         }
9379                         if (rc != 0)
9380                                 return (rc);
9381                         KASSERT(sc->tom_softc != NULL,
9382                             ("%s: TOM activated but softc NULL", __func__));
9383                         KASSERT(uld_active(sc, ULD_TOM),
9384                             ("%s: TOM activated but flag not set", __func__));
9385                 }
9386
9387                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
9388                 if (!uld_active(sc, ULD_IWARP))
9389                         (void) t4_activate_uld(sc, ULD_IWARP);
9390                 if (!uld_active(sc, ULD_ISCSI))
9391                         (void) t4_activate_uld(sc, ULD_ISCSI);
9392
9393                 pi->uld_vis++;
9394                 setbit(&sc->offload_map, pi->port_id);
9395         } else {
9396                 pi->uld_vis--;
9397
9398                 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
9399                         return (0);
9400
9401                 KASSERT(uld_active(sc, ULD_TOM),
9402                     ("%s: TOM never initialized?", __func__));
9403                 clrbit(&sc->offload_map, pi->port_id);
9404         }
9405
9406         return (0);
9407 }
9408
9409 /*
9410  * Add an upper layer driver to the global list.
9411  */
9412 int
9413 t4_register_uld(struct uld_info *ui)
9414 {
9415         int rc = 0;
9416         struct uld_info *u;
9417
9418         sx_xlock(&t4_uld_list_lock);
9419         SLIST_FOREACH(u, &t4_uld_list, link) {
9420             if (u->uld_id == ui->uld_id) {
9421                     rc = EEXIST;
9422                     goto done;
9423             }
9424         }
9425
9426         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
9427         ui->refcount = 0;
9428 done:
9429         sx_xunlock(&t4_uld_list_lock);
9430         return (rc);
9431 }
9432
9433 int
9434 t4_unregister_uld(struct uld_info *ui)
9435 {
9436         int rc = EINVAL;
9437         struct uld_info *u;
9438
9439         sx_xlock(&t4_uld_list_lock);
9440
9441         SLIST_FOREACH(u, &t4_uld_list, link) {
9442             if (u == ui) {
9443                     if (ui->refcount > 0) {
9444                             rc = EBUSY;
9445                             goto done;
9446                     }
9447
9448                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
9449                     rc = 0;
9450                     goto done;
9451             }
9452         }
9453 done:
9454         sx_xunlock(&t4_uld_list_lock);
9455         return (rc);
9456 }
9457
9458 int
9459 t4_activate_uld(struct adapter *sc, int id)
9460 {
9461         int rc;
9462         struct uld_info *ui;
9463
9464         ASSERT_SYNCHRONIZED_OP(sc);
9465
9466         if (id < 0 || id > ULD_MAX)
9467                 return (EINVAL);
9468         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
9469
9470         sx_slock(&t4_uld_list_lock);
9471
9472         SLIST_FOREACH(ui, &t4_uld_list, link) {
9473                 if (ui->uld_id == id) {
9474                         if (!(sc->flags & FULL_INIT_DONE)) {
9475                                 rc = adapter_full_init(sc);
9476                                 if (rc != 0)
9477                                         break;
9478                         }
9479
9480                         rc = ui->activate(sc);
9481                         if (rc == 0) {
9482                                 setbit(&sc->active_ulds, id);
9483                                 ui->refcount++;
9484                         }
9485                         break;
9486                 }
9487         }
9488
9489         sx_sunlock(&t4_uld_list_lock);
9490
9491         return (rc);
9492 }
9493
9494 int
9495 t4_deactivate_uld(struct adapter *sc, int id)
9496 {
9497         int rc;
9498         struct uld_info *ui;
9499
9500         ASSERT_SYNCHRONIZED_OP(sc);
9501
9502         if (id < 0 || id > ULD_MAX)
9503                 return (EINVAL);
9504         rc = ENXIO;
9505
9506         sx_slock(&t4_uld_list_lock);
9507
9508         SLIST_FOREACH(ui, &t4_uld_list, link) {
9509                 if (ui->uld_id == id) {
9510                         rc = ui->deactivate(sc);
9511                         if (rc == 0) {
9512                                 clrbit(&sc->active_ulds, id);
9513                                 ui->refcount--;
9514                         }
9515                         break;
9516                 }
9517         }
9518
9519         sx_sunlock(&t4_uld_list_lock);
9520
9521         return (rc);
9522 }
9523
9524 int
9525 uld_active(struct adapter *sc, int uld_id)
9526 {
9527
9528         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
9529
9530         return (isset(&sc->active_ulds, uld_id));
9531 }
9532 #endif
9533
9534 /*
9535  * t  = ptr to tunable.
9536  * nc = number of CPUs.
9537  * c  = compiled in default for that tunable.
9538  */
9539 static void
9540 calculate_nqueues(int *t, int nc, const int c)
9541 {
9542         int nq;
9543
9544         if (*t > 0)
9545                 return;
9546         nq = *t < 0 ? -*t : c;
9547         *t = min(nc, nq);
9548 }
9549
9550 /*
9551  * Come up with reasonable defaults for some of the tunables, provided they're
9552  * not set by the user (in which case we'll use the values as is).
9553  */
9554 static void
9555 tweak_tunables(void)
9556 {
9557         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
9558
9559         if (t4_ntxq10g < 1) {
9560 #ifdef RSS
9561                 t4_ntxq10g = rss_getnumbuckets();
9562 #else
9563                 calculate_nqueues(&t4_ntxq10g, nc, NTXQ_10G);
9564 #endif
9565         }
9566
9567         if (t4_ntxq1g < 1) {
9568 #ifdef RSS
9569                 /* XXX: way too many for 1GbE? */
9570                 t4_ntxq1g = rss_getnumbuckets();
9571 #else
9572                 calculate_nqueues(&t4_ntxq1g, nc, NTXQ_1G);
9573 #endif
9574         }
9575
9576         calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI);
9577
9578         if (t4_nrxq10g < 1) {
9579 #ifdef RSS
9580                 t4_nrxq10g = rss_getnumbuckets();
9581 #else
9582                 calculate_nqueues(&t4_nrxq10g, nc, NRXQ_10G);
9583 #endif
9584         }
9585
9586         if (t4_nrxq1g < 1) {
9587 #ifdef RSS
9588                 /* XXX: way too many for 1GbE? */
9589                 t4_nrxq1g = rss_getnumbuckets();
9590 #else
9591                 calculate_nqueues(&t4_nrxq1g, nc, NRXQ_1G);
9592 #endif
9593         }
9594
9595         calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI);
9596
9597 #ifdef TCP_OFFLOAD
9598         calculate_nqueues(&t4_nofldtxq10g, nc, NOFLDTXQ_10G);
9599         calculate_nqueues(&t4_nofldtxq1g, nc, NOFLDTXQ_1G);
9600         calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI);
9601         calculate_nqueues(&t4_nofldrxq10g, nc, NOFLDRXQ_10G);
9602         calculate_nqueues(&t4_nofldrxq1g, nc, NOFLDRXQ_1G);
9603         calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI);
9604
9605         if (t4_toecaps_allowed == -1)
9606                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
9607
9608         if (t4_rdmacaps_allowed == -1) {
9609                 t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP |
9610                     FW_CAPS_CONFIG_RDMA_RDMAC;
9611         }
9612
9613         if (t4_iscsicaps_allowed == -1) {
9614                 t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU |
9615                     FW_CAPS_CONFIG_ISCSI_TARGET_PDU |
9616                     FW_CAPS_CONFIG_ISCSI_T10DIF;
9617         }
9618 #else
9619         if (t4_toecaps_allowed == -1)
9620                 t4_toecaps_allowed = 0;
9621
9622         if (t4_rdmacaps_allowed == -1)
9623                 t4_rdmacaps_allowed = 0;
9624
9625         if (t4_iscsicaps_allowed == -1)
9626                 t4_iscsicaps_allowed = 0;
9627 #endif
9628
9629 #ifdef DEV_NETMAP
9630         calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI);
9631         calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI);
9632 #endif
9633
9634         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
9635                 t4_tmr_idx_10g = TMR_IDX_10G;
9636
9637         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
9638                 t4_pktc_idx_10g = PKTC_IDX_10G;
9639
9640         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
9641                 t4_tmr_idx_1g = TMR_IDX_1G;
9642
9643         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
9644                 t4_pktc_idx_1g = PKTC_IDX_1G;
9645
9646         if (t4_qsize_txq < 128)
9647                 t4_qsize_txq = 128;
9648
9649         if (t4_qsize_rxq < 128)
9650                 t4_qsize_rxq = 128;
9651         while (t4_qsize_rxq & 7)
9652                 t4_qsize_rxq++;
9653
9654         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
9655 }
9656
9657 #ifdef DDB
9658 static void
9659 t4_dump_tcb(struct adapter *sc, int tid)
9660 {
9661         uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos;
9662
9663         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2);
9664         save = t4_read_reg(sc, reg);
9665         base = sc->memwin[2].mw_base;
9666
9667         /* Dump TCB for the tid */
9668         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
9669         tcb_addr += tid * TCB_SIZE;
9670
9671         if (is_t4(sc)) {
9672                 pf = 0;
9673                 win_pos = tcb_addr & ~0xf;      /* start must be 16B aligned */
9674         } else {
9675                 pf = V_PFNUM(sc->pf);
9676                 win_pos = tcb_addr & ~0x7f;     /* start must be 128B aligned */
9677         }
9678         t4_write_reg(sc, reg, win_pos | pf);
9679         t4_read_reg(sc, reg);
9680
9681         off = tcb_addr - win_pos;
9682         for (i = 0; i < 4; i++) {
9683                 uint32_t buf[8];
9684                 for (j = 0; j < 8; j++, off += 4)
9685                         buf[j] = htonl(t4_read_reg(sc, base + off));
9686
9687                 db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n",
9688                     buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
9689                     buf[7]);
9690         }
9691
9692         t4_write_reg(sc, reg, save);
9693         t4_read_reg(sc, reg);
9694 }
9695
9696 static void
9697 t4_dump_devlog(struct adapter *sc)
9698 {
9699         struct devlog_params *dparams = &sc->params.devlog;
9700         struct fw_devlog_e e;
9701         int i, first, j, m, nentries, rc;
9702         uint64_t ftstamp = UINT64_MAX;
9703
9704         if (dparams->start == 0) {
9705                 db_printf("devlog params not valid\n");
9706                 return;
9707         }
9708
9709         nentries = dparams->size / sizeof(struct fw_devlog_e);
9710         m = fwmtype_to_hwmtype(dparams->memtype);
9711
9712         /* Find the first entry. */
9713         first = -1;
9714         for (i = 0; i < nentries && !db_pager_quit; i++) {
9715                 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
9716                     sizeof(e), (void *)&e);
9717                 if (rc != 0)
9718                         break;
9719
9720                 if (e.timestamp == 0)
9721                         break;
9722
9723                 e.timestamp = be64toh(e.timestamp);
9724                 if (e.timestamp < ftstamp) {
9725                         ftstamp = e.timestamp;
9726                         first = i;
9727                 }
9728         }
9729
9730         if (first == -1)
9731                 return;
9732
9733         i = first;
9734         do {
9735                 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
9736                     sizeof(e), (void *)&e);
9737                 if (rc != 0)
9738                         return;
9739
9740                 if (e.timestamp == 0)
9741                         return;
9742
9743                 e.timestamp = be64toh(e.timestamp);
9744                 e.seqno = be32toh(e.seqno);
9745                 for (j = 0; j < 8; j++)
9746                         e.params[j] = be32toh(e.params[j]);
9747
9748                 db_printf("%10d  %15ju  %8s  %8s  ",
9749                     e.seqno, e.timestamp,
9750                     (e.level < nitems(devlog_level_strings) ?
9751                         devlog_level_strings[e.level] : "UNKNOWN"),
9752                     (e.facility < nitems(devlog_facility_strings) ?
9753                         devlog_facility_strings[e.facility] : "UNKNOWN"));
9754                 db_printf(e.fmt, e.params[0], e.params[1], e.params[2],
9755                     e.params[3], e.params[4], e.params[5], e.params[6],
9756                     e.params[7]);
9757
9758                 if (++i == nentries)
9759                         i = 0;
9760         } while (i != first && !db_pager_quit);
9761 }
9762
9763 static struct command_table db_t4_table = LIST_HEAD_INITIALIZER(db_t4_table);
9764 _DB_SET(_show, t4, NULL, db_show_table, 0, &db_t4_table);
9765
9766 DB_FUNC(devlog, db_show_devlog, db_t4_table, CS_OWN, NULL)
9767 {
9768         device_t dev;
9769         int t;
9770         bool valid;
9771
9772         valid = false;
9773         t = db_read_token();
9774         if (t == tIDENT) {
9775                 dev = device_lookup_by_name(db_tok_string);
9776                 valid = true;
9777         }
9778         db_skip_to_eol();
9779         if (!valid) {
9780                 db_printf("usage: show t4 devlog <nexus>\n");
9781                 return;
9782         }
9783
9784         if (dev == NULL) {
9785                 db_printf("device not found\n");
9786                 return;
9787         }
9788
9789         t4_dump_devlog(device_get_softc(dev));
9790 }
9791
9792 DB_FUNC(tcb, db_show_t4tcb, db_t4_table, CS_OWN, NULL)
9793 {
9794         device_t dev;
9795         int radix, tid, t;
9796         bool valid;
9797
9798         valid = false;
9799         radix = db_radix;
9800         db_radix = 10;
9801         t = db_read_token();
9802         if (t == tIDENT) {
9803                 dev = device_lookup_by_name(db_tok_string);
9804                 t = db_read_token();
9805                 if (t == tNUMBER) {
9806                         tid = db_tok_number;
9807                         valid = true;
9808                 }
9809         }
9810         db_radix = radix;
9811         db_skip_to_eol();
9812         if (!valid) {
9813                 db_printf("usage: show t4 tcb <nexus> <tid>\n");
9814                 return;
9815         }
9816
9817         if (dev == NULL) {
9818                 db_printf("device not found\n");
9819                 return;
9820         }
9821         if (tid < 0) {
9822                 db_printf("invalid tid\n");
9823                 return;
9824         }
9825
9826         t4_dump_tcb(device_get_softc(dev), tid);
9827 }
9828 #endif
9829
9830 static struct sx mlu;   /* mod load unload */
9831 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
9832
9833 static int
9834 mod_event(module_t mod, int cmd, void *arg)
9835 {
9836         int rc = 0;
9837         static int loaded = 0;
9838
9839         switch (cmd) {
9840         case MOD_LOAD:
9841                 sx_xlock(&mlu);
9842                 if (loaded++ == 0) {
9843                         t4_sge_modload();
9844                         t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl);
9845                         t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl);
9846                         t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt);
9847                         t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt);
9848                         sx_init(&t4_list_lock, "T4/T5 adapters");
9849                         SLIST_INIT(&t4_list);
9850 #ifdef TCP_OFFLOAD
9851                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
9852                         SLIST_INIT(&t4_uld_list);
9853 #endif
9854                         t4_tracer_modload();
9855                         tweak_tunables();
9856                 }
9857                 sx_xunlock(&mlu);
9858                 break;
9859
9860         case MOD_UNLOAD:
9861                 sx_xlock(&mlu);
9862                 if (--loaded == 0) {
9863                         int tries;
9864
9865                         sx_slock(&t4_list_lock);
9866                         if (!SLIST_EMPTY(&t4_list)) {
9867                                 rc = EBUSY;
9868                                 sx_sunlock(&t4_list_lock);
9869                                 goto done_unload;
9870                         }
9871 #ifdef TCP_OFFLOAD
9872                         sx_slock(&t4_uld_list_lock);
9873                         if (!SLIST_EMPTY(&t4_uld_list)) {
9874                                 rc = EBUSY;
9875                                 sx_sunlock(&t4_uld_list_lock);
9876                                 sx_sunlock(&t4_list_lock);
9877                                 goto done_unload;
9878                         }
9879 #endif
9880                         tries = 0;
9881                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
9882                                 uprintf("%ju clusters with custom free routine "
9883                                     "still is use.\n", t4_sge_extfree_refs());
9884                                 pause("t4unload", 2 * hz);
9885                         }
9886 #ifdef TCP_OFFLOAD
9887                         sx_sunlock(&t4_uld_list_lock);
9888 #endif
9889                         sx_sunlock(&t4_list_lock);
9890
9891                         if (t4_sge_extfree_refs() == 0) {
9892                                 t4_tracer_modunload();
9893 #ifdef TCP_OFFLOAD
9894                                 sx_destroy(&t4_uld_list_lock);
9895 #endif
9896                                 sx_destroy(&t4_list_lock);
9897                                 t4_sge_modunload();
9898                                 loaded = 0;
9899                         } else {
9900                                 rc = EBUSY;
9901                                 loaded++;       /* undo earlier decrement */
9902                         }
9903                 }
9904 done_unload:
9905                 sx_xunlock(&mlu);
9906                 break;
9907         }
9908
9909         return (rc);
9910 }
9911
9912 static devclass_t t4_devclass, t5_devclass, t6_devclass;
9913 static devclass_t cxgbe_devclass, cxl_devclass, cc_devclass;
9914 static devclass_t vcxgbe_devclass, vcxl_devclass, vcc_devclass;
9915
9916 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
9917 MODULE_VERSION(t4nex, 1);
9918 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
9919 #ifdef DEV_NETMAP
9920 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
9921 #endif /* DEV_NETMAP */
9922
9923 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
9924 MODULE_VERSION(t5nex, 1);
9925 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
9926 #ifdef DEV_NETMAP
9927 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
9928 #endif /* DEV_NETMAP */
9929
9930 DRIVER_MODULE(t6nex, pci, t6_driver, t6_devclass, mod_event, 0);
9931 MODULE_VERSION(t6nex, 1);
9932 MODULE_DEPEND(t6nex, firmware, 1, 1, 1);
9933 #ifdef DEV_NETMAP
9934 MODULE_DEPEND(t6nex, netmap, 1, 1, 1);
9935 #endif /* DEV_NETMAP */
9936
9937 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
9938 MODULE_VERSION(cxgbe, 1);
9939
9940 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
9941 MODULE_VERSION(cxl, 1);
9942
9943 DRIVER_MODULE(cc, t6nex, cc_driver, cc_devclass, 0, 0);
9944 MODULE_VERSION(cc, 1);
9945
9946 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0);
9947 MODULE_VERSION(vcxgbe, 1);
9948
9949 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0);
9950 MODULE_VERSION(vcxl, 1);
9951
9952 DRIVER_MODULE(vcc, cc, vcc_driver, vcc_devclass, 0, 0);
9953 MODULE_VERSION(vcc, 1);