sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 285  * encouraged respectively).
 286  */
 287 static unsigned int t4_fw_install = 1;
 288 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 289
 290 /*
 291  * ASIC features that will be used.  Disable the ones you don't want so that the
 292  * chip resources aren't wasted on features that will not be used.
 293  */
 294 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 295 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 296
 297 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 298 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 299
 300 static int t4_toecaps_allowed = -1;
 301 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 302
 303 static int t4_rdmacaps_allowed = 0;
 304 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 305
 306 static int t4_iscsicaps_allowed = 0;
 307 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 308
 309 static int t4_fcoecaps_allowed = 0;
 310 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 311
 312 static int t5_write_combine = 0;
 313 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 314
 315 struct intrs_and_queues {
 316         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 317         uint16_t nirq;          /* Total # of vectors */
 318         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 319         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 320         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 321         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 322         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 323         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 324         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 325 #ifdef TCP_OFFLOAD
 326         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 327         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 328         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 329         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 330 #endif
 331 #ifdef DEV_NETMAP
 332         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 333         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 334         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 335         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 336 #endif
 337 };
 338
 339 struct filter_entry {
 340         uint32_t valid:1;       /* filter allocated and valid */
 341         uint32_t locked:1;      /* filter is administratively locked */
 342         uint32_t pending:1;     /* filter action is pending firmware reply */
 343         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 344         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 345
 346         struct t4_filter_specification fs;
 347 };
 348
 349 static int map_bars_0_and_4(struct adapter *);
 350 static int map_bar_2(struct adapter *);
 351 static void setup_memwin(struct adapter *);
 352 static int validate_mem_range(struct adapter *, uint32_t, int);
 353 static int fwmtype_to_hwmtype(int);
 354 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 355     uint32_t *);
 356 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 357 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 358 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 359     struct intrs_and_queues *);
 360 static int prep_firmware(struct adapter *);
 361 static int partition_resources(struct adapter *, const struct firmware *,
 362     const char *);
 363 static int get_params__pre_init(struct adapter *);
 364 static int get_params__post_init(struct adapter *);
 365 static int set_params__post_init(struct adapter *);
 366 static void t4_set_desc(struct adapter *);
 367 static void build_medialist(struct port_info *, struct ifmedia *);
 368 static int cxgbe_init_synchronized(struct port_info *);
 369 static int cxgbe_uninit_synchronized(struct port_info *);
 370 static int setup_intr_handlers(struct adapter *);
 371 static void quiesce_eq(struct adapter *, struct sge_eq *);
 372 static void quiesce_iq(struct adapter *, struct sge_iq *);
 373 static void quiesce_fl(struct adapter *, struct sge_fl *);
 374 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 375     driver_intr_t *, void *, char *);
 376 static int t4_free_irq(struct adapter *, struct irq *);
 377 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 378     unsigned int);
 379 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 380 static void cxgbe_tick(void *);
 381 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 382 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 383     struct mbuf *);
 384 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 385 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 386 static int t4_sysctls(struct adapter *);
 387 static int cxgbe_sysctls(struct port_info *);
 388 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 389 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 390 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 391 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 392 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 393 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 394 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 395 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 396 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 397 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 398 #ifdef SBUF_DRAIN
 399 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 424 #endif
 425 static inline void txq_start(struct ifnet *, struct sge_txq *);
 426 static uint32_t fconf_to_mode(uint32_t);
 427 static uint32_t mode_to_fconf(uint32_t);
 428 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 429 static int get_filter_mode(struct adapter *, uint32_t *);
 430 static int set_filter_mode(struct adapter *, uint32_t);
 431 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 432 static int get_filter(struct adapter *, struct t4_filter *);
 433 static int set_filter(struct adapter *, struct t4_filter *);
 434 static int del_filter(struct adapter *, struct t4_filter *);
 435 static void clear_filter(struct filter_entry *);
 436 static int set_filter_wr(struct adapter *, int);
 437 static int del_filter_wr(struct adapter *, int);
 438 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 439 static int load_fw(struct adapter *, struct t4_data *);
 440 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 441 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 442 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 443 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 444 #ifdef TCP_OFFLOAD
 445 static int toe_capability(struct port_info *, int);
 446 #endif
 447 static int mod_event(module_t, int, void *);
 448
 449 struct {
 450         uint16_t device;
 451         char *desc;
 452 } t4_pciids[] = {
 453         {0xa000, "Chelsio Terminator 4 FPGA"},
 454         {0x4400, "Chelsio T440-dbg"},
 455         {0x4401, "Chelsio T420-CR"},
 456         {0x4402, "Chelsio T422-CR"},
 457         {0x4403, "Chelsio T440-CR"},
 458         {0x4404, "Chelsio T420-BCH"},
 459         {0x4405, "Chelsio T440-BCH"},
 460         {0x4406, "Chelsio T440-CH"},
 461         {0x4407, "Chelsio T420-SO"},
 462         {0x4408, "Chelsio T420-CX"},
 463         {0x4409, "Chelsio T420-BT"},
 464         {0x440a, "Chelsio T404-BT"},
 465         {0x440e, "Chelsio T440-LP-CR"},
 466 }, t5_pciids[] = {
 467         {0xb000, "Chelsio Terminator 5 FPGA"},
 468         {0x5400, "Chelsio T580-dbg"},
 469         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 470         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 471         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 472         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 473         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 474         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 475         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 476         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 477         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 478         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 479         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 480         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 481 #ifdef notyet
 482         {0x5404,  "Chelsio T520-BCH"},
 483         {0x5405,  "Chelsio T540-BCH"},
 484         {0x5406,  "Chelsio T540-CH"},
 485         {0x5408,  "Chelsio T520-CX"},
 486         {0x540b,  "Chelsio B520-SR"},
 487         {0x540c,  "Chelsio B504-BT"},
 488         {0x540f,  "Chelsio Amsterdam"},
 489         {0x5413,  "Chelsio T580-CHR"},
 490 #endif
 491 };
 492
 493 #ifdef TCP_OFFLOAD
 494 /*
 495  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 496  * exactly the same for both rxq and ofld_rxq.
 497  */
 498 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 499 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 500 #endif
 501
 502 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 503 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 504 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 505
 506 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 507
 508 static int
 509 t4_probe(device_t dev)
 510 {
 511         int i;
 512         uint16_t v = pci_get_vendor(dev);
 513         uint16_t d = pci_get_device(dev);
 514         uint8_t f = pci_get_function(dev);
 515
 516         if (v != PCI_VENDOR_ID_CHELSIO)
 517                 return (ENXIO);
 518
 519         /* Attach only to PF0 of the FPGA */
 520         if (d == 0xa000 && f != 0)
 521                 return (ENXIO);
 522
 523         for (i = 0; i < nitems(t4_pciids); i++) {
 524                 if (d == t4_pciids[i].device) {
 525                         device_set_desc(dev, t4_pciids[i].desc);
 526                         return (BUS_PROBE_DEFAULT);
 527                 }
 528         }
 529
 530         return (ENXIO);
 531 }
 532
 533 static int
 534 t5_probe(device_t dev)
 535 {
 536         int i;
 537         uint16_t v = pci_get_vendor(dev);
 538         uint16_t d = pci_get_device(dev);
 539         uint8_t f = pci_get_function(dev);
 540
 541         if (v != PCI_VENDOR_ID_CHELSIO)
 542                 return (ENXIO);
 543
 544         /* Attach only to PF0 of the FPGA */
 545         if (d == 0xb000 && f != 0)
 546                 return (ENXIO);
 547
 548         for (i = 0; i < nitems(t5_pciids); i++) {
 549                 if (d == t5_pciids[i].device) {
 550                         device_set_desc(dev, t5_pciids[i].desc);
 551                         return (BUS_PROBE_DEFAULT);
 552                 }
 553         }
 554
 555         return (ENXIO);
 556 }
 557
 558 static int
 559 t4_attach(device_t dev)
 560 {
 561         struct adapter *sc;
 562         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 563         struct intrs_and_queues iaq;
 564         struct sge *s;
 565 #ifdef TCP_OFFLOAD
 566         int ofld_rqidx, ofld_tqidx;
 567 #endif
 568 #ifdef DEV_NETMAP
 569         int nm_rqidx, nm_tqidx;
 570 #endif
 571         const char *pcie_ts;
 572
 573         sc = device_get_softc(dev);
 574         sc->dev = dev;
 575
 576         pci_enable_busmaster(dev);
 577         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 578                 uint32_t v;
 579
 580                 pci_set_max_read_req(dev, 4096);
 581                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 582                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 583                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 584         }
 585
 586         sc->traceq = -1;
 587         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 588         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 589             device_get_nameunit(dev));
 590
 591         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 592             device_get_nameunit(dev));
 593         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 594         sx_xlock(&t4_list_lock);
 595         SLIST_INSERT_HEAD(&t4_list, sc, link);
 596         sx_xunlock(&t4_list_lock);
 597
 598         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 599         TAILQ_INIT(&sc->sfl);
 600         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 601
 602         rc = map_bars_0_and_4(sc);
 603         if (rc != 0)
 604                 goto done; /* error message displayed already */
 605
 606         /*
 607          * This is the real PF# to which we're attaching.  Works from within PCI
 608          * passthrough environments too, where pci_get_function() could return a
 609          * different PF# depending on the passthrough configuration.  We need to
 610          * use the real PF# in all our communication with the firmware.
 611          */
 612         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 613         sc->mbox = sc->pf;
 614
 615         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 616         sc->an_handler = an_not_handled;
 617         for (i = 0; i < nitems(sc->cpl_handler); i++)
 618                 sc->cpl_handler[i] = cpl_not_handled;
 619         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 620                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 621         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 622         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 623         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 624         t4_init_sge_cpl_handlers(sc);
 625
 626         /* Prepare the adapter for operation */
 627         rc = -t4_prep_adapter(sc);
 628         if (rc != 0) {
 629                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 630                 goto done;
 631         }
 632
 633         /*
 634          * Do this really early, with the memory windows set up even before the
 635          * character device.  The userland tool's register i/o and mem read
 636          * will work even in "recovery mode".
 637          */
 638         setup_memwin(sc);
 639         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 640             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 641             device_get_nameunit(dev));
 642         if (sc->cdev == NULL)
 643                 device_printf(dev, "failed to create nexus char device.\n");
 644         else
 645                 sc->cdev->si_drv1 = sc;
 646
 647         /* Go no further if recovery mode has been requested. */
 648         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 649                 device_printf(dev, "recovery mode.\n");
 650                 goto done;
 651         }
 652
 653         /* Prepare the firmware for operation */
 654         rc = prep_firmware(sc);
 655         if (rc != 0)
 656                 goto done; /* error message displayed already */
 657
 658         rc = get_params__post_init(sc);
 659         if (rc != 0)
 660                 goto done; /* error message displayed already */
 661
 662         rc = set_params__post_init(sc);
 663         if (rc != 0)
 664                 goto done; /* error message displayed already */
 665
 666         rc = map_bar_2(sc);
 667         if (rc != 0)
 668                 goto done; /* error message displayed already */
 669
 670         rc = t4_create_dma_tag(sc);
 671         if (rc != 0)
 672                 goto done; /* error message displayed already */
 673
 674         /*
 675          * First pass over all the ports - allocate VIs and initialize some
 676          * basic parameters like mac address, port type, etc.  We also figure
 677          * out whether a port is 10G or 1G and use that information when
 678          * calculating how many interrupts to attempt to allocate.
 679          */
 680         n10g = n1g = 0;
 681         for_each_port(sc, i) {
 682                 struct port_info *pi;
 683
 684                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 685                 sc->port[i] = pi;
 686
 687                 /* These must be set before t4_port_init */
 688                 pi->adapter = sc;
 689                 pi->port_id = i;
 690
 691                 /* Allocate the vi and initialize parameters like mac addr */
 692                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 693                 if (rc != 0) {
 694                         device_printf(dev, "unable to initialize port %d: %d\n",
 695                             i, rc);
 696                         free(pi, M_CXGBE);
 697                         sc->port[i] = NULL;
 698                         goto done;
 699                 }
 700                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 701                 if (rc != 0) {
 702                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 703                         free(pi, M_CXGBE);
 704                         sc->port[i] = NULL;
 705                         goto done;
 706                 }
 707
 708                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 709                     device_get_nameunit(dev), i);
 710                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 711                 sc->chan_map[pi->tx_chan] = i;
 712
 713                 if (is_10G_port(pi) || is_40G_port(pi)) {
 714                         n10g++;
 715                         pi->tmr_idx = t4_tmr_idx_10g;
 716                         pi->pktc_idx = t4_pktc_idx_10g;
 717                 } else {
 718                         n1g++;
 719                         pi->tmr_idx = t4_tmr_idx_1g;
 720                         pi->pktc_idx = t4_pktc_idx_1g;
 721                 }
 722
 723                 pi->xact_addr_filt = -1;
 724                 pi->linkdnrc = -1;
 725
 726                 pi->qsize_rxq = t4_qsize_rxq;
 727                 pi->qsize_txq = t4_qsize_txq;
 728
 729                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 730                 if (pi->dev == NULL) {
 731                         device_printf(dev,
 732                             "failed to add device for port %d.\n", i);
 733                         rc = ENXIO;
 734                         goto done;
 735                 }
 736                 device_set_softc(pi->dev, pi);
 737         }
 738
 739         /*
 740          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 741          */
 742         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 743         if (rc != 0)
 744                 goto done; /* error message displayed already */
 745
 746         sc->intr_type = iaq.intr_type;
 747         sc->intr_count = iaq.nirq;
 748
 749         s = &sc->sge;
 750         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 751         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 752         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 753         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 754         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 755 #ifdef TCP_OFFLOAD
 756         if (is_offload(sc)) {
 757                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 758                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 759                 s->neq += s->nofldtxq + s->nofldrxq;
 760                 s->niq += s->nofldrxq;
 761
 762                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 763                     M_CXGBE, M_ZERO | M_WAITOK);
 764                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 765                     M_CXGBE, M_ZERO | M_WAITOK);
 766         }
 767 #endif
 768 #ifdef DEV_NETMAP
 769         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 770         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 771         s->neq += s->nnmtxq + s->nnmrxq;
 772         s->niq += s->nnmrxq;
 773
 774         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 775             M_CXGBE, M_ZERO | M_WAITOK);
 776         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 777             M_CXGBE, M_ZERO | M_WAITOK);
 778 #endif
 779
 780         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 781             M_ZERO | M_WAITOK);
 782         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 783             M_ZERO | M_WAITOK);
 784         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 785             M_ZERO | M_WAITOK);
 786         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 787             M_ZERO | M_WAITOK);
 788         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 789             M_ZERO | M_WAITOK);
 790
 791         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 792             M_ZERO | M_WAITOK);
 793
 794         t4_init_l2t(sc, M_WAITOK);
 795
 796         /*
 797          * Second pass over the ports.  This time we know the number of rx and
 798          * tx queues that each port should get.
 799          */
 800         rqidx = tqidx = 0;
 801 #ifdef TCP_OFFLOAD
 802         ofld_rqidx = ofld_tqidx = 0;
 803 #endif
 804 #ifdef DEV_NETMAP
 805         nm_rqidx = nm_tqidx = 0;
 806 #endif
 807         for_each_port(sc, i) {
 808                 struct port_info *pi = sc->port[i];
 809
 810                 if (pi == NULL)
 811                         continue;
 812
 813                 pi->first_rxq = rqidx;
 814                 pi->first_txq = tqidx;
 815                 if (is_10G_port(pi) || is_40G_port(pi)) {
 816                         pi->flags |= iaq.intr_flags_10g;
 817                         pi->nrxq = iaq.nrxq10g;
 818                         pi->ntxq = iaq.ntxq10g;
 819                 } else {
 820                         pi->flags |= iaq.intr_flags_1g;
 821                         pi->nrxq = iaq.nrxq1g;
 822                         pi->ntxq = iaq.ntxq1g;
 823                 }
 824
 825                 if (pi->ntxq > 1)
 826                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 827                 else
 828                         pi->rsrv_noflowq = 0;
 829
 830                 rqidx += pi->nrxq;
 831                 tqidx += pi->ntxq;
 832 #ifdef TCP_OFFLOAD
 833                 if (is_offload(sc)) {
 834                         pi->first_ofld_rxq = ofld_rqidx;
 835                         pi->first_ofld_txq = ofld_tqidx;
 836                         if (is_10G_port(pi) || is_40G_port(pi)) {
 837                                 pi->nofldrxq = iaq.nofldrxq10g;
 838                                 pi->nofldtxq = iaq.nofldtxq10g;
 839                         } else {
 840                                 pi->nofldrxq = iaq.nofldrxq1g;
 841                                 pi->nofldtxq = iaq.nofldtxq1g;
 842                         }
 843                         ofld_rqidx += pi->nofldrxq;
 844                         ofld_tqidx += pi->nofldtxq;
 845                 }
 846 #endif
 847 #ifdef DEV_NETMAP
 848                 pi->first_nm_rxq = nm_rqidx;
 849                 pi->first_nm_txq = nm_tqidx;
 850                 if (is_10G_port(pi) || is_40G_port(pi)) {
 851                         pi->nnmrxq = iaq.nnmrxq10g;
 852                         pi->nnmtxq = iaq.nnmtxq10g;
 853                 } else {
 854                         pi->nnmrxq = iaq.nnmrxq1g;
 855                         pi->nnmtxq = iaq.nnmtxq1g;
 856                 }
 857                 nm_rqidx += pi->nnmrxq;
 858                 nm_tqidx += pi->nnmtxq;
 859 #endif
 860         }
 861
 862         rc = setup_intr_handlers(sc);
 863         if (rc != 0) {
 864                 device_printf(dev,
 865                     "failed to setup interrupt handlers: %d\n", rc);
 866                 goto done;
 867         }
 868
 869         rc = bus_generic_attach(dev);
 870         if (rc != 0) {
 871                 device_printf(dev,
 872                     "failed to attach all child ports: %d\n", rc);
 873                 goto done;
 874         }
 875
 876         switch (sc->params.pci.speed) {
 877                 case 0x1:
 878                         pcie_ts = "2.5";
 879                         break;
 880                 case 0x2:
 881                         pcie_ts = "5.0";
 882                         break;
 883                 case 0x3:
 884                         pcie_ts = "8.0";
 885                         break;
 886                 default:
 887                         pcie_ts = "??";
 888                         break;
 889         }
 890         device_printf(dev,
 891             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 892             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 893             sc->params.nports, sc->intr_count,
 894             sc->intr_type == INTR_MSIX ? "MSI-X" :
 895             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 896             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 897
 898         t4_set_desc(sc);
 899
 900 done:
 901         if (rc != 0 && sc->cdev) {
 902                 /* cdev was created and so cxgbetool works; recover that way. */
 903                 device_printf(dev,
 904                     "error during attach, adapter is now in recovery mode.\n");
 905                 rc = 0;
 906         }
 907
 908         if (rc != 0)
 909                 t4_detach(dev);
 910         else
 911                 t4_sysctls(sc);
 912
 913         return (rc);
 914 }
 915
 916 /*
 917  * Idempotent
 918  */
 919 static int
 920 t4_detach(device_t dev)
 921 {
 922         struct adapter *sc;
 923         struct port_info *pi;
 924         int i, rc;
 925
 926         sc = device_get_softc(dev);
 927
 928         if (sc->flags & FULL_INIT_DONE)
 929                 t4_intr_disable(sc);
 930
 931         if (sc->cdev) {
 932                 destroy_dev(sc->cdev);
 933                 sc->cdev = NULL;
 934         }
 935
 936         rc = bus_generic_detach(dev);
 937         if (rc) {
 938                 device_printf(dev,
 939                     "failed to detach child devices: %d\n", rc);
 940                 return (rc);
 941         }
 942
 943         for (i = 0; i < sc->intr_count; i++)
 944                 t4_free_irq(sc, &sc->irq[i]);
 945
 946         for (i = 0; i < MAX_NPORTS; i++) {
 947                 pi = sc->port[i];
 948                 if (pi) {
 949                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 950                         if (pi->dev)
 951                                 device_delete_child(dev, pi->dev);
 952
 953                         mtx_destroy(&pi->pi_lock);
 954                         free(pi, M_CXGBE);
 955                 }
 956         }
 957
 958         if (sc->flags & FULL_INIT_DONE)
 959                 adapter_full_uninit(sc);
 960
 961         if (sc->flags & FW_OK)
 962                 t4_fw_bye(sc, sc->mbox);
 963
 964         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 965                 pci_release_msi(dev);
 966
 967         if (sc->regs_res)
 968                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 969                     sc->regs_res);
 970
 971         if (sc->udbs_res)
 972                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 973                     sc->udbs_res);
 974
 975         if (sc->msix_res)
 976                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 977                     sc->msix_res);
 978
 979         if (sc->l2t)
 980                 t4_free_l2t(sc->l2t);
 981
 982 #ifdef TCP_OFFLOAD
 983         free(sc->sge.ofld_rxq, M_CXGBE);
 984         free(sc->sge.ofld_txq, M_CXGBE);
 985 #endif
 986 #ifdef DEV_NETMAP
 987         free(sc->sge.nm_rxq, M_CXGBE);
 988         free(sc->sge.nm_txq, M_CXGBE);
 989 #endif
 990         free(sc->irq, M_CXGBE);
 991         free(sc->sge.rxq, M_CXGBE);
 992         free(sc->sge.txq, M_CXGBE);
 993         free(sc->sge.ctrlq, M_CXGBE);
 994         free(sc->sge.iqmap, M_CXGBE);
 995         free(sc->sge.eqmap, M_CXGBE);
 996         free(sc->tids.ftid_tab, M_CXGBE);
 997         t4_destroy_dma_tag(sc);
 998         if (mtx_initialized(&sc->sc_lock)) {
 999                 sx_xlock(&t4_list_lock);
1000                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1001                 sx_xunlock(&t4_list_lock);
1002                 mtx_destroy(&sc->sc_lock);
1003         }
1004
1005         if (mtx_initialized(&sc->tids.ftid_lock))
1006                 mtx_destroy(&sc->tids.ftid_lock);
1007         if (mtx_initialized(&sc->sfl_lock))
1008                 mtx_destroy(&sc->sfl_lock);
1009         if (mtx_initialized(&sc->ifp_lock))
1010                 mtx_destroy(&sc->ifp_lock);
1011
1012         bzero(sc, sizeof(*sc));
1013
1014         return (0);
1015 }
1016
1017 static int
1018 cxgbe_probe(device_t dev)
1019 {
1020         char buf[128];
1021         struct port_info *pi = device_get_softc(dev);
1022
1023         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1024         device_set_desc_copy(dev, buf);
1025
1026         return (BUS_PROBE_DEFAULT);
1027 }
1028
1029 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1030     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1031     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1032 #define T4_CAP_ENABLE (T4_CAP)
1033
1034 static int
1035 cxgbe_attach(device_t dev)
1036 {
1037         struct port_info *pi = device_get_softc(dev);
1038         struct ifnet *ifp;
1039         char *s;
1040         int n, o;
1041
1042         /* Allocate an ifnet and set it up */
1043         ifp = if_alloc(IFT_ETHER);
1044         if (ifp == NULL) {
1045                 device_printf(dev, "Cannot allocate ifnet\n");
1046                 return (ENOMEM);
1047         }
1048         pi->ifp = ifp;
1049         ifp->if_softc = pi;
1050
1051         callout_init(&pi->tick, CALLOUT_MPSAFE);
1052
1053         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1054         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1055
1056         ifp->if_init = cxgbe_init;
1057         ifp->if_ioctl = cxgbe_ioctl;
1058         ifp->if_transmit = cxgbe_transmit;
1059         ifp->if_qflush = cxgbe_qflush;
1060
1061         ifp->if_capabilities = T4_CAP;
1062 #ifdef TCP_OFFLOAD
1063         if (is_offload(pi->adapter))
1064                 ifp->if_capabilities |= IFCAP_TOE;
1065 #endif
1066         ifp->if_capenable = T4_CAP_ENABLE;
1067         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1068             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1069
1070         /* Initialize ifmedia for this port */
1071         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1072             cxgbe_media_status);
1073         build_medialist(pi, &pi->media);
1074
1075         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1076             EVENTHANDLER_PRI_ANY);
1077
1078         ether_ifattach(ifp, pi->hw_addr);
1079
1080         n = 128;
1081         s = malloc(n, M_CXGBE, M_WAITOK);
1082         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1083         MPASS(n > o);
1084 #ifdef TCP_OFFLOAD
1085         if (is_offload(pi->adapter)) {
1086                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1087                     pi->nofldtxq, pi->nofldrxq);
1088                 MPASS(n > o);
1089         }
1090 #endif
1091 #ifdef DEV_NETMAP
1092         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1093             pi->nnmrxq);
1094         MPASS(n > o);
1095 #endif
1096         device_printf(dev, "%s\n", s);
1097         free(s, M_CXGBE);
1098
1099 #ifdef DEV_NETMAP
1100         /* nm_media handled here to keep implementation private to this file */
1101         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1102             cxgbe_media_status);
1103         build_medialist(pi, &pi->nm_media);
1104         create_netmap_ifnet(pi);        /* logs errors it something fails */
1105 #endif
1106         cxgbe_sysctls(pi);
1107
1108         return (0);
1109 }
1110
1111 static int
1112 cxgbe_detach(device_t dev)
1113 {
1114         struct port_info *pi = device_get_softc(dev);
1115         struct adapter *sc = pi->adapter;
1116         struct ifnet *ifp = pi->ifp;
1117
1118         /* Tell if_ioctl and if_init that the port is going away */
1119         ADAPTER_LOCK(sc);
1120         SET_DOOMED(pi);
1121         wakeup(&sc->flags);
1122         while (IS_BUSY(sc))
1123                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1124         SET_BUSY(sc);
1125 #ifdef INVARIANTS
1126         sc->last_op = "t4detach";
1127         sc->last_op_thr = curthread;
1128 #endif
1129         ADAPTER_UNLOCK(sc);
1130
1131         if (pi->flags & HAS_TRACEQ) {
1132                 sc->traceq = -1;        /* cloner should not create ifnet */
1133                 t4_tracer_port_detach(sc);
1134         }
1135
1136         if (pi->vlan_c)
1137                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1138
1139         PORT_LOCK(pi);
1140         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1141         callout_stop(&pi->tick);
1142         PORT_UNLOCK(pi);
1143         callout_drain(&pi->tick);
1144
1145         /* Let detach proceed even if these fail. */
1146         cxgbe_uninit_synchronized(pi);
1147         port_full_uninit(pi);
1148
1149         ifmedia_removeall(&pi->media);
1150         ether_ifdetach(pi->ifp);
1151         if_free(pi->ifp);
1152
1153 #ifdef DEV_NETMAP
1154         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1155         destroy_netmap_ifnet(pi);
1156 #endif
1157
1158         ADAPTER_LOCK(sc);
1159         CLR_BUSY(sc);
1160         wakeup(&sc->flags);
1161         ADAPTER_UNLOCK(sc);
1162
1163         return (0);
1164 }
1165
1166 static void
1167 cxgbe_init(void *arg)
1168 {
1169         struct port_info *pi = arg;
1170         struct adapter *sc = pi->adapter;
1171
1172         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1173                 return;
1174         cxgbe_init_synchronized(pi);
1175         end_synchronized_op(sc, 0);
1176 }
1177
1178 static int
1179 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1180 {
1181         int rc = 0, mtu, flags;
1182         struct port_info *pi = ifp->if_softc;
1183         struct adapter *sc = pi->adapter;
1184         struct ifreq *ifr = (struct ifreq *)data;
1185         uint32_t mask;
1186
1187         switch (cmd) {
1188         case SIOCSIFMTU:
1189                 mtu = ifr->ifr_mtu;
1190                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1191                         return (EINVAL);
1192
1193                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1194                 if (rc)
1195                         return (rc);
1196                 ifp->if_mtu = mtu;
1197                 if (pi->flags & PORT_INIT_DONE) {
1198                         t4_update_fl_bufsize(ifp);
1199                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1200                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1201                 }
1202                 end_synchronized_op(sc, 0);
1203                 break;
1204
1205         case SIOCSIFFLAGS:
1206                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
1207                 if (rc)
1208                         return (rc);
1209
1210                 if (ifp->if_flags & IFF_UP) {
1211                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1212                                 flags = pi->if_flags;
1213                                 if ((ifp->if_flags ^ flags) &
1214                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1215                                         rc = update_mac_settings(ifp,
1216                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1217                                 }
1218                         } else
1219                                 rc = cxgbe_init_synchronized(pi);
1220                         pi->if_flags = ifp->if_flags;
1221                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1222                         rc = cxgbe_uninit_synchronized(pi);
1223                 end_synchronized_op(sc, 0);
1224                 break;
1225
1226         case SIOCADDMULTI:
1227         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1228                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1229                 if (rc)
1230                         return (rc);
1231                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1232                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1233                 end_synchronized_op(sc, LOCK_HELD);
1234                 break;
1235
1236         case SIOCSIFCAP:
1237                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1238                 if (rc)
1239                         return (rc);
1240
1241                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1242                 if (mask & IFCAP_TXCSUM) {
1243                         ifp->if_capenable ^= IFCAP_TXCSUM;
1244                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1245
1246                         if (IFCAP_TSO4 & ifp->if_capenable &&
1247                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1248                                 ifp->if_capenable &= ~IFCAP_TSO4;
1249                                 if_printf(ifp,
1250                                     "tso4 disabled due to -txcsum.\n");
1251                         }
1252                 }
1253                 if (mask & IFCAP_TXCSUM_IPV6) {
1254                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1255                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1256
1257                         if (IFCAP_TSO6 & ifp->if_capenable &&
1258                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1259                                 ifp->if_capenable &= ~IFCAP_TSO6;
1260                                 if_printf(ifp,
1261                                     "tso6 disabled due to -txcsum6.\n");
1262                         }
1263                 }
1264                 if (mask & IFCAP_RXCSUM)
1265                         ifp->if_capenable ^= IFCAP_RXCSUM;
1266                 if (mask & IFCAP_RXCSUM_IPV6)
1267                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1268
1269                 /*
1270                  * Note that we leave CSUM_TSO alone (it is always set).  The
1271                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1272                  * sending a TSO request our way, so it's sufficient to toggle
1273                  * IFCAP_TSOx only.
1274                  */
1275                 if (mask & IFCAP_TSO4) {
1276                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1277                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1278                                 if_printf(ifp, "enable txcsum first.\n");
1279                                 rc = EAGAIN;
1280                                 goto fail;
1281                         }
1282                         ifp->if_capenable ^= IFCAP_TSO4;
1283                 }
1284                 if (mask & IFCAP_TSO6) {
1285                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1286                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1287                                 if_printf(ifp, "enable txcsum6 first.\n");
1288                                 rc = EAGAIN;
1289                                 goto fail;
1290                         }
1291                         ifp->if_capenable ^= IFCAP_TSO6;
1292                 }
1293                 if (mask & IFCAP_LRO) {
1294 #if defined(INET) || defined(INET6)
1295                         int i;
1296                         struct sge_rxq *rxq;
1297
1298                         ifp->if_capenable ^= IFCAP_LRO;
1299                         for_each_rxq(pi, i, rxq) {
1300                                 if (ifp->if_capenable & IFCAP_LRO)
1301                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1302                                 else
1303                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1304                         }
1305 #endif
1306                 }
1307 #ifdef TCP_OFFLOAD
1308                 if (mask & IFCAP_TOE) {
1309                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1310
1311                         rc = toe_capability(pi, enable);
1312                         if (rc != 0)
1313                                 goto fail;
1314
1315                         ifp->if_capenable ^= mask;
1316                 }
1317 #endif
1318                 if (mask & IFCAP_VLAN_HWTAGGING) {
1319                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1320                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1321                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1322                 }
1323                 if (mask & IFCAP_VLAN_MTU) {
1324                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1325
1326                         /* Need to find out how to disable auto-mtu-inflation */
1327                 }
1328                 if (mask & IFCAP_VLAN_HWTSO)
1329                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1330                 if (mask & IFCAP_VLAN_HWCSUM)
1331                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1332
1333 #ifdef VLAN_CAPABILITIES
1334                 VLAN_CAPABILITIES(ifp);
1335 #endif
1336 fail:
1337                 end_synchronized_op(sc, 0);
1338                 break;
1339
1340         case SIOCSIFMEDIA:
1341         case SIOCGIFMEDIA:
1342                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1343                 break;
1344
1345         default:
1346                 rc = ether_ioctl(ifp, cmd, data);
1347         }
1348
1349         return (rc);
1350 }
1351
1352 static int
1353 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1354 {
1355         struct port_info *pi = ifp->if_softc;
1356         struct adapter *sc = pi->adapter;
1357         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1358         struct buf_ring *br;
1359         int rc;
1360
1361         M_ASSERTPKTHDR(m);
1362
1363         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1364                 m_freem(m);
1365                 return (ENETDOWN);
1366         }
1367
1368         if (m->m_flags & M_FLOWID)
1369                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1370                     + pi->rsrv_noflowq);
1371         br = txq->br;
1372
1373         if (TXQ_TRYLOCK(txq) == 0) {
1374                 struct sge_eq *eq = &txq->eq;
1375
1376                 /*
1377                  * It is possible that t4_eth_tx finishes up and releases the
1378                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1379                  * need to make sure that this mbuf doesn't just sit there in
1380                  * the drbr.
1381                  */
1382
1383                 rc = drbr_enqueue(ifp, br, m);
1384                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1385                     !(eq->flags & EQ_DOOMED))
1386                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1387                 return (rc);
1388         }
1389
1390         /*
1391          * txq->m is the mbuf that is held up due to a temporary shortage of
1392          * resources and it should be put on the wire first.  Then what's in
1393          * drbr and finally the mbuf that was just passed in to us.
1394          *
1395          * Return code should indicate the fate of the mbuf that was passed in
1396          * this time.
1397          */
1398
1399         TXQ_LOCK_ASSERT_OWNED(txq);
1400         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1401
1402                 /* Queued for transmission. */
1403
1404                 rc = drbr_enqueue(ifp, br, m);
1405                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1406                 (void) t4_eth_tx(ifp, txq, m);
1407                 TXQ_UNLOCK(txq);
1408                 return (rc);
1409         }
1410
1411         /* Direct transmission. */
1412         rc = t4_eth_tx(ifp, txq, m);
1413         if (rc != 0 && txq->m)
1414                 rc = 0; /* held, will be transmitted soon (hopefully) */
1415
1416         TXQ_UNLOCK(txq);
1417         return (rc);
1418 }
1419
1420 static void
1421 cxgbe_qflush(struct ifnet *ifp)
1422 {
1423         struct port_info *pi = ifp->if_softc;
1424         struct sge_txq *txq;
1425         int i;
1426         struct mbuf *m;
1427
1428         /* queues do not exist if !PORT_INIT_DONE. */
1429         if (pi->flags & PORT_INIT_DONE) {
1430                 for_each_txq(pi, i, txq) {
1431                         TXQ_LOCK(txq);
1432                         m_freem(txq->m);
1433                         txq->m = NULL;
1434                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1435                                 m_freem(m);
1436                         TXQ_UNLOCK(txq);
1437                 }
1438         }
1439         if_qflush(ifp);
1440 }
1441
1442 static int
1443 cxgbe_media_change(struct ifnet *ifp)
1444 {
1445         struct port_info *pi = ifp->if_softc;
1446
1447         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1448
1449         return (EOPNOTSUPP);
1450 }
1451
1452 static void
1453 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1454 {
1455         struct port_info *pi = ifp->if_softc;
1456         struct ifmedia *media = NULL;
1457         struct ifmedia_entry *cur;
1458         int speed = pi->link_cfg.speed;
1459         int data = (pi->port_type << 8) | pi->mod_type;
1460
1461         if (ifp == pi->ifp)
1462                 media = &pi->media;
1463 #ifdef DEV_NETMAP
1464         else if (ifp == pi->nm_ifp)
1465                 media = &pi->nm_media;
1466 #endif
1467         MPASS(media != NULL);
1468
1469         cur = media->ifm_cur;
1470         if (cur->ifm_data != data) {
1471                 build_medialist(pi, media);
1472                 cur = media->ifm_cur;
1473         }
1474
1475         ifmr->ifm_status = IFM_AVALID;
1476         if (!pi->link_cfg.link_ok)
1477                 return;
1478
1479         ifmr->ifm_status |= IFM_ACTIVE;
1480
1481         /* active and current will differ iff current media is autoselect. */
1482         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1483                 return;
1484
1485         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1486         if (speed == SPEED_10000)
1487                 ifmr->ifm_active |= IFM_10G_T;
1488         else if (speed == SPEED_1000)
1489                 ifmr->ifm_active |= IFM_1000_T;
1490         else if (speed == SPEED_100)
1491                 ifmr->ifm_active |= IFM_100_TX;
1492         else if (speed == SPEED_10)
1493                 ifmr->ifm_active |= IFM_10_T;
1494         else
1495                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1496                             speed));
1497 }
1498
1499 void
1500 t4_fatal_err(struct adapter *sc)
1501 {
1502         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1503         t4_intr_disable(sc);
1504         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1505             device_get_nameunit(sc->dev));
1506 }
1507
1508 static int
1509 map_bars_0_and_4(struct adapter *sc)
1510 {
1511         sc->regs_rid = PCIR_BAR(0);
1512         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1513             &sc->regs_rid, RF_ACTIVE);
1514         if (sc->regs_res == NULL) {
1515                 device_printf(sc->dev, "cannot map registers.\n");
1516                 return (ENXIO);
1517         }
1518         sc->bt = rman_get_bustag(sc->regs_res);
1519         sc->bh = rman_get_bushandle(sc->regs_res);
1520         sc->mmio_len = rman_get_size(sc->regs_res);
1521         setbit(&sc->doorbells, DOORBELL_KDB);
1522
1523         sc->msix_rid = PCIR_BAR(4);
1524         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1525             &sc->msix_rid, RF_ACTIVE);
1526         if (sc->msix_res == NULL) {
1527                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1528                 return (ENXIO);
1529         }
1530
1531         return (0);
1532 }
1533
1534 static int
1535 map_bar_2(struct adapter *sc)
1536 {
1537
1538         /*
1539          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1540          * to map it if RDMA is disabled.
1541          */
1542         if (is_t4(sc) && sc->rdmacaps == 0)
1543                 return (0);
1544
1545         sc->udbs_rid = PCIR_BAR(2);
1546         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1547             &sc->udbs_rid, RF_ACTIVE);
1548         if (sc->udbs_res == NULL) {
1549                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1550                 return (ENXIO);
1551         }
1552         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1553
1554         if (is_t5(sc)) {
1555                 setbit(&sc->doorbells, DOORBELL_UDB);
1556 #if defined(__i386__) || defined(__amd64__)
1557                 if (t5_write_combine) {
1558                         int rc;
1559
1560                         /*
1561                          * Enable write combining on BAR2.  This is the
1562                          * userspace doorbell BAR and is split into 128B
1563                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1564                          * with an egress queue.  The first 64B has the doorbell
1565                          * and the second 64B can be used to submit a tx work
1566                          * request with an implicit doorbell.
1567                          */
1568
1569                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1570                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1571                         if (rc == 0) {
1572                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1573                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1574                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1575                         } else {
1576                                 device_printf(sc->dev,
1577                                     "couldn't enable write combining: %d\n",
1578                                     rc);
1579                         }
1580
1581                         t4_write_reg(sc, A_SGE_STAT_CFG,
1582                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1583                 }
1584 #endif
1585         }
1586
1587         return (0);
1588 }
1589
1590 static const struct memwin t4_memwin[] = {
1591         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1592         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1593         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1594 };
1595
1596 static const struct memwin t5_memwin[] = {
1597         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1598         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1599         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1600 };
1601
1602 static void
1603 setup_memwin(struct adapter *sc)
1604 {
1605         const struct memwin *mw;
1606         int i, n;
1607         uint32_t bar0;
1608
1609         if (is_t4(sc)) {
1610                 /*
1611                  * Read low 32b of bar0 indirectly via the hardware backdoor
1612                  * mechanism.  Works from within PCI passthrough environments
1613                  * too, where rman_get_start() can return a different value.  We
1614                  * need to program the T4 memory window decoders with the actual
1615                  * addresses that will be coming across the PCIe link.
1616                  */
1617                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1618                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1619
1620                 mw = &t4_memwin[0];
1621                 n = nitems(t4_memwin);
1622         } else {
1623                 /* T5 uses the relative offset inside the PCIe BAR */
1624                 bar0 = 0;
1625
1626                 mw = &t5_memwin[0];
1627                 n = nitems(t5_memwin);
1628         }
1629
1630         for (i = 0; i < n; i++, mw++) {
1631                 t4_write_reg(sc,
1632                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1633                     (mw->base + bar0) | V_BIR(0) |
1634                     V_WINDOW(ilog2(mw->aperture) - 10));
1635         }
1636
1637         /* flush */
1638         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1639 }
1640
1641 /*
1642  * Verify that the memory range specified by the addr/len pair is valid and lies
1643  * entirely within a single region (EDCx or MCx).
1644  */
1645 static int
1646 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1647 {
1648         uint32_t em, addr_len, maddr, mlen;
1649
1650         /* Memory can only be accessed in naturally aligned 4 byte units */
1651         if (addr & 3 || len & 3 || len == 0)
1652                 return (EINVAL);
1653
1654         /* Enabled memories */
1655         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1656         if (em & F_EDRAM0_ENABLE) {
1657                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1658                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1659                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1660                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1661                     addr + len <= maddr + mlen)
1662                         return (0);
1663         }
1664         if (em & F_EDRAM1_ENABLE) {
1665                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1666                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1667                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1668                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1669                     addr + len <= maddr + mlen)
1670                         return (0);
1671         }
1672         if (em & F_EXT_MEM_ENABLE) {
1673                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1674                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1675                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1676                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1677                     addr + len <= maddr + mlen)
1678                         return (0);
1679         }
1680         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1681                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1682                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1683                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1684                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1685                     addr + len <= maddr + mlen)
1686                         return (0);
1687         }
1688
1689         return (EFAULT);
1690 }
1691
1692 static int
1693 fwmtype_to_hwmtype(int mtype)
1694 {
1695
1696         switch (mtype) {
1697         case FW_MEMTYPE_EDC0:
1698                 return (MEM_EDC0);
1699         case FW_MEMTYPE_EDC1:
1700                 return (MEM_EDC1);
1701         case FW_MEMTYPE_EXTMEM:
1702                 return (MEM_MC0);
1703         case FW_MEMTYPE_EXTMEM1:
1704                 return (MEM_MC1);
1705         default:
1706                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1707         }
1708 }
1709
1710 /*
1711  * Verify that the memory range specified by the memtype/offset/len pair is
1712  * valid and lies entirely within the memtype specified.  The global address of
1713  * the start of the range is returned in addr.
1714  */
1715 static int
1716 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1717     uint32_t *addr)
1718 {
1719         uint32_t em, addr_len, maddr, mlen;
1720
1721         /* Memory can only be accessed in naturally aligned 4 byte units */
1722         if (off & 3 || len & 3 || len == 0)
1723                 return (EINVAL);
1724
1725         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1726         switch (fwmtype_to_hwmtype(mtype)) {
1727         case MEM_EDC0:
1728                 if (!(em & F_EDRAM0_ENABLE))
1729                         return (EINVAL);
1730                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1731                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1732                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1733                 break;
1734         case MEM_EDC1:
1735                 if (!(em & F_EDRAM1_ENABLE))
1736                         return (EINVAL);
1737                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1738                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1739                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1740                 break;
1741         case MEM_MC:
1742                 if (!(em & F_EXT_MEM_ENABLE))
1743                         return (EINVAL);
1744                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1745                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1746                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1747                 break;
1748         case MEM_MC1:
1749                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1750                         return (EINVAL);
1751                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1752                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1753                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1754                 break;
1755         default:
1756                 return (EINVAL);
1757         }
1758
1759         if (mlen > 0 && off < mlen && off + len <= mlen) {
1760                 *addr = maddr + off;    /* global address */
1761                 return (0);
1762         }
1763
1764         return (EFAULT);
1765 }
1766
1767 static void
1768 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1769 {
1770         const struct memwin *mw;
1771
1772         if (is_t4(sc)) {
1773                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1774                     ("%s: incorrect memwin# (%d)", __func__, win));
1775                 mw = &t4_memwin[win];
1776         } else {
1777                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1778                     ("%s: incorrect memwin# (%d)", __func__, win));
1779                 mw = &t5_memwin[win];
1780         }
1781
1782         if (base != NULL)
1783                 *base = mw->base;
1784         if (aperture != NULL)
1785                 *aperture = mw->aperture;
1786 }
1787
1788 /*
1789  * Positions the memory window such that it can be used to access the specified
1790  * address in the chip's address space.  The return value is the offset of addr
1791  * from the start of the window.
1792  */
1793 static uint32_t
1794 position_memwin(struct adapter *sc, int n, uint32_t addr)
1795 {
1796         uint32_t start, pf;
1797         uint32_t reg;
1798
1799         KASSERT(n >= 0 && n <= 3,
1800             ("%s: invalid window %d.", __func__, n));
1801         KASSERT((addr & 3) == 0,
1802             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1803
1804         if (is_t4(sc)) {
1805                 pf = 0;
1806                 start = addr & ~0xf;    /* start must be 16B aligned */
1807         } else {
1808                 pf = V_PFNUM(sc->pf);
1809                 start = addr & ~0x7f;   /* start must be 128B aligned */
1810         }
1811         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1812
1813         t4_write_reg(sc, reg, start | pf);
1814         t4_read_reg(sc, reg);
1815
1816         return (addr - start);
1817 }
1818
1819 static int
1820 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1821     struct intrs_and_queues *iaq)
1822 {
1823         int rc, itype, navail, nrxq10g, nrxq1g, n;
1824         int nofldrxq10g = 0, nofldrxq1g = 0;
1825         int nnmrxq10g = 0, nnmrxq1g = 0;
1826
1827         bzero(iaq, sizeof(*iaq));
1828
1829         iaq->ntxq10g = t4_ntxq10g;
1830         iaq->ntxq1g = t4_ntxq1g;
1831         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1832         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1833         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1834 #ifdef TCP_OFFLOAD
1835         if (is_offload(sc)) {
1836                 iaq->nofldtxq10g = t4_nofldtxq10g;
1837                 iaq->nofldtxq1g = t4_nofldtxq1g;
1838                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1839                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1840         }
1841 #endif
1842 #ifdef DEV_NETMAP
1843         iaq->nnmtxq10g = t4_nnmtxq10g;
1844         iaq->nnmtxq1g = t4_nnmtxq1g;
1845         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1846         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1847 #endif
1848
1849         for (itype = INTR_MSIX; itype; itype >>= 1) {
1850
1851                 if ((itype & t4_intr_types) == 0)
1852                         continue;       /* not allowed */
1853
1854                 if (itype == INTR_MSIX)
1855                         navail = pci_msix_count(sc->dev);
1856                 else if (itype == INTR_MSI)
1857                         navail = pci_msi_count(sc->dev);
1858                 else
1859                         navail = 1;
1860 restart:
1861                 if (navail == 0)
1862                         continue;
1863
1864                 iaq->intr_type = itype;
1865                 iaq->intr_flags_10g = 0;
1866                 iaq->intr_flags_1g = 0;
1867
1868                 /*
1869                  * Best option: an interrupt vector for errors, one for the
1870                  * firmware event queue, and one for every rxq (NIC, TOE, and
1871                  * netmap).
1872                  */
1873                 iaq->nirq = T4_EXTRA_INTR;
1874                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1875                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1876                 if (iaq->nirq <= navail &&
1877                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1878                         iaq->intr_flags_10g = INTR_ALL;
1879                         iaq->intr_flags_1g = INTR_ALL;
1880                         goto allocate;
1881                 }
1882
1883                 /*
1884                  * Second best option: a vector for errors, one for the firmware
1885                  * event queue, and vectors for either all the NIC rx queues or
1886                  * all the TOE rx queues.  The queues that don't get vectors
1887                  * will forward their interrupts to those that do.
1888                  *
1889                  * Note: netmap rx queues cannot be created early and so they
1890                  * can't be setup to receive forwarded interrupts for others.
1891                  */
1892                 iaq->nirq = T4_EXTRA_INTR;
1893                 if (nrxq10g >= nofldrxq10g) {
1894                         iaq->intr_flags_10g = INTR_RXQ;
1895                         iaq->nirq += n10g * nrxq10g;
1896 #ifdef DEV_NETMAP
1897                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1898 #endif
1899                 } else {
1900                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1901                         iaq->nirq += n10g * nofldrxq10g;
1902 #ifdef DEV_NETMAP
1903                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1904 #endif
1905                 }
1906                 if (nrxq1g >= nofldrxq1g) {
1907                         iaq->intr_flags_1g = INTR_RXQ;
1908                         iaq->nirq += n1g * nrxq1g;
1909 #ifdef DEV_NETMAP
1910                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1911 #endif
1912                 } else {
1913                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1914                         iaq->nirq += n1g * nofldrxq1g;
1915 #ifdef DEV_NETMAP
1916                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1917 #endif
1918                 }
1919                 if (iaq->nirq <= navail &&
1920                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1921                         goto allocate;
1922
1923                 /*
1924                  * Next best option: an interrupt vector for errors, one for the
1925                  * firmware event queue, and at least one per port.  At this
1926                  * point we know we'll have to downsize nrxq and/or nofldrxq
1927                  * and/or nnmrxq to fit what's available to us.
1928                  */
1929                 iaq->nirq = T4_EXTRA_INTR;
1930                 iaq->nirq += n10g + n1g;
1931                 if (iaq->nirq <= navail) {
1932                         int leftover = navail - iaq->nirq;
1933
1934                         if (n10g > 0) {
1935                                 int target = max(nrxq10g, nofldrxq10g);
1936
1937                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1938                                     INTR_RXQ : INTR_OFLD_RXQ;
1939
1940                                 n = 1;
1941                                 while (n < target && leftover >= n10g) {
1942                                         leftover -= n10g;
1943                                         iaq->nirq += n10g;
1944                                         n++;
1945                                 }
1946                                 iaq->nrxq10g = min(n, nrxq10g);
1947 #ifdef TCP_OFFLOAD
1948                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1949 #endif
1950 #ifdef DEV_NETMAP
1951                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1952 #endif
1953                         }
1954
1955                         if (n1g > 0) {
1956                                 int target = max(nrxq1g, nofldrxq1g);
1957
1958                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1959                                     INTR_RXQ : INTR_OFLD_RXQ;
1960
1961                                 n = 1;
1962                                 while (n < target && leftover >= n1g) {
1963                                         leftover -= n1g;
1964                                         iaq->nirq += n1g;
1965                                         n++;
1966                                 }
1967                                 iaq->nrxq1g = min(n, nrxq1g);
1968 #ifdef TCP_OFFLOAD
1969                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1970 #endif
1971 #ifdef DEV_NETMAP
1972                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
1973 #endif
1974                         }
1975
1976                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1977                                 goto allocate;
1978                 }
1979
1980                 /*
1981                  * Least desirable option: one interrupt vector for everything.
1982                  */
1983                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1984                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
1985 #ifdef TCP_OFFLOAD
1986                 if (is_offload(sc))
1987                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1988 #endif
1989 #ifdef DEV_NETMAP
1990                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
1991 #endif
1992
1993 allocate:
1994                 navail = iaq->nirq;
1995                 rc = 0;
1996                 if (itype == INTR_MSIX)
1997                         rc = pci_alloc_msix(sc->dev, &navail);
1998                 else if (itype == INTR_MSI)
1999                         rc = pci_alloc_msi(sc->dev, &navail);
2000
2001                 if (rc == 0) {
2002                         if (navail == iaq->nirq)
2003                                 return (0);
2004
2005                         /*
2006                          * Didn't get the number requested.  Use whatever number
2007                          * the kernel is willing to allocate (it's in navail).
2008                          */
2009                         device_printf(sc->dev, "fewer vectors than requested, "
2010                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2011                             itype, iaq->nirq, navail);
2012                         pci_release_msi(sc->dev);
2013                         goto restart;
2014                 }
2015
2016                 device_printf(sc->dev,
2017                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2018                     itype, rc, iaq->nirq, navail);
2019         }
2020
2021         device_printf(sc->dev,
2022             "failed to find a usable interrupt type.  "
2023             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2024             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2025
2026         return (ENXIO);
2027 }
2028
2029 #define FW_VERSION(chip) ( \
2030     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2031     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2032     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2033     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2034 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2035
2036 struct fw_info {
2037         uint8_t chip;
2038         char *kld_name;
2039         char *fw_mod_name;
2040         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2041 } fw_info[] = {
2042         {
2043                 .chip = CHELSIO_T4,
2044                 .kld_name = "t4fw_cfg",
2045                 .fw_mod_name = "t4fw",
2046                 .fw_hdr = {
2047                         .chip = FW_HDR_CHIP_T4,
2048                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2049                         .intfver_nic = FW_INTFVER(T4, NIC),
2050                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2051                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2052                         .intfver_ri = FW_INTFVER(T4, RI),
2053                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2054                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2055                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2056                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2057                 },
2058         }, {
2059                 .chip = CHELSIO_T5,
2060                 .kld_name = "t5fw_cfg",
2061                 .fw_mod_name = "t5fw",
2062                 .fw_hdr = {
2063                         .chip = FW_HDR_CHIP_T5,
2064                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2065                         .intfver_nic = FW_INTFVER(T5, NIC),
2066                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2067                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2068                         .intfver_ri = FW_INTFVER(T5, RI),
2069                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2070                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2071                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2072                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2073                 },
2074         }
2075 };
2076
2077 static struct fw_info *
2078 find_fw_info(int chip)
2079 {
2080         int i;
2081
2082         for (i = 0; i < nitems(fw_info); i++) {
2083                 if (fw_info[i].chip == chip)
2084                         return (&fw_info[i]);
2085         }
2086         return (NULL);
2087 }
2088
2089 /*
2090  * Is the given firmware API compatible with the one the driver was compiled
2091  * with?
2092  */
2093 static int
2094 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2095 {
2096
2097         /* short circuit if it's the exact same firmware version */
2098         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2099                 return (1);
2100
2101         /*
2102          * XXX: Is this too conservative?  Perhaps I should limit this to the
2103          * features that are supported in the driver.
2104          */
2105 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2106         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2107             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2108             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2109                 return (1);
2110 #undef SAME_INTF
2111
2112         return (0);
2113 }
2114
2115 /*
2116  * The firmware in the KLD is usable, but should it be installed?  This routine
2117  * explains itself in detail if it indicates the KLD firmware should be
2118  * installed.
2119  */
2120 static int
2121 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2122 {
2123         const char *reason;
2124
2125         if (!card_fw_usable) {
2126                 reason = "incompatible or unusable";
2127                 goto install;
2128         }
2129
2130         if (k > c) {
2131                 reason = "older than the version bundled with this driver";
2132                 goto install;
2133         }
2134
2135         if (t4_fw_install == 2 && k != c) {
2136                 reason = "different than the version bundled with this driver";
2137                 goto install;
2138         }
2139
2140         return (0);
2141
2142 install:
2143         if (t4_fw_install == 0) {
2144                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2145                     "but the driver is prohibited from installing a different "
2146                     "firmware on the card.\n",
2147                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2148                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2149
2150                 return (0);
2151         }
2152
2153         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2154             "installing firmware %u.%u.%u.%u on card.\n",
2155             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2156             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2157             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2158             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2159
2160         return (1);
2161 }
2162 /*
2163  * Establish contact with the firmware and determine if we are the master driver
2164  * or not, and whether we are responsible for chip initialization.
2165  */
2166 static int
2167 prep_firmware(struct adapter *sc)
2168 {
2169         const struct firmware *fw = NULL, *default_cfg;
2170         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2171         enum dev_state state;
2172         struct fw_info *fw_info;
2173         struct fw_hdr *card_fw;         /* fw on the card */
2174         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2175         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2176                                            against */
2177
2178         /* Contact firmware. */
2179         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2180         if (rc < 0 || state == DEV_STATE_ERR) {
2181                 rc = -rc;
2182                 device_printf(sc->dev,
2183                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2184                 return (rc);
2185         }
2186         pf = rc;
2187         if (pf == sc->mbox)
2188                 sc->flags |= MASTER_PF;
2189         else if (state == DEV_STATE_UNINIT) {
2190                 /*
2191                  * We didn't get to be the master so we definitely won't be
2192                  * configuring the chip.  It's a bug if someone else hasn't
2193                  * configured it already.
2194                  */
2195                 device_printf(sc->dev, "couldn't be master(%d), "
2196                     "device not already initialized either(%d).\n", rc, state);
2197                 return (EDOOFUS);
2198         }
2199
2200         /* This is the firmware whose headers the driver was compiled against */
2201         fw_info = find_fw_info(chip_id(sc));
2202         if (fw_info == NULL) {
2203                 device_printf(sc->dev,
2204                     "unable to look up firmware information for chip %d.\n",
2205                     chip_id(sc));
2206                 return (EINVAL);
2207         }
2208         drv_fw = &fw_info->fw_hdr;
2209
2210         /*
2211          * The firmware KLD contains many modules.  The KLD name is also the
2212          * name of the module that contains the default config file.
2213          */
2214         default_cfg = firmware_get(fw_info->kld_name);
2215
2216         /* Read the header of the firmware on the card */
2217         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2218         rc = -t4_read_flash(sc, FLASH_FW_START,
2219             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2220         if (rc == 0)
2221                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2222         else {
2223                 device_printf(sc->dev,
2224                     "Unable to read card's firmware header: %d\n", rc);
2225                 card_fw_usable = 0;
2226         }
2227
2228         /* This is the firmware in the KLD */
2229         fw = firmware_get(fw_info->fw_mod_name);
2230         if (fw != NULL) {
2231                 kld_fw = (const void *)fw->data;
2232                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2233         } else {
2234                 kld_fw = NULL;
2235                 kld_fw_usable = 0;
2236         }
2237
2238         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2239             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2240                 /*
2241                  * Common case: the firmware on the card is an exact match and
2242                  * the KLD is an exact match too, or the KLD is
2243                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2244                  * here -- use cxgbetool loadfw if you want to reinstall the
2245                  * same firmware as the one on the card.
2246                  */
2247         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2248             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2249             be32toh(card_fw->fw_ver))) {
2250
2251                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2252                 if (rc != 0) {
2253                         device_printf(sc->dev,
2254                             "failed to install firmware: %d\n", rc);
2255                         goto done;
2256                 }
2257
2258                 /* Installed successfully, update the cached header too. */
2259                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2260                 card_fw_usable = 1;
2261                 need_fw_reset = 0;      /* already reset as part of load_fw */
2262         }
2263
2264         if (!card_fw_usable) {
2265                 uint32_t d, c, k;
2266
2267                 d = ntohl(drv_fw->fw_ver);
2268                 c = ntohl(card_fw->fw_ver);
2269                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2270
2271                 device_printf(sc->dev, "Cannot find a usable firmware: "
2272                     "fw_install %d, chip state %d, "
2273                     "driver compiled with %d.%d.%d.%d, "
2274                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2275                     t4_fw_install, state,
2276                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2277                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2278                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2279                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2280                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2281                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2282                 rc = EINVAL;
2283                 goto done;
2284         }
2285
2286         /* We're using whatever's on the card and it's known to be good. */
2287         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2288         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2289             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2290             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2291             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2292             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2293         t4_get_tp_version(sc, &sc->params.tp_vers);
2294
2295         /* Reset device */
2296         if (need_fw_reset &&
2297             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2298                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2299                 if (rc != ETIMEDOUT && rc != EIO)
2300                         t4_fw_bye(sc, sc->mbox);
2301                 goto done;
2302         }
2303         sc->flags |= FW_OK;
2304
2305         rc = get_params__pre_init(sc);
2306         if (rc != 0)
2307                 goto done; /* error message displayed already */
2308
2309         /* Partition adapter resources as specified in the config file. */
2310         if (state == DEV_STATE_UNINIT) {
2311
2312                 KASSERT(sc->flags & MASTER_PF,
2313                     ("%s: trying to change chip settings when not master.",
2314                     __func__));
2315
2316                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2317                 if (rc != 0)
2318                         goto done;      /* error message displayed already */
2319
2320                 t4_tweak_chip_settings(sc);
2321
2322                 /* get basic stuff going */
2323                 rc = -t4_fw_initialize(sc, sc->mbox);
2324                 if (rc != 0) {
2325                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2326                         goto done;
2327                 }
2328         } else {
2329                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2330                 sc->cfcsum = 0;
2331         }
2332
2333 done:
2334         free(card_fw, M_CXGBE);
2335         if (fw != NULL)
2336                 firmware_put(fw, FIRMWARE_UNLOAD);
2337         if (default_cfg != NULL)
2338                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2339
2340         return (rc);
2341 }
2342
2343 #define FW_PARAM_DEV(param) \
2344         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2345          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2346 #define FW_PARAM_PFVF(param) \
2347         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2348          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2349
2350 /*
2351  * Partition chip resources for use between various PFs, VFs, etc.
2352  */
2353 static int
2354 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2355     const char *name_prefix)
2356 {
2357         const struct firmware *cfg = NULL;
2358         int rc = 0;
2359         struct fw_caps_config_cmd caps;
2360         uint32_t mtype, moff, finicsum, cfcsum;
2361
2362         /*
2363          * Figure out what configuration file to use.  Pick the default config
2364          * file for the card if the user hasn't specified one explicitly.
2365          */
2366         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2367         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2368                 /* Card specific overrides go here. */
2369                 if (pci_get_device(sc->dev) == 0x440a)
2370                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2371                 if (is_fpga(sc))
2372                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2373         }
2374
2375         /*
2376          * We need to load another module if the profile is anything except
2377          * "default" or "flash".
2378          */
2379         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2380             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2381                 char s[32];
2382
2383                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2384                 cfg = firmware_get(s);
2385                 if (cfg == NULL) {
2386                         if (default_cfg != NULL) {
2387                                 device_printf(sc->dev,
2388                                     "unable to load module \"%s\" for "
2389                                     "configuration profile \"%s\", will use "
2390                                     "the default config file instead.\n",
2391                                     s, sc->cfg_file);
2392                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2393                                     "%s", DEFAULT_CF);
2394                         } else {
2395                                 device_printf(sc->dev,
2396                                     "unable to load module \"%s\" for "
2397                                     "configuration profile \"%s\", will use "
2398                                     "the config file on the card's flash "
2399                                     "instead.\n", s, sc->cfg_file);
2400                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2401                                     "%s", FLASH_CF);
2402                         }
2403                 }
2404         }
2405
2406         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2407             default_cfg == NULL) {
2408                 device_printf(sc->dev,
2409                     "default config file not available, will use the config "
2410                     "file on the card's flash instead.\n");
2411                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2412         }
2413
2414         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2415                 u_int cflen, i, n;
2416                 const uint32_t *cfdata;
2417                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2418
2419                 KASSERT(cfg != NULL || default_cfg != NULL,
2420                     ("%s: no config to upload", __func__));
2421
2422                 /*
2423                  * Ask the firmware where it wants us to upload the config file.
2424                  */
2425                 param = FW_PARAM_DEV(CF);
2426                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2427                 if (rc != 0) {
2428                         /* No support for config file?  Shouldn't happen. */
2429                         device_printf(sc->dev,
2430                             "failed to query config file location: %d.\n", rc);
2431                         goto done;
2432                 }
2433                 mtype = G_FW_PARAMS_PARAM_Y(val);
2434                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2435
2436                 /*
2437                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2438                  * useless stuffing/comments at the end of the config file so
2439                  * it's ok to simply throw away the last remaining bytes when
2440                  * the config file is not an exact multiple of 4.  This also
2441                  * helps with the validate_mt_off_len check.
2442                  */
2443                 if (cfg != NULL) {
2444                         cflen = cfg->datasize & ~3;
2445                         cfdata = cfg->data;
2446                 } else {
2447                         cflen = default_cfg->datasize & ~3;
2448                         cfdata = default_cfg->data;
2449                 }
2450
2451                 if (cflen > FLASH_CFG_MAX_SIZE) {
2452                         device_printf(sc->dev,
2453                             "config file too long (%d, max allowed is %d).  "
2454                             "Will try to use the config on the card, if any.\n",
2455                             cflen, FLASH_CFG_MAX_SIZE);
2456                         goto use_config_on_flash;
2457                 }
2458
2459                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2460                 if (rc != 0) {
2461                         device_printf(sc->dev,
2462                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2463                             "Will try to use the config on the card, if any.\n",
2464                             __func__, mtype, moff, cflen, rc);
2465                         goto use_config_on_flash;
2466                 }
2467
2468                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2469                 while (cflen) {
2470                         off = position_memwin(sc, 2, addr);
2471                         n = min(cflen, mw_aperture - off);
2472                         for (i = 0; i < n; i += 4)
2473                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2474                         cflen -= n;
2475                         addr += n;
2476                 }
2477         } else {
2478 use_config_on_flash:
2479                 mtype = FW_MEMTYPE_FLASH;
2480                 moff = t4_flash_cfg_addr(sc);
2481         }
2482
2483         bzero(&caps, sizeof(caps));
2484         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2485             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2486         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2487             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2488             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2489         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2490         if (rc != 0) {
2491                 device_printf(sc->dev,
2492                     "failed to pre-process config file: %d "
2493                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2494                 goto done;
2495         }
2496
2497         finicsum = be32toh(caps.finicsum);
2498         cfcsum = be32toh(caps.cfcsum);
2499         if (finicsum != cfcsum) {
2500                 device_printf(sc->dev,
2501                     "WARNING: config file checksum mismatch: %08x %08x\n",
2502                     finicsum, cfcsum);
2503         }
2504         sc->cfcsum = cfcsum;
2505
2506 #define LIMIT_CAPS(x) do { \
2507         caps.x &= htobe16(t4_##x##_allowed); \
2508 } while (0)
2509
2510         /*
2511          * Let the firmware know what features will (not) be used so it can tune
2512          * things accordingly.
2513          */
2514         LIMIT_CAPS(linkcaps);
2515         LIMIT_CAPS(niccaps);
2516         LIMIT_CAPS(toecaps);
2517         LIMIT_CAPS(rdmacaps);
2518         LIMIT_CAPS(iscsicaps);
2519         LIMIT_CAPS(fcoecaps);
2520 #undef LIMIT_CAPS
2521
2522         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2523             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2524         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2525         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2526         if (rc != 0) {
2527                 device_printf(sc->dev,
2528                     "failed to process config file: %d.\n", rc);
2529         }
2530 done:
2531         if (cfg != NULL)
2532                 firmware_put(cfg, FIRMWARE_UNLOAD);
2533         return (rc);
2534 }
2535
2536 /*
2537  * Retrieve parameters that are needed (or nice to have) very early.
2538  */
2539 static int
2540 get_params__pre_init(struct adapter *sc)
2541 {
2542         int rc;
2543         uint32_t param[2], val[2];
2544         struct fw_devlog_cmd cmd;
2545         struct devlog_params *dlog = &sc->params.devlog;
2546
2547         param[0] = FW_PARAM_DEV(PORTVEC);
2548         param[1] = FW_PARAM_DEV(CCLK);
2549         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2550         if (rc != 0) {
2551                 device_printf(sc->dev,
2552                     "failed to query parameters (pre_init): %d.\n", rc);
2553                 return (rc);
2554         }
2555
2556         sc->params.portvec = val[0];
2557         sc->params.nports = bitcount32(val[0]);
2558         sc->params.vpd.cclk = val[1];
2559
2560         /* Read device log parameters. */
2561         bzero(&cmd, sizeof(cmd));
2562         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2563             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2564         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2565         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2566         if (rc != 0) {
2567                 device_printf(sc->dev,
2568                     "failed to get devlog parameters: %d.\n", rc);
2569                 bzero(dlog, sizeof (*dlog));
2570                 rc = 0; /* devlog isn't critical for device operation */
2571         } else {
2572                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2573                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2574                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2575                 dlog->size = be32toh(cmd.memsize_devlog);
2576         }
2577
2578         return (rc);
2579 }
2580
2581 /*
2582  * Retrieve various parameters that are of interest to the driver.  The device
2583  * has been initialized by the firmware at this point.
2584  */
2585 static int
2586 get_params__post_init(struct adapter *sc)
2587 {
2588         int rc;
2589         uint32_t param[7], val[7];
2590         struct fw_caps_config_cmd caps;
2591
2592         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2593         param[1] = FW_PARAM_PFVF(EQ_START);
2594         param[2] = FW_PARAM_PFVF(FILTER_START);
2595         param[3] = FW_PARAM_PFVF(FILTER_END);
2596         param[4] = FW_PARAM_PFVF(L2T_START);
2597         param[5] = FW_PARAM_PFVF(L2T_END);
2598         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2599         if (rc != 0) {
2600                 device_printf(sc->dev,
2601                     "failed to query parameters (post_init): %d.\n", rc);
2602                 return (rc);
2603         }
2604
2605         sc->sge.iq_start = val[0];
2606         sc->sge.eq_start = val[1];
2607         sc->tids.ftid_base = val[2];
2608         sc->tids.nftids = val[3] - val[2] + 1;
2609         sc->params.ftid_min = val[2];
2610         sc->params.ftid_max = val[3];
2611         sc->vres.l2t.start = val[4];
2612         sc->vres.l2t.size = val[5] - val[4] + 1;
2613         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2614             ("%s: L2 table size (%u) larger than expected (%u)",
2615             __func__, sc->vres.l2t.size, L2T_SIZE));
2616
2617         /* get capabilites */
2618         bzero(&caps, sizeof(caps));
2619         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2620             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2621         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2622         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2623         if (rc != 0) {
2624                 device_printf(sc->dev,
2625                     "failed to get card capabilities: %d.\n", rc);
2626                 return (rc);
2627         }
2628
2629 #define READ_CAPS(x) do { \
2630         sc->x = htobe16(caps.x); \
2631 } while (0)
2632         READ_CAPS(linkcaps);
2633         READ_CAPS(niccaps);
2634         READ_CAPS(toecaps);
2635         READ_CAPS(rdmacaps);
2636         READ_CAPS(iscsicaps);
2637         READ_CAPS(fcoecaps);
2638
2639         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2640                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2641                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2642                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2643                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2644                 if (rc != 0) {
2645                         device_printf(sc->dev,
2646                             "failed to query NIC parameters: %d.\n", rc);
2647                         return (rc);
2648                 }
2649                 sc->tids.etid_base = val[0];
2650                 sc->params.etid_min = val[0];
2651                 sc->tids.netids = val[1] - val[0] + 1;
2652                 sc->params.netids = sc->tids.netids;
2653                 sc->params.eo_wr_cred = val[2];
2654                 sc->params.ethoffload = 1;
2655         }
2656
2657         if (sc->toecaps) {
2658                 /* query offload-related parameters */
2659                 param[0] = FW_PARAM_DEV(NTID);
2660                 param[1] = FW_PARAM_PFVF(SERVER_START);
2661                 param[2] = FW_PARAM_PFVF(SERVER_END);
2662                 param[3] = FW_PARAM_PFVF(TDDP_START);
2663                 param[4] = FW_PARAM_PFVF(TDDP_END);
2664                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2665                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2666                 if (rc != 0) {
2667                         device_printf(sc->dev,
2668                             "failed to query TOE parameters: %d.\n", rc);
2669                         return (rc);
2670                 }
2671                 sc->tids.ntids = val[0];
2672                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2673                 sc->tids.stid_base = val[1];
2674                 sc->tids.nstids = val[2] - val[1] + 1;
2675                 sc->vres.ddp.start = val[3];
2676                 sc->vres.ddp.size = val[4] - val[3] + 1;
2677                 sc->params.ofldq_wr_cred = val[5];
2678                 sc->params.offload = 1;
2679         }
2680         if (sc->rdmacaps) {
2681                 param[0] = FW_PARAM_PFVF(STAG_START);
2682                 param[1] = FW_PARAM_PFVF(STAG_END);
2683                 param[2] = FW_PARAM_PFVF(RQ_START);
2684                 param[3] = FW_PARAM_PFVF(RQ_END);
2685                 param[4] = FW_PARAM_PFVF(PBL_START);
2686                 param[5] = FW_PARAM_PFVF(PBL_END);
2687                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2688                 if (rc != 0) {
2689                         device_printf(sc->dev,
2690                             "failed to query RDMA parameters(1): %d.\n", rc);
2691                         return (rc);
2692                 }
2693                 sc->vres.stag.start = val[0];
2694                 sc->vres.stag.size = val[1] - val[0] + 1;
2695                 sc->vres.rq.start = val[2];
2696                 sc->vres.rq.size = val[3] - val[2] + 1;
2697                 sc->vres.pbl.start = val[4];
2698                 sc->vres.pbl.size = val[5] - val[4] + 1;
2699
2700                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2701                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2702                 param[2] = FW_PARAM_PFVF(CQ_START);
2703                 param[3] = FW_PARAM_PFVF(CQ_END);
2704                 param[4] = FW_PARAM_PFVF(OCQ_START);
2705                 param[5] = FW_PARAM_PFVF(OCQ_END);
2706                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2707                 if (rc != 0) {
2708                         device_printf(sc->dev,
2709                             "failed to query RDMA parameters(2): %d.\n", rc);
2710                         return (rc);
2711                 }
2712                 sc->vres.qp.start = val[0];
2713                 sc->vres.qp.size = val[1] - val[0] + 1;
2714                 sc->vres.cq.start = val[2];
2715                 sc->vres.cq.size = val[3] - val[2] + 1;
2716                 sc->vres.ocq.start = val[4];
2717                 sc->vres.ocq.size = val[5] - val[4] + 1;
2718         }
2719         if (sc->iscsicaps) {
2720                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2721                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2722                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2723                 if (rc != 0) {
2724                         device_printf(sc->dev,
2725                             "failed to query iSCSI parameters: %d.\n", rc);
2726                         return (rc);
2727                 }
2728                 sc->vres.iscsi.start = val[0];
2729                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2730         }
2731
2732         /*
2733          * We've got the params we wanted to query via the firmware.  Now grab
2734          * some others directly from the chip.
2735          */
2736         rc = t4_read_chip_settings(sc);
2737
2738         return (rc);
2739 }
2740
2741 static int
2742 set_params__post_init(struct adapter *sc)
2743 {
2744         uint32_t param, val;
2745
2746         /* ask for encapsulated CPLs */
2747         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2748         val = 1;
2749         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2750
2751         return (0);
2752 }
2753
2754 #undef FW_PARAM_PFVF
2755 #undef FW_PARAM_DEV
2756
2757 static void
2758 t4_set_desc(struct adapter *sc)
2759 {
2760         char buf[128];
2761         struct adapter_params *p = &sc->params;
2762
2763         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2764             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2765             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2766
2767         device_set_desc_copy(sc->dev, buf);
2768 }
2769
2770 static void
2771 build_medialist(struct port_info *pi, struct ifmedia *media)
2772 {
2773         int data, m;
2774
2775         PORT_LOCK(pi);
2776
2777         ifmedia_removeall(media);
2778
2779         m = IFM_ETHER | IFM_FDX;
2780         data = (pi->port_type << 8) | pi->mod_type;
2781
2782         switch(pi->port_type) {
2783         case FW_PORT_TYPE_BT_XFI:
2784                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2785                 break;
2786
2787         case FW_PORT_TYPE_BT_XAUI:
2788                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2789                 /* fall through */
2790
2791         case FW_PORT_TYPE_BT_SGMII:
2792                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2793                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2794                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2795                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2796                 break;
2797
2798         case FW_PORT_TYPE_CX4:
2799                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2800                 ifmedia_set(media, m | IFM_10G_CX4);
2801                 break;
2802
2803         case FW_PORT_TYPE_QSFP_10G:
2804         case FW_PORT_TYPE_SFP:
2805         case FW_PORT_TYPE_FIBER_XFI:
2806         case FW_PORT_TYPE_FIBER_XAUI:
2807                 switch (pi->mod_type) {
2808
2809                 case FW_PORT_MOD_TYPE_LR:
2810                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2811                         ifmedia_set(media, m | IFM_10G_LR);
2812                         break;
2813
2814                 case FW_PORT_MOD_TYPE_SR:
2815                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2816                         ifmedia_set(media, m | IFM_10G_SR);
2817                         break;
2818
2819                 case FW_PORT_MOD_TYPE_LRM:
2820                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2821                         ifmedia_set(media, m | IFM_10G_LRM);
2822                         break;
2823
2824                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2825                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2826                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2827                         ifmedia_set(media, m | IFM_10G_TWINAX);
2828                         break;
2829
2830                 case FW_PORT_MOD_TYPE_NONE:
2831                         m &= ~IFM_FDX;
2832                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2833                         ifmedia_set(media, m | IFM_NONE);
2834                         break;
2835
2836                 case FW_PORT_MOD_TYPE_NA:
2837                 case FW_PORT_MOD_TYPE_ER:
2838                 default:
2839                         device_printf(pi->dev,
2840                             "unknown port_type (%d), mod_type (%d)\n",
2841                             pi->port_type, pi->mod_type);
2842                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2843                         ifmedia_set(media, m | IFM_UNKNOWN);
2844                         break;
2845                 }
2846                 break;
2847
2848         case FW_PORT_TYPE_QSFP:
2849                 switch (pi->mod_type) {
2850
2851                 case FW_PORT_MOD_TYPE_LR:
2852                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2853                         ifmedia_set(media, m | IFM_40G_LR4);
2854                         break;
2855
2856                 case FW_PORT_MOD_TYPE_SR:
2857                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2858                         ifmedia_set(media, m | IFM_40G_SR4);
2859                         break;
2860
2861                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2862                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2863                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2864                         ifmedia_set(media, m | IFM_40G_CR4);
2865                         break;
2866
2867                 case FW_PORT_MOD_TYPE_NONE:
2868                         m &= ~IFM_FDX;
2869                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2870                         ifmedia_set(media, m | IFM_NONE);
2871                         break;
2872
2873                 default:
2874                         device_printf(pi->dev,
2875                             "unknown port_type (%d), mod_type (%d)\n",
2876                             pi->port_type, pi->mod_type);
2877                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2878                         ifmedia_set(media, m | IFM_UNKNOWN);
2879                         break;
2880                 }
2881                 break;
2882
2883         default:
2884                 device_printf(pi->dev,
2885                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2886                     pi->mod_type);
2887                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2888                 ifmedia_set(media, m | IFM_UNKNOWN);
2889                 break;
2890         }
2891
2892         PORT_UNLOCK(pi);
2893 }
2894
2895 #define FW_MAC_EXACT_CHUNK      7
2896
2897 /*
2898  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2899  * indicates which parameters should be programmed (the rest are left alone).
2900  */
2901 int
2902 update_mac_settings(struct ifnet *ifp, int flags)
2903 {
2904         int rc = 0;
2905         struct port_info *pi = ifp->if_softc;
2906         struct adapter *sc = pi->adapter;
2907         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2908         uint16_t viid = 0xffff;
2909         int16_t *xact_addr_filt = NULL;
2910
2911         ASSERT_SYNCHRONIZED_OP(sc);
2912         KASSERT(flags, ("%s: not told what to update.", __func__));
2913
2914         if (ifp == pi->ifp) {
2915                 viid = pi->viid;
2916                 xact_addr_filt = &pi->xact_addr_filt;
2917         }
2918 #ifdef DEV_NETMAP
2919         else if (ifp == pi->nm_ifp) {
2920                 viid = pi->nm_viid;
2921                 xact_addr_filt = &pi->nm_xact_addr_filt;
2922         }
2923 #endif
2924         if (flags & XGMAC_MTU)
2925                 mtu = ifp->if_mtu;
2926
2927         if (flags & XGMAC_PROMISC)
2928                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2929
2930         if (flags & XGMAC_ALLMULTI)
2931                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2932
2933         if (flags & XGMAC_VLANEX)
2934                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2935
2936         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2937                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2938                     1, vlanex, false);
2939                 if (rc) {
2940                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2941                             rc);
2942                         return (rc);
2943                 }
2944         }
2945
2946         if (flags & XGMAC_UCADDR) {
2947                 uint8_t ucaddr[ETHER_ADDR_LEN];
2948
2949                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2950                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2951                     true, true);
2952                 if (rc < 0) {
2953                         rc = -rc;
2954                         if_printf(ifp, "change_mac failed: %d\n", rc);
2955                         return (rc);
2956                 } else {
2957                         *xact_addr_filt = rc;
2958                         rc = 0;
2959                 }
2960         }
2961
2962         if (flags & XGMAC_MCADDRS) {
2963                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2964                 int del = 1;
2965                 uint64_t hash = 0;
2966                 struct ifmultiaddr *ifma;
2967                 int i = 0, j;
2968
2969                 if_maddr_rlock(ifp);
2970                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2971                         if (ifma->ifma_addr->sa_family != AF_LINK)
2972                                 continue;
2973                         mcaddr[i++] =
2974                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2975
2976                         if (i == FW_MAC_EXACT_CHUNK) {
2977                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
2978                                     i, mcaddr, NULL, &hash, 0);
2979                                 if (rc < 0) {
2980                                         rc = -rc;
2981                                         for (j = 0; j < i; j++) {
2982                                                 if_printf(ifp,
2983                                                     "failed to add mc address"
2984                                                     " %02x:%02x:%02x:"
2985                                                     "%02x:%02x:%02x rc=%d\n",
2986                                                     mcaddr[j][0], mcaddr[j][1],
2987                                                     mcaddr[j][2], mcaddr[j][3],
2988                                                     mcaddr[j][4], mcaddr[j][5],
2989                                                     rc);
2990                                         }
2991                                         goto mcfail;
2992                                 }
2993                                 del = 0;
2994                                 i = 0;
2995                         }
2996                 }
2997                 if (i > 0) {
2998                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
2999                             mcaddr, NULL, &hash, 0);
3000                         if (rc < 0) {
3001                                 rc = -rc;
3002                                 for (j = 0; j < i; j++) {
3003                                         if_printf(ifp,
3004                                             "failed to add mc address"
3005                                             " %02x:%02x:%02x:"
3006                                             "%02x:%02x:%02x rc=%d\n",
3007                                             mcaddr[j][0], mcaddr[j][1],
3008                                             mcaddr[j][2], mcaddr[j][3],
3009                                             mcaddr[j][4], mcaddr[j][5],
3010                                             rc);
3011                                 }
3012                                 goto mcfail;
3013                         }
3014                 }
3015
3016                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3017                 if (rc != 0)
3018                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3019 mcfail:
3020                 if_maddr_runlock(ifp);
3021         }
3022
3023         return (rc);
3024 }
3025
3026 int
3027 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3028     char *wmesg)
3029 {
3030         int rc, pri;
3031
3032 #ifdef WITNESS
3033         /* the caller thinks it's ok to sleep, but is it really? */
3034         if (flags & SLEEP_OK)
3035                 pause("t4slptst", 1);
3036 #endif
3037
3038         if (INTR_OK)
3039                 pri = PCATCH;
3040         else
3041                 pri = 0;
3042
3043         ADAPTER_LOCK(sc);
3044         for (;;) {
3045
3046                 if (pi && IS_DOOMED(pi)) {
3047                         rc = ENXIO;
3048                         goto done;
3049                 }
3050
3051                 if (!IS_BUSY(sc)) {
3052                         rc = 0;
3053                         break;
3054                 }
3055
3056                 if (!(flags & SLEEP_OK)) {
3057                         rc = EBUSY;
3058                         goto done;
3059                 }
3060
3061                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3062                         rc = EINTR;
3063                         goto done;
3064                 }
3065         }
3066
3067         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3068         SET_BUSY(sc);
3069 #ifdef INVARIANTS
3070         sc->last_op = wmesg;
3071         sc->last_op_thr = curthread;
3072 #endif
3073
3074 done:
3075         if (!(flags & HOLD_LOCK) || rc)
3076                 ADAPTER_UNLOCK(sc);
3077
3078         return (rc);
3079 }
3080
3081 void
3082 end_synchronized_op(struct adapter *sc, int flags)
3083 {
3084
3085         if (flags & LOCK_HELD)
3086                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3087         else
3088                 ADAPTER_LOCK(sc);
3089
3090         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3091         CLR_BUSY(sc);
3092         wakeup(&sc->flags);
3093         ADAPTER_UNLOCK(sc);
3094 }
3095
3096 static int
3097 cxgbe_init_synchronized(struct port_info *pi)
3098 {
3099         struct adapter *sc = pi->adapter;
3100         struct ifnet *ifp = pi->ifp;
3101         int rc = 0;
3102
3103         ASSERT_SYNCHRONIZED_OP(sc);
3104
3105         if (isset(&sc->open_device_map, pi->port_id)) {
3106                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3107                     ("mismatch between open_device_map and if_drv_flags"));
3108                 return (0);     /* already running */
3109         }
3110
3111         if (!(sc->flags & FULL_INIT_DONE) &&
3112             ((rc = adapter_full_init(sc)) != 0))
3113                 return (rc);    /* error message displayed already */
3114
3115         if (!(pi->flags & PORT_INIT_DONE) &&
3116             ((rc = port_full_init(pi)) != 0))
3117                 return (rc); /* error message displayed already */
3118
3119         rc = update_mac_settings(ifp, XGMAC_ALL);
3120         if (rc)
3121                 goto done;      /* error message displayed already */
3122
3123         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3124         if (rc != 0) {
3125                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3126                 goto done;
3127         }
3128
3129         /*
3130          * The first iq of the first port to come up is used for tracing.
3131          */
3132         if (sc->traceq < 0) {
3133                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3134                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3135                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3136                     V_QUEUENUMBER(sc->traceq));
3137                 pi->flags |= HAS_TRACEQ;
3138         }
3139
3140         /* all ok */
3141         setbit(&sc->open_device_map, pi->port_id);
3142         PORT_LOCK(pi);
3143         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3144         PORT_UNLOCK(pi);
3145
3146         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3147 done:
3148         if (rc != 0)
3149                 cxgbe_uninit_synchronized(pi);
3150
3151         return (rc);
3152 }
3153
3154 /*
3155  * Idempotent.
3156  */
3157 static int
3158 cxgbe_uninit_synchronized(struct port_info *pi)
3159 {
3160         struct adapter *sc = pi->adapter;
3161         struct ifnet *ifp = pi->ifp;
3162         int rc;
3163
3164         ASSERT_SYNCHRONIZED_OP(sc);
3165
3166         /*
3167          * Disable the VI so that all its data in either direction is discarded
3168          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3169          * tick) intact as the TP can deliver negative advice or data that it's
3170          * holding in its RAM (for an offloaded connection) even after the VI is
3171          * disabled.
3172          */
3173         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3174         if (rc) {
3175                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3176                 return (rc);
3177         }
3178
3179         clrbit(&sc->open_device_map, pi->port_id);
3180         PORT_LOCK(pi);
3181         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3182         PORT_UNLOCK(pi);
3183
3184         pi->link_cfg.link_ok = 0;
3185         pi->link_cfg.speed = 0;
3186         pi->linkdnrc = -1;
3187         t4_os_link_changed(sc, pi->port_id, 0, -1);
3188
3189         return (0);
3190 }
3191
3192 /*
3193  * It is ok for this function to fail midway and return right away.  t4_detach
3194  * will walk the entire sc->irq list and clean up whatever is valid.
3195  */
3196 static int
3197 setup_intr_handlers(struct adapter *sc)
3198 {
3199         int rc, rid, p, q;
3200         char s[8];
3201         struct irq *irq;
3202         struct port_info *pi;
3203         struct sge_rxq *rxq;
3204 #ifdef TCP_OFFLOAD
3205         struct sge_ofld_rxq *ofld_rxq;
3206 #endif
3207 #ifdef DEV_NETMAP
3208         struct sge_nm_rxq *nm_rxq;
3209 #endif
3210
3211         /*
3212          * Setup interrupts.
3213          */
3214         irq = &sc->irq[0];
3215         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3216         if (sc->intr_count == 1)
3217                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3218
3219         /* Multiple interrupts. */
3220         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3221             ("%s: too few intr.", __func__));
3222
3223         /* The first one is always error intr */
3224         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3225         if (rc != 0)
3226                 return (rc);
3227         irq++;
3228         rid++;
3229
3230         /* The second one is always the firmware event queue */
3231         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3232         if (rc != 0)
3233                 return (rc);
3234         irq++;
3235         rid++;
3236
3237         for_each_port(sc, p) {
3238                 pi = sc->port[p];
3239
3240                 if (pi->flags & INTR_RXQ) {
3241                         for_each_rxq(pi, q, rxq) {
3242                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3243                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3244                                     s);
3245                                 if (rc != 0)
3246                                         return (rc);
3247                                 irq++;
3248                                 rid++;
3249                         }
3250                 }
3251 #ifdef TCP_OFFLOAD
3252                 if (pi->flags & INTR_OFLD_RXQ) {
3253                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3254                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3255                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3256                                     ofld_rxq, s);
3257                                 if (rc != 0)
3258                                         return (rc);
3259                                 irq++;
3260                                 rid++;
3261                         }
3262                 }
3263 #endif
3264 #ifdef DEV_NETMAP
3265                 if (pi->flags & INTR_NM_RXQ) {
3266                         for_each_nm_rxq(pi, q, nm_rxq) {
3267                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3268                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3269                                     nm_rxq, s);
3270                                 if (rc != 0)
3271                                         return (rc);
3272                                 irq++;
3273                                 rid++;
3274                         }
3275                 }
3276 #endif
3277         }
3278         MPASS(irq == &sc->irq[sc->intr_count]);
3279
3280         return (0);
3281 }
3282
3283 int
3284 adapter_full_init(struct adapter *sc)
3285 {
3286         int rc, i;
3287
3288         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3289         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3290             ("%s: FULL_INIT_DONE already", __func__));
3291
3292         /*
3293          * queues that belong to the adapter (not any particular port).
3294          */
3295         rc = t4_setup_adapter_queues(sc);
3296         if (rc != 0)
3297                 goto done;
3298
3299         for (i = 0; i < nitems(sc->tq); i++) {
3300                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3301                     taskqueue_thread_enqueue, &sc->tq[i]);
3302                 if (sc->tq[i] == NULL) {
3303                         device_printf(sc->dev,
3304                             "failed to allocate task queue %d\n", i);
3305                         rc = ENOMEM;
3306                         goto done;
3307                 }
3308                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3309                     device_get_nameunit(sc->dev), i);
3310         }
3311
3312         t4_intr_enable(sc);
3313         sc->flags |= FULL_INIT_DONE;
3314 done:
3315         if (rc != 0)
3316                 adapter_full_uninit(sc);
3317
3318         return (rc);
3319 }
3320
3321 int
3322 adapter_full_uninit(struct adapter *sc)
3323 {
3324         int i;
3325
3326         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3327
3328         t4_teardown_adapter_queues(sc);
3329
3330         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3331                 taskqueue_free(sc->tq[i]);
3332                 sc->tq[i] = NULL;
3333         }
3334
3335         sc->flags &= ~FULL_INIT_DONE;
3336
3337         return (0);
3338 }
3339
3340 int
3341 port_full_init(struct port_info *pi)
3342 {
3343         struct adapter *sc = pi->adapter;
3344         struct ifnet *ifp = pi->ifp;
3345         uint16_t *rss;
3346         struct sge_rxq *rxq;
3347         int rc, i, j;
3348
3349         ASSERT_SYNCHRONIZED_OP(sc);
3350         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3351             ("%s: PORT_INIT_DONE already", __func__));
3352
3353         sysctl_ctx_init(&pi->ctx);
3354         pi->flags |= PORT_SYSCTL_CTX;
3355
3356         /*
3357          * Allocate tx/rx/fl queues for this port.
3358          */
3359         rc = t4_setup_port_queues(pi);
3360         if (rc != 0)
3361                 goto done;      /* error message displayed already */
3362
3363         /*
3364          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3365          */
3366         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3367         for (i = 0; i < pi->rss_size;) {
3368                 for_each_rxq(pi, j, rxq) {
3369                         rss[i++] = rxq->iq.abs_id;
3370                         if (i == pi->rss_size)
3371                                 break;
3372                 }
3373         }
3374
3375         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3376             pi->rss_size);
3377         if (rc != 0) {
3378                 if_printf(ifp, "rss_config failed: %d\n", rc);
3379                 goto done;
3380         }
3381
3382         pi->rss = rss;
3383         pi->flags |= PORT_INIT_DONE;
3384 done:
3385         if (rc != 0)
3386                 port_full_uninit(pi);
3387
3388         return (rc);
3389 }
3390
3391 /*
3392  * Idempotent.
3393  */
3394 int
3395 port_full_uninit(struct port_info *pi)
3396 {
3397         struct adapter *sc = pi->adapter;
3398         int i;
3399         struct sge_rxq *rxq;
3400         struct sge_txq *txq;
3401 #ifdef TCP_OFFLOAD
3402         struct sge_ofld_rxq *ofld_rxq;
3403         struct sge_wrq *ofld_txq;
3404 #endif
3405
3406         if (pi->flags & PORT_INIT_DONE) {
3407
3408                 /* Need to quiesce queues.  XXX: ctrl queues? */
3409
3410                 for_each_txq(pi, i, txq) {
3411                         quiesce_eq(sc, &txq->eq);
3412                 }
3413
3414 #ifdef TCP_OFFLOAD
3415                 for_each_ofld_txq(pi, i, ofld_txq) {
3416                         quiesce_eq(sc, &ofld_txq->eq);
3417                 }
3418 #endif
3419
3420                 for_each_rxq(pi, i, rxq) {
3421                         quiesce_iq(sc, &rxq->iq);
3422                         quiesce_fl(sc, &rxq->fl);
3423                 }
3424
3425 #ifdef TCP_OFFLOAD
3426                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3427                         quiesce_iq(sc, &ofld_rxq->iq);
3428                         quiesce_fl(sc, &ofld_rxq->fl);
3429                 }
3430 #endif
3431                 free(pi->rss, M_CXGBE);
3432         }
3433
3434         t4_teardown_port_queues(pi);
3435         pi->flags &= ~PORT_INIT_DONE;
3436
3437         return (0);
3438 }
3439
3440 static void
3441 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3442 {
3443         EQ_LOCK(eq);
3444         eq->flags |= EQ_DOOMED;
3445
3446         /*
3447          * Wait for the response to a credit flush if one's
3448          * pending.
3449          */
3450         while (eq->flags & EQ_CRFLUSHED)
3451                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3452         EQ_UNLOCK(eq);
3453
3454         callout_drain(&eq->tx_callout); /* XXX: iffy */
3455         pause("callout", 10);           /* Still iffy */
3456
3457         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3458 }
3459
3460 static void
3461 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3462 {
3463         (void) sc;      /* unused */
3464
3465         /* Synchronize with the interrupt handler */
3466         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3467                 pause("iqfree", 1);
3468 }
3469
3470 static void
3471 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3472 {
3473         mtx_lock(&sc->sfl_lock);
3474         FL_LOCK(fl);
3475         fl->flags |= FL_DOOMED;
3476         FL_UNLOCK(fl);
3477         mtx_unlock(&sc->sfl_lock);
3478
3479         callout_drain(&sc->sfl_callout);
3480         KASSERT((fl->flags & FL_STARVING) == 0,
3481             ("%s: still starving", __func__));
3482 }
3483
3484 static int
3485 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3486     driver_intr_t *handler, void *arg, char *name)
3487 {
3488         int rc;
3489
3490         irq->rid = rid;
3491         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3492             RF_SHAREABLE | RF_ACTIVE);
3493         if (irq->res == NULL) {
3494                 device_printf(sc->dev,
3495                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3496                 return (ENOMEM);
3497         }
3498
3499         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3500             NULL, handler, arg, &irq->tag);
3501         if (rc != 0) {
3502                 device_printf(sc->dev,
3503                     "failed to setup interrupt for rid %d, name %s: %d\n",
3504                     rid, name, rc);
3505         } else if (name)
3506                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3507
3508         return (rc);
3509 }
3510
3511 static int
3512 t4_free_irq(struct adapter *sc, struct irq *irq)
3513 {
3514         if (irq->tag)
3515                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3516         if (irq->res)
3517                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3518
3519         bzero(irq, sizeof(*irq));
3520
3521         return (0);
3522 }
3523
3524 static void
3525 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3526     unsigned int end)
3527 {
3528         uint32_t *p = (uint32_t *)(buf + start);
3529
3530         for ( ; start <= end; start += sizeof(uint32_t))
3531                 *p++ = t4_read_reg(sc, start);
3532 }
3533
3534 static void
3535 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3536 {
3537         int i, n;
3538         const unsigned int *reg_ranges;
3539         static const unsigned int t4_reg_ranges[] = {
3540                 0x1008, 0x1108,
3541                 0x1180, 0x11b4,
3542                 0x11fc, 0x123c,
3543                 0x1300, 0x173c,
3544                 0x1800, 0x18fc,
3545                 0x3000, 0x30d8,
3546                 0x30e0, 0x5924,
3547                 0x5960, 0x59d4,
3548                 0x5a00, 0x5af8,
3549                 0x6000, 0x6098,
3550                 0x6100, 0x6150,
3551                 0x6200, 0x6208,
3552                 0x6240, 0x6248,
3553                 0x6280, 0x6338,
3554                 0x6370, 0x638c,
3555                 0x6400, 0x643c,
3556                 0x6500, 0x6524,
3557                 0x6a00, 0x6a38,
3558                 0x6a60, 0x6a78,
3559                 0x6b00, 0x6b84,
3560                 0x6bf0, 0x6c84,
3561                 0x6cf0, 0x6d84,
3562                 0x6df0, 0x6e84,
3563                 0x6ef0, 0x6f84,
3564                 0x6ff0, 0x7084,
3565                 0x70f0, 0x7184,
3566                 0x71f0, 0x7284,
3567                 0x72f0, 0x7384,
3568                 0x73f0, 0x7450,
3569                 0x7500, 0x7530,
3570                 0x7600, 0x761c,
3571                 0x7680, 0x76cc,
3572                 0x7700, 0x7798,
3573                 0x77c0, 0x77fc,
3574                 0x7900, 0x79fc,
3575                 0x7b00, 0x7c38,
3576                 0x7d00, 0x7efc,
3577                 0x8dc0, 0x8e1c,
3578                 0x8e30, 0x8e78,
3579                 0x8ea0, 0x8f6c,
3580                 0x8fc0, 0x9074,
3581                 0x90fc, 0x90fc,
3582                 0x9400, 0x9458,
3583                 0x9600, 0x96bc,
3584                 0x9800, 0x9808,
3585                 0x9820, 0x983c,
3586                 0x9850, 0x9864,
3587                 0x9c00, 0x9c6c,
3588                 0x9c80, 0x9cec,
3589                 0x9d00, 0x9d6c,
3590                 0x9d80, 0x9dec,
3591                 0x9e00, 0x9e6c,
3592                 0x9e80, 0x9eec,
3593                 0x9f00, 0x9f6c,
3594                 0x9f80, 0x9fec,
3595                 0xd004, 0xd03c,
3596                 0xdfc0, 0xdfe0,
3597                 0xe000, 0xea7c,
3598                 0xf000, 0x11110,
3599                 0x11118, 0x11190,
3600                 0x19040, 0x1906c,
3601                 0x19078, 0x19080,
3602                 0x1908c, 0x19124,
3603                 0x19150, 0x191b0,
3604                 0x191d0, 0x191e8,
3605                 0x19238, 0x1924c,
3606                 0x193f8, 0x19474,
3607                 0x19490, 0x194f8,
3608                 0x19800, 0x19f30,
3609                 0x1a000, 0x1a06c,
3610                 0x1a0b0, 0x1a120,
3611                 0x1a128, 0x1a138,
3612                 0x1a190, 0x1a1c4,
3613                 0x1a1fc, 0x1a1fc,
3614                 0x1e040, 0x1e04c,
3615                 0x1e284, 0x1e28c,
3616                 0x1e2c0, 0x1e2c0,
3617                 0x1e2e0, 0x1e2e0,
3618                 0x1e300, 0x1e384,
3619                 0x1e3c0, 0x1e3c8,
3620                 0x1e440, 0x1e44c,
3621                 0x1e684, 0x1e68c,
3622                 0x1e6c0, 0x1e6c0,
3623                 0x1e6e0, 0x1e6e0,
3624                 0x1e700, 0x1e784,
3625                 0x1e7c0, 0x1e7c8,
3626                 0x1e840, 0x1e84c,
3627                 0x1ea84, 0x1ea8c,
3628                 0x1eac0, 0x1eac0,
3629                 0x1eae0, 0x1eae0,
3630                 0x1eb00, 0x1eb84,
3631                 0x1ebc0, 0x1ebc8,
3632                 0x1ec40, 0x1ec4c,
3633                 0x1ee84, 0x1ee8c,
3634                 0x1eec0, 0x1eec0,
3635                 0x1eee0, 0x1eee0,
3636                 0x1ef00, 0x1ef84,
3637                 0x1efc0, 0x1efc8,
3638                 0x1f040, 0x1f04c,
3639                 0x1f284, 0x1f28c,
3640                 0x1f2c0, 0x1f2c0,
3641                 0x1f2e0, 0x1f2e0,
3642                 0x1f300, 0x1f384,
3643                 0x1f3c0, 0x1f3c8,
3644                 0x1f440, 0x1f44c,
3645                 0x1f684, 0x1f68c,
3646                 0x1f6c0, 0x1f6c0,
3647                 0x1f6e0, 0x1f6e0,
3648                 0x1f700, 0x1f784,
3649                 0x1f7c0, 0x1f7c8,
3650                 0x1f840, 0x1f84c,
3651                 0x1fa84, 0x1fa8c,
3652                 0x1fac0, 0x1fac0,
3653                 0x1fae0, 0x1fae0,
3654                 0x1fb00, 0x1fb84,
3655                 0x1fbc0, 0x1fbc8,
3656                 0x1fc40, 0x1fc4c,
3657                 0x1fe84, 0x1fe8c,
3658                 0x1fec0, 0x1fec0,
3659                 0x1fee0, 0x1fee0,
3660                 0x1ff00, 0x1ff84,
3661                 0x1ffc0, 0x1ffc8,
3662                 0x20000, 0x2002c,
3663                 0x20100, 0x2013c,
3664                 0x20190, 0x201c8,
3665                 0x20200, 0x20318,
3666                 0x20400, 0x20528,
3667                 0x20540, 0x20614,
3668                 0x21000, 0x21040,
3669                 0x2104c, 0x21060,
3670                 0x210c0, 0x210ec,
3671                 0x21200, 0x21268,
3672                 0x21270, 0x21284,
3673                 0x212fc, 0x21388,
3674                 0x21400, 0x21404,
3675                 0x21500, 0x21518,
3676                 0x2152c, 0x2153c,
3677                 0x21550, 0x21554,
3678                 0x21600, 0x21600,
3679                 0x21608, 0x21628,
3680                 0x21630, 0x2163c,
3681                 0x21700, 0x2171c,
3682                 0x21780, 0x2178c,
3683                 0x21800, 0x21c38,
3684                 0x21c80, 0x21d7c,
3685                 0x21e00, 0x21e04,
3686                 0x22000, 0x2202c,
3687                 0x22100, 0x2213c,
3688                 0x22190, 0x221c8,
3689                 0x22200, 0x22318,
3690                 0x22400, 0x22528,
3691                 0x22540, 0x22614,
3692                 0x23000, 0x23040,
3693                 0x2304c, 0x23060,
3694                 0x230c0, 0x230ec,
3695                 0x23200, 0x23268,
3696                 0x23270, 0x23284,
3697                 0x232fc, 0x23388,
3698                 0x23400, 0x23404,
3699                 0x23500, 0x23518,
3700                 0x2352c, 0x2353c,
3701                 0x23550, 0x23554,
3702                 0x23600, 0x23600,
3703                 0x23608, 0x23628,
3704                 0x23630, 0x2363c,
3705                 0x23700, 0x2371c,
3706                 0x23780, 0x2378c,
3707                 0x23800, 0x23c38,
3708                 0x23c80, 0x23d7c,
3709                 0x23e00, 0x23e04,
3710                 0x24000, 0x2402c,
3711                 0x24100, 0x2413c,
3712                 0x24190, 0x241c8,
3713                 0x24200, 0x24318,
3714                 0x24400, 0x24528,
3715                 0x24540, 0x24614,
3716                 0x25000, 0x25040,
3717                 0x2504c, 0x25060,
3718                 0x250c0, 0x250ec,
3719                 0x25200, 0x25268,
3720                 0x25270, 0x25284,
3721                 0x252fc, 0x25388,
3722                 0x25400, 0x25404,
3723                 0x25500, 0x25518,
3724                 0x2552c, 0x2553c,
3725                 0x25550, 0x25554,
3726                 0x25600, 0x25600,
3727                 0x25608, 0x25628,
3728                 0x25630, 0x2563c,
3729                 0x25700, 0x2571c,
3730                 0x25780, 0x2578c,
3731                 0x25800, 0x25c38,
3732                 0x25c80, 0x25d7c,
3733                 0x25e00, 0x25e04,
3734                 0x26000, 0x2602c,
3735                 0x26100, 0x2613c,
3736                 0x26190, 0x261c8,
3737                 0x26200, 0x26318,
3738                 0x26400, 0x26528,
3739                 0x26540, 0x26614,
3740                 0x27000, 0x27040,
3741                 0x2704c, 0x27060,
3742                 0x270c0, 0x270ec,
3743                 0x27200, 0x27268,
3744                 0x27270, 0x27284,
3745                 0x272fc, 0x27388,
3746                 0x27400, 0x27404,
3747                 0x27500, 0x27518,
3748                 0x2752c, 0x2753c,
3749                 0x27550, 0x27554,
3750                 0x27600, 0x27600,
3751                 0x27608, 0x27628,
3752                 0x27630, 0x2763c,
3753                 0x27700, 0x2771c,
3754                 0x27780, 0x2778c,
3755                 0x27800, 0x27c38,
3756                 0x27c80, 0x27d7c,
3757                 0x27e00, 0x27e04
3758         };
3759         static const unsigned int t5_reg_ranges[] = {
3760                 0x1008, 0x1148,
3761                 0x1180, 0x11b4,
3762                 0x11fc, 0x123c,
3763                 0x1280, 0x173c,
3764                 0x1800, 0x18fc,
3765                 0x3000, 0x3028,
3766                 0x3060, 0x30d8,
3767                 0x30e0, 0x30fc,
3768                 0x3140, 0x357c,
3769                 0x35a8, 0x35cc,
3770                 0x35ec, 0x35ec,
3771                 0x3600, 0x5624,
3772                 0x56cc, 0x575c,
3773                 0x580c, 0x5814,
3774                 0x5890, 0x58bc,
3775                 0x5940, 0x59dc,
3776                 0x59fc, 0x5a18,
3777                 0x5a60, 0x5a9c,
3778                 0x5b94, 0x5bfc,
3779                 0x6000, 0x6040,
3780                 0x6058, 0x614c,
3781                 0x7700, 0x7798,
3782                 0x77c0, 0x78fc,
3783                 0x7b00, 0x7c54,
3784                 0x7d00, 0x7efc,
3785                 0x8dc0, 0x8de0,
3786                 0x8df8, 0x8e84,
3787                 0x8ea0, 0x8f84,
3788                 0x8fc0, 0x90f8,
3789                 0x9400, 0x9470,
3790                 0x9600, 0x96f4,
3791                 0x9800, 0x9808,
3792                 0x9820, 0x983c,
3793                 0x9850, 0x9864,
3794                 0x9c00, 0x9c6c,
3795                 0x9c80, 0x9cec,
3796                 0x9d00, 0x9d6c,
3797                 0x9d80, 0x9dec,
3798                 0x9e00, 0x9e6c,
3799                 0x9e80, 0x9eec,
3800                 0x9f00, 0x9f6c,
3801                 0x9f80, 0xa020,
3802                 0xd004, 0xd03c,
3803                 0xdfc0, 0xdfe0,
3804                 0xe000, 0x11088,
3805                 0x1109c, 0x11110,
3806                 0x11118, 0x1117c,
3807                 0x11190, 0x11204,
3808                 0x19040, 0x1906c,
3809                 0x19078, 0x19080,
3810                 0x1908c, 0x19124,
3811                 0x19150, 0x191b0,
3812                 0x191d0, 0x191e8,
3813                 0x19238, 0x19290,
3814                 0x193f8, 0x19474,
3815                 0x19490, 0x194cc,
3816                 0x194f0, 0x194f8,
3817                 0x19c00, 0x19c60,
3818                 0x19c94, 0x19e10,
3819                 0x19e50, 0x19f34,
3820                 0x19f40, 0x19f50,
3821                 0x19f90, 0x19fe4,
3822                 0x1a000, 0x1a06c,
3823                 0x1a0b0, 0x1a120,
3824                 0x1a128, 0x1a138,
3825                 0x1a190, 0x1a1c4,
3826                 0x1a1fc, 0x1a1fc,
3827                 0x1e008, 0x1e00c,
3828                 0x1e040, 0x1e04c,
3829                 0x1e284, 0x1e290,
3830                 0x1e2c0, 0x1e2c0,
3831                 0x1e2e0, 0x1e2e0,
3832                 0x1e300, 0x1e384,
3833                 0x1e3c0, 0x1e3c8,
3834                 0x1e408, 0x1e40c,
3835                 0x1e440, 0x1e44c,
3836                 0x1e684, 0x1e690,
3837                 0x1e6c0, 0x1e6c0,
3838                 0x1e6e0, 0x1e6e0,
3839                 0x1e700, 0x1e784,
3840                 0x1e7c0, 0x1e7c8,
3841                 0x1e808, 0x1e80c,
3842                 0x1e840, 0x1e84c,
3843                 0x1ea84, 0x1ea90,
3844                 0x1eac0, 0x1eac0,
3845                 0x1eae0, 0x1eae0,
3846                 0x1eb00, 0x1eb84,
3847                 0x1ebc0, 0x1ebc8,
3848                 0x1ec08, 0x1ec0c,
3849                 0x1ec40, 0x1ec4c,
3850                 0x1ee84, 0x1ee90,
3851                 0x1eec0, 0x1eec0,
3852                 0x1eee0, 0x1eee0,
3853                 0x1ef00, 0x1ef84,
3854                 0x1efc0, 0x1efc8,
3855                 0x1f008, 0x1f00c,
3856                 0x1f040, 0x1f04c,
3857                 0x1f284, 0x1f290,
3858                 0x1f2c0, 0x1f2c0,
3859                 0x1f2e0, 0x1f2e0,
3860                 0x1f300, 0x1f384,
3861                 0x1f3c0, 0x1f3c8,
3862                 0x1f408, 0x1f40c,
3863                 0x1f440, 0x1f44c,
3864                 0x1f684, 0x1f690,
3865                 0x1f6c0, 0x1f6c0,
3866                 0x1f6e0, 0x1f6e0,
3867                 0x1f700, 0x1f784,
3868                 0x1f7c0, 0x1f7c8,
3869                 0x1f808, 0x1f80c,
3870                 0x1f840, 0x1f84c,
3871                 0x1fa84, 0x1fa90,
3872                 0x1fac0, 0x1fac0,
3873                 0x1fae0, 0x1fae0,
3874                 0x1fb00, 0x1fb84,
3875                 0x1fbc0, 0x1fbc8,
3876                 0x1fc08, 0x1fc0c,
3877                 0x1fc40, 0x1fc4c,
3878                 0x1fe84, 0x1fe90,
3879                 0x1fec0, 0x1fec0,
3880                 0x1fee0, 0x1fee0,
3881                 0x1ff00, 0x1ff84,
3882                 0x1ffc0, 0x1ffc8,
3883                 0x30000, 0x30030,
3884                 0x30100, 0x30144,
3885                 0x30190, 0x301d0,
3886                 0x30200, 0x30318,
3887                 0x30400, 0x3052c,
3888                 0x30540, 0x3061c,
3889                 0x30800, 0x30834,
3890                 0x308c0, 0x30908,
3891                 0x30910, 0x309ac,
3892                 0x30a00, 0x30a2c,
3893                 0x30a44, 0x30a50,
3894                 0x30a74, 0x30c24,
3895                 0x30d00, 0x30d00,
3896                 0x30d08, 0x30d14,
3897                 0x30d1c, 0x30d20,
3898                 0x30d3c, 0x30d50,
3899                 0x31200, 0x3120c,
3900                 0x31220, 0x31220,
3901                 0x31240, 0x31240,
3902                 0x31600, 0x3160c,
3903                 0x31a00, 0x31a1c,
3904                 0x31e00, 0x31e20,
3905                 0x31e38, 0x31e3c,
3906                 0x31e80, 0x31e80,
3907                 0x31e88, 0x31ea8,
3908                 0x31eb0, 0x31eb4,
3909                 0x31ec8, 0x31ed4,
3910                 0x31fb8, 0x32004,
3911                 0x32200, 0x32200,
3912                 0x32208, 0x32240,
3913                 0x32248, 0x32280,
3914                 0x32288, 0x322c0,
3915                 0x322c8, 0x322fc,
3916                 0x32600, 0x32630,
3917                 0x32a00, 0x32abc,
3918                 0x32b00, 0x32b70,
3919                 0x33000, 0x33048,
3920                 0x33060, 0x3309c,
3921                 0x330f0, 0x33148,
3922                 0x33160, 0x3319c,
3923                 0x331f0, 0x332e4,
3924                 0x332f8, 0x333e4,
3925                 0x333f8, 0x33448,
3926                 0x33460, 0x3349c,
3927                 0x334f0, 0x33548,
3928                 0x33560, 0x3359c,
3929                 0x335f0, 0x336e4,
3930                 0x336f8, 0x337e4,
3931                 0x337f8, 0x337fc,
3932                 0x33814, 0x33814,
3933                 0x3382c, 0x3382c,
3934                 0x33880, 0x3388c,
3935                 0x338e8, 0x338ec,
3936                 0x33900, 0x33948,
3937                 0x33960, 0x3399c,
3938                 0x339f0, 0x33ae4,
3939                 0x33af8, 0x33b10,
3940                 0x33b28, 0x33b28,
3941                 0x33b3c, 0x33b50,
3942                 0x33bf0, 0x33c10,
3943                 0x33c28, 0x33c28,
3944                 0x33c3c, 0x33c50,
3945                 0x33cf0, 0x33cfc,
3946                 0x34000, 0x34030,
3947                 0x34100, 0x34144,
3948                 0x34190, 0x341d0,
3949                 0x34200, 0x34318,
3950                 0x34400, 0x3452c,
3951                 0x34540, 0x3461c,
3952                 0x34800, 0x34834,
3953                 0x348c0, 0x34908,
3954                 0x34910, 0x349ac,
3955                 0x34a00, 0x34a2c,
3956                 0x34a44, 0x34a50,
3957                 0x34a74, 0x34c24,
3958                 0x34d00, 0x34d00,
3959                 0x34d08, 0x34d14,
3960                 0x34d1c, 0x34d20,
3961                 0x34d3c, 0x34d50,
3962                 0x35200, 0x3520c,
3963                 0x35220, 0x35220,
3964                 0x35240, 0x35240,
3965                 0x35600, 0x3560c,
3966                 0x35a00, 0x35a1c,
3967                 0x35e00, 0x35e20,
3968                 0x35e38, 0x35e3c,
3969                 0x35e80, 0x35e80,
3970                 0x35e88, 0x35ea8,
3971                 0x35eb0, 0x35eb4,
3972                 0x35ec8, 0x35ed4,
3973                 0x35fb8, 0x36004,
3974                 0x36200, 0x36200,
3975                 0x36208, 0x36240,
3976                 0x36248, 0x36280,
3977                 0x36288, 0x362c0,
3978                 0x362c8, 0x362fc,
3979                 0x36600, 0x36630,
3980                 0x36a00, 0x36abc,
3981                 0x36b00, 0x36b70,
3982                 0x37000, 0x37048,
3983                 0x37060, 0x3709c,
3984                 0x370f0, 0x37148,
3985                 0x37160, 0x3719c,
3986                 0x371f0, 0x372e4,
3987                 0x372f8, 0x373e4,
3988                 0x373f8, 0x37448,
3989                 0x37460, 0x3749c,
3990                 0x374f0, 0x37548,
3991                 0x37560, 0x3759c,
3992                 0x375f0, 0x376e4,
3993                 0x376f8, 0x377e4,
3994                 0x377f8, 0x377fc,
3995                 0x37814, 0x37814,
3996                 0x3782c, 0x3782c,
3997                 0x37880, 0x3788c,
3998                 0x378e8, 0x378ec,
3999                 0x37900, 0x37948,
4000                 0x37960, 0x3799c,
4001                 0x379f0, 0x37ae4,
4002                 0x37af8, 0x37b10,
4003                 0x37b28, 0x37b28,
4004                 0x37b3c, 0x37b50,
4005                 0x37bf0, 0x37c10,
4006                 0x37c28, 0x37c28,
4007                 0x37c3c, 0x37c50,
4008                 0x37cf0, 0x37cfc,
4009                 0x38000, 0x38030,
4010                 0x38100, 0x38144,
4011                 0x38190, 0x381d0,
4012                 0x38200, 0x38318,
4013                 0x38400, 0x3852c,
4014                 0x38540, 0x3861c,
4015                 0x38800, 0x38834,
4016                 0x388c0, 0x38908,
4017                 0x38910, 0x389ac,
4018                 0x38a00, 0x38a2c,
4019                 0x38a44, 0x38a50,
4020                 0x38a74, 0x38c24,
4021                 0x38d00, 0x38d00,
4022                 0x38d08, 0x38d14,
4023                 0x38d1c, 0x38d20,
4024                 0x38d3c, 0x38d50,
4025                 0x39200, 0x3920c,
4026                 0x39220, 0x39220,
4027                 0x39240, 0x39240,
4028                 0x39600, 0x3960c,
4029                 0x39a00, 0x39a1c,
4030                 0x39e00, 0x39e20,
4031                 0x39e38, 0x39e3c,
4032                 0x39e80, 0x39e80,
4033                 0x39e88, 0x39ea8,
4034                 0x39eb0, 0x39eb4,
4035                 0x39ec8, 0x39ed4,
4036                 0x39fb8, 0x3a004,
4037                 0x3a200, 0x3a200,
4038                 0x3a208, 0x3a240,
4039                 0x3a248, 0x3a280,
4040                 0x3a288, 0x3a2c0,
4041                 0x3a2c8, 0x3a2fc,
4042                 0x3a600, 0x3a630,
4043                 0x3aa00, 0x3aabc,
4044                 0x3ab00, 0x3ab70,
4045                 0x3b000, 0x3b048,
4046                 0x3b060, 0x3b09c,
4047                 0x3b0f0, 0x3b148,
4048                 0x3b160, 0x3b19c,
4049                 0x3b1f0, 0x3b2e4,
4050                 0x3b2f8, 0x3b3e4,
4051                 0x3b3f8, 0x3b448,
4052                 0x3b460, 0x3b49c,
4053                 0x3b4f0, 0x3b548,
4054                 0x3b560, 0x3b59c,
4055                 0x3b5f0, 0x3b6e4,
4056                 0x3b6f8, 0x3b7e4,
4057                 0x3b7f8, 0x3b7fc,
4058                 0x3b814, 0x3b814,
4059                 0x3b82c, 0x3b82c,
4060                 0x3b880, 0x3b88c,
4061                 0x3b8e8, 0x3b8ec,
4062                 0x3b900, 0x3b948,
4063                 0x3b960, 0x3b99c,
4064                 0x3b9f0, 0x3bae4,
4065                 0x3baf8, 0x3bb10,
4066                 0x3bb28, 0x3bb28,
4067                 0x3bb3c, 0x3bb50,
4068                 0x3bbf0, 0x3bc10,
4069                 0x3bc28, 0x3bc28,
4070                 0x3bc3c, 0x3bc50,
4071                 0x3bcf0, 0x3bcfc,
4072                 0x3c000, 0x3c030,
4073                 0x3c100, 0x3c144,
4074                 0x3c190, 0x3c1d0,
4075                 0x3c200, 0x3c318,
4076                 0x3c400, 0x3c52c,
4077                 0x3c540, 0x3c61c,
4078                 0x3c800, 0x3c834,
4079                 0x3c8c0, 0x3c908,
4080                 0x3c910, 0x3c9ac,
4081                 0x3ca00, 0x3ca2c,
4082                 0x3ca44, 0x3ca50,
4083                 0x3ca74, 0x3cc24,
4084                 0x3cd00, 0x3cd00,
4085                 0x3cd08, 0x3cd14,
4086                 0x3cd1c, 0x3cd20,
4087                 0x3cd3c, 0x3cd50,
4088                 0x3d200, 0x3d20c,
4089                 0x3d220, 0x3d220,
4090                 0x3d240, 0x3d240,
4091                 0x3d600, 0x3d60c,
4092                 0x3da00, 0x3da1c,
4093                 0x3de00, 0x3de20,
4094                 0x3de38, 0x3de3c,
4095                 0x3de80, 0x3de80,
4096                 0x3de88, 0x3dea8,
4097                 0x3deb0, 0x3deb4,
4098                 0x3dec8, 0x3ded4,
4099                 0x3dfb8, 0x3e004,
4100                 0x3e200, 0x3e200,
4101                 0x3e208, 0x3e240,
4102                 0x3e248, 0x3e280,
4103                 0x3e288, 0x3e2c0,
4104                 0x3e2c8, 0x3e2fc,
4105                 0x3e600, 0x3e630,
4106                 0x3ea00, 0x3eabc,
4107                 0x3eb00, 0x3eb70,
4108                 0x3f000, 0x3f048,
4109                 0x3f060, 0x3f09c,
4110                 0x3f0f0, 0x3f148,
4111                 0x3f160, 0x3f19c,
4112                 0x3f1f0, 0x3f2e4,
4113                 0x3f2f8, 0x3f3e4,
4114                 0x3f3f8, 0x3f448,
4115                 0x3f460, 0x3f49c,
4116                 0x3f4f0, 0x3f548,
4117                 0x3f560, 0x3f59c,
4118                 0x3f5f0, 0x3f6e4,
4119                 0x3f6f8, 0x3f7e4,
4120                 0x3f7f8, 0x3f7fc,
4121                 0x3f814, 0x3f814,
4122                 0x3f82c, 0x3f82c,
4123                 0x3f880, 0x3f88c,
4124                 0x3f8e8, 0x3f8ec,
4125                 0x3f900, 0x3f948,
4126                 0x3f960, 0x3f99c,
4127                 0x3f9f0, 0x3fae4,
4128                 0x3faf8, 0x3fb10,
4129                 0x3fb28, 0x3fb28,
4130                 0x3fb3c, 0x3fb50,
4131                 0x3fbf0, 0x3fc10,
4132                 0x3fc28, 0x3fc28,
4133                 0x3fc3c, 0x3fc50,
4134                 0x3fcf0, 0x3fcfc,
4135                 0x40000, 0x4000c,
4136                 0x40040, 0x40068,
4137                 0x4007c, 0x40144,
4138                 0x40180, 0x4018c,
4139                 0x40200, 0x40298,
4140                 0x402ac, 0x4033c,
4141                 0x403f8, 0x403fc,
4142                 0x41304, 0x413c4,
4143                 0x41400, 0x4141c,
4144                 0x41480, 0x414d0,
4145                 0x44000, 0x44078,
4146                 0x440c0, 0x44278,
4147                 0x442c0, 0x44478,
4148                 0x444c0, 0x44678,
4149                 0x446c0, 0x44878,
4150                 0x448c0, 0x449fc,
4151                 0x45000, 0x45068,
4152                 0x45080, 0x45084,
4153                 0x450a0, 0x450b0,
4154                 0x45200, 0x45268,
4155                 0x45280, 0x45284,
4156                 0x452a0, 0x452b0,
4157                 0x460c0, 0x460e4,
4158                 0x47000, 0x4708c,
4159                 0x47200, 0x47250,
4160                 0x47400, 0x47420,
4161                 0x47600, 0x47618,
4162                 0x47800, 0x47814,
4163                 0x48000, 0x4800c,
4164                 0x48040, 0x48068,
4165                 0x4807c, 0x48144,
4166                 0x48180, 0x4818c,
4167                 0x48200, 0x48298,
4168                 0x482ac, 0x4833c,
4169                 0x483f8, 0x483fc,
4170                 0x49304, 0x493c4,
4171                 0x49400, 0x4941c,
4172                 0x49480, 0x494d0,
4173                 0x4c000, 0x4c078,
4174                 0x4c0c0, 0x4c278,
4175                 0x4c2c0, 0x4c478,
4176                 0x4c4c0, 0x4c678,
4177                 0x4c6c0, 0x4c878,
4178                 0x4c8c0, 0x4c9fc,
4179                 0x4d000, 0x4d068,
4180                 0x4d080, 0x4d084,
4181                 0x4d0a0, 0x4d0b0,
4182                 0x4d200, 0x4d268,
4183                 0x4d280, 0x4d284,
4184                 0x4d2a0, 0x4d2b0,
4185                 0x4e0c0, 0x4e0e4,
4186                 0x4f000, 0x4f08c,
4187                 0x4f200, 0x4f250,
4188                 0x4f400, 0x4f420,
4189                 0x4f600, 0x4f618,
4190                 0x4f800, 0x4f814,
4191                 0x50000, 0x500cc,
4192                 0x50400, 0x50400,
4193                 0x50800, 0x508cc,
4194                 0x50c00, 0x50c00,
4195                 0x51000, 0x5101c,
4196                 0x51300, 0x51308,
4197         };
4198
4199         if (is_t4(sc)) {
4200                 reg_ranges = &t4_reg_ranges[0];
4201                 n = nitems(t4_reg_ranges);
4202         } else {
4203                 reg_ranges = &t5_reg_ranges[0];
4204                 n = nitems(t5_reg_ranges);
4205         }
4206
4207         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4208         for (i = 0; i < n; i += 2)
4209                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4210 }
4211
4212 static void
4213 cxgbe_tick(void *arg)
4214 {
4215         struct port_info *pi = arg;
4216         struct adapter *sc = pi->adapter;
4217         struct ifnet *ifp = pi->ifp;
4218         struct sge_txq *txq;
4219         int i, drops;
4220         struct port_stats *s = &pi->stats;
4221
4222         PORT_LOCK(pi);
4223         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4224                 PORT_UNLOCK(pi);
4225                 return; /* without scheduling another callout */
4226         }
4227
4228         t4_get_port_stats(sc, pi->tx_chan, s);
4229
4230         ifp->if_opackets = s->tx_frames - s->tx_pause;
4231         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4232         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4233         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4234         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4235         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4236         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4237             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4238             s->rx_trunc3;
4239         for (i = 0; i < 4; i++) {
4240                 if (pi->rx_chan_map & (1 << i)) {
4241                         uint32_t v;
4242
4243                         /*
4244                          * XXX: indirect reads from the same ADDR/DATA pair can
4245                          * race with each other.
4246                          */
4247                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4248                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4249                         ifp->if_iqdrops += v;
4250                 }
4251         }
4252
4253         drops = s->tx_drop;
4254         for_each_txq(pi, i, txq)
4255                 drops += txq->br->br_drops;
4256         ifp->if_oqdrops = drops;
4257
4258         ifp->if_oerrors = s->tx_error_frames;
4259         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4260             s->rx_fcs_err + s->rx_len_err;
4261
4262         callout_schedule(&pi->tick, hz);
4263         PORT_UNLOCK(pi);
4264 }
4265
4266 static void
4267 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4268 {
4269         struct ifnet *vlan;
4270
4271         if (arg != ifp || ifp->if_type != IFT_ETHER)
4272                 return;
4273
4274         vlan = VLAN_DEVAT(ifp, vid);
4275         VLAN_SETCOOKIE(vlan, ifp);
4276 }
4277
4278 static int
4279 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4280 {
4281
4282 #ifdef INVARIANTS
4283         panic("%s: opcode 0x%02x on iq %p with payload %p",
4284             __func__, rss->opcode, iq, m);
4285 #else
4286         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4287             __func__, rss->opcode, iq, m);
4288         m_freem(m);
4289 #endif
4290         return (EDOOFUS);
4291 }
4292
4293 int
4294 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4295 {
4296         uintptr_t *loc, new;
4297
4298         if (opcode >= nitems(sc->cpl_handler))
4299                 return (EINVAL);
4300
4301         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4302         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4303         atomic_store_rel_ptr(loc, new);
4304
4305         return (0);
4306 }
4307
4308 static int
4309 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4310 {
4311
4312 #ifdef INVARIANTS
4313         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4314 #else
4315         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4316             __func__, iq, ctrl);
4317 #endif
4318         return (EDOOFUS);
4319 }
4320
4321 int
4322 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4323 {
4324         uintptr_t *loc, new;
4325
4326         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4327         loc = (uintptr_t *) &sc->an_handler;
4328         atomic_store_rel_ptr(loc, new);
4329
4330         return (0);
4331 }
4332
4333 static int
4334 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4335 {
4336         const struct cpl_fw6_msg *cpl =
4337             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4338
4339 #ifdef INVARIANTS
4340         panic("%s: fw_msg type %d", __func__, cpl->type);
4341 #else
4342         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4343 #endif
4344         return (EDOOFUS);
4345 }
4346
4347 int
4348 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4349 {
4350         uintptr_t *loc, new;
4351
4352         if (type >= nitems(sc->fw_msg_handler))
4353                 return (EINVAL);
4354
4355         /*
4356          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4357          * handler dispatch table.  Reject any attempt to install a handler for
4358          * this subtype.
4359          */
4360         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4361                 return (EINVAL);
4362
4363         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4364         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4365         atomic_store_rel_ptr(loc, new);
4366
4367         return (0);
4368 }
4369
4370 static int
4371 t4_sysctls(struct adapter *sc)
4372 {
4373         struct sysctl_ctx_list *ctx;
4374         struct sysctl_oid *oid;
4375         struct sysctl_oid_list *children, *c0;
4376         static char *caps[] = {
4377                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4378                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4379                     "\6HASHFILTER\7ETHOFLD",
4380                 "\20\1TOE",                             /* caps[2] toecaps */
4381                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4382                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4383                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4384                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4385                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4386                     "\4PO_INITIAOR\5PO_TARGET"
4387         };
4388         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4389
4390         ctx = device_get_sysctl_ctx(sc->dev);
4391
4392         /*
4393          * dev.t4nex.X.
4394          */
4395         oid = device_get_sysctl_tree(sc->dev);
4396         c0 = children = SYSCTL_CHILDREN(oid);
4397
4398         sc->sc_do_rxcopy = 1;
4399         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4400             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4401
4402         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4403             sc->params.nports, "# of ports");
4404
4405         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4406             NULL, chip_rev(sc), "chip hardware revision");
4407
4408         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4409             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
4410
4411         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4412             CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
4413
4414         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4415             sc->cfcsum, "config file checksum");
4416
4417         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4418             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4419             sysctl_bitfield, "A", "available doorbells");
4420
4421         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4422             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4423             sysctl_bitfield, "A", "available link capabilities");
4424
4425         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4426             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4427             sysctl_bitfield, "A", "available NIC capabilities");
4428
4429         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4430             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4431             sysctl_bitfield, "A", "available TCP offload capabilities");
4432
4433         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4434             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4435             sysctl_bitfield, "A", "available RDMA capabilities");
4436
4437         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4438             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4439             sysctl_bitfield, "A", "available iSCSI capabilities");
4440
4441         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4442             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4443             sysctl_bitfield, "A", "available FCoE capabilities");
4444
4445         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4446             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4447
4448         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4449             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4450             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4451             "interrupt holdoff timer values (us)");
4452
4453         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4454             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4455             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4456             "interrupt holdoff packet counter values");
4457
4458         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4459             NULL, sc->tids.nftids, "number of filters");
4460
4461         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4462             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4463             "chip temperature (in Celsius)");
4464
4465         t4_sge_sysctls(sc, ctx, children);
4466
4467         sc->lro_timeout = 100;
4468         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4469             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4470
4471 #ifdef SBUF_DRAIN
4472         /*
4473          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4474          */
4475         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4476             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4477             "logs and miscellaneous information");
4478         children = SYSCTL_CHILDREN(oid);
4479
4480         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4481             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4482             sysctl_cctrl, "A", "congestion control");
4483
4484         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4485             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4486             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4487
4488         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4489             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4490             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4491
4492         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4493             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4494             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4495
4496         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4497             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4498             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4499
4500         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4501             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4502             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4503
4504         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4505             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4506             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4507
4508         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4509             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4510             sysctl_cim_la, "A", "CIM logic analyzer");
4511
4512         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4513             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4514             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4515
4516         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4517             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4518             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4519
4520         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4521             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4522             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4523
4524         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4525             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4526             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4527
4528         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4529             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4530             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4531
4532         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4533             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4534             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4535
4536         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4537             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4538             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4539
4540         if (is_t5(sc)) {
4541                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4542                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4543                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4544
4545                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4546                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4547                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4548         }
4549
4550         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4551             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4552             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4553
4554         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4555             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4556             sysctl_cim_qcfg, "A", "CIM queue configuration");
4557
4558         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4559             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4560             sysctl_cpl_stats, "A", "CPL statistics");
4561
4562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4563             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4564             sysctl_ddp_stats, "A", "DDP statistics");
4565
4566         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4567             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4568             sysctl_devlog, "A", "firmware's device log");
4569
4570         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4571             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4572             sysctl_fcoe_stats, "A", "FCoE statistics");
4573
4574         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4575             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4576             sysctl_hw_sched, "A", "hardware scheduler ");
4577
4578         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4579             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4580             sysctl_l2t, "A", "hardware L2 table");
4581
4582         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4583             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4584             sysctl_lb_stats, "A", "loopback statistics");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4588             sysctl_meminfo, "A", "memory regions");
4589
4590         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4591             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4592             sysctl_mps_tcam, "A", "MPS TCAM entries");
4593
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4595             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4596             sysctl_path_mtus, "A", "path MTUs");
4597
4598         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4599             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4600             sysctl_pm_stats, "A", "PM statistics");
4601
4602         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4603             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4604             sysctl_rdma_stats, "A", "RDMA statistics");
4605
4606         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4607             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4608             sysctl_tcp_stats, "A", "TCP statistics");
4609
4610         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4611             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4612             sysctl_tids, "A", "TID information");
4613
4614         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4615             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4616             sysctl_tp_err_stats, "A", "TP error statistics");
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4620             sysctl_tp_la, "A", "TP logic analyzer");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4624             sysctl_tx_rate, "A", "Tx rate");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4628             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4629
4630         if (is_t5(sc)) {
4631                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4632                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4633                     sysctl_wcwr_stats, "A", "write combined work requests");
4634         }
4635 #endif
4636
4637 #ifdef TCP_OFFLOAD
4638         if (is_offload(sc)) {
4639                 /*
4640                  * dev.t4nex.X.toe.
4641                  */
4642                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4643                     NULL, "TOE parameters");
4644                 children = SYSCTL_CHILDREN(oid);
4645
4646                 sc->tt.sndbuf = 256 * 1024;
4647                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4648                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4649
4650                 sc->tt.ddp = 0;
4651                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4652                     &sc->tt.ddp, 0, "DDP allowed");
4653
4654                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4655                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4656                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4657
4658                 sc->tt.ddp_thres =
4659                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4660                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4661                     &sc->tt.ddp_thres, 0, "DDP threshold");
4662
4663                 sc->tt.rx_coalesce = 1;
4664                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4665                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4666         }
4667 #endif
4668
4669
4670         return (0);
4671 }
4672
4673 static int
4674 cxgbe_sysctls(struct port_info *pi)
4675 {
4676         struct sysctl_ctx_list *ctx;
4677         struct sysctl_oid *oid;
4678         struct sysctl_oid_list *children;
4679         struct adapter *sc = pi->adapter;
4680
4681         ctx = device_get_sysctl_ctx(pi->dev);
4682
4683         /*
4684          * dev.cxgbe.X.
4685          */
4686         oid = device_get_sysctl_tree(pi->dev);
4687         children = SYSCTL_CHILDREN(oid);
4688
4689         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4690            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4691         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4692                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4693                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4694                     "PHY temperature (in Celsius)");
4695                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4696                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4697                     "PHY firmware version");
4698         }
4699         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4700             &pi->nrxq, 0, "# of rx queues");
4701         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4702             &pi->ntxq, 0, "# of tx queues");
4703         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4704             &pi->first_rxq, 0, "index of first rx queue");
4705         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4706             &pi->first_txq, 0, "index of first tx queue");
4707         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4708             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4709             "Reserve queue 0 for non-flowid packets");
4710
4711 #ifdef TCP_OFFLOAD
4712         if (is_offload(sc)) {
4713                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4714                     &pi->nofldrxq, 0,
4715                     "# of rx queues for offloaded TCP connections");
4716                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4717                     &pi->nofldtxq, 0,
4718                     "# of tx queues for offloaded TCP connections");
4719                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4720                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4721                     "index of first TOE rx queue");
4722                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4723                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4724                     "index of first TOE tx queue");
4725         }
4726 #endif
4727 #ifdef DEV_NETMAP
4728         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4729             &pi->nnmrxq, 0, "# of rx queues for netmap");
4730         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4731             &pi->nnmtxq, 0, "# of tx queues for netmap");
4732         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4733             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4734             "index of first netmap rx queue");
4735         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4736             CTLFLAG_RD, &pi->first_nm_txq, 0,
4737             "index of first netmap tx queue");
4738 #endif
4739
4740         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4741             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4742             "holdoff timer index");
4743         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4744             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4745             "holdoff packet counter index");
4746
4747         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4748             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4749             "rx queue size");
4750         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4751             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4752             "tx queue size");
4753
4754         /*
4755          * dev.cxgbe.X.stats.
4756          */
4757         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4758             NULL, "port statistics");
4759         children = SYSCTL_CHILDREN(oid);
4760
4761 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4762         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4763             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4764             sysctl_handle_t4_reg64, "QU", desc)
4765
4766         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4767             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4768         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4769             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4770         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4771             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4772         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4773             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4774         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4775             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4776         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4777             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4778         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4779             "# of tx frames in this range",
4780             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4781         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4782             "# of tx frames in this range",
4783             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4784         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4785             "# of tx frames in this range",
4786             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4787         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4788             "# of tx frames in this range",
4789             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4790         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4791             "# of tx frames in this range",
4792             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4793         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4794             "# of tx frames in this range",
4795             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4796         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4797             "# of tx frames in this range",
4798             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4799         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4800             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4801         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4802             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4803         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4804             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4805         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4806             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4807         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4808             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4809         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4810             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4811         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4812             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4813         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4814             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4815         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4816             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4817         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4818             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4819
4820         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4821             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4822         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4823             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4824         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4825             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4826         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4827             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4828         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4829             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4830         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4831             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4832         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4833             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4834         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4835             "# of frames received with bad FCS",
4836             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4837         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4838             "# of frames received with length error",
4839             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4840         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4841             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4842         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4843             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4844         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4845             "# of rx frames in this range",
4846             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4847         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4848             "# of rx frames in this range",
4849             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4850         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4851             "# of rx frames in this range",
4852             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4853         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4854             "# of rx frames in this range",
4855             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4856         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4857             "# of rx frames in this range",
4858             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4859         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4860             "# of rx frames in this range",
4861             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4862         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4863             "# of rx frames in this range",
4864             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4865         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4866             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4867         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4868             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4869         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4870             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4871         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4872             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4873         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4874             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4875         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4876             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4877         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4878             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4879         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4880             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4881         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4882             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4883
4884 #undef SYSCTL_ADD_T4_REG64
4885
4886 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4887         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4888             &pi->stats.name, desc)
4889
4890         /* We get these from port_stats and they may be stale by upto 1s */
4891         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4892             "# drops due to buffer-group 0 overflows");
4893         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4894             "# drops due to buffer-group 1 overflows");
4895         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4896             "# drops due to buffer-group 2 overflows");
4897         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4898             "# drops due to buffer-group 3 overflows");
4899         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4900             "# of buffer-group 0 truncated packets");
4901         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4902             "# of buffer-group 1 truncated packets");
4903         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4904             "# of buffer-group 2 truncated packets");
4905         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4906             "# of buffer-group 3 truncated packets");
4907
4908 #undef SYSCTL_ADD_T4_PORTSTAT
4909
4910         return (0);
4911 }
4912
4913 static int
4914 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4915 {
4916         int rc, *i;
4917         struct sbuf sb;
4918
4919         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4920         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4921                 sbuf_printf(&sb, "%d ", *i);
4922         sbuf_trim(&sb);
4923         sbuf_finish(&sb);
4924         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4925         sbuf_delete(&sb);
4926         return (rc);
4927 }
4928
4929 static int
4930 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4931 {
4932         int rc;
4933         struct sbuf *sb;
4934
4935         rc = sysctl_wire_old_buffer(req, 0);
4936         if (rc != 0)
4937                 return(rc);
4938
4939         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4940         if (sb == NULL)
4941                 return (ENOMEM);
4942
4943         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4944         rc = sbuf_finish(sb);
4945         sbuf_delete(sb);
4946
4947         return (rc);
4948 }
4949
4950 static int
4951 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4952 {
4953         struct port_info *pi = arg1;
4954         int op = arg2;
4955         struct adapter *sc = pi->adapter;
4956         u_int v;
4957         int rc;
4958
4959         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4960         if (rc)
4961                 return (rc);
4962         /* XXX: magic numbers */
4963         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4964             &v);
4965         end_synchronized_op(sc, 0);
4966         if (rc)
4967                 return (rc);
4968         if (op == 0)
4969                 v /= 256;
4970
4971         rc = sysctl_handle_int(oidp, &v, 0, req);
4972         return (rc);
4973 }
4974
4975 static int
4976 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
4977 {
4978         struct port_info *pi = arg1;
4979         int rc, val;
4980
4981         val = pi->rsrv_noflowq;
4982         rc = sysctl_handle_int(oidp, &val, 0, req);
4983         if (rc != 0 || req->newptr == NULL)
4984                 return (rc);
4985
4986         if ((val >= 1) && (pi->ntxq > 1))
4987                 pi->rsrv_noflowq = 1;
4988         else
4989                 pi->rsrv_noflowq = 0;
4990
4991         return (rc);
4992 }
4993
4994 static int
4995 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
4996 {
4997         struct port_info *pi = arg1;
4998         struct adapter *sc = pi->adapter;
4999         int idx, rc, i;
5000         struct sge_rxq *rxq;
5001 #ifdef TCP_OFFLOAD
5002         struct sge_ofld_rxq *ofld_rxq;
5003 #endif
5004         uint8_t v;
5005
5006         idx = pi->tmr_idx;
5007
5008         rc = sysctl_handle_int(oidp, &idx, 0, req);
5009         if (rc != 0 || req->newptr == NULL)
5010                 return (rc);
5011
5012         if (idx < 0 || idx >= SGE_NTIMERS)
5013                 return (EINVAL);
5014
5015         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5016             "t4tmr");
5017         if (rc)
5018                 return (rc);
5019
5020         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5021         for_each_rxq(pi, i, rxq) {
5022 #ifdef atomic_store_rel_8
5023                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5024 #else
5025                 rxq->iq.intr_params = v;
5026 #endif
5027         }
5028 #ifdef TCP_OFFLOAD
5029         for_each_ofld_rxq(pi, i, ofld_rxq) {
5030 #ifdef atomic_store_rel_8
5031                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5032 #else
5033                 ofld_rxq->iq.intr_params = v;
5034 #endif
5035         }
5036 #endif
5037         pi->tmr_idx = idx;
5038
5039         end_synchronized_op(sc, LOCK_HELD);
5040         return (0);
5041 }
5042
5043 static int
5044 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5045 {
5046         struct port_info *pi = arg1;
5047         struct adapter *sc = pi->adapter;
5048         int idx, rc;
5049
5050         idx = pi->pktc_idx;
5051
5052         rc = sysctl_handle_int(oidp, &idx, 0, req);
5053         if (rc != 0 || req->newptr == NULL)
5054                 return (rc);
5055
5056         if (idx < -1 || idx >= SGE_NCOUNTERS)
5057                 return (EINVAL);
5058
5059         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5060             "t4pktc");
5061         if (rc)
5062                 return (rc);
5063
5064         if (pi->flags & PORT_INIT_DONE)
5065                 rc = EBUSY; /* cannot be changed once the queues are created */
5066         else
5067                 pi->pktc_idx = idx;
5068
5069         end_synchronized_op(sc, LOCK_HELD);
5070         return (rc);
5071 }
5072
5073 static int
5074 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5075 {
5076         struct port_info *pi = arg1;
5077         struct adapter *sc = pi->adapter;
5078         int qsize, rc;
5079
5080         qsize = pi->qsize_rxq;
5081
5082         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5083         if (rc != 0 || req->newptr == NULL)
5084                 return (rc);
5085
5086         if (qsize < 128 || (qsize & 7))
5087                 return (EINVAL);
5088
5089         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5090             "t4rxqs");
5091         if (rc)
5092                 return (rc);
5093
5094         if (pi->flags & PORT_INIT_DONE)
5095                 rc = EBUSY; /* cannot be changed once the queues are created */
5096         else
5097                 pi->qsize_rxq = qsize;
5098
5099         end_synchronized_op(sc, LOCK_HELD);
5100         return (rc);
5101 }
5102
5103 static int
5104 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5105 {
5106         struct port_info *pi = arg1;
5107         struct adapter *sc = pi->adapter;
5108         int qsize, rc;
5109
5110         qsize = pi->qsize_txq;
5111
5112         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5113         if (rc != 0 || req->newptr == NULL)
5114                 return (rc);
5115
5116         /* bufring size must be powerof2 */
5117         if (qsize < 128 || !powerof2(qsize))
5118                 return (EINVAL);
5119
5120         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5121             "t4txqs");
5122         if (rc)
5123                 return (rc);
5124
5125         if (pi->flags & PORT_INIT_DONE)
5126                 rc = EBUSY; /* cannot be changed once the queues are created */
5127         else
5128                 pi->qsize_txq = qsize;
5129
5130         end_synchronized_op(sc, LOCK_HELD);
5131         return (rc);
5132 }
5133
5134 static int
5135 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5136 {
5137         struct adapter *sc = arg1;
5138         int reg = arg2;
5139         uint64_t val;
5140
5141         val = t4_read_reg64(sc, reg);
5142
5143         return (sysctl_handle_64(oidp, &val, 0, req));
5144 }
5145
5146 static int
5147 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5148 {
5149         struct adapter *sc = arg1;
5150         int rc, t;
5151         uint32_t param, val;
5152
5153         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5154         if (rc)
5155                 return (rc);
5156         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5157             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5158             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5159         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5160         end_synchronized_op(sc, 0);
5161         if (rc)
5162                 return (rc);
5163
5164         /* unknown is returned as 0 but we display -1 in that case */
5165         t = val == 0 ? -1 : val;
5166
5167         rc = sysctl_handle_int(oidp, &t, 0, req);
5168         return (rc);
5169 }
5170
5171 #ifdef SBUF_DRAIN
5172 static int
5173 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5174 {
5175         struct adapter *sc = arg1;
5176         struct sbuf *sb;
5177         int rc, i;
5178         uint16_t incr[NMTUS][NCCTRL_WIN];
5179         static const char *dec_fac[] = {
5180                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5181                 "0.9375"
5182         };
5183
5184         rc = sysctl_wire_old_buffer(req, 0);
5185         if (rc != 0)
5186                 return (rc);
5187
5188         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5189         if (sb == NULL)
5190                 return (ENOMEM);
5191
5192         t4_read_cong_tbl(sc, incr);
5193
5194         for (i = 0; i < NCCTRL_WIN; ++i) {
5195                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5196                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5197                     incr[5][i], incr[6][i], incr[7][i]);
5198                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5199                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5200                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5201                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5202         }
5203
5204         rc = sbuf_finish(sb);
5205         sbuf_delete(sb);
5206
5207         return (rc);
5208 }
5209
5210 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5211         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5212         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5213         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5214 };
5215
5216 static int
5217 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5218 {
5219         struct adapter *sc = arg1;
5220         struct sbuf *sb;
5221         int rc, i, n, qid = arg2;
5222         uint32_t *buf, *p;
5223         char *qtype;
5224         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5225
5226         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5227             ("%s: bad qid %d\n", __func__, qid));
5228
5229         if (qid < CIM_NUM_IBQ) {
5230                 /* inbound queue */
5231                 qtype = "IBQ";
5232                 n = 4 * CIM_IBQ_SIZE;
5233                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5234                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5235         } else {
5236                 /* outbound queue */
5237                 qtype = "OBQ";
5238                 qid -= CIM_NUM_IBQ;
5239                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5240                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5241                 rc = t4_read_cim_obq(sc, qid, buf, n);
5242         }
5243
5244         if (rc < 0) {
5245                 rc = -rc;
5246                 goto done;
5247         }
5248         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5249
5250         rc = sysctl_wire_old_buffer(req, 0);
5251         if (rc != 0)
5252                 goto done;
5253
5254         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5255         if (sb == NULL) {
5256                 rc = ENOMEM;
5257                 goto done;
5258         }
5259
5260         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5261         for (i = 0, p = buf; i < n; i += 16, p += 4)
5262                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5263                     p[2], p[3]);
5264
5265         rc = sbuf_finish(sb);
5266         sbuf_delete(sb);
5267 done:
5268         free(buf, M_CXGBE);
5269         return (rc);
5270 }
5271
5272 static int
5273 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5274 {
5275         struct adapter *sc = arg1;
5276         u_int cfg;
5277         struct sbuf *sb;
5278         uint32_t *buf, *p;
5279         int rc;
5280
5281         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5282         if (rc != 0)
5283                 return (rc);
5284
5285         rc = sysctl_wire_old_buffer(req, 0);
5286         if (rc != 0)
5287                 return (rc);
5288
5289         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5290         if (sb == NULL)
5291                 return (ENOMEM);
5292
5293         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5294             M_ZERO | M_WAITOK);
5295
5296         rc = -t4_cim_read_la(sc, buf, NULL);
5297         if (rc != 0)
5298                 goto done;
5299
5300         sbuf_printf(sb, "Status   Data      PC%s",
5301             cfg & F_UPDBGLACAPTPCONLY ? "" :
5302             "     LS0Stat  LS0Addr             LS0Data");
5303
5304         KASSERT((sc->params.cim_la_size & 7) == 0,
5305             ("%s: p will walk off the end of buf", __func__));
5306
5307         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5308                 if (cfg & F_UPDBGLACAPTPCONLY) {
5309                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5310                             p[6], p[7]);
5311                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5312                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5313                             p[4] & 0xff, p[5] >> 8);
5314                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5315                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5316                             p[1] & 0xf, p[2] >> 4);
5317                 } else {
5318                         sbuf_printf(sb,
5319                             "\n  %02x   %x%07x %x%07x %08x %08x "
5320                             "%08x%08x%08x%08x",
5321                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5322                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5323                             p[6], p[7]);
5324                 }
5325         }
5326
5327         rc = sbuf_finish(sb);
5328         sbuf_delete(sb);
5329 done:
5330         free(buf, M_CXGBE);
5331         return (rc);
5332 }
5333
5334 static int
5335 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5336 {
5337         struct adapter *sc = arg1;
5338         u_int i;
5339         struct sbuf *sb;
5340         uint32_t *buf, *p;
5341         int rc;
5342
5343         rc = sysctl_wire_old_buffer(req, 0);
5344         if (rc != 0)
5345                 return (rc);
5346
5347         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5348         if (sb == NULL)
5349                 return (ENOMEM);
5350
5351         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5352             M_ZERO | M_WAITOK);
5353
5354         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5355         p = buf;
5356
5357         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5358                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5359                     p[1], p[0]);
5360         }
5361
5362         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5363         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5364                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5365                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5366                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5367                     (p[1] >> 2) | ((p[2] & 3) << 30),
5368                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5369                     p[0] & 1);
5370         }
5371
5372         rc = sbuf_finish(sb);
5373         sbuf_delete(sb);
5374         free(buf, M_CXGBE);
5375         return (rc);
5376 }
5377
5378 static int
5379 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5380 {
5381         struct adapter *sc = arg1;
5382         u_int i;
5383         struct sbuf *sb;
5384         uint32_t *buf, *p;
5385         int rc;
5386
5387         rc = sysctl_wire_old_buffer(req, 0);
5388         if (rc != 0)
5389                 return (rc);
5390
5391         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5392         if (sb == NULL)
5393                 return (ENOMEM);
5394
5395         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5396             M_ZERO | M_WAITOK);
5397
5398         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5399         p = buf;
5400
5401         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5402         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5403                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5404                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5405                     p[4], p[3], p[2], p[1], p[0]);
5406         }
5407
5408         sbuf_printf(sb, "\n\nCntl ID               Data");
5409         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5410                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5411                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5412         }
5413
5414         rc = sbuf_finish(sb);
5415         sbuf_delete(sb);
5416         free(buf, M_CXGBE);
5417         return (rc);
5418 }
5419
5420 static int
5421 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5422 {
5423         struct adapter *sc = arg1;
5424         struct sbuf *sb;
5425         int rc, i;
5426         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5427         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5428         uint16_t thres[CIM_NUM_IBQ];
5429         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5430         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5431         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5432
5433         if (is_t4(sc)) {
5434                 cim_num_obq = CIM_NUM_OBQ;
5435                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5436                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5437         } else {
5438                 cim_num_obq = CIM_NUM_OBQ_T5;
5439                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5440                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5441         }
5442         nq = CIM_NUM_IBQ + cim_num_obq;
5443
5444         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5445         if (rc == 0)
5446                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5447         if (rc != 0)
5448                 return (rc);
5449
5450         t4_read_cimq_cfg(sc, base, size, thres);
5451
5452         rc = sysctl_wire_old_buffer(req, 0);
5453         if (rc != 0)
5454                 return (rc);
5455
5456         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5457         if (sb == NULL)
5458                 return (ENOMEM);
5459
5460         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5461
5462         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5463                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5464                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5465                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5466                     G_QUEREMFLITS(p[2]) * 16);
5467         for ( ; i < nq; i++, p += 4, wr += 2)
5468                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5469                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5470                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5471                     G_QUEREMFLITS(p[2]) * 16);
5472
5473         rc = sbuf_finish(sb);
5474         sbuf_delete(sb);
5475
5476         return (rc);
5477 }
5478
5479 static int
5480 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5481 {
5482         struct adapter *sc = arg1;
5483         struct sbuf *sb;
5484         int rc;
5485         struct tp_cpl_stats stats;
5486
5487         rc = sysctl_wire_old_buffer(req, 0);
5488         if (rc != 0)
5489                 return (rc);
5490
5491         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5492         if (sb == NULL)
5493                 return (ENOMEM);
5494
5495         t4_tp_get_cpl_stats(sc, &stats);
5496
5497         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5498             "channel 3\n");
5499         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5500                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5501         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5502                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5503
5504         rc = sbuf_finish(sb);
5505         sbuf_delete(sb);
5506
5507         return (rc);
5508 }
5509
5510 static int
5511 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5512 {
5513         struct adapter *sc = arg1;
5514         struct sbuf *sb;
5515         int rc;
5516         struct tp_usm_stats stats;
5517
5518         rc = sysctl_wire_old_buffer(req, 0);
5519         if (rc != 0)
5520                 return(rc);
5521
5522         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5523         if (sb == NULL)
5524                 return (ENOMEM);
5525
5526         t4_get_usm_stats(sc, &stats);
5527
5528         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5529         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5530         sbuf_printf(sb, "Drops:  %u", stats.drops);
5531
5532         rc = sbuf_finish(sb);
5533         sbuf_delete(sb);
5534
5535         return (rc);
5536 }
5537
5538 const char *devlog_level_strings[] = {
5539         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5540         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5541         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5542         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5543         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5544         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5545 };
5546
5547 const char *devlog_facility_strings[] = {
5548         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5549         [FW_DEVLOG_FACILITY_CF]         = "CF",
5550         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5551         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5552         [FW_DEVLOG_FACILITY_RES]        = "RES",
5553         [FW_DEVLOG_FACILITY_HW]         = "HW",
5554         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5555         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5556         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5557         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5558         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5559         [FW_DEVLOG_FACILITY_VI]         = "VI",
5560         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5561         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5562         [FW_DEVLOG_FACILITY_TM]         = "TM",
5563         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5564         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5565         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5566         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5567         [FW_DEVLOG_FACILITY_RI]         = "RI",
5568         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5569         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5570         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5571         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5572 };
5573
5574 static int
5575 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5576 {
5577         struct adapter *sc = arg1;
5578         struct devlog_params *dparams = &sc->params.devlog;
5579         struct fw_devlog_e *buf, *e;
5580         int i, j, rc, nentries, first = 0, m;
5581         struct sbuf *sb;
5582         uint64_t ftstamp = UINT64_MAX;
5583
5584         if (dparams->start == 0) {
5585                 dparams->memtype = FW_MEMTYPE_EDC0;
5586                 dparams->start = 0x84000;
5587                 dparams->size = 32768;
5588         }
5589
5590         nentries = dparams->size / sizeof(struct fw_devlog_e);
5591
5592         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5593         if (buf == NULL)
5594                 return (ENOMEM);
5595
5596         m = fwmtype_to_hwmtype(dparams->memtype);
5597         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5598         if (rc != 0)
5599                 goto done;
5600
5601         for (i = 0; i < nentries; i++) {
5602                 e = &buf[i];
5603
5604                 if (e->timestamp == 0)
5605                         break;  /* end */
5606
5607                 e->timestamp = be64toh(e->timestamp);
5608                 e->seqno = be32toh(e->seqno);
5609                 for (j = 0; j < 8; j++)
5610                         e->params[j] = be32toh(e->params[j]);
5611
5612                 if (e->timestamp < ftstamp) {
5613                         ftstamp = e->timestamp;
5614                         first = i;
5615                 }
5616         }
5617
5618         if (buf[first].timestamp == 0)
5619                 goto done;      /* nothing in the log */
5620
5621         rc = sysctl_wire_old_buffer(req, 0);
5622         if (rc != 0)
5623                 goto done;
5624
5625         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5626         if (sb == NULL) {
5627                 rc = ENOMEM;
5628                 goto done;
5629         }
5630         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5631             "Seq#", "Tstamp", "Level", "Facility", "Message");
5632
5633         i = first;
5634         do {
5635                 e = &buf[i];
5636                 if (e->timestamp == 0)
5637                         break;  /* end */
5638
5639                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5640                     e->seqno, e->timestamp,
5641                     (e->level < nitems(devlog_level_strings) ?
5642                         devlog_level_strings[e->level] : "UNKNOWN"),
5643                     (e->facility < nitems(devlog_facility_strings) ?
5644                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5645                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5646                     e->params[2], e->params[3], e->params[4],
5647                     e->params[5], e->params[6], e->params[7]);
5648
5649                 if (++i == nentries)
5650                         i = 0;
5651         } while (i != first);
5652
5653         rc = sbuf_finish(sb);
5654         sbuf_delete(sb);
5655 done:
5656         free(buf, M_CXGBE);
5657         return (rc);
5658 }
5659
5660 static int
5661 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5662 {
5663         struct adapter *sc = arg1;
5664         struct sbuf *sb;
5665         int rc;
5666         struct tp_fcoe_stats stats[4];
5667
5668         rc = sysctl_wire_old_buffer(req, 0);
5669         if (rc != 0)
5670                 return (rc);
5671
5672         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5673         if (sb == NULL)
5674                 return (ENOMEM);
5675
5676         t4_get_fcoe_stats(sc, 0, &stats[0]);
5677         t4_get_fcoe_stats(sc, 1, &stats[1]);
5678         t4_get_fcoe_stats(sc, 2, &stats[2]);
5679         t4_get_fcoe_stats(sc, 3, &stats[3]);
5680
5681         sbuf_printf(sb, "                   channel 0        channel 1        "
5682             "channel 2        channel 3\n");
5683         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5684             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5685             stats[3].octetsDDP);
5686         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5687             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5688         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5689             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5690             stats[3].framesDrop);
5691
5692         rc = sbuf_finish(sb);
5693         sbuf_delete(sb);
5694
5695         return (rc);
5696 }
5697
5698 static int
5699 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5700 {
5701         struct adapter *sc = arg1;
5702         struct sbuf *sb;
5703         int rc, i;
5704         unsigned int map, kbps, ipg, mode;
5705         unsigned int pace_tab[NTX_SCHED];
5706
5707         rc = sysctl_wire_old_buffer(req, 0);
5708         if (rc != 0)
5709                 return (rc);
5710
5711         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5712         if (sb == NULL)
5713                 return (ENOMEM);
5714
5715         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5716         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5717         t4_read_pace_tbl(sc, pace_tab);
5718
5719         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5720             "Class IPG (0.1 ns)   Flow IPG (us)");
5721
5722         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5723                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5724                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5725                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5726                 if (kbps)
5727                         sbuf_printf(sb, "%9u     ", kbps);
5728                 else
5729                         sbuf_printf(sb, " disabled     ");
5730
5731                 if (ipg)
5732                         sbuf_printf(sb, "%13u        ", ipg);
5733                 else
5734                         sbuf_printf(sb, "     disabled        ");
5735
5736                 if (pace_tab[i])
5737                         sbuf_printf(sb, "%10u", pace_tab[i]);
5738                 else
5739                         sbuf_printf(sb, "  disabled");
5740         }
5741
5742         rc = sbuf_finish(sb);
5743         sbuf_delete(sb);
5744
5745         return (rc);
5746 }
5747
5748 static int
5749 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5750 {
5751         struct adapter *sc = arg1;
5752         struct sbuf *sb;
5753         int rc, i, j;
5754         uint64_t *p0, *p1;
5755         struct lb_port_stats s[2];
5756         static const char *stat_name[] = {
5757                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5758                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5759                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5760                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5761                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5762                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5763                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5764         };
5765
5766         rc = sysctl_wire_old_buffer(req, 0);
5767         if (rc != 0)
5768                 return (rc);
5769
5770         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5771         if (sb == NULL)
5772                 return (ENOMEM);
5773
5774         memset(s, 0, sizeof(s));
5775
5776         for (i = 0; i < 4; i += 2) {
5777                 t4_get_lb_stats(sc, i, &s[0]);
5778                 t4_get_lb_stats(sc, i + 1, &s[1]);
5779
5780                 p0 = &s[0].octets;
5781                 p1 = &s[1].octets;
5782                 sbuf_printf(sb, "%s                       Loopback %u"
5783                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5784
5785                 for (j = 0; j < nitems(stat_name); j++)
5786                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5787                                    *p0++, *p1++);
5788         }
5789
5790         rc = sbuf_finish(sb);
5791         sbuf_delete(sb);
5792
5793         return (rc);
5794 }
5795
5796 static int
5797 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5798 {
5799         int rc = 0;
5800         struct port_info *pi = arg1;
5801         struct sbuf *sb;
5802         static const char *linkdnreasons[] = {
5803                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5804                 "PHY overheated", "unknown", "rx los", "reserved7"
5805         };
5806
5807         rc = sysctl_wire_old_buffer(req, 0);
5808         if (rc != 0)
5809                 return(rc);
5810         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5811         if (sb == NULL)
5812                 return (ENOMEM);
5813
5814         if (pi->linkdnrc < 0)
5815                 sbuf_printf(sb, "n/a");
5816         else if (pi->linkdnrc < nitems(linkdnreasons))
5817                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5818         else
5819                 sbuf_printf(sb, "%d", pi->linkdnrc);
5820
5821         rc = sbuf_finish(sb);
5822         sbuf_delete(sb);
5823
5824         return (rc);
5825 }
5826
5827 struct mem_desc {
5828         unsigned int base;
5829         unsigned int limit;
5830         unsigned int idx;
5831 };
5832
5833 static int
5834 mem_desc_cmp(const void *a, const void *b)
5835 {
5836         return ((const struct mem_desc *)a)->base -
5837                ((const struct mem_desc *)b)->base;
5838 }
5839
5840 static void
5841 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5842     unsigned int to)
5843 {
5844         unsigned int size;
5845
5846         size = to - from + 1;
5847         if (size == 0)
5848                 return;
5849
5850         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5851         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5852 }
5853
5854 static int
5855 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5856 {
5857         struct adapter *sc = arg1;
5858         struct sbuf *sb;
5859         int rc, i, n;
5860         uint32_t lo, hi, used, alloc;
5861         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5862         static const char *region[] = {
5863                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5864                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5865                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5866                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5867                 "RQUDP region:", "PBL region:", "TXPBL region:",
5868                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5869                 "On-chip queues:"
5870         };
5871         struct mem_desc avail[4];
5872         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5873         struct mem_desc *md = mem;
5874
5875         rc = sysctl_wire_old_buffer(req, 0);
5876         if (rc != 0)
5877                 return (rc);
5878
5879         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5880         if (sb == NULL)
5881                 return (ENOMEM);
5882
5883         for (i = 0; i < nitems(mem); i++) {
5884                 mem[i].limit = 0;
5885                 mem[i].idx = i;
5886         }
5887
5888         /* Find and sort the populated memory ranges */
5889         i = 0;
5890         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5891         if (lo & F_EDRAM0_ENABLE) {
5892                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5893                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5894                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5895                 avail[i].idx = 0;
5896                 i++;
5897         }
5898         if (lo & F_EDRAM1_ENABLE) {
5899                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5900                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
5901                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5902                 avail[i].idx = 1;
5903                 i++;
5904         }
5905         if (lo & F_EXT_MEM_ENABLE) {
5906                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5907                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5908                 avail[i].limit = avail[i].base +
5909                     (G_EXT_MEM_SIZE(hi) << 20);
5910                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
5911                 i++;
5912         }
5913         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
5914                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
5915                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
5916                 avail[i].limit = avail[i].base +
5917                     (G_EXT_MEM1_SIZE(hi) << 20);
5918                 avail[i].idx = 4;
5919                 i++;
5920         }
5921         if (!i)                                    /* no memory available */
5922                 return 0;
5923         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
5924
5925         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
5926         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
5927         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
5928         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
5929         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
5930         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
5931         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
5932         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
5933         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
5934
5935         /* the next few have explicit upper bounds */
5936         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
5937         md->limit = md->base - 1 +
5938                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
5939                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
5940         md++;
5941
5942         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
5943         md->limit = md->base - 1 +
5944                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
5945                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
5946         md++;
5947
5948         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
5949                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
5950                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
5951                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
5952         } else {
5953                 md->base = 0;
5954                 md->idx = nitems(region);  /* hide it */
5955         }
5956         md++;
5957
5958 #define ulp_region(reg) \
5959         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
5960         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
5961
5962         ulp_region(RX_ISCSI);
5963         ulp_region(RX_TDDP);
5964         ulp_region(TX_TPT);
5965         ulp_region(RX_STAG);
5966         ulp_region(RX_RQ);
5967         ulp_region(RX_RQUDP);
5968         ulp_region(RX_PBL);
5969         ulp_region(TX_PBL);
5970 #undef ulp_region
5971
5972         md->base = 0;
5973         md->idx = nitems(region);
5974         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
5975                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
5976                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
5977                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
5978         }
5979         md++;
5980
5981         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
5982         md->limit = md->base + sc->tids.ntids - 1;
5983         md++;
5984         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
5985         md->limit = md->base + sc->tids.ntids - 1;
5986         md++;
5987
5988         md->base = sc->vres.ocq.start;
5989         if (sc->vres.ocq.size)
5990                 md->limit = md->base + sc->vres.ocq.size - 1;
5991         else
5992                 md->idx = nitems(region);  /* hide it */
5993         md++;
5994
5995         /* add any address-space holes, there can be up to 3 */
5996         for (n = 0; n < i - 1; n++)
5997                 if (avail[n].limit < avail[n + 1].base)
5998                         (md++)->base = avail[n].limit;
5999         if (avail[n].limit)
6000                 (md++)->base = avail[n].limit;
6001
6002         n = md - mem;
6003         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6004
6005         for (lo = 0; lo < i; lo++)
6006                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6007                                 avail[lo].limit - 1);
6008
6009         sbuf_printf(sb, "\n");
6010         for (i = 0; i < n; i++) {
6011                 if (mem[i].idx >= nitems(region))
6012                         continue;                        /* skip holes */
6013                 if (!mem[i].limit)
6014                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6015                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6016                                 mem[i].limit);
6017         }
6018
6019         sbuf_printf(sb, "\n");
6020         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6021         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6022         mem_region_show(sb, "uP RAM:", lo, hi);
6023
6024         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6025         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6026         mem_region_show(sb, "uP Extmem2:", lo, hi);
6027
6028         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6029         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6030                    G_PMRXMAXPAGE(lo),
6031                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6032                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6033
6034         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6035         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6036         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6037                    G_PMTXMAXPAGE(lo),
6038                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6039                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6040         sbuf_printf(sb, "%u p-structs\n",
6041                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6042
6043         for (i = 0; i < 4; i++) {
6044                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6045                 if (is_t4(sc)) {
6046                         used = G_USED(lo);
6047                         alloc = G_ALLOC(lo);
6048                 } else {
6049                         used = G_T5_USED(lo);
6050                         alloc = G_T5_ALLOC(lo);
6051                 }
6052                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6053                            i, used, alloc);
6054         }
6055         for (i = 0; i < 4; i++) {
6056                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6057                 if (is_t4(sc)) {
6058                         used = G_USED(lo);
6059                         alloc = G_ALLOC(lo);
6060                 } else {
6061                         used = G_T5_USED(lo);
6062                         alloc = G_T5_ALLOC(lo);
6063                 }
6064                 sbuf_printf(sb,
6065                            "\nLoopback %d using %u pages out of %u allocated",
6066                            i, used, alloc);
6067         }
6068
6069         rc = sbuf_finish(sb);
6070         sbuf_delete(sb);
6071
6072         return (rc);
6073 }
6074
6075 static inline void
6076 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6077 {
6078         *mask = x | y;
6079         y = htobe64(y);
6080         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6081 }
6082
6083 static int
6084 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6085 {
6086         struct adapter *sc = arg1;
6087         struct sbuf *sb;
6088         int rc, i, n;
6089
6090         rc = sysctl_wire_old_buffer(req, 0);
6091         if (rc != 0)
6092                 return (rc);
6093
6094         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6095         if (sb == NULL)
6096                 return (ENOMEM);
6097
6098         sbuf_printf(sb,
6099             "Idx  Ethernet address     Mask     Vld Ports PF"
6100             "  VF              Replication             P0 P1 P2 P3  ML");
6101         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6102             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6103         for (i = 0; i < n; i++) {
6104                 uint64_t tcamx, tcamy, mask;
6105                 uint32_t cls_lo, cls_hi;
6106                 uint8_t addr[ETHER_ADDR_LEN];
6107
6108                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6109                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6110                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6111                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6112
6113                 if (tcamx & tcamy)
6114                         continue;
6115
6116                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6117                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6118                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6119                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6120                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6121                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6122                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6123
6124                 if (cls_lo & F_REPLICATE) {
6125                         struct fw_ldst_cmd ldst_cmd;
6126
6127                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6128                         ldst_cmd.op_to_addrspace =
6129                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6130                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6131                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6132                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6133                         ldst_cmd.u.mps.fid_ctl =
6134                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6135                                 V_FW_LDST_CMD_CTL(i));
6136
6137                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6138                             "t4mps");
6139                         if (rc)
6140                                 break;
6141                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6142                             sizeof(ldst_cmd), &ldst_cmd);
6143                         end_synchronized_op(sc, 0);
6144
6145                         if (rc != 0) {
6146                                 sbuf_printf(sb,
6147                                     " ------------ error %3u ------------", rc);
6148                                 rc = 0;
6149                         } else {
6150                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6151                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6152                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6153                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6154                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6155                         }
6156                 } else
6157                         sbuf_printf(sb, "%36s", "");
6158
6159                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6160                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6161                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6162         }
6163
6164         if (rc)
6165                 (void) sbuf_finish(sb);
6166         else
6167                 rc = sbuf_finish(sb);
6168         sbuf_delete(sb);
6169
6170         return (rc);
6171 }
6172
6173 static int
6174 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6175 {
6176         struct adapter *sc = arg1;
6177         struct sbuf *sb;
6178         int rc;
6179         uint16_t mtus[NMTUS];
6180
6181         rc = sysctl_wire_old_buffer(req, 0);
6182         if (rc != 0)
6183                 return (rc);
6184
6185         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6186         if (sb == NULL)
6187                 return (ENOMEM);
6188
6189         t4_read_mtu_tbl(sc, mtus, NULL);
6190
6191         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6192             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6193             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6194             mtus[14], mtus[15]);
6195
6196         rc = sbuf_finish(sb);
6197         sbuf_delete(sb);
6198
6199         return (rc);
6200 }
6201
6202 static int
6203 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6204 {
6205         struct adapter *sc = arg1;
6206         struct sbuf *sb;
6207         int rc, i;
6208         uint32_t cnt[PM_NSTATS];
6209         uint64_t cyc[PM_NSTATS];
6210         static const char *rx_stats[] = {
6211                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6212         };
6213         static const char *tx_stats[] = {
6214                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6215         };
6216
6217         rc = sysctl_wire_old_buffer(req, 0);
6218         if (rc != 0)
6219                 return (rc);
6220
6221         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6222         if (sb == NULL)
6223                 return (ENOMEM);
6224
6225         t4_pmtx_get_stats(sc, cnt, cyc);
6226         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6227         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6228                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6229                     cyc[i]);
6230
6231         t4_pmrx_get_stats(sc, cnt, cyc);
6232         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6233         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6234                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6235                     cyc[i]);
6236
6237         rc = sbuf_finish(sb);
6238         sbuf_delete(sb);
6239
6240         return (rc);
6241 }
6242
6243 static int
6244 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6245 {
6246         struct adapter *sc = arg1;
6247         struct sbuf *sb;
6248         int rc;
6249         struct tp_rdma_stats stats;
6250
6251         rc = sysctl_wire_old_buffer(req, 0);
6252         if (rc != 0)
6253                 return (rc);
6254
6255         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6256         if (sb == NULL)
6257                 return (ENOMEM);
6258
6259         t4_tp_get_rdma_stats(sc, &stats);
6260         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6261         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6262
6263         rc = sbuf_finish(sb);
6264         sbuf_delete(sb);
6265
6266         return (rc);
6267 }
6268
6269 static int
6270 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6271 {
6272         struct adapter *sc = arg1;
6273         struct sbuf *sb;
6274         int rc;
6275         struct tp_tcp_stats v4, v6;
6276
6277         rc = sysctl_wire_old_buffer(req, 0);
6278         if (rc != 0)
6279                 return (rc);
6280
6281         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6282         if (sb == NULL)
6283                 return (ENOMEM);
6284
6285         t4_tp_get_tcp_stats(sc, &v4, &v6);
6286         sbuf_printf(sb,
6287             "                                IP                 IPv6\n");
6288         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6289             v4.tcpOutRsts, v6.tcpOutRsts);
6290         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6291             v4.tcpInSegs, v6.tcpInSegs);
6292         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6293             v4.tcpOutSegs, v6.tcpOutSegs);
6294         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6295             v4.tcpRetransSegs, v6.tcpRetransSegs);
6296
6297         rc = sbuf_finish(sb);
6298         sbuf_delete(sb);
6299
6300         return (rc);
6301 }
6302
6303 static int
6304 sysctl_tids(SYSCTL_HANDLER_ARGS)
6305 {
6306         struct adapter *sc = arg1;
6307         struct sbuf *sb;
6308         int rc;
6309         struct tid_info *t = &sc->tids;
6310
6311         rc = sysctl_wire_old_buffer(req, 0);
6312         if (rc != 0)
6313                 return (rc);
6314
6315         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6316         if (sb == NULL)
6317                 return (ENOMEM);
6318
6319         if (t->natids) {
6320                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6321                     t->atids_in_use);
6322         }
6323
6324         if (t->ntids) {
6325                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6326                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6327
6328                         if (b) {
6329                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6330                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6331                                     t->ntids - 1);
6332                         } else {
6333                                 sbuf_printf(sb, "TID range: %u-%u",
6334                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6335                                     t->ntids - 1);
6336                         }
6337                 } else
6338                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6339                 sbuf_printf(sb, ", in use: %u\n",
6340                     atomic_load_acq_int(&t->tids_in_use));
6341         }
6342
6343         if (t->nstids) {
6344                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6345                     t->stid_base + t->nstids - 1, t->stids_in_use);
6346         }
6347
6348         if (t->nftids) {
6349                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6350                     t->ftid_base + t->nftids - 1);
6351         }
6352
6353         if (t->netids) {
6354                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6355                     t->etid_base + t->netids - 1);
6356         }
6357
6358         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6359             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6360             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6361
6362         rc = sbuf_finish(sb);
6363         sbuf_delete(sb);
6364
6365         return (rc);
6366 }
6367
6368 static int
6369 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6370 {
6371         struct adapter *sc = arg1;
6372         struct sbuf *sb;
6373         int rc;
6374         struct tp_err_stats stats;
6375
6376         rc = sysctl_wire_old_buffer(req, 0);
6377         if (rc != 0)
6378                 return (rc);
6379
6380         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6381         if (sb == NULL)
6382                 return (ENOMEM);
6383
6384         t4_tp_get_err_stats(sc, &stats);
6385
6386         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6387                       "channel 3\n");
6388         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6389             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6390             stats.macInErrs[3]);
6391         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6392             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6393             stats.hdrInErrs[3]);
6394         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6395             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6396             stats.tcpInErrs[3]);
6397         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6398             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6399             stats.tcp6InErrs[3]);
6400         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6401             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6402             stats.tnlCongDrops[3]);
6403         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6404             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6405             stats.tnlTxDrops[3]);
6406         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6407             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6408             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6409         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6410             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6411             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6412         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6413             stats.ofldNoNeigh, stats.ofldCongDefer);
6414
6415         rc = sbuf_finish(sb);
6416         sbuf_delete(sb);
6417
6418         return (rc);
6419 }
6420
6421 struct field_desc {
6422         const char *name;
6423         u_int start;
6424         u_int width;
6425 };
6426
6427 static void
6428 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6429 {
6430         char buf[32];
6431         int line_size = 0;
6432
6433         while (f->name) {
6434                 uint64_t mask = (1ULL << f->width) - 1;
6435                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6436                     ((uintmax_t)v >> f->start) & mask);
6437
6438                 if (line_size + len >= 79) {
6439                         line_size = 8;
6440                         sbuf_printf(sb, "\n        ");
6441                 }
6442                 sbuf_printf(sb, "%s ", buf);
6443                 line_size += len + 1;
6444                 f++;
6445         }
6446         sbuf_printf(sb, "\n");
6447 }
6448
6449 static struct field_desc tp_la0[] = {
6450         { "RcfOpCodeOut", 60, 4 },
6451         { "State", 56, 4 },
6452         { "WcfState", 52, 4 },
6453         { "RcfOpcSrcOut", 50, 2 },
6454         { "CRxError", 49, 1 },
6455         { "ERxError", 48, 1 },
6456         { "SanityFailed", 47, 1 },
6457         { "SpuriousMsg", 46, 1 },
6458         { "FlushInputMsg", 45, 1 },
6459         { "FlushInputCpl", 44, 1 },
6460         { "RssUpBit", 43, 1 },
6461         { "RssFilterHit", 42, 1 },
6462         { "Tid", 32, 10 },
6463         { "InitTcb", 31, 1 },
6464         { "LineNumber", 24, 7 },
6465         { "Emsg", 23, 1 },
6466         { "EdataOut", 22, 1 },
6467         { "Cmsg", 21, 1 },
6468         { "CdataOut", 20, 1 },
6469         { "EreadPdu", 19, 1 },
6470         { "CreadPdu", 18, 1 },
6471         { "TunnelPkt", 17, 1 },
6472         { "RcfPeerFin", 16, 1 },
6473         { "RcfReasonOut", 12, 4 },
6474         { "TxCchannel", 10, 2 },
6475         { "RcfTxChannel", 8, 2 },
6476         { "RxEchannel", 6, 2 },
6477         { "RcfRxChannel", 5, 1 },
6478         { "RcfDataOutSrdy", 4, 1 },
6479         { "RxDvld", 3, 1 },
6480         { "RxOoDvld", 2, 1 },
6481         { "RxCongestion", 1, 1 },
6482         { "TxCongestion", 0, 1 },
6483         { NULL }
6484 };
6485
6486 static struct field_desc tp_la1[] = {
6487         { "CplCmdIn", 56, 8 },
6488         { "CplCmdOut", 48, 8 },
6489         { "ESynOut", 47, 1 },
6490         { "EAckOut", 46, 1 },
6491         { "EFinOut", 45, 1 },
6492         { "ERstOut", 44, 1 },
6493         { "SynIn", 43, 1 },
6494         { "AckIn", 42, 1 },
6495         { "FinIn", 41, 1 },
6496         { "RstIn", 40, 1 },
6497         { "DataIn", 39, 1 },
6498         { "DataInVld", 38, 1 },
6499         { "PadIn", 37, 1 },
6500         { "RxBufEmpty", 36, 1 },
6501         { "RxDdp", 35, 1 },
6502         { "RxFbCongestion", 34, 1 },
6503         { "TxFbCongestion", 33, 1 },
6504         { "TxPktSumSrdy", 32, 1 },
6505         { "RcfUlpType", 28, 4 },
6506         { "Eread", 27, 1 },
6507         { "Ebypass", 26, 1 },
6508         { "Esave", 25, 1 },
6509         { "Static0", 24, 1 },
6510         { "Cread", 23, 1 },
6511         { "Cbypass", 22, 1 },
6512         { "Csave", 21, 1 },
6513         { "CPktOut", 20, 1 },
6514         { "RxPagePoolFull", 18, 2 },
6515         { "RxLpbkPkt", 17, 1 },
6516         { "TxLpbkPkt", 16, 1 },
6517         { "RxVfValid", 15, 1 },
6518         { "SynLearned", 14, 1 },
6519         { "SetDelEntry", 13, 1 },
6520         { "SetInvEntry", 12, 1 },
6521         { "CpcmdDvld", 11, 1 },
6522         { "CpcmdSave", 10, 1 },
6523         { "RxPstructsFull", 8, 2 },
6524         { "EpcmdDvld", 7, 1 },
6525         { "EpcmdFlush", 6, 1 },
6526         { "EpcmdTrimPrefix", 5, 1 },
6527         { "EpcmdTrimPostfix", 4, 1 },
6528         { "ERssIp4Pkt", 3, 1 },
6529         { "ERssIp6Pkt", 2, 1 },
6530         { "ERssTcpUdpPkt", 1, 1 },
6531         { "ERssFceFipPkt", 0, 1 },
6532         { NULL }
6533 };
6534
6535 static struct field_desc tp_la2[] = {
6536         { "CplCmdIn", 56, 8 },
6537         { "MpsVfVld", 55, 1 },
6538         { "MpsPf", 52, 3 },
6539         { "MpsVf", 44, 8 },
6540         { "SynIn", 43, 1 },
6541         { "AckIn", 42, 1 },
6542         { "FinIn", 41, 1 },
6543         { "RstIn", 40, 1 },
6544         { "DataIn", 39, 1 },
6545         { "DataInVld", 38, 1 },
6546         { "PadIn", 37, 1 },
6547         { "RxBufEmpty", 36, 1 },
6548         { "RxDdp", 35, 1 },
6549         { "RxFbCongestion", 34, 1 },
6550         { "TxFbCongestion", 33, 1 },
6551         { "TxPktSumSrdy", 32, 1 },
6552         { "RcfUlpType", 28, 4 },
6553         { "Eread", 27, 1 },
6554         { "Ebypass", 26, 1 },
6555         { "Esave", 25, 1 },
6556         { "Static0", 24, 1 },
6557         { "Cread", 23, 1 },
6558         { "Cbypass", 22, 1 },
6559         { "Csave", 21, 1 },
6560         { "CPktOut", 20, 1 },
6561         { "RxPagePoolFull", 18, 2 },
6562         { "RxLpbkPkt", 17, 1 },
6563         { "TxLpbkPkt", 16, 1 },
6564         { "RxVfValid", 15, 1 },
6565         { "SynLearned", 14, 1 },
6566         { "SetDelEntry", 13, 1 },
6567         { "SetInvEntry", 12, 1 },
6568         { "CpcmdDvld", 11, 1 },
6569         { "CpcmdSave", 10, 1 },
6570         { "RxPstructsFull", 8, 2 },
6571         { "EpcmdDvld", 7, 1 },
6572         { "EpcmdFlush", 6, 1 },
6573         { "EpcmdTrimPrefix", 5, 1 },
6574         { "EpcmdTrimPostfix", 4, 1 },
6575         { "ERssIp4Pkt", 3, 1 },
6576         { "ERssIp6Pkt", 2, 1 },
6577         { "ERssTcpUdpPkt", 1, 1 },
6578         { "ERssFceFipPkt", 0, 1 },
6579         { NULL }
6580 };
6581
6582 static void
6583 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6584 {
6585
6586         field_desc_show(sb, *p, tp_la0);
6587 }
6588
6589 static void
6590 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6591 {
6592
6593         if (idx)
6594                 sbuf_printf(sb, "\n");
6595         field_desc_show(sb, p[0], tp_la0);
6596         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6597                 field_desc_show(sb, p[1], tp_la0);
6598 }
6599
6600 static void
6601 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6602 {
6603
6604         if (idx)
6605                 sbuf_printf(sb, "\n");
6606         field_desc_show(sb, p[0], tp_la0);
6607         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6608                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6609 }
6610
6611 static int
6612 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6613 {
6614         struct adapter *sc = arg1;
6615         struct sbuf *sb;
6616         uint64_t *buf, *p;
6617         int rc;
6618         u_int i, inc;
6619         void (*show_func)(struct sbuf *, uint64_t *, int);
6620
6621         rc = sysctl_wire_old_buffer(req, 0);
6622         if (rc != 0)
6623                 return (rc);
6624
6625         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6626         if (sb == NULL)
6627                 return (ENOMEM);
6628
6629         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6630
6631         t4_tp_read_la(sc, buf, NULL);
6632         p = buf;
6633
6634         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6635         case 2:
6636                 inc = 2;
6637                 show_func = tp_la_show2;
6638                 break;
6639         case 3:
6640                 inc = 2;
6641                 show_func = tp_la_show3;
6642                 break;
6643         default:
6644                 inc = 1;
6645                 show_func = tp_la_show;
6646         }
6647
6648         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6649                 (*show_func)(sb, p, i);
6650
6651         rc = sbuf_finish(sb);
6652         sbuf_delete(sb);
6653         free(buf, M_CXGBE);
6654         return (rc);
6655 }
6656
6657 static int
6658 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6659 {
6660         struct adapter *sc = arg1;
6661         struct sbuf *sb;
6662         int rc;
6663         u64 nrate[NCHAN], orate[NCHAN];
6664
6665         rc = sysctl_wire_old_buffer(req, 0);
6666         if (rc != 0)
6667                 return (rc);
6668
6669         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6670         if (sb == NULL)
6671                 return (ENOMEM);
6672
6673         t4_get_chan_txrate(sc, nrate, orate);
6674         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6675                  "channel 3\n");
6676         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6677             nrate[0], nrate[1], nrate[2], nrate[3]);
6678         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6679             orate[0], orate[1], orate[2], orate[3]);
6680
6681         rc = sbuf_finish(sb);
6682         sbuf_delete(sb);
6683
6684         return (rc);
6685 }
6686
6687 static int
6688 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6689 {
6690         struct adapter *sc = arg1;
6691         struct sbuf *sb;
6692         uint32_t *buf, *p;
6693         int rc, i;
6694
6695         rc = sysctl_wire_old_buffer(req, 0);
6696         if (rc != 0)
6697                 return (rc);
6698
6699         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6700         if (sb == NULL)
6701                 return (ENOMEM);
6702
6703         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6704             M_ZERO | M_WAITOK);
6705
6706         t4_ulprx_read_la(sc, buf);
6707         p = buf;
6708
6709         sbuf_printf(sb, "      Pcmd        Type   Message"
6710             "                Data");
6711         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6712                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6713                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6714         }
6715
6716         rc = sbuf_finish(sb);
6717         sbuf_delete(sb);
6718         free(buf, M_CXGBE);
6719         return (rc);
6720 }
6721
6722 static int
6723 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6724 {
6725         struct adapter *sc = arg1;
6726         struct sbuf *sb;
6727         int rc, v;
6728
6729         rc = sysctl_wire_old_buffer(req, 0);
6730         if (rc != 0)
6731                 return (rc);
6732
6733         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6734         if (sb == NULL)
6735                 return (ENOMEM);
6736
6737         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6738         if (G_STATSOURCE_T5(v) == 7) {
6739                 if (G_STATMODE(v) == 0) {
6740                         sbuf_printf(sb, "total %d, incomplete %d",
6741                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6742                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6743                 } else if (G_STATMODE(v) == 1) {
6744                         sbuf_printf(sb, "total %d, data overflow %d",
6745                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6746                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6747                 }
6748         }
6749         rc = sbuf_finish(sb);
6750         sbuf_delete(sb);
6751
6752         return (rc);
6753 }
6754 #endif
6755
6756 static inline void
6757 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6758 {
6759         struct buf_ring *br;
6760         struct mbuf *m;
6761
6762         TXQ_LOCK_ASSERT_OWNED(txq);
6763
6764         br = txq->br;
6765         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6766         if (m)
6767                 t4_eth_tx(ifp, txq, m);
6768 }
6769
6770 void
6771 t4_tx_callout(void *arg)
6772 {
6773         struct sge_eq *eq = arg;
6774         struct adapter *sc;
6775
6776         if (EQ_TRYLOCK(eq) == 0)
6777                 goto reschedule;
6778
6779         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6780                 EQ_UNLOCK(eq);
6781 reschedule:
6782                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6783                         callout_schedule(&eq->tx_callout, 1);
6784                 return;
6785         }
6786
6787         EQ_LOCK_ASSERT_OWNED(eq);
6788
6789         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6790
6791                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6792                         struct sge_txq *txq = arg;
6793                         struct port_info *pi = txq->ifp->if_softc;
6794
6795                         sc = pi->adapter;
6796                 } else {
6797                         struct sge_wrq *wrq = arg;
6798
6799                         sc = wrq->adapter;
6800                 }
6801
6802                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6803         }
6804
6805         EQ_UNLOCK(eq);
6806 }
6807
6808 void
6809 t4_tx_task(void *arg, int count)
6810 {
6811         struct sge_eq *eq = arg;
6812
6813         EQ_LOCK(eq);
6814         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6815                 struct sge_txq *txq = arg;
6816                 txq_start(txq->ifp, txq);
6817         } else {
6818                 struct sge_wrq *wrq = arg;
6819                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6820         }
6821         EQ_UNLOCK(eq);
6822 }
6823
6824 static uint32_t
6825 fconf_to_mode(uint32_t fconf)
6826 {
6827         uint32_t mode;
6828
6829         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6830             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6831
6832         if (fconf & F_FRAGMENTATION)
6833                 mode |= T4_FILTER_IP_FRAGMENT;
6834
6835         if (fconf & F_MPSHITTYPE)
6836                 mode |= T4_FILTER_MPS_HIT_TYPE;
6837
6838         if (fconf & F_MACMATCH)
6839                 mode |= T4_FILTER_MAC_IDX;
6840
6841         if (fconf & F_ETHERTYPE)
6842                 mode |= T4_FILTER_ETH_TYPE;
6843
6844         if (fconf & F_PROTOCOL)
6845                 mode |= T4_FILTER_IP_PROTO;
6846
6847         if (fconf & F_TOS)
6848                 mode |= T4_FILTER_IP_TOS;
6849
6850         if (fconf & F_VLAN)
6851                 mode |= T4_FILTER_VLAN;
6852
6853         if (fconf & F_VNIC_ID)
6854                 mode |= T4_FILTER_VNIC;
6855
6856         if (fconf & F_PORT)
6857                 mode |= T4_FILTER_PORT;
6858
6859         if (fconf & F_FCOE)
6860                 mode |= T4_FILTER_FCoE;
6861
6862         return (mode);
6863 }
6864
6865 static uint32_t
6866 mode_to_fconf(uint32_t mode)
6867 {
6868         uint32_t fconf = 0;
6869
6870         if (mode & T4_FILTER_IP_FRAGMENT)
6871                 fconf |= F_FRAGMENTATION;
6872
6873         if (mode & T4_FILTER_MPS_HIT_TYPE)
6874                 fconf |= F_MPSHITTYPE;
6875
6876         if (mode & T4_FILTER_MAC_IDX)
6877                 fconf |= F_MACMATCH;
6878
6879         if (mode & T4_FILTER_ETH_TYPE)
6880                 fconf |= F_ETHERTYPE;
6881
6882         if (mode & T4_FILTER_IP_PROTO)
6883                 fconf |= F_PROTOCOL;
6884
6885         if (mode & T4_FILTER_IP_TOS)
6886                 fconf |= F_TOS;
6887
6888         if (mode & T4_FILTER_VLAN)
6889                 fconf |= F_VLAN;
6890
6891         if (mode & T4_FILTER_VNIC)
6892                 fconf |= F_VNIC_ID;
6893
6894         if (mode & T4_FILTER_PORT)
6895                 fconf |= F_PORT;
6896
6897         if (mode & T4_FILTER_FCoE)
6898                 fconf |= F_FCOE;
6899
6900         return (fconf);
6901 }
6902
6903 static uint32_t
6904 fspec_to_fconf(struct t4_filter_specification *fs)
6905 {
6906         uint32_t fconf = 0;
6907
6908         if (fs->val.frag || fs->mask.frag)
6909                 fconf |= F_FRAGMENTATION;
6910
6911         if (fs->val.matchtype || fs->mask.matchtype)
6912                 fconf |= F_MPSHITTYPE;
6913
6914         if (fs->val.macidx || fs->mask.macidx)
6915                 fconf |= F_MACMATCH;
6916
6917         if (fs->val.ethtype || fs->mask.ethtype)
6918                 fconf |= F_ETHERTYPE;
6919
6920         if (fs->val.proto || fs->mask.proto)
6921                 fconf |= F_PROTOCOL;
6922
6923         if (fs->val.tos || fs->mask.tos)
6924                 fconf |= F_TOS;
6925
6926         if (fs->val.vlan_vld || fs->mask.vlan_vld)
6927                 fconf |= F_VLAN;
6928
6929         if (fs->val.vnic_vld || fs->mask.vnic_vld)
6930                 fconf |= F_VNIC_ID;
6931
6932         if (fs->val.iport || fs->mask.iport)
6933                 fconf |= F_PORT;
6934
6935         if (fs->val.fcoe || fs->mask.fcoe)
6936                 fconf |= F_FCOE;
6937
6938         return (fconf);
6939 }
6940
6941 static int
6942 get_filter_mode(struct adapter *sc, uint32_t *mode)
6943 {
6944         int rc;
6945         uint32_t fconf;
6946
6947         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6948             "t4getfm");
6949         if (rc)
6950                 return (rc);
6951
6952         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
6953             A_TP_VLAN_PRI_MAP);
6954
6955         if (sc->params.tp.vlan_pri_map != fconf) {
6956                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
6957                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
6958                     fconf);
6959                 sc->params.tp.vlan_pri_map = fconf;
6960         }
6961
6962         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
6963
6964         end_synchronized_op(sc, LOCK_HELD);
6965         return (0);
6966 }
6967
6968 static int
6969 set_filter_mode(struct adapter *sc, uint32_t mode)
6970 {
6971         uint32_t fconf;
6972         int rc;
6973
6974         fconf = mode_to_fconf(mode);
6975
6976         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6977             "t4setfm");
6978         if (rc)
6979                 return (rc);
6980
6981         if (sc->tids.ftids_in_use > 0) {
6982                 rc = EBUSY;
6983                 goto done;
6984         }
6985
6986 #ifdef TCP_OFFLOAD
6987         if (sc->offload_map) {
6988                 rc = EBUSY;
6989                 goto done;
6990         }
6991 #endif
6992
6993 #ifdef notyet
6994         rc = -t4_set_filter_mode(sc, fconf);
6995         if (rc == 0)
6996                 sc->filter_mode = fconf;
6997 #else
6998         rc = ENOTSUP;
6999 #endif
7000
7001 done:
7002         end_synchronized_op(sc, LOCK_HELD);
7003         return (rc);
7004 }
7005
7006 static inline uint64_t
7007 get_filter_hits(struct adapter *sc, uint32_t fid)
7008 {
7009         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7010         uint64_t hits;
7011
7012         memwin_info(sc, 0, &mw_base, NULL);
7013         off = position_memwin(sc, 0,
7014             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7015         if (is_t4(sc)) {
7016                 hits = t4_read_reg64(sc, mw_base + off + 16);
7017                 hits = be64toh(hits);
7018         } else {
7019                 hits = t4_read_reg(sc, mw_base + off + 24);
7020                 hits = be32toh(hits);
7021         }
7022
7023         return (hits);
7024 }
7025
7026 static int
7027 get_filter(struct adapter *sc, struct t4_filter *t)
7028 {
7029         int i, rc, nfilters = sc->tids.nftids;
7030         struct filter_entry *f;
7031
7032         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7033             "t4getf");
7034         if (rc)
7035                 return (rc);
7036
7037         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7038             t->idx >= nfilters) {
7039                 t->idx = 0xffffffff;
7040                 goto done;
7041         }
7042
7043         f = &sc->tids.ftid_tab[t->idx];
7044         for (i = t->idx; i < nfilters; i++, f++) {
7045                 if (f->valid) {
7046                         t->idx = i;
7047                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7048                         t->smtidx = f->smtidx;
7049                         if (f->fs.hitcnts)
7050                                 t->hits = get_filter_hits(sc, t->idx);
7051                         else
7052                                 t->hits = UINT64_MAX;
7053                         t->fs = f->fs;
7054
7055                         goto done;
7056                 }
7057         }
7058
7059         t->idx = 0xffffffff;
7060 done:
7061         end_synchronized_op(sc, LOCK_HELD);
7062         return (0);
7063 }
7064
7065 static int
7066 set_filter(struct adapter *sc, struct t4_filter *t)
7067 {
7068         unsigned int nfilters, nports;
7069         struct filter_entry *f;
7070         int i, rc;
7071
7072         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7073         if (rc)
7074                 return (rc);
7075
7076         nfilters = sc->tids.nftids;
7077         nports = sc->params.nports;
7078
7079         if (nfilters == 0) {
7080                 rc = ENOTSUP;
7081                 goto done;
7082         }
7083
7084         if (!(sc->flags & FULL_INIT_DONE)) {
7085                 rc = EAGAIN;
7086                 goto done;
7087         }
7088
7089         if (t->idx >= nfilters) {
7090                 rc = EINVAL;
7091                 goto done;
7092         }
7093
7094         /* Validate against the global filter mode */
7095         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7096             sc->params.tp.vlan_pri_map) {
7097                 rc = E2BIG;
7098                 goto done;
7099         }
7100
7101         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7102                 rc = EINVAL;
7103                 goto done;
7104         }
7105
7106         if (t->fs.val.iport >= nports) {
7107                 rc = EINVAL;
7108                 goto done;
7109         }
7110
7111         /* Can't specify an iq if not steering to it */
7112         if (!t->fs.dirsteer && t->fs.iq) {
7113                 rc = EINVAL;
7114                 goto done;
7115         }
7116
7117         /* IPv6 filter idx must be 4 aligned */
7118         if (t->fs.type == 1 &&
7119             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7120                 rc = EINVAL;
7121                 goto done;
7122         }
7123
7124         if (sc->tids.ftid_tab == NULL) {
7125                 KASSERT(sc->tids.ftids_in_use == 0,
7126                     ("%s: no memory allocated but filters_in_use > 0",
7127                     __func__));
7128
7129                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7130                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7131                 if (sc->tids.ftid_tab == NULL) {
7132                         rc = ENOMEM;
7133                         goto done;
7134                 }
7135                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7136         }
7137
7138         for (i = 0; i < 4; i++) {
7139                 f = &sc->tids.ftid_tab[t->idx + i];
7140
7141                 if (f->pending || f->valid) {
7142                         rc = EBUSY;
7143                         goto done;
7144                 }
7145                 if (f->locked) {
7146                         rc = EPERM;
7147                         goto done;
7148                 }
7149
7150                 if (t->fs.type == 0)
7151                         break;
7152         }
7153
7154         f = &sc->tids.ftid_tab[t->idx];
7155         f->fs = t->fs;
7156
7157         rc = set_filter_wr(sc, t->idx);
7158 done:
7159         end_synchronized_op(sc, 0);
7160
7161         if (rc == 0) {
7162                 mtx_lock(&sc->tids.ftid_lock);
7163                 for (;;) {
7164                         if (f->pending == 0) {
7165                                 rc = f->valid ? 0 : EIO;
7166                                 break;
7167                         }
7168
7169                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7170                             PCATCH, "t4setfw", 0)) {
7171                                 rc = EINPROGRESS;
7172                                 break;
7173                         }
7174                 }
7175                 mtx_unlock(&sc->tids.ftid_lock);
7176         }
7177         return (rc);
7178 }
7179
7180 static int
7181 del_filter(struct adapter *sc, struct t4_filter *t)
7182 {
7183         unsigned int nfilters;
7184         struct filter_entry *f;
7185         int rc;
7186
7187         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7188         if (rc)
7189                 return (rc);
7190
7191         nfilters = sc->tids.nftids;
7192
7193         if (nfilters == 0) {
7194                 rc = ENOTSUP;
7195                 goto done;
7196         }
7197
7198         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7199             t->idx >= nfilters) {
7200                 rc = EINVAL;
7201                 goto done;
7202         }
7203
7204         if (!(sc->flags & FULL_INIT_DONE)) {
7205                 rc = EAGAIN;
7206                 goto done;
7207         }
7208
7209         f = &sc->tids.ftid_tab[t->idx];
7210
7211         if (f->pending) {
7212                 rc = EBUSY;
7213                 goto done;
7214         }
7215         if (f->locked) {
7216                 rc = EPERM;
7217                 goto done;
7218         }
7219
7220         if (f->valid) {
7221                 t->fs = f->fs;  /* extra info for the caller */
7222                 rc = del_filter_wr(sc, t->idx);
7223         }
7224
7225 done:
7226         end_synchronized_op(sc, 0);
7227
7228         if (rc == 0) {
7229                 mtx_lock(&sc->tids.ftid_lock);
7230                 for (;;) {
7231                         if (f->pending == 0) {
7232                                 rc = f->valid ? EIO : 0;
7233                                 break;
7234                         }
7235
7236                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7237                             PCATCH, "t4delfw", 0)) {
7238                                 rc = EINPROGRESS;
7239                                 break;
7240                         }
7241                 }
7242                 mtx_unlock(&sc->tids.ftid_lock);
7243         }
7244
7245         return (rc);
7246 }
7247
7248 static void
7249 clear_filter(struct filter_entry *f)
7250 {
7251         if (f->l2t)
7252                 t4_l2t_release(f->l2t);
7253
7254         bzero(f, sizeof (*f));
7255 }
7256
7257 static int
7258 set_filter_wr(struct adapter *sc, int fidx)
7259 {
7260         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7261         struct wrqe *wr;
7262         struct fw_filter_wr *fwr;
7263         unsigned int ftid;
7264
7265         ASSERT_SYNCHRONIZED_OP(sc);
7266
7267         if (f->fs.newdmac || f->fs.newvlan) {
7268                 /* This filter needs an L2T entry; allocate one. */
7269                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7270                 if (f->l2t == NULL)
7271                         return (EAGAIN);
7272                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7273                     f->fs.dmac)) {
7274                         t4_l2t_release(f->l2t);
7275                         f->l2t = NULL;
7276                         return (ENOMEM);
7277                 }
7278         }
7279
7280         ftid = sc->tids.ftid_base + fidx;
7281
7282         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7283         if (wr == NULL)
7284                 return (ENOMEM);
7285
7286         fwr = wrtod(wr);
7287         bzero(fwr, sizeof (*fwr));
7288
7289         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7290         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7291         fwr->tid_to_iq =
7292             htobe32(V_FW_FILTER_WR_TID(ftid) |
7293                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7294                 V_FW_FILTER_WR_NOREPLY(0) |
7295                 V_FW_FILTER_WR_IQ(f->fs.iq));
7296         fwr->del_filter_to_l2tix =
7297             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7298                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7299                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7300                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7301                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7302                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7303                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7304                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7305                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7306                     f->fs.newvlan == VLAN_REWRITE) |
7307                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7308                     f->fs.newvlan == VLAN_REWRITE) |
7309                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7310                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7311                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7312                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7313         fwr->ethtype = htobe16(f->fs.val.ethtype);
7314         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7315         fwr->frag_to_ovlan_vldm =
7316             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7317                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7318                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7319                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7320                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7321                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7322         fwr->smac_sel = 0;
7323         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7324             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7325         fwr->maci_to_matchtypem =
7326             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7327                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7328                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7329                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7330                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7331                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7332                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7333                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7334         fwr->ptcl = f->fs.val.proto;
7335         fwr->ptclm = f->fs.mask.proto;
7336         fwr->ttyp = f->fs.val.tos;
7337         fwr->ttypm = f->fs.mask.tos;
7338         fwr->ivlan = htobe16(f->fs.val.vlan);
7339         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7340         fwr->ovlan = htobe16(f->fs.val.vnic);
7341         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7342         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7343         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7344         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7345         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7346         fwr->lp = htobe16(f->fs.val.dport);
7347         fwr->lpm = htobe16(f->fs.mask.dport);
7348         fwr->fp = htobe16(f->fs.val.sport);
7349         fwr->fpm = htobe16(f->fs.mask.sport);
7350         if (f->fs.newsmac)
7351                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7352
7353         f->pending = 1;
7354         sc->tids.ftids_in_use++;
7355
7356         t4_wrq_tx(sc, wr);
7357         return (0);
7358 }
7359
7360 static int
7361 del_filter_wr(struct adapter *sc, int fidx)
7362 {
7363         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7364         struct wrqe *wr;
7365         struct fw_filter_wr *fwr;
7366         unsigned int ftid;
7367
7368         ftid = sc->tids.ftid_base + fidx;
7369
7370         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7371         if (wr == NULL)
7372                 return (ENOMEM);
7373         fwr = wrtod(wr);
7374         bzero(fwr, sizeof (*fwr));
7375
7376         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7377
7378         f->pending = 1;
7379         t4_wrq_tx(sc, wr);
7380         return (0);
7381 }
7382
7383 int
7384 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7385 {
7386         struct adapter *sc = iq->adapter;
7387         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7388         unsigned int idx = GET_TID(rpl);
7389         unsigned int rc;
7390         struct filter_entry *f;
7391
7392         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7393             rss->opcode));
7394
7395         if (is_ftid(sc, idx)) {
7396
7397                 idx -= sc->tids.ftid_base;
7398                 f = &sc->tids.ftid_tab[idx];
7399                 rc = G_COOKIE(rpl->cookie);
7400
7401                 mtx_lock(&sc->tids.ftid_lock);
7402                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7403                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7404                             __func__, idx));
7405                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7406                         f->pending = 0;  /* asynchronous setup completed */
7407                         f->valid = 1;
7408                 } else {
7409                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7410                                 /* Add or delete failed, display an error */
7411                                 log(LOG_ERR,
7412                                     "filter %u setup failed with error %u\n",
7413                                     idx, rc);
7414                         }
7415
7416                         clear_filter(f);
7417                         sc->tids.ftids_in_use--;
7418                 }
7419                 wakeup(&sc->tids.ftid_tab);
7420                 mtx_unlock(&sc->tids.ftid_lock);
7421         }
7422
7423         return (0);
7424 }
7425
7426 static int
7427 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7428 {
7429         int rc;
7430
7431         if (cntxt->cid > M_CTXTQID)
7432                 return (EINVAL);
7433
7434         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7435             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7436                 return (EINVAL);
7437
7438         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7439         if (rc)
7440                 return (rc);
7441
7442         if (sc->flags & FW_OK) {
7443                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7444                     &cntxt->data[0]);
7445                 if (rc == 0)
7446                         goto done;
7447         }
7448
7449         /*
7450          * Read via firmware failed or wasn't even attempted.  Read directly via
7451          * the backdoor.
7452          */
7453         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7454 done:
7455         end_synchronized_op(sc, 0);
7456         return (rc);
7457 }
7458
7459 static int
7460 load_fw(struct adapter *sc, struct t4_data *fw)
7461 {
7462         int rc;
7463         uint8_t *fw_data;
7464
7465         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7466         if (rc)
7467                 return (rc);
7468
7469         if (sc->flags & FULL_INIT_DONE) {
7470                 rc = EBUSY;
7471                 goto done;
7472         }
7473
7474         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7475         if (fw_data == NULL) {
7476                 rc = ENOMEM;
7477                 goto done;
7478         }
7479
7480         rc = copyin(fw->data, fw_data, fw->len);
7481         if (rc == 0)
7482                 rc = -t4_load_fw(sc, fw_data, fw->len);
7483
7484         free(fw_data, M_CXGBE);
7485 done:
7486         end_synchronized_op(sc, 0);
7487         return (rc);
7488 }
7489
7490 static int
7491 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7492 {
7493         uint32_t addr, off, remaining, i, n;
7494         uint32_t *buf, *b;
7495         uint32_t mw_base, mw_aperture;
7496         int rc;
7497         uint8_t *dst;
7498
7499         rc = validate_mem_range(sc, mr->addr, mr->len);
7500         if (rc != 0)
7501                 return (rc);
7502
7503         memwin_info(sc, win, &mw_base, &mw_aperture);
7504         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7505         addr = mr->addr;
7506         remaining = mr->len;
7507         dst = (void *)mr->data;
7508
7509         while (remaining) {
7510                 off = position_memwin(sc, win, addr);
7511
7512                 /* number of bytes that we'll copy in the inner loop */
7513                 n = min(remaining, mw_aperture - off);
7514                 for (i = 0; i < n; i += 4)
7515                         *b++ = t4_read_reg(sc, mw_base + off + i);
7516
7517                 rc = copyout(buf, dst, n);
7518                 if (rc != 0)
7519                         break;
7520
7521                 b = buf;
7522                 dst += n;
7523                 remaining -= n;
7524                 addr += n;
7525         }
7526
7527         free(buf, M_CXGBE);
7528         return (rc);
7529 }
7530
7531 static int
7532 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7533 {
7534         int rc;
7535
7536         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7537                 return (EINVAL);
7538
7539         if (i2cd->len > sizeof(i2cd->data))
7540                 return (EFBIG);
7541
7542         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7543         if (rc)
7544                 return (rc);
7545         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7546             i2cd->offset, i2cd->len, &i2cd->data[0]);
7547         end_synchronized_op(sc, 0);
7548
7549         return (rc);
7550 }
7551
7552 static int
7553 in_range(int val, int lo, int hi)
7554 {
7555
7556         return (val < 0 || (val <= hi && val >= lo));
7557 }
7558
7559 static int
7560 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7561 {
7562         int fw_subcmd, fw_type, rc;
7563
7564         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7565         if (rc)
7566                 return (rc);
7567
7568         if (!(sc->flags & FULL_INIT_DONE)) {
7569                 rc = EAGAIN;
7570                 goto done;
7571         }
7572
7573         /*
7574          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7575          * sub-command and type are in common locations.)
7576          */
7577         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7578                 fw_subcmd = FW_SCHED_SC_CONFIG;
7579         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7580                 fw_subcmd = FW_SCHED_SC_PARAMS;
7581         else {
7582                 rc = EINVAL;
7583                 goto done;
7584         }
7585         if (p->type == SCHED_CLASS_TYPE_PACKET)
7586                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7587         else {
7588                 rc = EINVAL;
7589                 goto done;
7590         }
7591
7592         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7593                 /* Vet our parameters ..*/
7594                 if (p->u.config.minmax < 0) {
7595                         rc = EINVAL;
7596                         goto done;
7597                 }
7598
7599                 /* And pass the request to the firmware ...*/
7600                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax);
7601                 goto done;
7602         }
7603
7604         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7605                 int fw_level;
7606                 int fw_mode;
7607                 int fw_rateunit;
7608                 int fw_ratemode;
7609
7610                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7611                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7612                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7613                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7614                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7615                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7616                 else {
7617                         rc = EINVAL;
7618                         goto done;
7619                 }
7620
7621                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7622                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7623                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7624                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7625                 else {
7626                         rc = EINVAL;
7627                         goto done;
7628                 }
7629
7630                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7631                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7632                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7633                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7634                 else {
7635                         rc = EINVAL;
7636                         goto done;
7637                 }
7638
7639                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7640                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7641                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7642                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7643                 else {
7644                         rc = EINVAL;
7645                         goto done;
7646                 }
7647
7648                 /* Vet our parameters ... */
7649                 if (!in_range(p->u.params.channel, 0, 3) ||
7650                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7651                     !in_range(p->u.params.minrate, 0, 10000000) ||
7652                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7653                     !in_range(p->u.params.weight, 0, 100)) {
7654                         rc = ERANGE;
7655                         goto done;
7656                 }
7657
7658                 /*
7659                  * Translate any unset parameters into the firmware's
7660                  * nomenclature and/or fail the call if the parameters
7661                  * are required ...
7662                  */
7663                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7664                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7665                         rc = EINVAL;
7666                         goto done;
7667                 }
7668                 if (p->u.params.minrate < 0)
7669                         p->u.params.minrate = 0;
7670                 if (p->u.params.maxrate < 0) {
7671                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7672                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7673                                 rc = EINVAL;
7674                                 goto done;
7675                         } else
7676                                 p->u.params.maxrate = 0;
7677                 }
7678                 if (p->u.params.weight < 0) {
7679                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7680                                 rc = EINVAL;
7681                                 goto done;
7682                         } else
7683                                 p->u.params.weight = 0;
7684                 }
7685                 if (p->u.params.pktsize < 0) {
7686                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7687                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7688                                 rc = EINVAL;
7689                                 goto done;
7690                         } else
7691                                 p->u.params.pktsize = 0;
7692                 }
7693
7694                 /* See what the firmware thinks of the request ... */
7695                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7696                     fw_rateunit, fw_ratemode, p->u.params.channel,
7697                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7698                     p->u.params.weight, p->u.params.pktsize);
7699                 goto done;
7700         }
7701
7702         rc = EINVAL;
7703 done:
7704         end_synchronized_op(sc, 0);
7705         return (rc);
7706 }
7707
7708 static int
7709 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7710 {
7711         struct port_info *pi = NULL;
7712         struct sge_txq *txq;
7713         uint32_t fw_mnem, fw_queue, fw_class;
7714         int i, rc;
7715
7716         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7717         if (rc)
7718                 return (rc);
7719
7720         if (!(sc->flags & FULL_INIT_DONE)) {
7721                 rc = EAGAIN;
7722                 goto done;
7723         }
7724
7725         if (p->port >= sc->params.nports) {
7726                 rc = EINVAL;
7727                 goto done;
7728         }
7729
7730         pi = sc->port[p->port];
7731         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7732                 rc = EINVAL;
7733                 goto done;
7734         }
7735
7736         /*
7737          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7738          * Scheduling Class in this case).
7739          */
7740         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7741             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7742         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7743
7744         /*
7745          * If op.queue is non-negative, then we're only changing the scheduling
7746          * on a single specified TX queue.
7747          */
7748         if (p->queue >= 0) {
7749                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7750                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7751                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7752                     &fw_class);
7753                 goto done;
7754         }
7755
7756         /*
7757          * Change the scheduling on all the TX queues for the
7758          * interface.
7759          */
7760         for_each_txq(pi, i, txq) {
7761                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7762                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7763                     &fw_class);
7764                 if (rc)
7765                         goto done;
7766         }
7767
7768         rc = 0;
7769 done:
7770         end_synchronized_op(sc, 0);
7771         return (rc);
7772 }
7773
7774 int
7775 t4_os_find_pci_capability(struct adapter *sc, int cap)
7776 {
7777         int i;
7778
7779         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7780 }
7781
7782 int
7783 t4_os_pci_save_state(struct adapter *sc)
7784 {
7785         device_t dev;
7786         struct pci_devinfo *dinfo;
7787
7788         dev = sc->dev;
7789         dinfo = device_get_ivars(dev);
7790
7791         pci_cfg_save(dev, dinfo, 0);
7792         return (0);
7793 }
7794
7795 int
7796 t4_os_pci_restore_state(struct adapter *sc)
7797 {
7798         device_t dev;
7799         struct pci_devinfo *dinfo;
7800
7801         dev = sc->dev;
7802         dinfo = device_get_ivars(dev);
7803
7804         pci_cfg_restore(dev, dinfo);
7805         return (0);
7806 }
7807
7808 void
7809 t4_os_portmod_changed(const struct adapter *sc, int idx)
7810 {
7811         struct port_info *pi = sc->port[idx];
7812         static const char *mod_str[] = {
7813                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7814         };
7815
7816         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7817                 if_printf(pi->ifp, "transceiver unplugged.\n");
7818         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7819                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7820         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7821                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7822         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7823                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7824                     mod_str[pi->mod_type]);
7825         } else {
7826                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7827                     pi->mod_type);
7828         }
7829 }
7830
7831 void
7832 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7833 {
7834         struct port_info *pi = sc->port[idx];
7835         struct ifnet *ifp = pi->ifp;
7836
7837         if (link_stat) {
7838                 pi->linkdnrc = -1;
7839                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7840                 if_link_state_change(ifp, LINK_STATE_UP);
7841         } else {
7842                 if (reason >= 0)
7843                         pi->linkdnrc = reason;
7844                 if_link_state_change(ifp, LINK_STATE_DOWN);
7845         }
7846 }
7847
7848 void
7849 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7850 {
7851         struct adapter *sc;
7852
7853         sx_slock(&t4_list_lock);
7854         SLIST_FOREACH(sc, &t4_list, link) {
7855                 /*
7856                  * func should not make any assumptions about what state sc is
7857                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7858                  */
7859                 func(sc, arg);
7860         }
7861         sx_sunlock(&t4_list_lock);
7862 }
7863
7864 static int
7865 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7866 {
7867        return (0);
7868 }
7869
7870 static int
7871 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7872 {
7873        return (0);
7874 }
7875
7876 static int
7877 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7878     struct thread *td)
7879 {
7880         int rc;
7881         struct adapter *sc = dev->si_drv1;
7882
7883         rc = priv_check(td, PRIV_DRIVER);
7884         if (rc != 0)
7885                 return (rc);
7886
7887         switch (cmd) {
7888         case CHELSIO_T4_GETREG: {
7889                 struct t4_reg *edata = (struct t4_reg *)data;
7890
7891                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7892                         return (EFAULT);
7893
7894                 if (edata->size == 4)
7895                         edata->val = t4_read_reg(sc, edata->addr);
7896                 else if (edata->size == 8)
7897                         edata->val = t4_read_reg64(sc, edata->addr);
7898                 else
7899                         return (EINVAL);
7900
7901                 break;
7902         }
7903         case CHELSIO_T4_SETREG: {
7904                 struct t4_reg *edata = (struct t4_reg *)data;
7905
7906                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7907                         return (EFAULT);
7908
7909                 if (edata->size == 4) {
7910                         if (edata->val & 0xffffffff00000000)
7911                                 return (EINVAL);
7912                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7913                 } else if (edata->size == 8)
7914                         t4_write_reg64(sc, edata->addr, edata->val);
7915                 else
7916                         return (EINVAL);
7917                 break;
7918         }
7919         case CHELSIO_T4_REGDUMP: {
7920                 struct t4_regdump *regs = (struct t4_regdump *)data;
7921                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7922                 uint8_t *buf;
7923
7924                 if (regs->len < reglen) {
7925                         regs->len = reglen; /* hint to the caller */
7926                         return (ENOBUFS);
7927                 }
7928
7929                 regs->len = reglen;
7930                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
7931                 t4_get_regs(sc, regs, buf);
7932                 rc = copyout(buf, regs->data, reglen);
7933                 free(buf, M_CXGBE);
7934                 break;
7935         }
7936         case CHELSIO_T4_GET_FILTER_MODE:
7937                 rc = get_filter_mode(sc, (uint32_t *)data);
7938                 break;
7939         case CHELSIO_T4_SET_FILTER_MODE:
7940                 rc = set_filter_mode(sc, *(uint32_t *)data);
7941                 break;
7942         case CHELSIO_T4_GET_FILTER:
7943                 rc = get_filter(sc, (struct t4_filter *)data);
7944                 break;
7945         case CHELSIO_T4_SET_FILTER:
7946                 rc = set_filter(sc, (struct t4_filter *)data);
7947                 break;
7948         case CHELSIO_T4_DEL_FILTER:
7949                 rc = del_filter(sc, (struct t4_filter *)data);
7950                 break;
7951         case CHELSIO_T4_GET_SGE_CONTEXT:
7952                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
7953                 break;
7954         case CHELSIO_T4_LOAD_FW:
7955                 rc = load_fw(sc, (struct t4_data *)data);
7956                 break;
7957         case CHELSIO_T4_GET_MEM:
7958                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
7959                 break;
7960         case CHELSIO_T4_GET_I2C:
7961                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
7962                 break;
7963         case CHELSIO_T4_CLEAR_STATS: {
7964                 int i;
7965                 u_int port_id = *(uint32_t *)data;
7966                 struct port_info *pi;
7967
7968                 if (port_id >= sc->params.nports)
7969                         return (EINVAL);
7970                 pi = sc->port[port_id];
7971
7972                 /* MAC stats */
7973                 t4_clr_port_stats(sc, pi->tx_chan);
7974
7975                 if (pi->flags & PORT_INIT_DONE) {
7976                         struct sge_rxq *rxq;
7977                         struct sge_txq *txq;
7978                         struct sge_wrq *wrq;
7979
7980                         for_each_rxq(pi, i, rxq) {
7981 #if defined(INET) || defined(INET6)
7982                                 rxq->lro.lro_queued = 0;
7983                                 rxq->lro.lro_flushed = 0;
7984 #endif
7985                                 rxq->rxcsum = 0;
7986                                 rxq->vlan_extraction = 0;
7987                         }
7988
7989                         for_each_txq(pi, i, txq) {
7990                                 txq->txcsum = 0;
7991                                 txq->tso_wrs = 0;
7992                                 txq->vlan_insertion = 0;
7993                                 txq->imm_wrs = 0;
7994                                 txq->sgl_wrs = 0;
7995                                 txq->txpkt_wrs = 0;
7996                                 txq->txpkts_wrs = 0;
7997                                 txq->txpkts_pkts = 0;
7998                                 txq->br->br_drops = 0;
7999                                 txq->no_dmamap = 0;
8000                                 txq->no_desc = 0;
8001                         }
8002
8003 #ifdef TCP_OFFLOAD
8004                         /* nothing to clear for each ofld_rxq */
8005
8006                         for_each_ofld_txq(pi, i, wrq) {
8007                                 wrq->tx_wrs = 0;
8008                                 wrq->no_desc = 0;
8009                         }
8010 #endif
8011                         wrq = &sc->sge.ctrlq[pi->port_id];
8012                         wrq->tx_wrs = 0;
8013                         wrq->no_desc = 0;
8014                 }
8015                 break;
8016         }
8017         case CHELSIO_T4_SCHED_CLASS:
8018                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8019                 break;
8020         case CHELSIO_T4_SCHED_QUEUE:
8021                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8022                 break;
8023         case CHELSIO_T4_GET_TRACER:
8024                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8025                 break;
8026         case CHELSIO_T4_SET_TRACER:
8027                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8028                 break;
8029         default:
8030                 rc = EINVAL;
8031         }
8032
8033         return (rc);
8034 }
8035
8036 #ifdef TCP_OFFLOAD
8037 static int
8038 toe_capability(struct port_info *pi, int enable)
8039 {
8040         int rc;
8041         struct adapter *sc = pi->adapter;
8042
8043         ASSERT_SYNCHRONIZED_OP(sc);
8044
8045         if (!is_offload(sc))
8046                 return (ENODEV);
8047
8048         if (enable) {
8049                 if (!(sc->flags & FULL_INIT_DONE)) {
8050                         rc = cxgbe_init_synchronized(pi);
8051                         if (rc)
8052                                 return (rc);
8053                 }
8054
8055                 if (isset(&sc->offload_map, pi->port_id))
8056                         return (0);
8057
8058                 if (!(sc->flags & TOM_INIT_DONE)) {
8059                         rc = t4_activate_uld(sc, ULD_TOM);
8060                         if (rc == EAGAIN) {
8061                                 log(LOG_WARNING,
8062                                     "You must kldload t4_tom.ko before trying "
8063                                     "to enable TOE on a cxgbe interface.\n");
8064                         }
8065                         if (rc != 0)
8066                                 return (rc);
8067                         KASSERT(sc->tom_softc != NULL,
8068                             ("%s: TOM activated but softc NULL", __func__));
8069                         KASSERT(sc->flags & TOM_INIT_DONE,
8070                             ("%s: TOM activated but flag not set", __func__));
8071                 }
8072
8073                 setbit(&sc->offload_map, pi->port_id);
8074         } else {
8075                 if (!isset(&sc->offload_map, pi->port_id))
8076                         return (0);
8077
8078                 KASSERT(sc->flags & TOM_INIT_DONE,
8079                     ("%s: TOM never initialized?", __func__));
8080                 clrbit(&sc->offload_map, pi->port_id);
8081         }
8082
8083         return (0);
8084 }
8085
8086 /*
8087  * Add an upper layer driver to the global list.
8088  */
8089 int
8090 t4_register_uld(struct uld_info *ui)
8091 {
8092         int rc = 0;
8093         struct uld_info *u;
8094
8095         sx_xlock(&t4_uld_list_lock);
8096         SLIST_FOREACH(u, &t4_uld_list, link) {
8097             if (u->uld_id == ui->uld_id) {
8098                     rc = EEXIST;
8099                     goto done;
8100             }
8101         }
8102
8103         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8104         ui->refcount = 0;
8105 done:
8106         sx_xunlock(&t4_uld_list_lock);
8107         return (rc);
8108 }
8109
8110 int
8111 t4_unregister_uld(struct uld_info *ui)
8112 {
8113         int rc = EINVAL;
8114         struct uld_info *u;
8115
8116         sx_xlock(&t4_uld_list_lock);
8117
8118         SLIST_FOREACH(u, &t4_uld_list, link) {
8119             if (u == ui) {
8120                     if (ui->refcount > 0) {
8121                             rc = EBUSY;
8122                             goto done;
8123                     }
8124
8125                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8126                     rc = 0;
8127                     goto done;
8128             }
8129         }
8130 done:
8131         sx_xunlock(&t4_uld_list_lock);
8132         return (rc);
8133 }
8134
8135 int
8136 t4_activate_uld(struct adapter *sc, int id)
8137 {
8138         int rc = EAGAIN;
8139         struct uld_info *ui;
8140
8141         ASSERT_SYNCHRONIZED_OP(sc);
8142
8143         sx_slock(&t4_uld_list_lock);
8144
8145         SLIST_FOREACH(ui, &t4_uld_list, link) {
8146                 if (ui->uld_id == id) {
8147                         rc = ui->activate(sc);
8148                         if (rc == 0)
8149                                 ui->refcount++;
8150                         goto done;
8151                 }
8152         }
8153 done:
8154         sx_sunlock(&t4_uld_list_lock);
8155
8156         return (rc);
8157 }
8158
8159 int
8160 t4_deactivate_uld(struct adapter *sc, int id)
8161 {
8162         int rc = EINVAL;
8163         struct uld_info *ui;
8164
8165         ASSERT_SYNCHRONIZED_OP(sc);
8166
8167         sx_slock(&t4_uld_list_lock);
8168
8169         SLIST_FOREACH(ui, &t4_uld_list, link) {
8170                 if (ui->uld_id == id) {
8171                         rc = ui->deactivate(sc);
8172                         if (rc == 0)
8173                                 ui->refcount--;
8174                         goto done;
8175                 }
8176         }
8177 done:
8178         sx_sunlock(&t4_uld_list_lock);
8179
8180         return (rc);
8181 }
8182 #endif
8183
8184 /*
8185  * Come up with reasonable defaults for some of the tunables, provided they're
8186  * not set by the user (in which case we'll use the values as is).
8187  */
8188 static void
8189 tweak_tunables(void)
8190 {
8191         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8192
8193         if (t4_ntxq10g < 1)
8194                 t4_ntxq10g = min(nc, NTXQ_10G);
8195
8196         if (t4_ntxq1g < 1)
8197                 t4_ntxq1g = min(nc, NTXQ_1G);
8198
8199         if (t4_nrxq10g < 1)
8200                 t4_nrxq10g = min(nc, NRXQ_10G);
8201
8202         if (t4_nrxq1g < 1)
8203                 t4_nrxq1g = min(nc, NRXQ_1G);
8204
8205 #ifdef TCP_OFFLOAD
8206         if (t4_nofldtxq10g < 1)
8207                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8208
8209         if (t4_nofldtxq1g < 1)
8210                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8211
8212         if (t4_nofldrxq10g < 1)
8213                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8214
8215         if (t4_nofldrxq1g < 1)
8216                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8217
8218         if (t4_toecaps_allowed == -1)
8219                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8220 #else
8221         if (t4_toecaps_allowed == -1)
8222                 t4_toecaps_allowed = 0;
8223 #endif
8224
8225 #ifdef DEV_NETMAP
8226         if (t4_nnmtxq10g < 1)
8227                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8228
8229         if (t4_nnmtxq1g < 1)
8230                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8231
8232         if (t4_nnmrxq10g < 1)
8233                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8234
8235         if (t4_nnmrxq1g < 1)
8236                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8237 #endif
8238
8239         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8240                 t4_tmr_idx_10g = TMR_IDX_10G;
8241
8242         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8243                 t4_pktc_idx_10g = PKTC_IDX_10G;
8244
8245         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8246                 t4_tmr_idx_1g = TMR_IDX_1G;
8247
8248         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8249                 t4_pktc_idx_1g = PKTC_IDX_1G;
8250
8251         if (t4_qsize_txq < 128)
8252                 t4_qsize_txq = 128;
8253
8254         if (t4_qsize_rxq < 128)
8255                 t4_qsize_rxq = 128;
8256         while (t4_qsize_rxq & 7)
8257                 t4_qsize_rxq++;
8258
8259         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8260 }
8261
8262 static int
8263 mod_event(module_t mod, int cmd, void *arg)
8264 {
8265         int rc = 0;
8266         static int loaded = 0;
8267
8268         switch (cmd) {
8269         case MOD_LOAD:
8270                 if (atomic_fetchadd_int(&loaded, 1))
8271                         break;
8272                 t4_sge_modload();
8273                 sx_init(&t4_list_lock, "T4/T5 adapters");
8274                 SLIST_INIT(&t4_list);
8275 #ifdef TCP_OFFLOAD
8276                 sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8277                 SLIST_INIT(&t4_uld_list);
8278 #endif
8279                 t4_tracer_modload();
8280                 tweak_tunables();
8281                 break;
8282
8283         case MOD_UNLOAD:
8284                 if (atomic_fetchadd_int(&loaded, -1) > 1)
8285                         break;
8286                 t4_tracer_modunload();
8287 #ifdef TCP_OFFLOAD
8288                 sx_slock(&t4_uld_list_lock);
8289                 if (!SLIST_EMPTY(&t4_uld_list)) {
8290                         rc = EBUSY;
8291                         sx_sunlock(&t4_uld_list_lock);
8292                         break;
8293                 }
8294                 sx_sunlock(&t4_uld_list_lock);
8295                 sx_destroy(&t4_uld_list_lock);
8296 #endif
8297                 sx_slock(&t4_list_lock);
8298                 if (!SLIST_EMPTY(&t4_list)) {
8299                         rc = EBUSY;
8300                         sx_sunlock(&t4_list_lock);
8301                         break;
8302                 }
8303                 sx_sunlock(&t4_list_lock);
8304                 sx_destroy(&t4_list_lock);
8305                 break;
8306         }
8307
8308         return (rc);
8309 }
8310
8311 static devclass_t t4_devclass, t5_devclass;
8312 static devclass_t cxgbe_devclass, cxl_devclass;
8313
8314 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8315 MODULE_VERSION(t4nex, 1);
8316 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8317
8318 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8319 MODULE_VERSION(t5nex, 1);
8320 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8321
8322 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8323 MODULE_VERSION(cxgbe, 1);
8324
8325 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8326 MODULE_VERSION(cxl, 1);