sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 /*
 222  * Holdoff parameters for 10G and 1G ports.
 223  */
 224 #define TMR_IDX_10G 1
 225 static int t4_tmr_idx_10g = TMR_IDX_10G;
 226 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 227
 228 #define PKTC_IDX_10G (-1)
 229 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 230 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 231
 232 #define TMR_IDX_1G 1
 233 static int t4_tmr_idx_1g = TMR_IDX_1G;
 234 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 235
 236 #define PKTC_IDX_1G (-1)
 237 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 238 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 239
 240 /*
 241  * Size (# of entries) of each tx and rx queue.
 242  */
 243 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 244 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 245
 246 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 247 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 248
 249 /*
 250  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 251  */
 252 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 253 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 254
 255 /*
 256  * Configuration file.
 257  */
 258 #define DEFAULT_CF      "default"
 259 #define FLASH_CF        "flash"
 260 #define UWIRE_CF        "uwire"
 261 #define FPGA_CF         "fpga"
 262 static char t4_cfg_file[32] = DEFAULT_CF;
 263 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 264
 265 /*
 266  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 267  * encouraged respectively).
 268  */
 269 static unsigned int t4_fw_install = 1;
 270 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 271
 272 /*
 273  * ASIC features that will be used.  Disable the ones you don't want so that the
 274  * chip resources aren't wasted on features that will not be used.
 275  */
 276 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 277 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 278
 279 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 280 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 281
 282 static int t4_toecaps_allowed = -1;
 283 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 284
 285 static int t4_rdmacaps_allowed = 0;
 286 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 287
 288 static int t4_iscsicaps_allowed = 0;
 289 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 290
 291 static int t4_fcoecaps_allowed = 0;
 292 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 293
 294 static int t5_write_combine = 0;
 295 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 296
 297 struct intrs_and_queues {
 298         int intr_type;          /* INTx, MSI, or MSI-X */
 299         int nirq;               /* Number of vectors */
 300         int intr_flags;
 301         int ntxq10g;            /* # of NIC txq's for each 10G port */
 302         int nrxq10g;            /* # of NIC rxq's for each 10G port */
 303         int ntxq1g;             /* # of NIC txq's for each 1G port */
 304         int nrxq1g;             /* # of NIC rxq's for each 1G port */
 305         int rsrv_noflowq;       /* Flag whether to reserve queue 0 */
 306 #ifdef TCP_OFFLOAD
 307         int nofldtxq10g;        /* # of TOE txq's for each 10G port */
 308         int nofldrxq10g;        /* # of TOE rxq's for each 10G port */
 309         int nofldtxq1g;         /* # of TOE txq's for each 1G port */
 310         int nofldrxq1g;         /* # of TOE rxq's for each 1G port */
 311 #endif
 312 };
 313
 314 struct filter_entry {
 315         uint32_t valid:1;       /* filter allocated and valid */
 316         uint32_t locked:1;      /* filter is administratively locked */
 317         uint32_t pending:1;     /* filter action is pending firmware reply */
 318         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 319         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 320
 321         struct t4_filter_specification fs;
 322 };
 323
 324 enum {
 325         XGMAC_MTU       = (1 << 0),
 326         XGMAC_PROMISC   = (1 << 1),
 327         XGMAC_ALLMULTI  = (1 << 2),
 328         XGMAC_VLANEX    = (1 << 3),
 329         XGMAC_UCADDR    = (1 << 4),
 330         XGMAC_MCADDRS   = (1 << 5),
 331
 332         XGMAC_ALL       = 0xffff
 333 };
 334
 335 static int map_bars_0_and_4(struct adapter *);
 336 static int map_bar_2(struct adapter *);
 337 static void setup_memwin(struct adapter *);
 338 static int validate_mem_range(struct adapter *, uint32_t, int);
 339 static int fwmtype_to_hwmtype(int);
 340 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 341     uint32_t *);
 342 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 343 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 344 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 345     struct intrs_and_queues *);
 346 static int prep_firmware(struct adapter *);
 347 static int partition_resources(struct adapter *, const struct firmware *,
 348     const char *);
 349 static int get_params__pre_init(struct adapter *);
 350 static int get_params__post_init(struct adapter *);
 351 static int set_params__post_init(struct adapter *);
 352 static void t4_set_desc(struct adapter *);
 353 static void build_medialist(struct port_info *);
 354 static int update_mac_settings(struct port_info *, int);
 355 static int cxgbe_init_synchronized(struct port_info *);
 356 static int cxgbe_uninit_synchronized(struct port_info *);
 357 static int setup_intr_handlers(struct adapter *);
 358 static int adapter_full_init(struct adapter *);
 359 static int adapter_full_uninit(struct adapter *);
 360 static int port_full_init(struct port_info *);
 361 static int port_full_uninit(struct port_info *);
 362 static void quiesce_eq(struct adapter *, struct sge_eq *);
 363 static void quiesce_iq(struct adapter *, struct sge_iq *);
 364 static void quiesce_fl(struct adapter *, struct sge_fl *);
 365 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 366     driver_intr_t *, void *, char *);
 367 static int t4_free_irq(struct adapter *, struct irq *);
 368 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 369     unsigned int);
 370 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 371 static void cxgbe_tick(void *);
 372 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 373 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 374     struct mbuf *);
 375 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 376 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 377 static int t4_sysctls(struct adapter *);
 378 static int cxgbe_sysctls(struct port_info *);
 379 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 380 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 381 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 382 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 383 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 384 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 385 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 386 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 387 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 388 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 389 #ifdef SBUF_DRAIN
 390 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 391 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 392 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 393 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 394 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 395 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 396 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 397 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 415 #endif
 416 static inline void txq_start(struct ifnet *, struct sge_txq *);
 417 static uint32_t fconf_to_mode(uint32_t);
 418 static uint32_t mode_to_fconf(uint32_t);
 419 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 420 static int get_filter_mode(struct adapter *, uint32_t *);
 421 static int set_filter_mode(struct adapter *, uint32_t);
 422 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 423 static int get_filter(struct adapter *, struct t4_filter *);
 424 static int set_filter(struct adapter *, struct t4_filter *);
 425 static int del_filter(struct adapter *, struct t4_filter *);
 426 static void clear_filter(struct filter_entry *);
 427 static int set_filter_wr(struct adapter *, int);
 428 static int del_filter_wr(struct adapter *, int);
 429 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 430 static int load_fw(struct adapter *, struct t4_data *);
 431 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 432 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 433 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 434 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 435 #ifdef TCP_OFFLOAD
 436 static int toe_capability(struct port_info *, int);
 437 #endif
 438 static int mod_event(module_t, int, void *);
 439
 440 struct {
 441         uint16_t device;
 442         char *desc;
 443 } t4_pciids[] = {
 444         {0xa000, "Chelsio Terminator 4 FPGA"},
 445         {0x4400, "Chelsio T440-dbg"},
 446         {0x4401, "Chelsio T420-CR"},
 447         {0x4402, "Chelsio T422-CR"},
 448         {0x4403, "Chelsio T440-CR"},
 449         {0x4404, "Chelsio T420-BCH"},
 450         {0x4405, "Chelsio T440-BCH"},
 451         {0x4406, "Chelsio T440-CH"},
 452         {0x4407, "Chelsio T420-SO"},
 453         {0x4408, "Chelsio T420-CX"},
 454         {0x4409, "Chelsio T420-BT"},
 455         {0x440a, "Chelsio T404-BT"},
 456         {0x440e, "Chelsio T440-LP-CR"},
 457 }, t5_pciids[] = {
 458         {0xb000, "Chelsio Terminator 5 FPGA"},
 459         {0x5400, "Chelsio T580-dbg"},
 460         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 461         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 462         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 463         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 464         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 465         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 466         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 467         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 468         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 469         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 470         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 471         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 472 #ifdef notyet
 473         {0x5404,  "Chelsio T520-BCH"},
 474         {0x5405,  "Chelsio T540-BCH"},
 475         {0x5406,  "Chelsio T540-CH"},
 476         {0x5408,  "Chelsio T520-CX"},
 477         {0x540b,  "Chelsio B520-SR"},
 478         {0x540c,  "Chelsio B504-BT"},
 479         {0x540f,  "Chelsio Amsterdam"},
 480         {0x5413,  "Chelsio T580-CHR"},
 481 #endif
 482 };
 483
 484 #ifdef TCP_OFFLOAD
 485 /*
 486  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 487  * exactly the same for both rxq and ofld_rxq.
 488  */
 489 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 490 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 491 #endif
 492
 493 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 494 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 495 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 496
 497 static int
 498 t4_probe(device_t dev)
 499 {
 500         int i;
 501         uint16_t v = pci_get_vendor(dev);
 502         uint16_t d = pci_get_device(dev);
 503         uint8_t f = pci_get_function(dev);
 504
 505         if (v != PCI_VENDOR_ID_CHELSIO)
 506                 return (ENXIO);
 507
 508         /* Attach only to PF0 of the FPGA */
 509         if (d == 0xa000 && f != 0)
 510                 return (ENXIO);
 511
 512         for (i = 0; i < nitems(t4_pciids); i++) {
 513                 if (d == t4_pciids[i].device) {
 514                         device_set_desc(dev, t4_pciids[i].desc);
 515                         return (BUS_PROBE_DEFAULT);
 516                 }
 517         }
 518
 519         return (ENXIO);
 520 }
 521
 522 static int
 523 t5_probe(device_t dev)
 524 {
 525         int i;
 526         uint16_t v = pci_get_vendor(dev);
 527         uint16_t d = pci_get_device(dev);
 528         uint8_t f = pci_get_function(dev);
 529
 530         if (v != PCI_VENDOR_ID_CHELSIO)
 531                 return (ENXIO);
 532
 533         /* Attach only to PF0 of the FPGA */
 534         if (d == 0xb000 && f != 0)
 535                 return (ENXIO);
 536
 537         for (i = 0; i < nitems(t5_pciids); i++) {
 538                 if (d == t5_pciids[i].device) {
 539                         device_set_desc(dev, t5_pciids[i].desc);
 540                         return (BUS_PROBE_DEFAULT);
 541                 }
 542         }
 543
 544         return (ENXIO);
 545 }
 546
 547 static int
 548 t4_attach(device_t dev)
 549 {
 550         struct adapter *sc;
 551         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 552         struct intrs_and_queues iaq;
 553         struct sge *s;
 554 #ifdef TCP_OFFLOAD
 555         int ofld_rqidx, ofld_tqidx;
 556 #endif
 557         const char *pcie_ts;
 558
 559         sc = device_get_softc(dev);
 560         sc->dev = dev;
 561
 562         pci_enable_busmaster(dev);
 563         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 564                 uint32_t v;
 565
 566                 pci_set_max_read_req(dev, 4096);
 567                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 568                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 569                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 570         }
 571
 572         sc->traceq = -1;
 573         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 574         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 575             device_get_nameunit(dev));
 576
 577         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 578             device_get_nameunit(dev));
 579         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 580         sx_xlock(&t4_list_lock);
 581         SLIST_INSERT_HEAD(&t4_list, sc, link);
 582         sx_xunlock(&t4_list_lock);
 583
 584         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 585         TAILQ_INIT(&sc->sfl);
 586         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 587
 588         rc = map_bars_0_and_4(sc);
 589         if (rc != 0)
 590                 goto done; /* error message displayed already */
 591
 592         /*
 593          * This is the real PF# to which we're attaching.  Works from within PCI
 594          * passthrough environments too, where pci_get_function() could return a
 595          * different PF# depending on the passthrough configuration.  We need to
 596          * use the real PF# in all our communication with the firmware.
 597          */
 598         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 599         sc->mbox = sc->pf;
 600
 601         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 602         sc->an_handler = an_not_handled;
 603         for (i = 0; i < nitems(sc->cpl_handler); i++)
 604                 sc->cpl_handler[i] = cpl_not_handled;
 605         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 606                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 607         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 608         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 609         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 610         t4_init_sge_cpl_handlers(sc);
 611
 612         /* Prepare the adapter for operation */
 613         rc = -t4_prep_adapter(sc);
 614         if (rc != 0) {
 615                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 616                 goto done;
 617         }
 618
 619         /*
 620          * Do this really early, with the memory windows set up even before the
 621          * character device.  The userland tool's register i/o and mem read
 622          * will work even in "recovery mode".
 623          */
 624         setup_memwin(sc);
 625         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 626             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 627             device_get_nameunit(dev));
 628         if (sc->cdev == NULL)
 629                 device_printf(dev, "failed to create nexus char device.\n");
 630         else
 631                 sc->cdev->si_drv1 = sc;
 632
 633         /* Go no further if recovery mode has been requested. */
 634         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 635                 device_printf(dev, "recovery mode.\n");
 636                 goto done;
 637         }
 638
 639         /* Prepare the firmware for operation */
 640         rc = prep_firmware(sc);
 641         if (rc != 0)
 642                 goto done; /* error message displayed already */
 643
 644         rc = get_params__post_init(sc);
 645         if (rc != 0)
 646                 goto done; /* error message displayed already */
 647
 648         rc = set_params__post_init(sc);
 649         if (rc != 0)
 650                 goto done; /* error message displayed already */
 651
 652         rc = map_bar_2(sc);
 653         if (rc != 0)
 654                 goto done; /* error message displayed already */
 655
 656         rc = t4_create_dma_tag(sc);
 657         if (rc != 0)
 658                 goto done; /* error message displayed already */
 659
 660         /*
 661          * First pass over all the ports - allocate VIs and initialize some
 662          * basic parameters like mac address, port type, etc.  We also figure
 663          * out whether a port is 10G or 1G and use that information when
 664          * calculating how many interrupts to attempt to allocate.
 665          */
 666         n10g = n1g = 0;
 667         for_each_port(sc, i) {
 668                 struct port_info *pi;
 669
 670                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 671                 sc->port[i] = pi;
 672
 673                 /* These must be set before t4_port_init */
 674                 pi->adapter = sc;
 675                 pi->port_id = i;
 676
 677                 /* Allocate the vi and initialize parameters like mac addr */
 678                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 679                 if (rc != 0) {
 680                         device_printf(dev, "unable to initialize port %d: %d\n",
 681                             i, rc);
 682                         free(pi, M_CXGBE);
 683                         sc->port[i] = NULL;
 684                         goto done;
 685                 }
 686
 687                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 688                     device_get_nameunit(dev), i);
 689                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 690                 sc->chan_map[pi->tx_chan] = i;
 691
 692                 if (is_10G_port(pi) || is_40G_port(pi)) {
 693                         n10g++;
 694                         pi->tmr_idx = t4_tmr_idx_10g;
 695                         pi->pktc_idx = t4_pktc_idx_10g;
 696                 } else {
 697                         n1g++;
 698                         pi->tmr_idx = t4_tmr_idx_1g;
 699                         pi->pktc_idx = t4_pktc_idx_1g;
 700                 }
 701
 702                 pi->xact_addr_filt = -1;
 703                 pi->linkdnrc = -1;
 704
 705                 pi->qsize_rxq = t4_qsize_rxq;
 706                 pi->qsize_txq = t4_qsize_txq;
 707
 708                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 709                 if (pi->dev == NULL) {
 710                         device_printf(dev,
 711                             "failed to add device for port %d.\n", i);
 712                         rc = ENXIO;
 713                         goto done;
 714                 }
 715                 device_set_softc(pi->dev, pi);
 716         }
 717
 718         /*
 719          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 720          */
 721         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 722         if (rc != 0)
 723                 goto done; /* error message displayed already */
 724
 725         sc->intr_type = iaq.intr_type;
 726         sc->intr_count = iaq.nirq;
 727         sc->flags |= iaq.intr_flags;
 728
 729         s = &sc->sge;
 730         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 731         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 732         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 733         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 734         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 735
 736 #ifdef TCP_OFFLOAD
 737         if (is_offload(sc)) {
 738
 739                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 740                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 741                 s->neq += s->nofldtxq + s->nofldrxq;
 742                 s->niq += s->nofldrxq;
 743
 744                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 745                     M_CXGBE, M_ZERO | M_WAITOK);
 746                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 747                     M_CXGBE, M_ZERO | M_WAITOK);
 748         }
 749 #endif
 750
 751         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 752             M_ZERO | M_WAITOK);
 753         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 754             M_ZERO | M_WAITOK);
 755         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 756             M_ZERO | M_WAITOK);
 757         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 758             M_ZERO | M_WAITOK);
 759         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 760             M_ZERO | M_WAITOK);
 761
 762         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 763             M_ZERO | M_WAITOK);
 764
 765         t4_init_l2t(sc, M_WAITOK);
 766
 767         /*
 768          * Second pass over the ports.  This time we know the number of rx and
 769          * tx queues that each port should get.
 770          */
 771         rqidx = tqidx = 0;
 772 #ifdef TCP_OFFLOAD
 773         ofld_rqidx = ofld_tqidx = 0;
 774 #endif
 775         for_each_port(sc, i) {
 776                 struct port_info *pi = sc->port[i];
 777
 778                 if (pi == NULL)
 779                         continue;
 780
 781                 pi->first_rxq = rqidx;
 782                 pi->first_txq = tqidx;
 783                 if (is_10G_port(pi) || is_40G_port(pi)) {
 784                         pi->nrxq = iaq.nrxq10g;
 785                         pi->ntxq = iaq.ntxq10g;
 786                 } else {
 787                         pi->nrxq = iaq.nrxq1g;
 788                         pi->ntxq = iaq.ntxq1g;
 789                 }
 790
 791                 if (pi->ntxq > 1)
 792                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 793                 else
 794                         pi->rsrv_noflowq = 0;
 795
 796                 rqidx += pi->nrxq;
 797                 tqidx += pi->ntxq;
 798
 799 #ifdef TCP_OFFLOAD
 800                 if (is_offload(sc)) {
 801                         pi->first_ofld_rxq = ofld_rqidx;
 802                         pi->first_ofld_txq = ofld_tqidx;
 803                         if (is_10G_port(pi) || is_40G_port(pi)) {
 804                                 pi->nofldrxq = iaq.nofldrxq10g;
 805                                 pi->nofldtxq = iaq.nofldtxq10g;
 806                         } else {
 807                                 pi->nofldrxq = iaq.nofldrxq1g;
 808                                 pi->nofldtxq = iaq.nofldtxq1g;
 809                         }
 810                         ofld_rqidx += pi->nofldrxq;
 811                         ofld_tqidx += pi->nofldtxq;
 812                 }
 813 #endif
 814         }
 815
 816         rc = setup_intr_handlers(sc);
 817         if (rc != 0) {
 818                 device_printf(dev,
 819                     "failed to setup interrupt handlers: %d\n", rc);
 820                 goto done;
 821         }
 822
 823         rc = bus_generic_attach(dev);
 824         if (rc != 0) {
 825                 device_printf(dev,
 826                     "failed to attach all child ports: %d\n", rc);
 827                 goto done;
 828         }
 829
 830         switch (sc->params.pci.speed) {
 831                 case 0x1:
 832                         pcie_ts = "2.5";
 833                         break;
 834                 case 0x2:
 835                         pcie_ts = "5.0";
 836                         break;
 837                 case 0x3:
 838                         pcie_ts = "8.0";
 839                         break;
 840                 default:
 841                         pcie_ts = "??";
 842                         break;
 843         }
 844         device_printf(dev,
 845             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 846             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 847             sc->params.nports, sc->intr_count,
 848             sc->intr_type == INTR_MSIX ? "MSI-X" :
 849             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 850             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 851
 852         t4_set_desc(sc);
 853
 854 done:
 855         if (rc != 0 && sc->cdev) {
 856                 /* cdev was created and so cxgbetool works; recover that way. */
 857                 device_printf(dev,
 858                     "error during attach, adapter is now in recovery mode.\n");
 859                 rc = 0;
 860         }
 861
 862         if (rc != 0)
 863                 t4_detach(dev);
 864         else
 865                 t4_sysctls(sc);
 866
 867         return (rc);
 868 }
 869
 870 /*
 871  * Idempotent
 872  */
 873 static int
 874 t4_detach(device_t dev)
 875 {
 876         struct adapter *sc;
 877         struct port_info *pi;
 878         int i, rc;
 879
 880         sc = device_get_softc(dev);
 881
 882         if (sc->flags & FULL_INIT_DONE)
 883                 t4_intr_disable(sc);
 884
 885         if (sc->cdev) {
 886                 destroy_dev(sc->cdev);
 887                 sc->cdev = NULL;
 888         }
 889
 890         rc = bus_generic_detach(dev);
 891         if (rc) {
 892                 device_printf(dev,
 893                     "failed to detach child devices: %d\n", rc);
 894                 return (rc);
 895         }
 896
 897         for (i = 0; i < sc->intr_count; i++)
 898                 t4_free_irq(sc, &sc->irq[i]);
 899
 900         for (i = 0; i < MAX_NPORTS; i++) {
 901                 pi = sc->port[i];
 902                 if (pi) {
 903                         t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
 904                         if (pi->dev)
 905                                 device_delete_child(dev, pi->dev);
 906
 907                         mtx_destroy(&pi->pi_lock);
 908                         free(pi, M_CXGBE);
 909                 }
 910         }
 911
 912         if (sc->flags & FULL_INIT_DONE)
 913                 adapter_full_uninit(sc);
 914
 915         if (sc->flags & FW_OK)
 916                 t4_fw_bye(sc, sc->mbox);
 917
 918         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 919                 pci_release_msi(dev);
 920
 921         if (sc->regs_res)
 922                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 923                     sc->regs_res);
 924
 925         if (sc->udbs_res)
 926                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 927                     sc->udbs_res);
 928
 929         if (sc->msix_res)
 930                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 931                     sc->msix_res);
 932
 933         if (sc->l2t)
 934                 t4_free_l2t(sc->l2t);
 935
 936 #ifdef TCP_OFFLOAD
 937         free(sc->sge.ofld_rxq, M_CXGBE);
 938         free(sc->sge.ofld_txq, M_CXGBE);
 939 #endif
 940         free(sc->irq, M_CXGBE);
 941         free(sc->sge.rxq, M_CXGBE);
 942         free(sc->sge.txq, M_CXGBE);
 943         free(sc->sge.ctrlq, M_CXGBE);
 944         free(sc->sge.iqmap, M_CXGBE);
 945         free(sc->sge.eqmap, M_CXGBE);
 946         free(sc->tids.ftid_tab, M_CXGBE);
 947         t4_destroy_dma_tag(sc);
 948         if (mtx_initialized(&sc->sc_lock)) {
 949                 sx_xlock(&t4_list_lock);
 950                 SLIST_REMOVE(&t4_list, sc, adapter, link);
 951                 sx_xunlock(&t4_list_lock);
 952                 mtx_destroy(&sc->sc_lock);
 953         }
 954
 955         if (mtx_initialized(&sc->tids.ftid_lock))
 956                 mtx_destroy(&sc->tids.ftid_lock);
 957         if (mtx_initialized(&sc->sfl_lock))
 958                 mtx_destroy(&sc->sfl_lock);
 959         if (mtx_initialized(&sc->ifp_lock))
 960                 mtx_destroy(&sc->ifp_lock);
 961
 962         bzero(sc, sizeof(*sc));
 963
 964         return (0);
 965 }
 966
 967
 968 static int
 969 cxgbe_probe(device_t dev)
 970 {
 971         char buf[128];
 972         struct port_info *pi = device_get_softc(dev);
 973
 974         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
 975         device_set_desc_copy(dev, buf);
 976
 977         return (BUS_PROBE_DEFAULT);
 978 }
 979
 980 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
 981     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
 982     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
 983 #define T4_CAP_ENABLE (T4_CAP)
 984
 985 static int
 986 cxgbe_attach(device_t dev)
 987 {
 988         struct port_info *pi = device_get_softc(dev);
 989         struct ifnet *ifp;
 990
 991         /* Allocate an ifnet and set it up */
 992         ifp = if_alloc(IFT_ETHER);
 993         if (ifp == NULL) {
 994                 device_printf(dev, "Cannot allocate ifnet\n");
 995                 return (ENOMEM);
 996         }
 997         pi->ifp = ifp;
 998         ifp->if_softc = pi;
 999
1000         callout_init(&pi->tick, CALLOUT_MPSAFE);
1001
1002         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1003         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1004
1005         ifp->if_init = cxgbe_init;
1006         ifp->if_ioctl = cxgbe_ioctl;
1007         ifp->if_transmit = cxgbe_transmit;
1008         ifp->if_qflush = cxgbe_qflush;
1009
1010         ifp->if_capabilities = T4_CAP;
1011 #ifdef TCP_OFFLOAD
1012         if (is_offload(pi->adapter))
1013                 ifp->if_capabilities |= IFCAP_TOE;
1014 #endif
1015         ifp->if_capenable = T4_CAP_ENABLE;
1016         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1017             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1018
1019         /* Initialize ifmedia for this port */
1020         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1021             cxgbe_media_status);
1022         build_medialist(pi);
1023
1024         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1025             EVENTHANDLER_PRI_ANY);
1026
1027         ether_ifattach(ifp, pi->hw_addr);
1028
1029 #ifdef TCP_OFFLOAD
1030         if (is_offload(pi->adapter)) {
1031                 device_printf(dev,
1032                     "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
1033                     pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
1034         } else
1035 #endif
1036                 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
1037
1038         cxgbe_sysctls(pi);
1039
1040         return (0);
1041 }
1042
1043 static int
1044 cxgbe_detach(device_t dev)
1045 {
1046         struct port_info *pi = device_get_softc(dev);
1047         struct adapter *sc = pi->adapter;
1048         struct ifnet *ifp = pi->ifp;
1049
1050         /* Tell if_ioctl and if_init that the port is going away */
1051         ADAPTER_LOCK(sc);
1052         SET_DOOMED(pi);
1053         wakeup(&sc->flags);
1054         while (IS_BUSY(sc))
1055                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1056         SET_BUSY(sc);
1057 #ifdef INVARIANTS
1058         sc->last_op = "t4detach";
1059         sc->last_op_thr = curthread;
1060 #endif
1061         ADAPTER_UNLOCK(sc);
1062
1063         if (pi->flags & HAS_TRACEQ) {
1064                 sc->traceq = -1;        /* cloner should not create ifnet */
1065                 t4_tracer_port_detach(sc);
1066         }
1067
1068         if (pi->vlan_c)
1069                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1070
1071         PORT_LOCK(pi);
1072         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1073         callout_stop(&pi->tick);
1074         PORT_UNLOCK(pi);
1075         callout_drain(&pi->tick);
1076
1077         /* Let detach proceed even if these fail. */
1078         cxgbe_uninit_synchronized(pi);
1079         port_full_uninit(pi);
1080
1081         ifmedia_removeall(&pi->media);
1082         ether_ifdetach(pi->ifp);
1083         if_free(pi->ifp);
1084
1085         ADAPTER_LOCK(sc);
1086         CLR_BUSY(sc);
1087         wakeup(&sc->flags);
1088         ADAPTER_UNLOCK(sc);
1089
1090         return (0);
1091 }
1092
1093 static void
1094 cxgbe_init(void *arg)
1095 {
1096         struct port_info *pi = arg;
1097         struct adapter *sc = pi->adapter;
1098
1099         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1100                 return;
1101         cxgbe_init_synchronized(pi);
1102         end_synchronized_op(sc, 0);
1103 }
1104
1105 static int
1106 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1107 {
1108         int rc = 0, mtu, flags;
1109         struct port_info *pi = ifp->if_softc;
1110         struct adapter *sc = pi->adapter;
1111         struct ifreq *ifr = (struct ifreq *)data;
1112         uint32_t mask;
1113
1114         switch (cmd) {
1115         case SIOCSIFMTU:
1116                 mtu = ifr->ifr_mtu;
1117                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1118                         return (EINVAL);
1119
1120                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1121                 if (rc)
1122                         return (rc);
1123                 ifp->if_mtu = mtu;
1124                 if (pi->flags & PORT_INIT_DONE) {
1125                         t4_update_fl_bufsize(ifp);
1126                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1127                                 rc = update_mac_settings(pi, XGMAC_MTU);
1128                 }
1129                 end_synchronized_op(sc, 0);
1130                 break;
1131
1132         case SIOCSIFFLAGS:
1133                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
1134                 if (rc)
1135                         return (rc);
1136
1137                 if (ifp->if_flags & IFF_UP) {
1138                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1139                                 flags = pi->if_flags;
1140                                 if ((ifp->if_flags ^ flags) &
1141                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1142                                         rc = update_mac_settings(pi,
1143                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1144                                 }
1145                         } else
1146                                 rc = cxgbe_init_synchronized(pi);
1147                         pi->if_flags = ifp->if_flags;
1148                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1149                         rc = cxgbe_uninit_synchronized(pi);
1150                 end_synchronized_op(sc, 0);
1151                 break;
1152
1153         case SIOCADDMULTI:
1154         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1155                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1156                 if (rc)
1157                         return (rc);
1158                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1159                         rc = update_mac_settings(pi, XGMAC_MCADDRS);
1160                 end_synchronized_op(sc, LOCK_HELD);
1161                 break;
1162
1163         case SIOCSIFCAP:
1164                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1165                 if (rc)
1166                         return (rc);
1167
1168                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1169                 if (mask & IFCAP_TXCSUM) {
1170                         ifp->if_capenable ^= IFCAP_TXCSUM;
1171                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1172
1173                         if (IFCAP_TSO4 & ifp->if_capenable &&
1174                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1175                                 ifp->if_capenable &= ~IFCAP_TSO4;
1176                                 if_printf(ifp,
1177                                     "tso4 disabled due to -txcsum.\n");
1178                         }
1179                 }
1180                 if (mask & IFCAP_TXCSUM_IPV6) {
1181                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1182                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1183
1184                         if (IFCAP_TSO6 & ifp->if_capenable &&
1185                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1186                                 ifp->if_capenable &= ~IFCAP_TSO6;
1187                                 if_printf(ifp,
1188                                     "tso6 disabled due to -txcsum6.\n");
1189                         }
1190                 }
1191                 if (mask & IFCAP_RXCSUM)
1192                         ifp->if_capenable ^= IFCAP_RXCSUM;
1193                 if (mask & IFCAP_RXCSUM_IPV6)
1194                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1195
1196                 /*
1197                  * Note that we leave CSUM_TSO alone (it is always set).  The
1198                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1199                  * sending a TSO request our way, so it's sufficient to toggle
1200                  * IFCAP_TSOx only.
1201                  */
1202                 if (mask & IFCAP_TSO4) {
1203                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1204                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1205                                 if_printf(ifp, "enable txcsum first.\n");
1206                                 rc = EAGAIN;
1207                                 goto fail;
1208                         }
1209                         ifp->if_capenable ^= IFCAP_TSO4;
1210                 }
1211                 if (mask & IFCAP_TSO6) {
1212                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1213                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1214                                 if_printf(ifp, "enable txcsum6 first.\n");
1215                                 rc = EAGAIN;
1216                                 goto fail;
1217                         }
1218                         ifp->if_capenable ^= IFCAP_TSO6;
1219                 }
1220                 if (mask & IFCAP_LRO) {
1221 #if defined(INET) || defined(INET6)
1222                         int i;
1223                         struct sge_rxq *rxq;
1224
1225                         ifp->if_capenable ^= IFCAP_LRO;
1226                         for_each_rxq(pi, i, rxq) {
1227                                 if (ifp->if_capenable & IFCAP_LRO)
1228                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1229                                 else
1230                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1231                         }
1232 #endif
1233                 }
1234 #ifdef TCP_OFFLOAD
1235                 if (mask & IFCAP_TOE) {
1236                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1237
1238                         rc = toe_capability(pi, enable);
1239                         if (rc != 0)
1240                                 goto fail;
1241
1242                         ifp->if_capenable ^= mask;
1243                 }
1244 #endif
1245                 if (mask & IFCAP_VLAN_HWTAGGING) {
1246                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1247                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1248                                 rc = update_mac_settings(pi, XGMAC_VLANEX);
1249                 }
1250                 if (mask & IFCAP_VLAN_MTU) {
1251                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1252
1253                         /* Need to find out how to disable auto-mtu-inflation */
1254                 }
1255                 if (mask & IFCAP_VLAN_HWTSO)
1256                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1257                 if (mask & IFCAP_VLAN_HWCSUM)
1258                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1259
1260 #ifdef VLAN_CAPABILITIES
1261                 VLAN_CAPABILITIES(ifp);
1262 #endif
1263 fail:
1264                 end_synchronized_op(sc, 0);
1265                 break;
1266
1267         case SIOCSIFMEDIA:
1268         case SIOCGIFMEDIA:
1269                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1270                 break;
1271
1272         default:
1273                 rc = ether_ioctl(ifp, cmd, data);
1274         }
1275
1276         return (rc);
1277 }
1278
1279 static int
1280 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1281 {
1282         struct port_info *pi = ifp->if_softc;
1283         struct adapter *sc = pi->adapter;
1284         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1285         struct buf_ring *br;
1286         int rc;
1287
1288         M_ASSERTPKTHDR(m);
1289
1290         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1291                 m_freem(m);
1292                 return (ENETDOWN);
1293         }
1294
1295         if (m->m_flags & M_FLOWID)
1296                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1297                     + pi->rsrv_noflowq);
1298         br = txq->br;
1299
1300         if (TXQ_TRYLOCK(txq) == 0) {
1301                 struct sge_eq *eq = &txq->eq;
1302
1303                 /*
1304                  * It is possible that t4_eth_tx finishes up and releases the
1305                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1306                  * need to make sure that this mbuf doesn't just sit there in
1307                  * the drbr.
1308                  */
1309
1310                 rc = drbr_enqueue(ifp, br, m);
1311                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1312                     !(eq->flags & EQ_DOOMED))
1313                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1314                 return (rc);
1315         }
1316
1317         /*
1318          * txq->m is the mbuf that is held up due to a temporary shortage of
1319          * resources and it should be put on the wire first.  Then what's in
1320          * drbr and finally the mbuf that was just passed in to us.
1321          *
1322          * Return code should indicate the fate of the mbuf that was passed in
1323          * this time.
1324          */
1325
1326         TXQ_LOCK_ASSERT_OWNED(txq);
1327         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1328
1329                 /* Queued for transmission. */
1330
1331                 rc = drbr_enqueue(ifp, br, m);
1332                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1333                 (void) t4_eth_tx(ifp, txq, m);
1334                 TXQ_UNLOCK(txq);
1335                 return (rc);
1336         }
1337
1338         /* Direct transmission. */
1339         rc = t4_eth_tx(ifp, txq, m);
1340         if (rc != 0 && txq->m)
1341                 rc = 0; /* held, will be transmitted soon (hopefully) */
1342
1343         TXQ_UNLOCK(txq);
1344         return (rc);
1345 }
1346
1347 static void
1348 cxgbe_qflush(struct ifnet *ifp)
1349 {
1350         struct port_info *pi = ifp->if_softc;
1351         struct sge_txq *txq;
1352         int i;
1353         struct mbuf *m;
1354
1355         /* queues do not exist if !PORT_INIT_DONE. */
1356         if (pi->flags & PORT_INIT_DONE) {
1357                 for_each_txq(pi, i, txq) {
1358                         TXQ_LOCK(txq);
1359                         m_freem(txq->m);
1360                         txq->m = NULL;
1361                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1362                                 m_freem(m);
1363                         TXQ_UNLOCK(txq);
1364                 }
1365         }
1366         if_qflush(ifp);
1367 }
1368
1369 static int
1370 cxgbe_media_change(struct ifnet *ifp)
1371 {
1372         struct port_info *pi = ifp->if_softc;
1373
1374         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1375
1376         return (EOPNOTSUPP);
1377 }
1378
1379 static void
1380 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1381 {
1382         struct port_info *pi = ifp->if_softc;
1383         struct ifmedia_entry *cur = pi->media.ifm_cur;
1384         int speed = pi->link_cfg.speed;
1385         int data = (pi->port_type << 8) | pi->mod_type;
1386
1387         if (cur->ifm_data != data) {
1388                 build_medialist(pi);
1389                 cur = pi->media.ifm_cur;
1390         }
1391
1392         ifmr->ifm_status = IFM_AVALID;
1393         if (!pi->link_cfg.link_ok)
1394                 return;
1395
1396         ifmr->ifm_status |= IFM_ACTIVE;
1397
1398         /* active and current will differ iff current media is autoselect. */
1399         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1400                 return;
1401
1402         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1403         if (speed == SPEED_10000)
1404                 ifmr->ifm_active |= IFM_10G_T;
1405         else if (speed == SPEED_1000)
1406                 ifmr->ifm_active |= IFM_1000_T;
1407         else if (speed == SPEED_100)
1408                 ifmr->ifm_active |= IFM_100_TX;
1409         else if (speed == SPEED_10)
1410                 ifmr->ifm_active |= IFM_10_T;
1411         else
1412                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1413                             speed));
1414 }
1415
1416 void
1417 t4_fatal_err(struct adapter *sc)
1418 {
1419         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1420         t4_intr_disable(sc);
1421         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1422             device_get_nameunit(sc->dev));
1423 }
1424
1425 static int
1426 map_bars_0_and_4(struct adapter *sc)
1427 {
1428         sc->regs_rid = PCIR_BAR(0);
1429         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1430             &sc->regs_rid, RF_ACTIVE);
1431         if (sc->regs_res == NULL) {
1432                 device_printf(sc->dev, "cannot map registers.\n");
1433                 return (ENXIO);
1434         }
1435         sc->bt = rman_get_bustag(sc->regs_res);
1436         sc->bh = rman_get_bushandle(sc->regs_res);
1437         sc->mmio_len = rman_get_size(sc->regs_res);
1438         setbit(&sc->doorbells, DOORBELL_KDB);
1439
1440         sc->msix_rid = PCIR_BAR(4);
1441         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1442             &sc->msix_rid, RF_ACTIVE);
1443         if (sc->msix_res == NULL) {
1444                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1445                 return (ENXIO);
1446         }
1447
1448         return (0);
1449 }
1450
1451 static int
1452 map_bar_2(struct adapter *sc)
1453 {
1454
1455         /*
1456          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1457          * to map it if RDMA is disabled.
1458          */
1459         if (is_t4(sc) && sc->rdmacaps == 0)
1460                 return (0);
1461
1462         sc->udbs_rid = PCIR_BAR(2);
1463         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1464             &sc->udbs_rid, RF_ACTIVE);
1465         if (sc->udbs_res == NULL) {
1466                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1467                 return (ENXIO);
1468         }
1469         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1470
1471         if (is_t5(sc)) {
1472                 setbit(&sc->doorbells, DOORBELL_UDB);
1473 #if defined(__i386__) || defined(__amd64__)
1474                 if (t5_write_combine) {
1475                         int rc;
1476
1477                         /*
1478                          * Enable write combining on BAR2.  This is the
1479                          * userspace doorbell BAR and is split into 128B
1480                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1481                          * with an egress queue.  The first 64B has the doorbell
1482                          * and the second 64B can be used to submit a tx work
1483                          * request with an implicit doorbell.
1484                          */
1485
1486                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1487                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1488                         if (rc == 0) {
1489                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1490                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1491                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1492                         } else {
1493                                 device_printf(sc->dev,
1494                                     "couldn't enable write combining: %d\n",
1495                                     rc);
1496                         }
1497
1498                         t4_write_reg(sc, A_SGE_STAT_CFG,
1499                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1500                 }
1501 #endif
1502         }
1503
1504         return (0);
1505 }
1506
1507 static const struct memwin t4_memwin[] = {
1508         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1509         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1510         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1511 };
1512
1513 static const struct memwin t5_memwin[] = {
1514         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1515         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1516         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1517 };
1518
1519 static void
1520 setup_memwin(struct adapter *sc)
1521 {
1522         const struct memwin *mw;
1523         int i, n;
1524         uint32_t bar0;
1525
1526         if (is_t4(sc)) {
1527                 /*
1528                  * Read low 32b of bar0 indirectly via the hardware backdoor
1529                  * mechanism.  Works from within PCI passthrough environments
1530                  * too, where rman_get_start() can return a different value.  We
1531                  * need to program the T4 memory window decoders with the actual
1532                  * addresses that will be coming across the PCIe link.
1533                  */
1534                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1535                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1536
1537                 mw = &t4_memwin[0];
1538                 n = nitems(t4_memwin);
1539         } else {
1540                 /* T5 uses the relative offset inside the PCIe BAR */
1541                 bar0 = 0;
1542
1543                 mw = &t5_memwin[0];
1544                 n = nitems(t5_memwin);
1545         }
1546
1547         for (i = 0; i < n; i++, mw++) {
1548                 t4_write_reg(sc,
1549                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1550                     (mw->base + bar0) | V_BIR(0) |
1551                     V_WINDOW(ilog2(mw->aperture) - 10));
1552         }
1553
1554         /* flush */
1555         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1556 }
1557
1558 /*
1559  * Verify that the memory range specified by the addr/len pair is valid and lies
1560  * entirely within a single region (EDCx or MCx).
1561  */
1562 static int
1563 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1564 {
1565         uint32_t em, addr_len, maddr, mlen;
1566
1567         /* Memory can only be accessed in naturally aligned 4 byte units */
1568         if (addr & 3 || len & 3 || len == 0)
1569                 return (EINVAL);
1570
1571         /* Enabled memories */
1572         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1573         if (em & F_EDRAM0_ENABLE) {
1574                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1575                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1576                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1577                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1578                     addr + len <= maddr + mlen)
1579                         return (0);
1580         }
1581         if (em & F_EDRAM1_ENABLE) {
1582                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1583                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1584                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1585                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1586                     addr + len <= maddr + mlen)
1587                         return (0);
1588         }
1589         if (em & F_EXT_MEM_ENABLE) {
1590                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1591                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1592                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1593                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1594                     addr + len <= maddr + mlen)
1595                         return (0);
1596         }
1597         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1598                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1599                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1600                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1601                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1602                     addr + len <= maddr + mlen)
1603                         return (0);
1604         }
1605
1606         return (EFAULT);
1607 }
1608
1609 static int
1610 fwmtype_to_hwmtype(int mtype)
1611 {
1612
1613         switch (mtype) {
1614         case FW_MEMTYPE_EDC0:
1615                 return (MEM_EDC0);
1616         case FW_MEMTYPE_EDC1:
1617                 return (MEM_EDC1);
1618         case FW_MEMTYPE_EXTMEM:
1619                 return (MEM_MC0);
1620         case FW_MEMTYPE_EXTMEM1:
1621                 return (MEM_MC1);
1622         default:
1623                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1624         }
1625 }
1626
1627 /*
1628  * Verify that the memory range specified by the memtype/offset/len pair is
1629  * valid and lies entirely within the memtype specified.  The global address of
1630  * the start of the range is returned in addr.
1631  */
1632 static int
1633 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1634     uint32_t *addr)
1635 {
1636         uint32_t em, addr_len, maddr, mlen;
1637
1638         /* Memory can only be accessed in naturally aligned 4 byte units */
1639         if (off & 3 || len & 3 || len == 0)
1640                 return (EINVAL);
1641
1642         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1643         switch (fwmtype_to_hwmtype(mtype)) {
1644         case MEM_EDC0:
1645                 if (!(em & F_EDRAM0_ENABLE))
1646                         return (EINVAL);
1647                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1648                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1649                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1650                 break;
1651         case MEM_EDC1:
1652                 if (!(em & F_EDRAM1_ENABLE))
1653                         return (EINVAL);
1654                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1655                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1656                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1657                 break;
1658         case MEM_MC:
1659                 if (!(em & F_EXT_MEM_ENABLE))
1660                         return (EINVAL);
1661                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1662                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1663                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1664                 break;
1665         case MEM_MC1:
1666                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1667                         return (EINVAL);
1668                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1669                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1670                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1671                 break;
1672         default:
1673                 return (EINVAL);
1674         }
1675
1676         if (mlen > 0 && off < mlen && off + len <= mlen) {
1677                 *addr = maddr + off;    /* global address */
1678                 return (0);
1679         }
1680
1681         return (EFAULT);
1682 }
1683
1684 static void
1685 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1686 {
1687         const struct memwin *mw;
1688
1689         if (is_t4(sc)) {
1690                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1691                     ("%s: incorrect memwin# (%d)", __func__, win));
1692                 mw = &t4_memwin[win];
1693         } else {
1694                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1695                     ("%s: incorrect memwin# (%d)", __func__, win));
1696                 mw = &t5_memwin[win];
1697         }
1698
1699         if (base != NULL)
1700                 *base = mw->base;
1701         if (aperture != NULL)
1702                 *aperture = mw->aperture;
1703 }
1704
1705 /*
1706  * Positions the memory window such that it can be used to access the specified
1707  * address in the chip's address space.  The return value is the offset of addr
1708  * from the start of the window.
1709  */
1710 static uint32_t
1711 position_memwin(struct adapter *sc, int n, uint32_t addr)
1712 {
1713         uint32_t start, pf;
1714         uint32_t reg;
1715
1716         KASSERT(n >= 0 && n <= 3,
1717             ("%s: invalid window %d.", __func__, n));
1718         KASSERT((addr & 3) == 0,
1719             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1720
1721         if (is_t4(sc)) {
1722                 pf = 0;
1723                 start = addr & ~0xf;    /* start must be 16B aligned */
1724         } else {
1725                 pf = V_PFNUM(sc->pf);
1726                 start = addr & ~0x7f;   /* start must be 128B aligned */
1727         }
1728         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1729
1730         t4_write_reg(sc, reg, start | pf);
1731         t4_read_reg(sc, reg);
1732
1733         return (addr - start);
1734 }
1735
1736 static int
1737 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1738     struct intrs_and_queues *iaq)
1739 {
1740         int rc, itype, navail, nrxq10g, nrxq1g, n;
1741         int nofldrxq10g = 0, nofldrxq1g = 0;
1742
1743         bzero(iaq, sizeof(*iaq));
1744
1745         iaq->ntxq10g = t4_ntxq10g;
1746         iaq->ntxq1g = t4_ntxq1g;
1747         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1748         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1749         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1750 #ifdef TCP_OFFLOAD
1751         if (is_offload(sc)) {
1752                 iaq->nofldtxq10g = t4_nofldtxq10g;
1753                 iaq->nofldtxq1g = t4_nofldtxq1g;
1754                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1755                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1756         }
1757 #endif
1758
1759         for (itype = INTR_MSIX; itype; itype >>= 1) {
1760
1761                 if ((itype & t4_intr_types) == 0)
1762                         continue;       /* not allowed */
1763
1764                 if (itype == INTR_MSIX)
1765                         navail = pci_msix_count(sc->dev);
1766                 else if (itype == INTR_MSI)
1767                         navail = pci_msi_count(sc->dev);
1768                 else
1769                         navail = 1;
1770 restart:
1771                 if (navail == 0)
1772                         continue;
1773
1774                 iaq->intr_type = itype;
1775                 iaq->intr_flags = 0;
1776
1777                 /*
1778                  * Best option: an interrupt vector for errors, one for the
1779                  * firmware event queue, and one each for each rxq (NIC as well
1780                  * as offload).
1781                  */
1782                 iaq->nirq = T4_EXTRA_INTR;
1783                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1784                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1785                 if (iaq->nirq <= navail &&
1786                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1787                         iaq->intr_flags |= INTR_DIRECT;
1788                         goto allocate;
1789                 }
1790
1791                 /*
1792                  * Second best option: an interrupt vector for errors, one for
1793                  * the firmware event queue, and one each for either NIC or
1794                  * offload rxq's.
1795                  */
1796                 iaq->nirq = T4_EXTRA_INTR;
1797                 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1798                 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1799                 if (iaq->nirq <= navail &&
1800                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1801                         goto allocate;
1802
1803                 /*
1804                  * Next best option: an interrupt vector for errors, one for the
1805                  * firmware event queue, and at least one per port.  At this
1806                  * point we know we'll have to downsize nrxq or nofldrxq to fit
1807                  * what's available to us.
1808                  */
1809                 iaq->nirq = T4_EXTRA_INTR;
1810                 iaq->nirq += n10g + n1g;
1811                 if (iaq->nirq <= navail) {
1812                         int leftover = navail - iaq->nirq;
1813
1814                         if (n10g > 0) {
1815                                 int target = max(nrxq10g, nofldrxq10g);
1816
1817                                 n = 1;
1818                                 while (n < target && leftover >= n10g) {
1819                                         leftover -= n10g;
1820                                         iaq->nirq += n10g;
1821                                         n++;
1822                                 }
1823                                 iaq->nrxq10g = min(n, nrxq10g);
1824 #ifdef TCP_OFFLOAD
1825                                 if (is_offload(sc))
1826                                         iaq->nofldrxq10g = min(n, nofldrxq10g);
1827 #endif
1828                         }
1829
1830                         if (n1g > 0) {
1831                                 int target = max(nrxq1g, nofldrxq1g);
1832
1833                                 n = 1;
1834                                 while (n < target && leftover >= n1g) {
1835                                         leftover -= n1g;
1836                                         iaq->nirq += n1g;
1837                                         n++;
1838                                 }
1839                                 iaq->nrxq1g = min(n, nrxq1g);
1840 #ifdef TCP_OFFLOAD
1841                                 if (is_offload(sc))
1842                                         iaq->nofldrxq1g = min(n, nofldrxq1g);
1843 #endif
1844                         }
1845
1846                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1847                                 goto allocate;
1848                 }
1849
1850                 /*
1851                  * Least desirable option: one interrupt vector for everything.
1852                  */
1853                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1854 #ifdef TCP_OFFLOAD
1855                 if (is_offload(sc))
1856                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1857 #endif
1858
1859 allocate:
1860                 navail = iaq->nirq;
1861                 rc = 0;
1862                 if (itype == INTR_MSIX)
1863                         rc = pci_alloc_msix(sc->dev, &navail);
1864                 else if (itype == INTR_MSI)
1865                         rc = pci_alloc_msi(sc->dev, &navail);
1866
1867                 if (rc == 0) {
1868                         if (navail == iaq->nirq)
1869                                 return (0);
1870
1871                         /*
1872                          * Didn't get the number requested.  Use whatever number
1873                          * the kernel is willing to allocate (it's in navail).
1874                          */
1875                         device_printf(sc->dev, "fewer vectors than requested, "
1876                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1877                             itype, iaq->nirq, navail);
1878                         pci_release_msi(sc->dev);
1879                         goto restart;
1880                 }
1881
1882                 device_printf(sc->dev,
1883                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1884                     itype, rc, iaq->nirq, navail);
1885         }
1886
1887         device_printf(sc->dev,
1888             "failed to find a usable interrupt type.  "
1889             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1890             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1891
1892         return (ENXIO);
1893 }
1894
1895 #define FW_VERSION(chip) ( \
1896     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
1897     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
1898     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
1899     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
1900 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
1901
1902 struct fw_info {
1903         uint8_t chip;
1904         char *kld_name;
1905         char *fw_mod_name;
1906         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
1907 } fw_info[] = {
1908         {
1909                 .chip = CHELSIO_T4,
1910                 .kld_name = "t4fw_cfg",
1911                 .fw_mod_name = "t4fw",
1912                 .fw_hdr = {
1913                         .chip = FW_HDR_CHIP_T4,
1914                         .fw_ver = htobe32_const(FW_VERSION(T4)),
1915                         .intfver_nic = FW_INTFVER(T4, NIC),
1916                         .intfver_vnic = FW_INTFVER(T4, VNIC),
1917                         .intfver_ofld = FW_INTFVER(T4, OFLD),
1918                         .intfver_ri = FW_INTFVER(T4, RI),
1919                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
1920                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
1921                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
1922                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
1923                 },
1924         }, {
1925                 .chip = CHELSIO_T5,
1926                 .kld_name = "t5fw_cfg",
1927                 .fw_mod_name = "t5fw",
1928                 .fw_hdr = {
1929                         .chip = FW_HDR_CHIP_T5,
1930                         .fw_ver = htobe32_const(FW_VERSION(T5)),
1931                         .intfver_nic = FW_INTFVER(T5, NIC),
1932                         .intfver_vnic = FW_INTFVER(T5, VNIC),
1933                         .intfver_ofld = FW_INTFVER(T5, OFLD),
1934                         .intfver_ri = FW_INTFVER(T5, RI),
1935                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
1936                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
1937                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
1938                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
1939                 },
1940         }
1941 };
1942
1943 static struct fw_info *
1944 find_fw_info(int chip)
1945 {
1946         int i;
1947
1948         for (i = 0; i < nitems(fw_info); i++) {
1949                 if (fw_info[i].chip == chip)
1950                         return (&fw_info[i]);
1951         }
1952         return (NULL);
1953 }
1954
1955 /*
1956  * Is the given firmware API compatible with the one the driver was compiled
1957  * with?
1958  */
1959 static int
1960 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
1961 {
1962
1963         /* short circuit if it's the exact same firmware version */
1964         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
1965                 return (1);
1966
1967         /*
1968          * XXX: Is this too conservative?  Perhaps I should limit this to the
1969          * features that are supported in the driver.
1970          */
1971 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
1972         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
1973             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
1974             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
1975                 return (1);
1976 #undef SAME_INTF
1977
1978         return (0);
1979 }
1980
1981 /*
1982  * The firmware in the KLD is usable, but should it be installed?  This routine
1983  * explains itself in detail if it indicates the KLD firmware should be
1984  * installed.
1985  */
1986 static int
1987 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
1988 {
1989         const char *reason;
1990
1991         if (!card_fw_usable) {
1992                 reason = "incompatible or unusable";
1993                 goto install;
1994         }
1995
1996         if (k > c) {
1997                 reason = "older than the version bundled with this driver";
1998                 goto install;
1999         }
2000
2001         if (t4_fw_install == 2 && k != c) {
2002                 reason = "different than the version bundled with this driver";
2003                 goto install;
2004         }
2005
2006         return (0);
2007
2008 install:
2009         if (t4_fw_install == 0) {
2010                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2011                     "but the driver is prohibited from installing a different "
2012                     "firmware on the card.\n",
2013                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2014                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2015
2016                 return (0);
2017         }
2018
2019         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2020             "installing firmware %u.%u.%u.%u on card.\n",
2021             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2022             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2023             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2024             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2025
2026         return (1);
2027 }
2028 /*
2029  * Establish contact with the firmware and determine if we are the master driver
2030  * or not, and whether we are responsible for chip initialization.
2031  */
2032 static int
2033 prep_firmware(struct adapter *sc)
2034 {
2035         const struct firmware *fw = NULL, *default_cfg;
2036         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2037         enum dev_state state;
2038         struct fw_info *fw_info;
2039         struct fw_hdr *card_fw;         /* fw on the card */
2040         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2041         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2042                                            against */
2043
2044         /* Contact firmware. */
2045         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2046         if (rc < 0 || state == DEV_STATE_ERR) {
2047                 rc = -rc;
2048                 device_printf(sc->dev,
2049                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2050                 return (rc);
2051         }
2052         pf = rc;
2053         if (pf == sc->mbox)
2054                 sc->flags |= MASTER_PF;
2055         else if (state == DEV_STATE_UNINIT) {
2056                 /*
2057                  * We didn't get to be the master so we definitely won't be
2058                  * configuring the chip.  It's a bug if someone else hasn't
2059                  * configured it already.
2060                  */
2061                 device_printf(sc->dev, "couldn't be master(%d), "
2062                     "device not already initialized either(%d).\n", rc, state);
2063                 return (EDOOFUS);
2064         }
2065
2066         /* This is the firmware whose headers the driver was compiled against */
2067         fw_info = find_fw_info(chip_id(sc));
2068         if (fw_info == NULL) {
2069                 device_printf(sc->dev,
2070                     "unable to look up firmware information for chip %d.\n",
2071                     chip_id(sc));
2072                 return (EINVAL);
2073         }
2074         drv_fw = &fw_info->fw_hdr;
2075
2076         /*
2077          * The firmware KLD contains many modules.  The KLD name is also the
2078          * name of the module that contains the default config file.
2079          */
2080         default_cfg = firmware_get(fw_info->kld_name);
2081
2082         /* Read the header of the firmware on the card */
2083         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2084         rc = -t4_read_flash(sc, FLASH_FW_START,
2085             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2086         if (rc == 0)
2087                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2088         else {
2089                 device_printf(sc->dev,
2090                     "Unable to read card's firmware header: %d\n", rc);
2091                 card_fw_usable = 0;
2092         }
2093
2094         /* This is the firmware in the KLD */
2095         fw = firmware_get(fw_info->fw_mod_name);
2096         if (fw != NULL) {
2097                 kld_fw = (const void *)fw->data;
2098                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2099         } else {
2100                 kld_fw = NULL;
2101                 kld_fw_usable = 0;
2102         }
2103
2104         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2105             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2106                 /*
2107                  * Common case: the firmware on the card is an exact match and
2108                  * the KLD is an exact match too, or the KLD is
2109                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2110                  * here -- use cxgbetool loadfw if you want to reinstall the
2111                  * same firmware as the one on the card.
2112                  */
2113         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2114             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2115             be32toh(card_fw->fw_ver))) {
2116
2117                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2118                 if (rc != 0) {
2119                         device_printf(sc->dev,
2120                             "failed to install firmware: %d\n", rc);
2121                         goto done;
2122                 }
2123
2124                 /* Installed successfully, update the cached header too. */
2125                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2126                 card_fw_usable = 1;
2127                 need_fw_reset = 0;      /* already reset as part of load_fw */
2128         }
2129
2130         if (!card_fw_usable) {
2131                 uint32_t d, c, k;
2132
2133                 d = ntohl(drv_fw->fw_ver);
2134                 c = ntohl(card_fw->fw_ver);
2135                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2136
2137                 device_printf(sc->dev, "Cannot find a usable firmware: "
2138                     "fw_install %d, chip state %d, "
2139                     "driver compiled with %d.%d.%d.%d, "
2140                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2141                     t4_fw_install, state,
2142                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2143                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2144                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2145                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2146                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2147                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2148                 rc = EINVAL;
2149                 goto done;
2150         }
2151
2152         /* We're using whatever's on the card and it's known to be good. */
2153         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2154         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2155             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2156             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2157             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2158             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2159         t4_get_tp_version(sc, &sc->params.tp_vers);
2160
2161         /* Reset device */
2162         if (need_fw_reset &&
2163             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2164                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2165                 if (rc != ETIMEDOUT && rc != EIO)
2166                         t4_fw_bye(sc, sc->mbox);
2167                 goto done;
2168         }
2169         sc->flags |= FW_OK;
2170
2171         rc = get_params__pre_init(sc);
2172         if (rc != 0)
2173                 goto done; /* error message displayed already */
2174
2175         /* Partition adapter resources as specified in the config file. */
2176         if (state == DEV_STATE_UNINIT) {
2177
2178                 KASSERT(sc->flags & MASTER_PF,
2179                     ("%s: trying to change chip settings when not master.",
2180                     __func__));
2181
2182                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2183                 if (rc != 0)
2184                         goto done;      /* error message displayed already */
2185
2186                 t4_tweak_chip_settings(sc);
2187
2188                 /* get basic stuff going */
2189                 rc = -t4_fw_initialize(sc, sc->mbox);
2190                 if (rc != 0) {
2191                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2192                         goto done;
2193                 }
2194         } else {
2195                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2196                 sc->cfcsum = 0;
2197         }
2198
2199 done:
2200         free(card_fw, M_CXGBE);
2201         if (fw != NULL)
2202                 firmware_put(fw, FIRMWARE_UNLOAD);
2203         if (default_cfg != NULL)
2204                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2205
2206         return (rc);
2207 }
2208
2209 #define FW_PARAM_DEV(param) \
2210         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2211          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2212 #define FW_PARAM_PFVF(param) \
2213         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2214          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2215
2216 /*
2217  * Partition chip resources for use between various PFs, VFs, etc.
2218  */
2219 static int
2220 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2221     const char *name_prefix)
2222 {
2223         const struct firmware *cfg = NULL;
2224         int rc = 0;
2225         struct fw_caps_config_cmd caps;
2226         uint32_t mtype, moff, finicsum, cfcsum;
2227
2228         /*
2229          * Figure out what configuration file to use.  Pick the default config
2230          * file for the card if the user hasn't specified one explicitly.
2231          */
2232         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2233         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2234                 /* Card specific overrides go here. */
2235                 if (pci_get_device(sc->dev) == 0x440a)
2236                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2237                 if (is_fpga(sc))
2238                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2239         }
2240
2241         /*
2242          * We need to load another module if the profile is anything except
2243          * "default" or "flash".
2244          */
2245         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2246             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2247                 char s[32];
2248
2249                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2250                 cfg = firmware_get(s);
2251                 if (cfg == NULL) {
2252                         if (default_cfg != NULL) {
2253                                 device_printf(sc->dev,
2254                                     "unable to load module \"%s\" for "
2255                                     "configuration profile \"%s\", will use "
2256                                     "the default config file instead.\n",
2257                                     s, sc->cfg_file);
2258                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2259                                     "%s", DEFAULT_CF);
2260                         } else {
2261                                 device_printf(sc->dev,
2262                                     "unable to load module \"%s\" for "
2263                                     "configuration profile \"%s\", will use "
2264                                     "the config file on the card's flash "
2265                                     "instead.\n", s, sc->cfg_file);
2266                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2267                                     "%s", FLASH_CF);
2268                         }
2269                 }
2270         }
2271
2272         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2273             default_cfg == NULL) {
2274                 device_printf(sc->dev,
2275                     "default config file not available, will use the config "
2276                     "file on the card's flash instead.\n");
2277                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2278         }
2279
2280         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2281                 u_int cflen, i, n;
2282                 const uint32_t *cfdata;
2283                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2284
2285                 KASSERT(cfg != NULL || default_cfg != NULL,
2286                     ("%s: no config to upload", __func__));
2287
2288                 /*
2289                  * Ask the firmware where it wants us to upload the config file.
2290                  */
2291                 param = FW_PARAM_DEV(CF);
2292                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2293                 if (rc != 0) {
2294                         /* No support for config file?  Shouldn't happen. */
2295                         device_printf(sc->dev,
2296                             "failed to query config file location: %d.\n", rc);
2297                         goto done;
2298                 }
2299                 mtype = G_FW_PARAMS_PARAM_Y(val);
2300                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2301
2302                 /*
2303                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2304                  * useless stuffing/comments at the end of the config file so
2305                  * it's ok to simply throw away the last remaining bytes when
2306                  * the config file is not an exact multiple of 4.  This also
2307                  * helps with the validate_mt_off_len check.
2308                  */
2309                 if (cfg != NULL) {
2310                         cflen = cfg->datasize & ~3;
2311                         cfdata = cfg->data;
2312                 } else {
2313                         cflen = default_cfg->datasize & ~3;
2314                         cfdata = default_cfg->data;
2315                 }
2316
2317                 if (cflen > FLASH_CFG_MAX_SIZE) {
2318                         device_printf(sc->dev,
2319                             "config file too long (%d, max allowed is %d).  "
2320                             "Will try to use the config on the card, if any.\n",
2321                             cflen, FLASH_CFG_MAX_SIZE);
2322                         goto use_config_on_flash;
2323                 }
2324
2325                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2326                 if (rc != 0) {
2327                         device_printf(sc->dev,
2328                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2329                             "Will try to use the config on the card, if any.\n",
2330                             __func__, mtype, moff, cflen, rc);
2331                         goto use_config_on_flash;
2332                 }
2333
2334                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2335                 while (cflen) {
2336                         off = position_memwin(sc, 2, addr);
2337                         n = min(cflen, mw_aperture - off);
2338                         for (i = 0; i < n; i += 4)
2339                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2340                         cflen -= n;
2341                         addr += n;
2342                 }
2343         } else {
2344 use_config_on_flash:
2345                 mtype = FW_MEMTYPE_FLASH;
2346                 moff = t4_flash_cfg_addr(sc);
2347         }
2348
2349         bzero(&caps, sizeof(caps));
2350         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2351             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2352         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2353             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2354             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2355         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2356         if (rc != 0) {
2357                 device_printf(sc->dev,
2358                     "failed to pre-process config file: %d "
2359                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2360                 goto done;
2361         }
2362
2363         finicsum = be32toh(caps.finicsum);
2364         cfcsum = be32toh(caps.cfcsum);
2365         if (finicsum != cfcsum) {
2366                 device_printf(sc->dev,
2367                     "WARNING: config file checksum mismatch: %08x %08x\n",
2368                     finicsum, cfcsum);
2369         }
2370         sc->cfcsum = cfcsum;
2371
2372 #define LIMIT_CAPS(x) do { \
2373         caps.x &= htobe16(t4_##x##_allowed); \
2374 } while (0)
2375
2376         /*
2377          * Let the firmware know what features will (not) be used so it can tune
2378          * things accordingly.
2379          */
2380         LIMIT_CAPS(linkcaps);
2381         LIMIT_CAPS(niccaps);
2382         LIMIT_CAPS(toecaps);
2383         LIMIT_CAPS(rdmacaps);
2384         LIMIT_CAPS(iscsicaps);
2385         LIMIT_CAPS(fcoecaps);
2386 #undef LIMIT_CAPS
2387
2388         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2389             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2390         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2391         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2392         if (rc != 0) {
2393                 device_printf(sc->dev,
2394                     "failed to process config file: %d.\n", rc);
2395         }
2396 done:
2397         if (cfg != NULL)
2398                 firmware_put(cfg, FIRMWARE_UNLOAD);
2399         return (rc);
2400 }
2401
2402 /*
2403  * Retrieve parameters that are needed (or nice to have) very early.
2404  */
2405 static int
2406 get_params__pre_init(struct adapter *sc)
2407 {
2408         int rc;
2409         uint32_t param[2], val[2];
2410         struct fw_devlog_cmd cmd;
2411         struct devlog_params *dlog = &sc->params.devlog;
2412
2413         param[0] = FW_PARAM_DEV(PORTVEC);
2414         param[1] = FW_PARAM_DEV(CCLK);
2415         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2416         if (rc != 0) {
2417                 device_printf(sc->dev,
2418                     "failed to query parameters (pre_init): %d.\n", rc);
2419                 return (rc);
2420         }
2421
2422         sc->params.portvec = val[0];
2423         sc->params.nports = bitcount32(val[0]);
2424         sc->params.vpd.cclk = val[1];
2425
2426         /* Read device log parameters. */
2427         bzero(&cmd, sizeof(cmd));
2428         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2429             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2430         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2431         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2432         if (rc != 0) {
2433                 device_printf(sc->dev,
2434                     "failed to get devlog parameters: %d.\n", rc);
2435                 bzero(dlog, sizeof (*dlog));
2436                 rc = 0; /* devlog isn't critical for device operation */
2437         } else {
2438                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2439                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2440                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2441                 dlog->size = be32toh(cmd.memsize_devlog);
2442         }
2443
2444         return (rc);
2445 }
2446
2447 /*
2448  * Retrieve various parameters that are of interest to the driver.  The device
2449  * has been initialized by the firmware at this point.
2450  */
2451 static int
2452 get_params__post_init(struct adapter *sc)
2453 {
2454         int rc;
2455         uint32_t param[7], val[7];
2456         struct fw_caps_config_cmd caps;
2457
2458         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2459         param[1] = FW_PARAM_PFVF(EQ_START);
2460         param[2] = FW_PARAM_PFVF(FILTER_START);
2461         param[3] = FW_PARAM_PFVF(FILTER_END);
2462         param[4] = FW_PARAM_PFVF(L2T_START);
2463         param[5] = FW_PARAM_PFVF(L2T_END);
2464         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2465         if (rc != 0) {
2466                 device_printf(sc->dev,
2467                     "failed to query parameters (post_init): %d.\n", rc);
2468                 return (rc);
2469         }
2470
2471         sc->sge.iq_start = val[0];
2472         sc->sge.eq_start = val[1];
2473         sc->tids.ftid_base = val[2];
2474         sc->tids.nftids = val[3] - val[2] + 1;
2475         sc->params.ftid_min = val[2];
2476         sc->params.ftid_max = val[3];
2477         sc->vres.l2t.start = val[4];
2478         sc->vres.l2t.size = val[5] - val[4] + 1;
2479         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2480             ("%s: L2 table size (%u) larger than expected (%u)",
2481             __func__, sc->vres.l2t.size, L2T_SIZE));
2482
2483         /* get capabilites */
2484         bzero(&caps, sizeof(caps));
2485         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2486             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2487         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2488         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2489         if (rc != 0) {
2490                 device_printf(sc->dev,
2491                     "failed to get card capabilities: %d.\n", rc);
2492                 return (rc);
2493         }
2494
2495 #define READ_CAPS(x) do { \
2496         sc->x = htobe16(caps.x); \
2497 } while (0)
2498         READ_CAPS(linkcaps);
2499         READ_CAPS(niccaps);
2500         READ_CAPS(toecaps);
2501         READ_CAPS(rdmacaps);
2502         READ_CAPS(iscsicaps);
2503         READ_CAPS(fcoecaps);
2504
2505         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2506                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2507                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2508                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2509                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2510                 if (rc != 0) {
2511                         device_printf(sc->dev,
2512                             "failed to query NIC parameters: %d.\n", rc);
2513                         return (rc);
2514                 }
2515                 sc->tids.etid_base = val[0];
2516                 sc->params.etid_min = val[0];
2517                 sc->tids.netids = val[1] - val[0] + 1;
2518                 sc->params.netids = sc->tids.netids;
2519                 sc->params.eo_wr_cred = val[2];
2520                 sc->params.ethoffload = 1;
2521         }
2522
2523         if (sc->toecaps) {
2524                 /* query offload-related parameters */
2525                 param[0] = FW_PARAM_DEV(NTID);
2526                 param[1] = FW_PARAM_PFVF(SERVER_START);
2527                 param[2] = FW_PARAM_PFVF(SERVER_END);
2528                 param[3] = FW_PARAM_PFVF(TDDP_START);
2529                 param[4] = FW_PARAM_PFVF(TDDP_END);
2530                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2531                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2532                 if (rc != 0) {
2533                         device_printf(sc->dev,
2534                             "failed to query TOE parameters: %d.\n", rc);
2535                         return (rc);
2536                 }
2537                 sc->tids.ntids = val[0];
2538                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2539                 sc->tids.stid_base = val[1];
2540                 sc->tids.nstids = val[2] - val[1] + 1;
2541                 sc->vres.ddp.start = val[3];
2542                 sc->vres.ddp.size = val[4] - val[3] + 1;
2543                 sc->params.ofldq_wr_cred = val[5];
2544                 sc->params.offload = 1;
2545         }
2546         if (sc->rdmacaps) {
2547                 param[0] = FW_PARAM_PFVF(STAG_START);
2548                 param[1] = FW_PARAM_PFVF(STAG_END);
2549                 param[2] = FW_PARAM_PFVF(RQ_START);
2550                 param[3] = FW_PARAM_PFVF(RQ_END);
2551                 param[4] = FW_PARAM_PFVF(PBL_START);
2552                 param[5] = FW_PARAM_PFVF(PBL_END);
2553                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2554                 if (rc != 0) {
2555                         device_printf(sc->dev,
2556                             "failed to query RDMA parameters(1): %d.\n", rc);
2557                         return (rc);
2558                 }
2559                 sc->vres.stag.start = val[0];
2560                 sc->vres.stag.size = val[1] - val[0] + 1;
2561                 sc->vres.rq.start = val[2];
2562                 sc->vres.rq.size = val[3] - val[2] + 1;
2563                 sc->vres.pbl.start = val[4];
2564                 sc->vres.pbl.size = val[5] - val[4] + 1;
2565
2566                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2567                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2568                 param[2] = FW_PARAM_PFVF(CQ_START);
2569                 param[3] = FW_PARAM_PFVF(CQ_END);
2570                 param[4] = FW_PARAM_PFVF(OCQ_START);
2571                 param[5] = FW_PARAM_PFVF(OCQ_END);
2572                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2573                 if (rc != 0) {
2574                         device_printf(sc->dev,
2575                             "failed to query RDMA parameters(2): %d.\n", rc);
2576                         return (rc);
2577                 }
2578                 sc->vres.qp.start = val[0];
2579                 sc->vres.qp.size = val[1] - val[0] + 1;
2580                 sc->vres.cq.start = val[2];
2581                 sc->vres.cq.size = val[3] - val[2] + 1;
2582                 sc->vres.ocq.start = val[4];
2583                 sc->vres.ocq.size = val[5] - val[4] + 1;
2584         }
2585         if (sc->iscsicaps) {
2586                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2587                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2588                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2589                 if (rc != 0) {
2590                         device_printf(sc->dev,
2591                             "failed to query iSCSI parameters: %d.\n", rc);
2592                         return (rc);
2593                 }
2594                 sc->vres.iscsi.start = val[0];
2595                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2596         }
2597
2598         /*
2599          * We've got the params we wanted to query via the firmware.  Now grab
2600          * some others directly from the chip.
2601          */
2602         rc = t4_read_chip_settings(sc);
2603
2604         return (rc);
2605 }
2606
2607 static int
2608 set_params__post_init(struct adapter *sc)
2609 {
2610         uint32_t param, val;
2611
2612         /* ask for encapsulated CPLs */
2613         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2614         val = 1;
2615         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2616
2617         return (0);
2618 }
2619
2620 #undef FW_PARAM_PFVF
2621 #undef FW_PARAM_DEV
2622
2623 static void
2624 t4_set_desc(struct adapter *sc)
2625 {
2626         char buf[128];
2627         struct adapter_params *p = &sc->params;
2628
2629         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2630             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2631             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2632
2633         device_set_desc_copy(sc->dev, buf);
2634 }
2635
2636 static void
2637 build_medialist(struct port_info *pi)
2638 {
2639         struct ifmedia *media = &pi->media;
2640         int data, m;
2641
2642         PORT_LOCK(pi);
2643
2644         ifmedia_removeall(media);
2645
2646         m = IFM_ETHER | IFM_FDX;
2647         data = (pi->port_type << 8) | pi->mod_type;
2648
2649         switch(pi->port_type) {
2650         case FW_PORT_TYPE_BT_XFI:
2651                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2652                 break;
2653
2654         case FW_PORT_TYPE_BT_XAUI:
2655                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2656                 /* fall through */
2657
2658         case FW_PORT_TYPE_BT_SGMII:
2659                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2660                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2661                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2662                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2663                 break;
2664
2665         case FW_PORT_TYPE_CX4:
2666                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2667                 ifmedia_set(media, m | IFM_10G_CX4);
2668                 break;
2669
2670         case FW_PORT_TYPE_SFP:
2671         case FW_PORT_TYPE_FIBER_XFI:
2672         case FW_PORT_TYPE_FIBER_XAUI:
2673                 switch (pi->mod_type) {
2674
2675                 case FW_PORT_MOD_TYPE_LR:
2676                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2677                         ifmedia_set(media, m | IFM_10G_LR);
2678                         break;
2679
2680                 case FW_PORT_MOD_TYPE_SR:
2681                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2682                         ifmedia_set(media, m | IFM_10G_SR);
2683                         break;
2684
2685                 case FW_PORT_MOD_TYPE_LRM:
2686                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2687                         ifmedia_set(media, m | IFM_10G_LRM);
2688                         break;
2689
2690                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2691                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2692                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2693                         ifmedia_set(media, m | IFM_10G_TWINAX);
2694                         break;
2695
2696                 case FW_PORT_MOD_TYPE_NONE:
2697                         m &= ~IFM_FDX;
2698                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2699                         ifmedia_set(media, m | IFM_NONE);
2700                         break;
2701
2702                 case FW_PORT_MOD_TYPE_NA:
2703                 case FW_PORT_MOD_TYPE_ER:
2704                 default:
2705                         device_printf(pi->dev,
2706                             "unknown port_type (%d), mod_type (%d)\n",
2707                             pi->port_type, pi->mod_type);
2708                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2709                         ifmedia_set(media, m | IFM_UNKNOWN);
2710                         break;
2711                 }
2712                 break;
2713
2714         case FW_PORT_TYPE_QSFP:
2715                 switch (pi->mod_type) {
2716
2717                 case FW_PORT_MOD_TYPE_LR:
2718                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2719                         ifmedia_set(media, m | IFM_40G_LR4);
2720                         break;
2721
2722                 case FW_PORT_MOD_TYPE_SR:
2723                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2724                         ifmedia_set(media, m | IFM_40G_SR4);
2725                         break;
2726
2727                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2728                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2729                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2730                         ifmedia_set(media, m | IFM_40G_CR4);
2731                         break;
2732
2733                 case FW_PORT_MOD_TYPE_NONE:
2734                         m &= ~IFM_FDX;
2735                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2736                         ifmedia_set(media, m | IFM_NONE);
2737                         break;
2738
2739                 default:
2740                         device_printf(pi->dev,
2741                             "unknown port_type (%d), mod_type (%d)\n",
2742                             pi->port_type, pi->mod_type);
2743                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2744                         ifmedia_set(media, m | IFM_UNKNOWN);
2745                         break;
2746                 }
2747                 break;
2748
2749         default:
2750                 device_printf(pi->dev,
2751                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2752                     pi->mod_type);
2753                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2754                 ifmedia_set(media, m | IFM_UNKNOWN);
2755                 break;
2756         }
2757
2758         PORT_UNLOCK(pi);
2759 }
2760
2761 #define FW_MAC_EXACT_CHUNK      7
2762
2763 /*
2764  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2765  * indicates which parameters should be programmed (the rest are left alone).
2766  */
2767 static int
2768 update_mac_settings(struct port_info *pi, int flags)
2769 {
2770         int rc;
2771         struct ifnet *ifp = pi->ifp;
2772         struct adapter *sc = pi->adapter;
2773         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2774
2775         ASSERT_SYNCHRONIZED_OP(sc);
2776         KASSERT(flags, ("%s: not told what to update.", __func__));
2777
2778         if (flags & XGMAC_MTU)
2779                 mtu = ifp->if_mtu;
2780
2781         if (flags & XGMAC_PROMISC)
2782                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2783
2784         if (flags & XGMAC_ALLMULTI)
2785                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2786
2787         if (flags & XGMAC_VLANEX)
2788                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2789
2790         rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2791             vlanex, false);
2792         if (rc) {
2793                 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2794                 return (rc);
2795         }
2796
2797         if (flags & XGMAC_UCADDR) {
2798                 uint8_t ucaddr[ETHER_ADDR_LEN];
2799
2800                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2801                 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2802                     ucaddr, true, true);
2803                 if (rc < 0) {
2804                         rc = -rc;
2805                         if_printf(ifp, "change_mac failed: %d\n", rc);
2806                         return (rc);
2807                 } else {
2808                         pi->xact_addr_filt = rc;
2809                         rc = 0;
2810                 }
2811         }
2812
2813         if (flags & XGMAC_MCADDRS) {
2814                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2815                 int del = 1;
2816                 uint64_t hash = 0;
2817                 struct ifmultiaddr *ifma;
2818                 int i = 0, j;
2819
2820                 if_maddr_rlock(ifp);
2821                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2822                         if (ifma->ifma_addr->sa_family != AF_LINK)
2823                                 continue;
2824                         mcaddr[i++] =
2825                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2826
2827                         if (i == FW_MAC_EXACT_CHUNK) {
2828                                 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2829                                     del, i, mcaddr, NULL, &hash, 0);
2830                                 if (rc < 0) {
2831                                         rc = -rc;
2832                                         for (j = 0; j < i; j++) {
2833                                                 if_printf(ifp,
2834                                                     "failed to add mc address"
2835                                                     " %02x:%02x:%02x:"
2836                                                     "%02x:%02x:%02x rc=%d\n",
2837                                                     mcaddr[j][0], mcaddr[j][1],
2838                                                     mcaddr[j][2], mcaddr[j][3],
2839                                                     mcaddr[j][4], mcaddr[j][5],
2840                                                     rc);
2841                                         }
2842                                         goto mcfail;
2843                                 }
2844                                 del = 0;
2845                                 i = 0;
2846                         }
2847                 }
2848                 if (i > 0) {
2849                         rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2850                             del, i, mcaddr, NULL, &hash, 0);
2851                         if (rc < 0) {
2852                                 rc = -rc;
2853                                 for (j = 0; j < i; j++) {
2854                                         if_printf(ifp,
2855                                             "failed to add mc address"
2856                                             " %02x:%02x:%02x:"
2857                                             "%02x:%02x:%02x rc=%d\n",
2858                                             mcaddr[j][0], mcaddr[j][1],
2859                                             mcaddr[j][2], mcaddr[j][3],
2860                                             mcaddr[j][4], mcaddr[j][5],
2861                                             rc);
2862                                 }
2863                                 goto mcfail;
2864                         }
2865                 }
2866
2867                 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2868                 if (rc != 0)
2869                         if_printf(ifp, "failed to set mc address hash: %d", rc);
2870 mcfail:
2871                 if_maddr_runlock(ifp);
2872         }
2873
2874         return (rc);
2875 }
2876
2877 int
2878 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
2879     char *wmesg)
2880 {
2881         int rc, pri;
2882
2883 #ifdef WITNESS
2884         /* the caller thinks it's ok to sleep, but is it really? */
2885         if (flags & SLEEP_OK)
2886                 pause("t4slptst", 1);
2887 #endif
2888
2889         if (INTR_OK)
2890                 pri = PCATCH;
2891         else
2892                 pri = 0;
2893
2894         ADAPTER_LOCK(sc);
2895         for (;;) {
2896
2897                 if (pi && IS_DOOMED(pi)) {
2898                         rc = ENXIO;
2899                         goto done;
2900                 }
2901
2902                 if (!IS_BUSY(sc)) {
2903                         rc = 0;
2904                         break;
2905                 }
2906
2907                 if (!(flags & SLEEP_OK)) {
2908                         rc = EBUSY;
2909                         goto done;
2910                 }
2911
2912                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
2913                         rc = EINTR;
2914                         goto done;
2915                 }
2916         }
2917
2918         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2919         SET_BUSY(sc);
2920 #ifdef INVARIANTS
2921         sc->last_op = wmesg;
2922         sc->last_op_thr = curthread;
2923 #endif
2924
2925 done:
2926         if (!(flags & HOLD_LOCK) || rc)
2927                 ADAPTER_UNLOCK(sc);
2928
2929         return (rc);
2930 }
2931
2932 void
2933 end_synchronized_op(struct adapter *sc, int flags)
2934 {
2935
2936         if (flags & LOCK_HELD)
2937                 ADAPTER_LOCK_ASSERT_OWNED(sc);
2938         else
2939                 ADAPTER_LOCK(sc);
2940
2941         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2942         CLR_BUSY(sc);
2943         wakeup(&sc->flags);
2944         ADAPTER_UNLOCK(sc);
2945 }
2946
2947 static int
2948 cxgbe_init_synchronized(struct port_info *pi)
2949 {
2950         struct adapter *sc = pi->adapter;
2951         struct ifnet *ifp = pi->ifp;
2952         int rc = 0;
2953
2954         ASSERT_SYNCHRONIZED_OP(sc);
2955
2956         if (isset(&sc->open_device_map, pi->port_id)) {
2957                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2958                     ("mismatch between open_device_map and if_drv_flags"));
2959                 return (0);     /* already running */
2960         }
2961
2962         if (!(sc->flags & FULL_INIT_DONE) &&
2963             ((rc = adapter_full_init(sc)) != 0))
2964                 return (rc);    /* error message displayed already */
2965
2966         if (!(pi->flags & PORT_INIT_DONE) &&
2967             ((rc = port_full_init(pi)) != 0))
2968                 return (rc); /* error message displayed already */
2969
2970         rc = update_mac_settings(pi, XGMAC_ALL);
2971         if (rc)
2972                 goto done;      /* error message displayed already */
2973
2974         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2975         if (rc != 0) {
2976                 if_printf(ifp, "start_link failed: %d\n", rc);
2977                 goto done;
2978         }
2979
2980         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2981         if (rc != 0) {
2982                 if_printf(ifp, "enable_vi failed: %d\n", rc);
2983                 goto done;
2984         }
2985
2986         /*
2987          * The first iq of the first port to come up is used for tracing.
2988          */
2989         if (sc->traceq < 0) {
2990                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
2991                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
2992                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
2993                     V_QUEUENUMBER(sc->traceq));
2994                 pi->flags |= HAS_TRACEQ;
2995         }
2996
2997         /* all ok */
2998         setbit(&sc->open_device_map, pi->port_id);
2999         PORT_LOCK(pi);
3000         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3001         PORT_UNLOCK(pi);
3002
3003         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3004 done:
3005         if (rc != 0)
3006                 cxgbe_uninit_synchronized(pi);
3007
3008         return (rc);
3009 }
3010
3011 /*
3012  * Idempotent.
3013  */
3014 static int
3015 cxgbe_uninit_synchronized(struct port_info *pi)
3016 {
3017         struct adapter *sc = pi->adapter;
3018         struct ifnet *ifp = pi->ifp;
3019         int rc;
3020
3021         ASSERT_SYNCHRONIZED_OP(sc);
3022
3023         /*
3024          * Disable the VI so that all its data in either direction is discarded
3025          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3026          * tick) intact as the TP can deliver negative advice or data that it's
3027          * holding in its RAM (for an offloaded connection) even after the VI is
3028          * disabled.
3029          */
3030         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3031         if (rc) {
3032                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3033                 return (rc);
3034         }
3035
3036         clrbit(&sc->open_device_map, pi->port_id);
3037         PORT_LOCK(pi);
3038         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3039         PORT_UNLOCK(pi);
3040
3041         pi->link_cfg.link_ok = 0;
3042         pi->link_cfg.speed = 0;
3043         pi->linkdnrc = -1;
3044         t4_os_link_changed(sc, pi->port_id, 0, -1);
3045
3046         return (0);
3047 }
3048
3049 /*
3050  * It is ok for this function to fail midway and return right away.  t4_detach
3051  * will walk the entire sc->irq list and clean up whatever is valid.
3052  */
3053 static int
3054 setup_intr_handlers(struct adapter *sc)
3055 {
3056         int rc, rid, p, q;
3057         char s[8];
3058         struct irq *irq;
3059         struct port_info *pi;
3060         struct sge_rxq *rxq;
3061 #ifdef TCP_OFFLOAD
3062         struct sge_ofld_rxq *ofld_rxq;
3063 #endif
3064
3065         /*
3066          * Setup interrupts.
3067          */
3068         irq = &sc->irq[0];
3069         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3070         if (sc->intr_count == 1) {
3071                 KASSERT(!(sc->flags & INTR_DIRECT),
3072                     ("%s: single interrupt && INTR_DIRECT?", __func__));
3073
3074                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
3075                 if (rc != 0)
3076                         return (rc);
3077         } else {
3078                 /* Multiple interrupts. */
3079                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3080                     ("%s: too few intr.", __func__));
3081
3082                 /* The first one is always error intr */
3083                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3084                 if (rc != 0)
3085                         return (rc);
3086                 irq++;
3087                 rid++;
3088
3089                 /* The second one is always the firmware event queue */
3090                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
3091                     "evt");
3092                 if (rc != 0)
3093                         return (rc);
3094                 irq++;
3095                 rid++;
3096
3097                 /*
3098                  * Note that if INTR_DIRECT is not set then either the NIC rx
3099                  * queues or (exclusive or) the TOE rx queueus will be taking
3100                  * direct interrupts.
3101                  *
3102                  * There is no need to check for is_offload(sc) as nofldrxq
3103                  * will be 0 if offload is disabled.
3104                  */
3105                 for_each_port(sc, p) {
3106                         pi = sc->port[p];
3107
3108 #ifdef TCP_OFFLOAD
3109                         /*
3110                          * Skip over the NIC queues if they aren't taking direct
3111                          * interrupts.
3112                          */
3113                         if (!(sc->flags & INTR_DIRECT) &&
3114                             pi->nofldrxq > pi->nrxq)
3115                                 goto ofld_queues;
3116 #endif
3117                         rxq = &sc->sge.rxq[pi->first_rxq];
3118                         for (q = 0; q < pi->nrxq; q++, rxq++) {
3119                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3120                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3121                                     s);
3122                                 if (rc != 0)
3123                                         return (rc);
3124                                 irq++;
3125                                 rid++;
3126                         }
3127
3128 #ifdef TCP_OFFLOAD
3129                         /*
3130                          * Skip over the offload queues if they aren't taking
3131                          * direct interrupts.
3132                          */
3133                         if (!(sc->flags & INTR_DIRECT))
3134                                 continue;
3135 ofld_queues:
3136                         ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
3137                         for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
3138                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3139                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3140                                     ofld_rxq, s);
3141                                 if (rc != 0)
3142                                         return (rc);
3143                                 irq++;
3144                                 rid++;
3145                         }
3146 #endif
3147                 }
3148         }
3149
3150         return (0);
3151 }
3152
3153 static int
3154 adapter_full_init(struct adapter *sc)
3155 {
3156         int rc, i;
3157
3158         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3159         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3160             ("%s: FULL_INIT_DONE already", __func__));
3161
3162         /*
3163          * queues that belong to the adapter (not any particular port).
3164          */
3165         rc = t4_setup_adapter_queues(sc);
3166         if (rc != 0)
3167                 goto done;
3168
3169         for (i = 0; i < nitems(sc->tq); i++) {
3170                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3171                     taskqueue_thread_enqueue, &sc->tq[i]);
3172                 if (sc->tq[i] == NULL) {
3173                         device_printf(sc->dev,
3174                             "failed to allocate task queue %d\n", i);
3175                         rc = ENOMEM;
3176                         goto done;
3177                 }
3178                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3179                     device_get_nameunit(sc->dev), i);
3180         }
3181
3182         t4_intr_enable(sc);
3183         sc->flags |= FULL_INIT_DONE;
3184 done:
3185         if (rc != 0)
3186                 adapter_full_uninit(sc);
3187
3188         return (rc);
3189 }
3190
3191 static int
3192 adapter_full_uninit(struct adapter *sc)
3193 {
3194         int i;
3195
3196         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3197
3198         t4_teardown_adapter_queues(sc);
3199
3200         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3201                 taskqueue_free(sc->tq[i]);
3202                 sc->tq[i] = NULL;
3203         }
3204
3205         sc->flags &= ~FULL_INIT_DONE;
3206
3207         return (0);
3208 }
3209
3210 static int
3211 port_full_init(struct port_info *pi)
3212 {
3213         struct adapter *sc = pi->adapter;
3214         struct ifnet *ifp = pi->ifp;
3215         uint16_t *rss;
3216         struct sge_rxq *rxq;
3217         int rc, i, j;
3218
3219         ASSERT_SYNCHRONIZED_OP(sc);
3220         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3221             ("%s: PORT_INIT_DONE already", __func__));
3222
3223         sysctl_ctx_init(&pi->ctx);
3224         pi->flags |= PORT_SYSCTL_CTX;
3225
3226         /*
3227          * Allocate tx/rx/fl queues for this port.
3228          */
3229         rc = t4_setup_port_queues(pi);
3230         if (rc != 0)
3231                 goto done;      /* error message displayed already */
3232
3233         /*
3234          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3235          */
3236         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3237         for (i = 0; i < pi->rss_size;) {
3238                 for_each_rxq(pi, j, rxq) {
3239                         rss[i++] = rxq->iq.abs_id;
3240                         if (i == pi->rss_size)
3241                                 break;
3242                 }
3243         }
3244
3245         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3246             pi->rss_size);
3247         if (rc != 0) {
3248                 if_printf(ifp, "rss_config failed: %d\n", rc);
3249                 goto done;
3250         }
3251
3252         pi->rss = rss;
3253         pi->flags |= PORT_INIT_DONE;
3254 done:
3255         if (rc != 0)
3256                 port_full_uninit(pi);
3257
3258         return (rc);
3259 }
3260
3261 /*
3262  * Idempotent.
3263  */
3264 static int
3265 port_full_uninit(struct port_info *pi)
3266 {
3267         struct adapter *sc = pi->adapter;
3268         int i;
3269         struct sge_rxq *rxq;
3270         struct sge_txq *txq;
3271 #ifdef TCP_OFFLOAD
3272         struct sge_ofld_rxq *ofld_rxq;
3273         struct sge_wrq *ofld_txq;
3274 #endif
3275
3276         if (pi->flags & PORT_INIT_DONE) {
3277
3278                 /* Need to quiesce queues.  XXX: ctrl queues? */
3279
3280                 for_each_txq(pi, i, txq) {
3281                         quiesce_eq(sc, &txq->eq);
3282                 }
3283
3284 #ifdef TCP_OFFLOAD
3285                 for_each_ofld_txq(pi, i, ofld_txq) {
3286                         quiesce_eq(sc, &ofld_txq->eq);
3287                 }
3288 #endif
3289
3290                 for_each_rxq(pi, i, rxq) {
3291                         quiesce_iq(sc, &rxq->iq);
3292                         quiesce_fl(sc, &rxq->fl);
3293                 }
3294
3295 #ifdef TCP_OFFLOAD
3296                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3297                         quiesce_iq(sc, &ofld_rxq->iq);
3298                         quiesce_fl(sc, &ofld_rxq->fl);
3299                 }
3300 #endif
3301                 free(pi->rss, M_CXGBE);
3302         }
3303
3304         t4_teardown_port_queues(pi);
3305         pi->flags &= ~PORT_INIT_DONE;
3306
3307         return (0);
3308 }
3309
3310 static void
3311 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3312 {
3313         EQ_LOCK(eq);
3314         eq->flags |= EQ_DOOMED;
3315
3316         /*
3317          * Wait for the response to a credit flush if one's
3318          * pending.
3319          */
3320         while (eq->flags & EQ_CRFLUSHED)
3321                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3322         EQ_UNLOCK(eq);
3323
3324         callout_drain(&eq->tx_callout); /* XXX: iffy */
3325         pause("callout", 10);           /* Still iffy */
3326
3327         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3328 }
3329
3330 static void
3331 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3332 {
3333         (void) sc;      /* unused */
3334
3335         /* Synchronize with the interrupt handler */
3336         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3337                 pause("iqfree", 1);
3338 }
3339
3340 static void
3341 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3342 {
3343         mtx_lock(&sc->sfl_lock);
3344         FL_LOCK(fl);
3345         fl->flags |= FL_DOOMED;
3346         FL_UNLOCK(fl);
3347         mtx_unlock(&sc->sfl_lock);
3348
3349         callout_drain(&sc->sfl_callout);
3350         KASSERT((fl->flags & FL_STARVING) == 0,
3351             ("%s: still starving", __func__));
3352 }
3353
3354 static int
3355 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3356     driver_intr_t *handler, void *arg, char *name)
3357 {
3358         int rc;
3359
3360         irq->rid = rid;
3361         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3362             RF_SHAREABLE | RF_ACTIVE);
3363         if (irq->res == NULL) {
3364                 device_printf(sc->dev,
3365                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3366                 return (ENOMEM);
3367         }
3368
3369         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3370             NULL, handler, arg, &irq->tag);
3371         if (rc != 0) {
3372                 device_printf(sc->dev,
3373                     "failed to setup interrupt for rid %d, name %s: %d\n",
3374                     rid, name, rc);
3375         } else if (name)
3376                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3377
3378         return (rc);
3379 }
3380
3381 static int
3382 t4_free_irq(struct adapter *sc, struct irq *irq)
3383 {
3384         if (irq->tag)
3385                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3386         if (irq->res)
3387                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3388
3389         bzero(irq, sizeof(*irq));
3390
3391         return (0);
3392 }
3393
3394 static void
3395 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3396     unsigned int end)
3397 {
3398         uint32_t *p = (uint32_t *)(buf + start);
3399
3400         for ( ; start <= end; start += sizeof(uint32_t))
3401                 *p++ = t4_read_reg(sc, start);
3402 }
3403
3404 static void
3405 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3406 {
3407         int i, n;
3408         const unsigned int *reg_ranges;
3409         static const unsigned int t4_reg_ranges[] = {
3410                 0x1008, 0x1108,
3411                 0x1180, 0x11b4,
3412                 0x11fc, 0x123c,
3413                 0x1300, 0x173c,
3414                 0x1800, 0x18fc,
3415                 0x3000, 0x30d8,
3416                 0x30e0, 0x5924,
3417                 0x5960, 0x59d4,
3418                 0x5a00, 0x5af8,
3419                 0x6000, 0x6098,
3420                 0x6100, 0x6150,
3421                 0x6200, 0x6208,
3422                 0x6240, 0x6248,
3423                 0x6280, 0x6338,
3424                 0x6370, 0x638c,
3425                 0x6400, 0x643c,
3426                 0x6500, 0x6524,
3427                 0x6a00, 0x6a38,
3428                 0x6a60, 0x6a78,
3429                 0x6b00, 0x6b84,
3430                 0x6bf0, 0x6c84,
3431                 0x6cf0, 0x6d84,
3432                 0x6df0, 0x6e84,
3433                 0x6ef0, 0x6f84,
3434                 0x6ff0, 0x7084,
3435                 0x70f0, 0x7184,
3436                 0x71f0, 0x7284,
3437                 0x72f0, 0x7384,
3438                 0x73f0, 0x7450,
3439                 0x7500, 0x7530,
3440                 0x7600, 0x761c,
3441                 0x7680, 0x76cc,
3442                 0x7700, 0x7798,
3443                 0x77c0, 0x77fc,
3444                 0x7900, 0x79fc,
3445                 0x7b00, 0x7c38,
3446                 0x7d00, 0x7efc,
3447                 0x8dc0, 0x8e1c,
3448                 0x8e30, 0x8e78,
3449                 0x8ea0, 0x8f6c,
3450                 0x8fc0, 0x9074,
3451                 0x90fc, 0x90fc,
3452                 0x9400, 0x9458,
3453                 0x9600, 0x96bc,
3454                 0x9800, 0x9808,
3455                 0x9820, 0x983c,
3456                 0x9850, 0x9864,
3457                 0x9c00, 0x9c6c,
3458                 0x9c80, 0x9cec,
3459                 0x9d00, 0x9d6c,
3460                 0x9d80, 0x9dec,
3461                 0x9e00, 0x9e6c,
3462                 0x9e80, 0x9eec,
3463                 0x9f00, 0x9f6c,
3464                 0x9f80, 0x9fec,
3465                 0xd004, 0xd03c,
3466                 0xdfc0, 0xdfe0,
3467                 0xe000, 0xea7c,
3468                 0xf000, 0x11110,
3469                 0x11118, 0x11190,
3470                 0x19040, 0x1906c,
3471                 0x19078, 0x19080,
3472                 0x1908c, 0x19124,
3473                 0x19150, 0x191b0,
3474                 0x191d0, 0x191e8,
3475                 0x19238, 0x1924c,
3476                 0x193f8, 0x19474,
3477                 0x19490, 0x194f8,
3478                 0x19800, 0x19f30,
3479                 0x1a000, 0x1a06c,
3480                 0x1a0b0, 0x1a120,
3481                 0x1a128, 0x1a138,
3482                 0x1a190, 0x1a1c4,
3483                 0x1a1fc, 0x1a1fc,
3484                 0x1e040, 0x1e04c,
3485                 0x1e284, 0x1e28c,
3486                 0x1e2c0, 0x1e2c0,
3487                 0x1e2e0, 0x1e2e0,
3488                 0x1e300, 0x1e384,
3489                 0x1e3c0, 0x1e3c8,
3490                 0x1e440, 0x1e44c,
3491                 0x1e684, 0x1e68c,
3492                 0x1e6c0, 0x1e6c0,
3493                 0x1e6e0, 0x1e6e0,
3494                 0x1e700, 0x1e784,
3495                 0x1e7c0, 0x1e7c8,
3496                 0x1e840, 0x1e84c,
3497                 0x1ea84, 0x1ea8c,
3498                 0x1eac0, 0x1eac0,
3499                 0x1eae0, 0x1eae0,
3500                 0x1eb00, 0x1eb84,
3501                 0x1ebc0, 0x1ebc8,
3502                 0x1ec40, 0x1ec4c,
3503                 0x1ee84, 0x1ee8c,
3504                 0x1eec0, 0x1eec0,
3505                 0x1eee0, 0x1eee0,
3506                 0x1ef00, 0x1ef84,
3507                 0x1efc0, 0x1efc8,
3508                 0x1f040, 0x1f04c,
3509                 0x1f284, 0x1f28c,
3510                 0x1f2c0, 0x1f2c0,
3511                 0x1f2e0, 0x1f2e0,
3512                 0x1f300, 0x1f384,
3513                 0x1f3c0, 0x1f3c8,
3514                 0x1f440, 0x1f44c,
3515                 0x1f684, 0x1f68c,
3516                 0x1f6c0, 0x1f6c0,
3517                 0x1f6e0, 0x1f6e0,
3518                 0x1f700, 0x1f784,
3519                 0x1f7c0, 0x1f7c8,
3520                 0x1f840, 0x1f84c,
3521                 0x1fa84, 0x1fa8c,
3522                 0x1fac0, 0x1fac0,
3523                 0x1fae0, 0x1fae0,
3524                 0x1fb00, 0x1fb84,
3525                 0x1fbc0, 0x1fbc8,
3526                 0x1fc40, 0x1fc4c,
3527                 0x1fe84, 0x1fe8c,
3528                 0x1fec0, 0x1fec0,
3529                 0x1fee0, 0x1fee0,
3530                 0x1ff00, 0x1ff84,
3531                 0x1ffc0, 0x1ffc8,
3532                 0x20000, 0x2002c,
3533                 0x20100, 0x2013c,
3534                 0x20190, 0x201c8,
3535                 0x20200, 0x20318,
3536                 0x20400, 0x20528,
3537                 0x20540, 0x20614,
3538                 0x21000, 0x21040,
3539                 0x2104c, 0x21060,
3540                 0x210c0, 0x210ec,
3541                 0x21200, 0x21268,
3542                 0x21270, 0x21284,
3543                 0x212fc, 0x21388,
3544                 0x21400, 0x21404,
3545                 0x21500, 0x21518,
3546                 0x2152c, 0x2153c,
3547                 0x21550, 0x21554,
3548                 0x21600, 0x21600,
3549                 0x21608, 0x21628,
3550                 0x21630, 0x2163c,
3551                 0x21700, 0x2171c,
3552                 0x21780, 0x2178c,
3553                 0x21800, 0x21c38,
3554                 0x21c80, 0x21d7c,
3555                 0x21e00, 0x21e04,
3556                 0x22000, 0x2202c,
3557                 0x22100, 0x2213c,
3558                 0x22190, 0x221c8,
3559                 0x22200, 0x22318,
3560                 0x22400, 0x22528,
3561                 0x22540, 0x22614,
3562                 0x23000, 0x23040,
3563                 0x2304c, 0x23060,
3564                 0x230c0, 0x230ec,
3565                 0x23200, 0x23268,
3566                 0x23270, 0x23284,
3567                 0x232fc, 0x23388,
3568                 0x23400, 0x23404,
3569                 0x23500, 0x23518,
3570                 0x2352c, 0x2353c,
3571                 0x23550, 0x23554,
3572                 0x23600, 0x23600,
3573                 0x23608, 0x23628,
3574                 0x23630, 0x2363c,
3575                 0x23700, 0x2371c,
3576                 0x23780, 0x2378c,
3577                 0x23800, 0x23c38,
3578                 0x23c80, 0x23d7c,
3579                 0x23e00, 0x23e04,
3580                 0x24000, 0x2402c,
3581                 0x24100, 0x2413c,
3582                 0x24190, 0x241c8,
3583                 0x24200, 0x24318,
3584                 0x24400, 0x24528,
3585                 0x24540, 0x24614,
3586                 0x25000, 0x25040,
3587                 0x2504c, 0x25060,
3588                 0x250c0, 0x250ec,
3589                 0x25200, 0x25268,
3590                 0x25270, 0x25284,
3591                 0x252fc, 0x25388,
3592                 0x25400, 0x25404,
3593                 0x25500, 0x25518,
3594                 0x2552c, 0x2553c,
3595                 0x25550, 0x25554,
3596                 0x25600, 0x25600,
3597                 0x25608, 0x25628,
3598                 0x25630, 0x2563c,
3599                 0x25700, 0x2571c,
3600                 0x25780, 0x2578c,
3601                 0x25800, 0x25c38,
3602                 0x25c80, 0x25d7c,
3603                 0x25e00, 0x25e04,
3604                 0x26000, 0x2602c,
3605                 0x26100, 0x2613c,
3606                 0x26190, 0x261c8,
3607                 0x26200, 0x26318,
3608                 0x26400, 0x26528,
3609                 0x26540, 0x26614,
3610                 0x27000, 0x27040,
3611                 0x2704c, 0x27060,
3612                 0x270c0, 0x270ec,
3613                 0x27200, 0x27268,
3614                 0x27270, 0x27284,
3615                 0x272fc, 0x27388,
3616                 0x27400, 0x27404,
3617                 0x27500, 0x27518,
3618                 0x2752c, 0x2753c,
3619                 0x27550, 0x27554,
3620                 0x27600, 0x27600,
3621                 0x27608, 0x27628,
3622                 0x27630, 0x2763c,
3623                 0x27700, 0x2771c,
3624                 0x27780, 0x2778c,
3625                 0x27800, 0x27c38,
3626                 0x27c80, 0x27d7c,
3627                 0x27e00, 0x27e04
3628         };
3629         static const unsigned int t5_reg_ranges[] = {
3630                 0x1008, 0x1148,
3631                 0x1180, 0x11b4,
3632                 0x11fc, 0x123c,
3633                 0x1280, 0x173c,
3634                 0x1800, 0x18fc,
3635                 0x3000, 0x3028,
3636                 0x3060, 0x30d8,
3637                 0x30e0, 0x30fc,
3638                 0x3140, 0x357c,
3639                 0x35a8, 0x35cc,
3640                 0x35ec, 0x35ec,
3641                 0x3600, 0x5624,
3642                 0x56cc, 0x575c,
3643                 0x580c, 0x5814,
3644                 0x5890, 0x58bc,
3645                 0x5940, 0x59dc,
3646                 0x59fc, 0x5a18,
3647                 0x5a60, 0x5a9c,
3648                 0x5b94, 0x5bfc,
3649                 0x6000, 0x6040,
3650                 0x6058, 0x614c,
3651                 0x7700, 0x7798,
3652                 0x77c0, 0x78fc,
3653                 0x7b00, 0x7c54,
3654                 0x7d00, 0x7efc,
3655                 0x8dc0, 0x8de0,
3656                 0x8df8, 0x8e84,
3657                 0x8ea0, 0x8f84,
3658                 0x8fc0, 0x90f8,
3659                 0x9400, 0x9470,
3660                 0x9600, 0x96f4,
3661                 0x9800, 0x9808,
3662                 0x9820, 0x983c,
3663                 0x9850, 0x9864,
3664                 0x9c00, 0x9c6c,
3665                 0x9c80, 0x9cec,
3666                 0x9d00, 0x9d6c,
3667                 0x9d80, 0x9dec,
3668                 0x9e00, 0x9e6c,
3669                 0x9e80, 0x9eec,
3670                 0x9f00, 0x9f6c,
3671                 0x9f80, 0xa020,
3672                 0xd004, 0xd03c,
3673                 0xdfc0, 0xdfe0,
3674                 0xe000, 0x11088,
3675                 0x1109c, 0x11110,
3676                 0x11118, 0x1117c,
3677                 0x11190, 0x11204,
3678                 0x19040, 0x1906c,
3679                 0x19078, 0x19080,
3680                 0x1908c, 0x19124,
3681                 0x19150, 0x191b0,
3682                 0x191d0, 0x191e8,
3683                 0x19238, 0x19290,
3684                 0x193f8, 0x19474,
3685                 0x19490, 0x194cc,
3686                 0x194f0, 0x194f8,
3687                 0x19c00, 0x19c60,
3688                 0x19c94, 0x19e10,
3689                 0x19e50, 0x19f34,
3690                 0x19f40, 0x19f50,
3691                 0x19f90, 0x19fe4,
3692                 0x1a000, 0x1a06c,
3693                 0x1a0b0, 0x1a120,
3694                 0x1a128, 0x1a138,
3695                 0x1a190, 0x1a1c4,
3696                 0x1a1fc, 0x1a1fc,
3697                 0x1e008, 0x1e00c,
3698                 0x1e040, 0x1e04c,
3699                 0x1e284, 0x1e290,
3700                 0x1e2c0, 0x1e2c0,
3701                 0x1e2e0, 0x1e2e0,
3702                 0x1e300, 0x1e384,
3703                 0x1e3c0, 0x1e3c8,
3704                 0x1e408, 0x1e40c,
3705                 0x1e440, 0x1e44c,
3706                 0x1e684, 0x1e690,
3707                 0x1e6c0, 0x1e6c0,
3708                 0x1e6e0, 0x1e6e0,
3709                 0x1e700, 0x1e784,
3710                 0x1e7c0, 0x1e7c8,
3711                 0x1e808, 0x1e80c,
3712                 0x1e840, 0x1e84c,
3713                 0x1ea84, 0x1ea90,
3714                 0x1eac0, 0x1eac0,
3715                 0x1eae0, 0x1eae0,
3716                 0x1eb00, 0x1eb84,
3717                 0x1ebc0, 0x1ebc8,
3718                 0x1ec08, 0x1ec0c,
3719                 0x1ec40, 0x1ec4c,
3720                 0x1ee84, 0x1ee90,
3721                 0x1eec0, 0x1eec0,
3722                 0x1eee0, 0x1eee0,
3723                 0x1ef00, 0x1ef84,
3724                 0x1efc0, 0x1efc8,
3725                 0x1f008, 0x1f00c,
3726                 0x1f040, 0x1f04c,
3727                 0x1f284, 0x1f290,
3728                 0x1f2c0, 0x1f2c0,
3729                 0x1f2e0, 0x1f2e0,
3730                 0x1f300, 0x1f384,
3731                 0x1f3c0, 0x1f3c8,
3732                 0x1f408, 0x1f40c,
3733                 0x1f440, 0x1f44c,
3734                 0x1f684, 0x1f690,
3735                 0x1f6c0, 0x1f6c0,
3736                 0x1f6e0, 0x1f6e0,
3737                 0x1f700, 0x1f784,
3738                 0x1f7c0, 0x1f7c8,
3739                 0x1f808, 0x1f80c,
3740                 0x1f840, 0x1f84c,
3741                 0x1fa84, 0x1fa90,
3742                 0x1fac0, 0x1fac0,
3743                 0x1fae0, 0x1fae0,
3744                 0x1fb00, 0x1fb84,
3745                 0x1fbc0, 0x1fbc8,
3746                 0x1fc08, 0x1fc0c,
3747                 0x1fc40, 0x1fc4c,
3748                 0x1fe84, 0x1fe90,
3749                 0x1fec0, 0x1fec0,
3750                 0x1fee0, 0x1fee0,
3751                 0x1ff00, 0x1ff84,
3752                 0x1ffc0, 0x1ffc8,
3753                 0x30000, 0x30030,
3754                 0x30100, 0x30144,
3755                 0x30190, 0x301d0,
3756                 0x30200, 0x30318,
3757                 0x30400, 0x3052c,
3758                 0x30540, 0x3061c,
3759                 0x30800, 0x30834,
3760                 0x308c0, 0x30908,
3761                 0x30910, 0x309ac,
3762                 0x30a00, 0x30a2c,
3763                 0x30a44, 0x30a50,
3764                 0x30a74, 0x30c24,
3765                 0x30d00, 0x30d00,
3766                 0x30d08, 0x30d14,
3767                 0x30d1c, 0x30d20,
3768                 0x30d3c, 0x30d50,
3769                 0x31200, 0x3120c,
3770                 0x31220, 0x31220,
3771                 0x31240, 0x31240,
3772                 0x31600, 0x3160c,
3773                 0x31a00, 0x31a1c,
3774                 0x31e00, 0x31e20,
3775                 0x31e38, 0x31e3c,
3776                 0x31e80, 0x31e80,
3777                 0x31e88, 0x31ea8,
3778                 0x31eb0, 0x31eb4,
3779                 0x31ec8, 0x31ed4,
3780                 0x31fb8, 0x32004,
3781                 0x32200, 0x32200,
3782                 0x32208, 0x32240,
3783                 0x32248, 0x32280,
3784                 0x32288, 0x322c0,
3785                 0x322c8, 0x322fc,
3786                 0x32600, 0x32630,
3787                 0x32a00, 0x32abc,
3788                 0x32b00, 0x32b70,
3789                 0x33000, 0x33048,
3790                 0x33060, 0x3309c,
3791                 0x330f0, 0x33148,
3792                 0x33160, 0x3319c,
3793                 0x331f0, 0x332e4,
3794                 0x332f8, 0x333e4,
3795                 0x333f8, 0x33448,
3796                 0x33460, 0x3349c,
3797                 0x334f0, 0x33548,
3798                 0x33560, 0x3359c,
3799                 0x335f0, 0x336e4,
3800                 0x336f8, 0x337e4,
3801                 0x337f8, 0x337fc,
3802                 0x33814, 0x33814,
3803                 0x3382c, 0x3382c,
3804                 0x33880, 0x3388c,
3805                 0x338e8, 0x338ec,
3806                 0x33900, 0x33948,
3807                 0x33960, 0x3399c,
3808                 0x339f0, 0x33ae4,
3809                 0x33af8, 0x33b10,
3810                 0x33b28, 0x33b28,
3811                 0x33b3c, 0x33b50,
3812                 0x33bf0, 0x33c10,
3813                 0x33c28, 0x33c28,
3814                 0x33c3c, 0x33c50,
3815                 0x33cf0, 0x33cfc,
3816                 0x34000, 0x34030,
3817                 0x34100, 0x34144,
3818                 0x34190, 0x341d0,
3819                 0x34200, 0x34318,
3820                 0x34400, 0x3452c,
3821                 0x34540, 0x3461c,
3822                 0x34800, 0x34834,
3823                 0x348c0, 0x34908,
3824                 0x34910, 0x349ac,
3825                 0x34a00, 0x34a2c,
3826                 0x34a44, 0x34a50,
3827                 0x34a74, 0x34c24,
3828                 0x34d00, 0x34d00,
3829                 0x34d08, 0x34d14,
3830                 0x34d1c, 0x34d20,
3831                 0x34d3c, 0x34d50,
3832                 0x35200, 0x3520c,
3833                 0x35220, 0x35220,
3834                 0x35240, 0x35240,
3835                 0x35600, 0x3560c,
3836                 0x35a00, 0x35a1c,
3837                 0x35e00, 0x35e20,
3838                 0x35e38, 0x35e3c,
3839                 0x35e80, 0x35e80,
3840                 0x35e88, 0x35ea8,
3841                 0x35eb0, 0x35eb4,
3842                 0x35ec8, 0x35ed4,
3843                 0x35fb8, 0x36004,
3844                 0x36200, 0x36200,
3845                 0x36208, 0x36240,
3846                 0x36248, 0x36280,
3847                 0x36288, 0x362c0,
3848                 0x362c8, 0x362fc,
3849                 0x36600, 0x36630,
3850                 0x36a00, 0x36abc,
3851                 0x36b00, 0x36b70,
3852                 0x37000, 0x37048,
3853                 0x37060, 0x3709c,
3854                 0x370f0, 0x37148,
3855                 0x37160, 0x3719c,
3856                 0x371f0, 0x372e4,
3857                 0x372f8, 0x373e4,
3858                 0x373f8, 0x37448,
3859                 0x37460, 0x3749c,
3860                 0x374f0, 0x37548,
3861                 0x37560, 0x3759c,
3862                 0x375f0, 0x376e4,
3863                 0x376f8, 0x377e4,
3864                 0x377f8, 0x377fc,
3865                 0x37814, 0x37814,
3866                 0x3782c, 0x3782c,
3867                 0x37880, 0x3788c,
3868                 0x378e8, 0x378ec,
3869                 0x37900, 0x37948,
3870                 0x37960, 0x3799c,
3871                 0x379f0, 0x37ae4,
3872                 0x37af8, 0x37b10,
3873                 0x37b28, 0x37b28,
3874                 0x37b3c, 0x37b50,
3875                 0x37bf0, 0x37c10,
3876                 0x37c28, 0x37c28,
3877                 0x37c3c, 0x37c50,
3878                 0x37cf0, 0x37cfc,
3879                 0x38000, 0x38030,
3880                 0x38100, 0x38144,
3881                 0x38190, 0x381d0,
3882                 0x38200, 0x38318,
3883                 0x38400, 0x3852c,
3884                 0x38540, 0x3861c,
3885                 0x38800, 0x38834,
3886                 0x388c0, 0x38908,
3887                 0x38910, 0x389ac,
3888                 0x38a00, 0x38a2c,
3889                 0x38a44, 0x38a50,
3890                 0x38a74, 0x38c24,
3891                 0x38d00, 0x38d00,
3892                 0x38d08, 0x38d14,
3893                 0x38d1c, 0x38d20,
3894                 0x38d3c, 0x38d50,
3895                 0x39200, 0x3920c,
3896                 0x39220, 0x39220,
3897                 0x39240, 0x39240,
3898                 0x39600, 0x3960c,
3899                 0x39a00, 0x39a1c,
3900                 0x39e00, 0x39e20,
3901                 0x39e38, 0x39e3c,
3902                 0x39e80, 0x39e80,
3903                 0x39e88, 0x39ea8,
3904                 0x39eb0, 0x39eb4,
3905                 0x39ec8, 0x39ed4,
3906                 0x39fb8, 0x3a004,
3907                 0x3a200, 0x3a200,
3908                 0x3a208, 0x3a240,
3909                 0x3a248, 0x3a280,
3910                 0x3a288, 0x3a2c0,
3911                 0x3a2c8, 0x3a2fc,
3912                 0x3a600, 0x3a630,
3913                 0x3aa00, 0x3aabc,
3914                 0x3ab00, 0x3ab70,
3915                 0x3b000, 0x3b048,
3916                 0x3b060, 0x3b09c,
3917                 0x3b0f0, 0x3b148,
3918                 0x3b160, 0x3b19c,
3919                 0x3b1f0, 0x3b2e4,
3920                 0x3b2f8, 0x3b3e4,
3921                 0x3b3f8, 0x3b448,
3922                 0x3b460, 0x3b49c,
3923                 0x3b4f0, 0x3b548,
3924                 0x3b560, 0x3b59c,
3925                 0x3b5f0, 0x3b6e4,
3926                 0x3b6f8, 0x3b7e4,
3927                 0x3b7f8, 0x3b7fc,
3928                 0x3b814, 0x3b814,
3929                 0x3b82c, 0x3b82c,
3930                 0x3b880, 0x3b88c,
3931                 0x3b8e8, 0x3b8ec,
3932                 0x3b900, 0x3b948,
3933                 0x3b960, 0x3b99c,
3934                 0x3b9f0, 0x3bae4,
3935                 0x3baf8, 0x3bb10,
3936                 0x3bb28, 0x3bb28,
3937                 0x3bb3c, 0x3bb50,
3938                 0x3bbf0, 0x3bc10,
3939                 0x3bc28, 0x3bc28,
3940                 0x3bc3c, 0x3bc50,
3941                 0x3bcf0, 0x3bcfc,
3942                 0x3c000, 0x3c030,
3943                 0x3c100, 0x3c144,
3944                 0x3c190, 0x3c1d0,
3945                 0x3c200, 0x3c318,
3946                 0x3c400, 0x3c52c,
3947                 0x3c540, 0x3c61c,
3948                 0x3c800, 0x3c834,
3949                 0x3c8c0, 0x3c908,
3950                 0x3c910, 0x3c9ac,
3951                 0x3ca00, 0x3ca2c,
3952                 0x3ca44, 0x3ca50,
3953                 0x3ca74, 0x3cc24,
3954                 0x3cd00, 0x3cd00,
3955                 0x3cd08, 0x3cd14,
3956                 0x3cd1c, 0x3cd20,
3957                 0x3cd3c, 0x3cd50,
3958                 0x3d200, 0x3d20c,
3959                 0x3d220, 0x3d220,
3960                 0x3d240, 0x3d240,
3961                 0x3d600, 0x3d60c,
3962                 0x3da00, 0x3da1c,
3963                 0x3de00, 0x3de20,
3964                 0x3de38, 0x3de3c,
3965                 0x3de80, 0x3de80,
3966                 0x3de88, 0x3dea8,
3967                 0x3deb0, 0x3deb4,
3968                 0x3dec8, 0x3ded4,
3969                 0x3dfb8, 0x3e004,
3970                 0x3e200, 0x3e200,
3971                 0x3e208, 0x3e240,
3972                 0x3e248, 0x3e280,
3973                 0x3e288, 0x3e2c0,
3974                 0x3e2c8, 0x3e2fc,
3975                 0x3e600, 0x3e630,
3976                 0x3ea00, 0x3eabc,
3977                 0x3eb00, 0x3eb70,
3978                 0x3f000, 0x3f048,
3979                 0x3f060, 0x3f09c,
3980                 0x3f0f0, 0x3f148,
3981                 0x3f160, 0x3f19c,
3982                 0x3f1f0, 0x3f2e4,
3983                 0x3f2f8, 0x3f3e4,
3984                 0x3f3f8, 0x3f448,
3985                 0x3f460, 0x3f49c,
3986                 0x3f4f0, 0x3f548,
3987                 0x3f560, 0x3f59c,
3988                 0x3f5f0, 0x3f6e4,
3989                 0x3f6f8, 0x3f7e4,
3990                 0x3f7f8, 0x3f7fc,
3991                 0x3f814, 0x3f814,
3992                 0x3f82c, 0x3f82c,
3993                 0x3f880, 0x3f88c,
3994                 0x3f8e8, 0x3f8ec,
3995                 0x3f900, 0x3f948,
3996                 0x3f960, 0x3f99c,
3997                 0x3f9f0, 0x3fae4,
3998                 0x3faf8, 0x3fb10,
3999                 0x3fb28, 0x3fb28,
4000                 0x3fb3c, 0x3fb50,
4001                 0x3fbf0, 0x3fc10,
4002                 0x3fc28, 0x3fc28,
4003                 0x3fc3c, 0x3fc50,
4004                 0x3fcf0, 0x3fcfc,
4005                 0x40000, 0x4000c,
4006                 0x40040, 0x40068,
4007                 0x4007c, 0x40144,
4008                 0x40180, 0x4018c,
4009                 0x40200, 0x40298,
4010                 0x402ac, 0x4033c,
4011                 0x403f8, 0x403fc,
4012                 0x41304, 0x413c4,
4013                 0x41400, 0x4141c,
4014                 0x41480, 0x414d0,
4015                 0x44000, 0x44078,
4016                 0x440c0, 0x44278,
4017                 0x442c0, 0x44478,
4018                 0x444c0, 0x44678,
4019                 0x446c0, 0x44878,
4020                 0x448c0, 0x449fc,
4021                 0x45000, 0x45068,
4022                 0x45080, 0x45084,
4023                 0x450a0, 0x450b0,
4024                 0x45200, 0x45268,
4025                 0x45280, 0x45284,
4026                 0x452a0, 0x452b0,
4027                 0x460c0, 0x460e4,
4028                 0x47000, 0x4708c,
4029                 0x47200, 0x47250,
4030                 0x47400, 0x47420,
4031                 0x47600, 0x47618,
4032                 0x47800, 0x47814,
4033                 0x48000, 0x4800c,
4034                 0x48040, 0x48068,
4035                 0x4807c, 0x48144,
4036                 0x48180, 0x4818c,
4037                 0x48200, 0x48298,
4038                 0x482ac, 0x4833c,
4039                 0x483f8, 0x483fc,
4040                 0x49304, 0x493c4,
4041                 0x49400, 0x4941c,
4042                 0x49480, 0x494d0,
4043                 0x4c000, 0x4c078,
4044                 0x4c0c0, 0x4c278,
4045                 0x4c2c0, 0x4c478,
4046                 0x4c4c0, 0x4c678,
4047                 0x4c6c0, 0x4c878,
4048                 0x4c8c0, 0x4c9fc,
4049                 0x4d000, 0x4d068,
4050                 0x4d080, 0x4d084,
4051                 0x4d0a0, 0x4d0b0,
4052                 0x4d200, 0x4d268,
4053                 0x4d280, 0x4d284,
4054                 0x4d2a0, 0x4d2b0,
4055                 0x4e0c0, 0x4e0e4,
4056                 0x4f000, 0x4f08c,
4057                 0x4f200, 0x4f250,
4058                 0x4f400, 0x4f420,
4059                 0x4f600, 0x4f618,
4060                 0x4f800, 0x4f814,
4061                 0x50000, 0x500cc,
4062                 0x50400, 0x50400,
4063                 0x50800, 0x508cc,
4064                 0x50c00, 0x50c00,
4065                 0x51000, 0x5101c,
4066                 0x51300, 0x51308,
4067         };
4068
4069         if (is_t4(sc)) {
4070                 reg_ranges = &t4_reg_ranges[0];
4071                 n = nitems(t4_reg_ranges);
4072         } else {
4073                 reg_ranges = &t5_reg_ranges[0];
4074                 n = nitems(t5_reg_ranges);
4075         }
4076
4077         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4078         for (i = 0; i < n; i += 2)
4079                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4080 }
4081
4082 static void
4083 cxgbe_tick(void *arg)
4084 {
4085         struct port_info *pi = arg;
4086         struct ifnet *ifp = pi->ifp;
4087         struct sge_txq *txq;
4088         int i, drops;
4089         struct port_stats *s = &pi->stats;
4090
4091         PORT_LOCK(pi);
4092         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4093                 PORT_UNLOCK(pi);
4094                 return; /* without scheduling another callout */
4095         }
4096
4097         t4_get_port_stats(pi->adapter, pi->tx_chan, s);
4098
4099         ifp->if_opackets = s->tx_frames - s->tx_pause;
4100         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4101         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4102         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4103         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4104         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4105         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4106             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4107             s->rx_trunc3;
4108
4109         drops = s->tx_drop;
4110         for_each_txq(pi, i, txq)
4111                 drops += txq->br->br_drops;
4112         ifp->if_snd.ifq_drops = drops;
4113
4114         ifp->if_oerrors = s->tx_error_frames;
4115         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4116             s->rx_fcs_err + s->rx_len_err;
4117
4118         callout_schedule(&pi->tick, hz);
4119         PORT_UNLOCK(pi);
4120 }
4121
4122 static void
4123 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4124 {
4125         struct ifnet *vlan;
4126
4127         if (arg != ifp || ifp->if_type != IFT_ETHER)
4128                 return;
4129
4130         vlan = VLAN_DEVAT(ifp, vid);
4131         VLAN_SETCOOKIE(vlan, ifp);
4132 }
4133
4134 static int
4135 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4136 {
4137
4138 #ifdef INVARIANTS
4139         panic("%s: opcode 0x%02x on iq %p with payload %p",
4140             __func__, rss->opcode, iq, m);
4141 #else
4142         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4143             __func__, rss->opcode, iq, m);
4144         m_freem(m);
4145 #endif
4146         return (EDOOFUS);
4147 }
4148
4149 int
4150 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4151 {
4152         uintptr_t *loc, new;
4153
4154         if (opcode >= nitems(sc->cpl_handler))
4155                 return (EINVAL);
4156
4157         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4158         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4159         atomic_store_rel_ptr(loc, new);
4160
4161         return (0);
4162 }
4163
4164 static int
4165 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4166 {
4167
4168 #ifdef INVARIANTS
4169         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4170 #else
4171         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4172             __func__, iq, ctrl);
4173 #endif
4174         return (EDOOFUS);
4175 }
4176
4177 int
4178 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4179 {
4180         uintptr_t *loc, new;
4181
4182         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4183         loc = (uintptr_t *) &sc->an_handler;
4184         atomic_store_rel_ptr(loc, new);
4185
4186         return (0);
4187 }
4188
4189 static int
4190 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4191 {
4192         const struct cpl_fw6_msg *cpl =
4193             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4194
4195 #ifdef INVARIANTS
4196         panic("%s: fw_msg type %d", __func__, cpl->type);
4197 #else
4198         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4199 #endif
4200         return (EDOOFUS);
4201 }
4202
4203 int
4204 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4205 {
4206         uintptr_t *loc, new;
4207
4208         if (type >= nitems(sc->fw_msg_handler))
4209                 return (EINVAL);
4210
4211         /*
4212          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4213          * handler dispatch table.  Reject any attempt to install a handler for
4214          * this subtype.
4215          */
4216         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4217                 return (EINVAL);
4218
4219         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4220         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4221         atomic_store_rel_ptr(loc, new);
4222
4223         return (0);
4224 }
4225
4226 static int
4227 t4_sysctls(struct adapter *sc)
4228 {
4229         struct sysctl_ctx_list *ctx;
4230         struct sysctl_oid *oid;
4231         struct sysctl_oid_list *children, *c0;
4232         static char *caps[] = {
4233                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4234                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4235                     "\6HASHFILTER\7ETHOFLD",
4236                 "\20\1TOE",                             /* caps[2] toecaps */
4237                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4238                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4239                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4240                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4241                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4242                     "\4PO_INITIAOR\5PO_TARGET"
4243         };
4244         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4245
4246         ctx = device_get_sysctl_ctx(sc->dev);
4247
4248         /*
4249          * dev.t4nex.X.
4250          */
4251         oid = device_get_sysctl_tree(sc->dev);
4252         c0 = children = SYSCTL_CHILDREN(oid);
4253
4254         sc->sc_do_rxcopy = 1;
4255         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4256             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4257
4258         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4259             sc->params.nports, "# of ports");
4260
4261         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4262             NULL, chip_rev(sc), "chip hardware revision");
4263
4264         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4265             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
4266
4267         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4268             CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
4269
4270         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4271             sc->cfcsum, "config file checksum");
4272
4273         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4274             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4275             sysctl_bitfield, "A", "available doorbells");
4276
4277         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4278             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4279             sysctl_bitfield, "A", "available link capabilities");
4280
4281         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4282             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4283             sysctl_bitfield, "A", "available NIC capabilities");
4284
4285         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4286             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4287             sysctl_bitfield, "A", "available TCP offload capabilities");
4288
4289         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4290             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4291             sysctl_bitfield, "A", "available RDMA capabilities");
4292
4293         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4294             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4295             sysctl_bitfield, "A", "available iSCSI capabilities");
4296
4297         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4298             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4299             sysctl_bitfield, "A", "available FCoE capabilities");
4300
4301         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4302             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4303
4304         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4305             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4306             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4307             "interrupt holdoff timer values (us)");
4308
4309         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4310             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4311             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4312             "interrupt holdoff packet counter values");
4313
4314         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4315             NULL, sc->tids.nftids, "number of filters");
4316
4317         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4318             CTLFLAG_RD, sc, 0, sysctl_temperature, "A",
4319             "chip temperature (in Celsius)");
4320
4321         t4_sge_sysctls(sc, ctx, children);
4322
4323         sc->lro_timeout = 100;
4324         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4325             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4326
4327 #ifdef SBUF_DRAIN
4328         /*
4329          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4330          */
4331         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4332             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4333             "logs and miscellaneous information");
4334         children = SYSCTL_CHILDREN(oid);
4335
4336         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4337             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4338             sysctl_cctrl, "A", "congestion control");
4339
4340         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4341             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4342             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4343
4344         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4345             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4346             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4347
4348         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4349             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4350             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4351
4352         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4353             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4354             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4355
4356         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4357             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4358             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4359
4360         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4361             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4362             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4363
4364         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4365             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4366             sysctl_cim_la, "A", "CIM logic analyzer");
4367
4368         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4369             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4370             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4371
4372         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4373             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4374             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4375
4376         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4377             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4378             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4379
4380         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4381             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4382             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4383
4384         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4385             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4386             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4387
4388         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4389             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4390             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4391
4392         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4393             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4394             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4395
4396         if (is_t5(sc)) {
4397                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4398                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4399                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4400
4401                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4402                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4403                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4404         }
4405
4406         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4407             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4408             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4409
4410         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4411             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4412             sysctl_cim_qcfg, "A", "CIM queue configuration");
4413
4414         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4415             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4416             sysctl_cpl_stats, "A", "CPL statistics");
4417
4418         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4419             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4420             sysctl_ddp_stats, "A", "DDP statistics");
4421
4422         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4423             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4424             sysctl_devlog, "A", "firmware's device log");
4425
4426         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4427             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4428             sysctl_fcoe_stats, "A", "FCoE statistics");
4429
4430         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4431             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4432             sysctl_hw_sched, "A", "hardware scheduler ");
4433
4434         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4435             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4436             sysctl_l2t, "A", "hardware L2 table");
4437
4438         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4439             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4440             sysctl_lb_stats, "A", "loopback statistics");
4441
4442         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4443             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4444             sysctl_meminfo, "A", "memory regions");
4445
4446         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4447             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4448             sysctl_mps_tcam, "A", "MPS TCAM entries");
4449
4450         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4451             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4452             sysctl_path_mtus, "A", "path MTUs");
4453
4454         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4455             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4456             sysctl_pm_stats, "A", "PM statistics");
4457
4458         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4459             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4460             sysctl_rdma_stats, "A", "RDMA statistics");
4461
4462         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4463             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4464             sysctl_tcp_stats, "A", "TCP statistics");
4465
4466         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4467             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4468             sysctl_tids, "A", "TID information");
4469
4470         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4471             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4472             sysctl_tp_err_stats, "A", "TP error statistics");
4473
4474         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4475             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4476             sysctl_tp_la, "A", "TP logic analyzer");
4477
4478         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4479             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4480             sysctl_tx_rate, "A", "Tx rate");
4481
4482         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4483             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4484             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4485
4486         if (is_t5(sc)) {
4487                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4488                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4489                     sysctl_wcwr_stats, "A", "write combined work requests");
4490         }
4491 #endif
4492
4493 #ifdef TCP_OFFLOAD
4494         if (is_offload(sc)) {
4495                 /*
4496                  * dev.t4nex.X.toe.
4497                  */
4498                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4499                     NULL, "TOE parameters");
4500                 children = SYSCTL_CHILDREN(oid);
4501
4502                 sc->tt.sndbuf = 256 * 1024;
4503                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4504                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4505
4506                 sc->tt.ddp = 0;
4507                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4508                     &sc->tt.ddp, 0, "DDP allowed");
4509
4510                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4511                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4512                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4513
4514                 sc->tt.ddp_thres =
4515                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4516                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4517                     &sc->tt.ddp_thres, 0, "DDP threshold");
4518
4519                 sc->tt.rx_coalesce = 1;
4520                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4521                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4522         }
4523 #endif
4524
4525
4526         return (0);
4527 }
4528
4529 static int
4530 cxgbe_sysctls(struct port_info *pi)
4531 {
4532         struct sysctl_ctx_list *ctx;
4533         struct sysctl_oid *oid;
4534         struct sysctl_oid_list *children;
4535         struct adapter *sc = pi->adapter;
4536
4537         ctx = device_get_sysctl_ctx(pi->dev);
4538
4539         /*
4540          * dev.cxgbe.X.
4541          */
4542         oid = device_get_sysctl_tree(pi->dev);
4543         children = SYSCTL_CHILDREN(oid);
4544
4545         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4546            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4547         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4548                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4549                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4550                     "PHY temperature (in Celsius)");
4551                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4552                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4553                     "PHY firmware version");
4554         }
4555         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4556             &pi->nrxq, 0, "# of rx queues");
4557         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4558             &pi->ntxq, 0, "# of tx queues");
4559         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4560             &pi->first_rxq, 0, "index of first rx queue");
4561         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4562             &pi->first_txq, 0, "index of first tx queue");
4563         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4564             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4565             "Reserve queue 0 for non-flowid packets");
4566
4567 #ifdef TCP_OFFLOAD
4568         if (is_offload(sc)) {
4569                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4570                     &pi->nofldrxq, 0,
4571                     "# of rx queues for offloaded TCP connections");
4572                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4573                     &pi->nofldtxq, 0,
4574                     "# of tx queues for offloaded TCP connections");
4575                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4576                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4577                     "index of first TOE rx queue");
4578                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4579                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4580                     "index of first TOE tx queue");
4581         }
4582 #endif
4583
4584         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4585             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4586             "holdoff timer index");
4587         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4588             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4589             "holdoff packet counter index");
4590
4591         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4592             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4593             "rx queue size");
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4595             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4596             "tx queue size");
4597
4598         /*
4599          * dev.cxgbe.X.stats.
4600          */
4601         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4602             NULL, "port statistics");
4603         children = SYSCTL_CHILDREN(oid);
4604
4605 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4606         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4607             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4608             sysctl_handle_t4_reg64, "QU", desc)
4609
4610         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4611             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4612         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4613             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4614         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4615             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4616         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4617             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4618         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4619             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4620         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4621             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4622         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4623             "# of tx frames in this range",
4624             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4625         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4626             "# of tx frames in this range",
4627             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4628         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4629             "# of tx frames in this range",
4630             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4631         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4632             "# of tx frames in this range",
4633             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4634         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4635             "# of tx frames in this range",
4636             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4637         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4638             "# of tx frames in this range",
4639             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4640         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4641             "# of tx frames in this range",
4642             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4643         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4644             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4645         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4646             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4647         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4648             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4649         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4650             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4651         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4652             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4653         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4654             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4655         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4656             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4657         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4658             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4659         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4660             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4661         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4662             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4663
4664         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4665             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4666         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4667             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4668         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4669             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4670         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4671             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4672         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4673             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4674         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4675             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4676         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4677             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4678         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4679             "# of frames received with bad FCS",
4680             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4681         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4682             "# of frames received with length error",
4683             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4684         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4685             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4686         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4687             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4688         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4689             "# of rx frames in this range",
4690             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4691         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4692             "# of rx frames in this range",
4693             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4694         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4695             "# of rx frames in this range",
4696             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4697         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4698             "# of rx frames in this range",
4699             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4700         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4701             "# of rx frames in this range",
4702             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4703         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4704             "# of rx frames in this range",
4705             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4706         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4707             "# of rx frames in this range",
4708             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4709         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4710             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4711         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4712             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4713         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4714             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4715         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4716             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4717         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4718             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4719         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4720             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4721         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4722             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4723         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4724             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4725         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4726             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4727
4728 #undef SYSCTL_ADD_T4_REG64
4729
4730 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4731         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4732             &pi->stats.name, desc)
4733
4734         /* We get these from port_stats and they may be stale by upto 1s */
4735         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4736             "# drops due to buffer-group 0 overflows");
4737         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4738             "# drops due to buffer-group 1 overflows");
4739         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4740             "# drops due to buffer-group 2 overflows");
4741         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4742             "# drops due to buffer-group 3 overflows");
4743         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4744             "# of buffer-group 0 truncated packets");
4745         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4746             "# of buffer-group 1 truncated packets");
4747         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4748             "# of buffer-group 2 truncated packets");
4749         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4750             "# of buffer-group 3 truncated packets");
4751
4752 #undef SYSCTL_ADD_T4_PORTSTAT
4753
4754         return (0);
4755 }
4756
4757 static int
4758 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4759 {
4760         int rc, *i;
4761         struct sbuf sb;
4762
4763         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4764         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4765                 sbuf_printf(&sb, "%d ", *i);
4766         sbuf_trim(&sb);
4767         sbuf_finish(&sb);
4768         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4769         sbuf_delete(&sb);
4770         return (rc);
4771 }
4772
4773 static int
4774 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4775 {
4776         int rc;
4777         struct sbuf *sb;
4778
4779         rc = sysctl_wire_old_buffer(req, 0);
4780         if (rc != 0)
4781                 return(rc);
4782
4783         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4784         if (sb == NULL)
4785                 return (ENOMEM);
4786
4787         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4788         rc = sbuf_finish(sb);
4789         sbuf_delete(sb);
4790
4791         return (rc);
4792 }
4793
4794 static int
4795 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4796 {
4797         struct port_info *pi = arg1;
4798         int op = arg2;
4799         struct adapter *sc = pi->adapter;
4800         u_int v;
4801         int rc;
4802
4803         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4804         if (rc)
4805                 return (rc);
4806         /* XXX: magic numbers */
4807         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4808             &v);
4809         end_synchronized_op(sc, 0);
4810         if (rc)
4811                 return (rc);
4812         if (op == 0)
4813                 v /= 256;
4814
4815         rc = sysctl_handle_int(oidp, &v, 0, req);
4816         return (rc);
4817 }
4818
4819 static int
4820 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
4821 {
4822         struct port_info *pi = arg1;
4823         int rc, val;
4824
4825         val = pi->rsrv_noflowq;
4826         rc = sysctl_handle_int(oidp, &val, 0, req);
4827         if (rc != 0 || req->newptr == NULL)
4828                 return (rc);
4829
4830         if ((val >= 1) && (pi->ntxq > 1))
4831                 pi->rsrv_noflowq = 1;
4832         else
4833                 pi->rsrv_noflowq = 0;
4834
4835         return (rc);
4836 }
4837
4838 static int
4839 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
4840 {
4841         struct port_info *pi = arg1;
4842         struct adapter *sc = pi->adapter;
4843         int idx, rc, i;
4844         struct sge_rxq *rxq;
4845 #ifdef TCP_OFFLOAD
4846         struct sge_ofld_rxq *ofld_rxq;
4847 #endif
4848         uint8_t v;
4849
4850         idx = pi->tmr_idx;
4851
4852         rc = sysctl_handle_int(oidp, &idx, 0, req);
4853         if (rc != 0 || req->newptr == NULL)
4854                 return (rc);
4855
4856         if (idx < 0 || idx >= SGE_NTIMERS)
4857                 return (EINVAL);
4858
4859         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4860             "t4tmr");
4861         if (rc)
4862                 return (rc);
4863
4864         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
4865         for_each_rxq(pi, i, rxq) {
4866 #ifdef atomic_store_rel_8
4867                 atomic_store_rel_8(&rxq->iq.intr_params, v);
4868 #else
4869                 rxq->iq.intr_params = v;
4870 #endif
4871         }
4872 #ifdef TCP_OFFLOAD
4873         for_each_ofld_rxq(pi, i, ofld_rxq) {
4874 #ifdef atomic_store_rel_8
4875                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
4876 #else
4877                 ofld_rxq->iq.intr_params = v;
4878 #endif
4879         }
4880 #endif
4881         pi->tmr_idx = idx;
4882
4883         end_synchronized_op(sc, LOCK_HELD);
4884         return (0);
4885 }
4886
4887 static int
4888 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
4889 {
4890         struct port_info *pi = arg1;
4891         struct adapter *sc = pi->adapter;
4892         int idx, rc;
4893
4894         idx = pi->pktc_idx;
4895
4896         rc = sysctl_handle_int(oidp, &idx, 0, req);
4897         if (rc != 0 || req->newptr == NULL)
4898                 return (rc);
4899
4900         if (idx < -1 || idx >= SGE_NCOUNTERS)
4901                 return (EINVAL);
4902
4903         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4904             "t4pktc");
4905         if (rc)
4906                 return (rc);
4907
4908         if (pi->flags & PORT_INIT_DONE)
4909                 rc = EBUSY; /* cannot be changed once the queues are created */
4910         else
4911                 pi->pktc_idx = idx;
4912
4913         end_synchronized_op(sc, LOCK_HELD);
4914         return (rc);
4915 }
4916
4917 static int
4918 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
4919 {
4920         struct port_info *pi = arg1;
4921         struct adapter *sc = pi->adapter;
4922         int qsize, rc;
4923
4924         qsize = pi->qsize_rxq;
4925
4926         rc = sysctl_handle_int(oidp, &qsize, 0, req);
4927         if (rc != 0 || req->newptr == NULL)
4928                 return (rc);
4929
4930         if (qsize < 128 || (qsize & 7))
4931                 return (EINVAL);
4932
4933         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4934             "t4rxqs");
4935         if (rc)
4936                 return (rc);
4937
4938         if (pi->flags & PORT_INIT_DONE)
4939                 rc = EBUSY; /* cannot be changed once the queues are created */
4940         else
4941                 pi->qsize_rxq = qsize;
4942
4943         end_synchronized_op(sc, LOCK_HELD);
4944         return (rc);
4945 }
4946
4947 static int
4948 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
4949 {
4950         struct port_info *pi = arg1;
4951         struct adapter *sc = pi->adapter;
4952         int qsize, rc;
4953
4954         qsize = pi->qsize_txq;
4955
4956         rc = sysctl_handle_int(oidp, &qsize, 0, req);
4957         if (rc != 0 || req->newptr == NULL)
4958                 return (rc);
4959
4960         /* bufring size must be powerof2 */
4961         if (qsize < 128 || !powerof2(qsize))
4962                 return (EINVAL);
4963
4964         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4965             "t4txqs");
4966         if (rc)
4967                 return (rc);
4968
4969         if (pi->flags & PORT_INIT_DONE)
4970                 rc = EBUSY; /* cannot be changed once the queues are created */
4971         else
4972                 pi->qsize_txq = qsize;
4973
4974         end_synchronized_op(sc, LOCK_HELD);
4975         return (rc);
4976 }
4977
4978 static int
4979 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
4980 {
4981         struct adapter *sc = arg1;
4982         int reg = arg2;
4983         uint64_t val;
4984
4985         val = t4_read_reg64(sc, reg);
4986
4987         return (sysctl_handle_64(oidp, &val, 0, req));
4988 }
4989
4990 static int
4991 sysctl_temperature(SYSCTL_HANDLER_ARGS)
4992 {
4993         struct adapter *sc = arg1;
4994         int rc, t;
4995         uint32_t param, val;
4996
4997         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
4998         if (rc)
4999                 return (rc);
5000         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5001             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5002             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5003         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5004         end_synchronized_op(sc, 0);
5005         if (rc)
5006                 return (rc);
5007
5008         /* unknown is returned as 0 but we display -1 in that case */
5009         t = val == 0 ? -1 : val;
5010
5011         rc = sysctl_handle_int(oidp, &t, 0, req);
5012         return (rc);
5013 }
5014
5015 #ifdef SBUF_DRAIN
5016 static int
5017 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5018 {
5019         struct adapter *sc = arg1;
5020         struct sbuf *sb;
5021         int rc, i;
5022         uint16_t incr[NMTUS][NCCTRL_WIN];
5023         static const char *dec_fac[] = {
5024                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5025                 "0.9375"
5026         };
5027
5028         rc = sysctl_wire_old_buffer(req, 0);
5029         if (rc != 0)
5030                 return (rc);
5031
5032         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5033         if (sb == NULL)
5034                 return (ENOMEM);
5035
5036         t4_read_cong_tbl(sc, incr);
5037
5038         for (i = 0; i < NCCTRL_WIN; ++i) {
5039                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5040                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5041                     incr[5][i], incr[6][i], incr[7][i]);
5042                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5043                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5044                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5045                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5046         }
5047
5048         rc = sbuf_finish(sb);
5049         sbuf_delete(sb);
5050
5051         return (rc);
5052 }
5053
5054 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5055         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5056         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5057         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5058 };
5059
5060 static int
5061 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5062 {
5063         struct adapter *sc = arg1;
5064         struct sbuf *sb;
5065         int rc, i, n, qid = arg2;
5066         uint32_t *buf, *p;
5067         char *qtype;
5068         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5069
5070         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5071             ("%s: bad qid %d\n", __func__, qid));
5072
5073         if (qid < CIM_NUM_IBQ) {
5074                 /* inbound queue */
5075                 qtype = "IBQ";
5076                 n = 4 * CIM_IBQ_SIZE;
5077                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5078                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5079         } else {
5080                 /* outbound queue */
5081                 qtype = "OBQ";
5082                 qid -= CIM_NUM_IBQ;
5083                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5084                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5085                 rc = t4_read_cim_obq(sc, qid, buf, n);
5086         }
5087
5088         if (rc < 0) {
5089                 rc = -rc;
5090                 goto done;
5091         }
5092         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5093
5094         rc = sysctl_wire_old_buffer(req, 0);
5095         if (rc != 0)
5096                 goto done;
5097
5098         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5099         if (sb == NULL) {
5100                 rc = ENOMEM;
5101                 goto done;
5102         }
5103
5104         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5105         for (i = 0, p = buf; i < n; i += 16, p += 4)
5106                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5107                     p[2], p[3]);
5108
5109         rc = sbuf_finish(sb);
5110         sbuf_delete(sb);
5111 done:
5112         free(buf, M_CXGBE);
5113         return (rc);
5114 }
5115
5116 static int
5117 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5118 {
5119         struct adapter *sc = arg1;
5120         u_int cfg;
5121         struct sbuf *sb;
5122         uint32_t *buf, *p;
5123         int rc;
5124
5125         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5126         if (rc != 0)
5127                 return (rc);
5128
5129         rc = sysctl_wire_old_buffer(req, 0);
5130         if (rc != 0)
5131                 return (rc);
5132
5133         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5134         if (sb == NULL)
5135                 return (ENOMEM);
5136
5137         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5138             M_ZERO | M_WAITOK);
5139
5140         rc = -t4_cim_read_la(sc, buf, NULL);
5141         if (rc != 0)
5142                 goto done;
5143
5144         sbuf_printf(sb, "Status   Data      PC%s",
5145             cfg & F_UPDBGLACAPTPCONLY ? "" :
5146             "     LS0Stat  LS0Addr             LS0Data");
5147
5148         KASSERT((sc->params.cim_la_size & 7) == 0,
5149             ("%s: p will walk off the end of buf", __func__));
5150
5151         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5152                 if (cfg & F_UPDBGLACAPTPCONLY) {
5153                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5154                             p[6], p[7]);
5155                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5156                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5157                             p[4] & 0xff, p[5] >> 8);
5158                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5159                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5160                             p[1] & 0xf, p[2] >> 4);
5161                 } else {
5162                         sbuf_printf(sb,
5163                             "\n  %02x   %x%07x %x%07x %08x %08x "
5164                             "%08x%08x%08x%08x",
5165                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5166                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5167                             p[6], p[7]);
5168                 }
5169         }
5170
5171         rc = sbuf_finish(sb);
5172         sbuf_delete(sb);
5173 done:
5174         free(buf, M_CXGBE);
5175         return (rc);
5176 }
5177
5178 static int
5179 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5180 {
5181         struct adapter *sc = arg1;
5182         u_int i;
5183         struct sbuf *sb;
5184         uint32_t *buf, *p;
5185         int rc;
5186
5187         rc = sysctl_wire_old_buffer(req, 0);
5188         if (rc != 0)
5189                 return (rc);
5190
5191         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5192         if (sb == NULL)
5193                 return (ENOMEM);
5194
5195         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5196             M_ZERO | M_WAITOK);
5197
5198         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5199         p = buf;
5200
5201         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5202                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5203                     p[1], p[0]);
5204         }
5205
5206         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5207         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5208                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5209                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5210                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5211                     (p[1] >> 2) | ((p[2] & 3) << 30),
5212                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5213                     p[0] & 1);
5214         }
5215
5216         rc = sbuf_finish(sb);
5217         sbuf_delete(sb);
5218         free(buf, M_CXGBE);
5219         return (rc);
5220 }
5221
5222 static int
5223 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5224 {
5225         struct adapter *sc = arg1;
5226         u_int i;
5227         struct sbuf *sb;
5228         uint32_t *buf, *p;
5229         int rc;
5230
5231         rc = sysctl_wire_old_buffer(req, 0);
5232         if (rc != 0)
5233                 return (rc);
5234
5235         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5236         if (sb == NULL)
5237                 return (ENOMEM);
5238
5239         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5240             M_ZERO | M_WAITOK);
5241
5242         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5243         p = buf;
5244
5245         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5246         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5247                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5248                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5249                     p[4], p[3], p[2], p[1], p[0]);
5250         }
5251
5252         sbuf_printf(sb, "\n\nCntl ID               Data");
5253         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5254                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5255                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5256         }
5257
5258         rc = sbuf_finish(sb);
5259         sbuf_delete(sb);
5260         free(buf, M_CXGBE);
5261         return (rc);
5262 }
5263
5264 static int
5265 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5266 {
5267         struct adapter *sc = arg1;
5268         struct sbuf *sb;
5269         int rc, i;
5270         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5271         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5272         uint16_t thres[CIM_NUM_IBQ];
5273         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5274         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5275         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5276
5277         if (is_t4(sc)) {
5278                 cim_num_obq = CIM_NUM_OBQ;
5279                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5280                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5281         } else {
5282                 cim_num_obq = CIM_NUM_OBQ_T5;
5283                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5284                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5285         }
5286         nq = CIM_NUM_IBQ + cim_num_obq;
5287
5288         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5289         if (rc == 0)
5290                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5291         if (rc != 0)
5292                 return (rc);
5293
5294         t4_read_cimq_cfg(sc, base, size, thres);
5295
5296         rc = sysctl_wire_old_buffer(req, 0);
5297         if (rc != 0)
5298                 return (rc);
5299
5300         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5301         if (sb == NULL)
5302                 return (ENOMEM);
5303
5304         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5305
5306         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5307                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5308                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5309                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5310                     G_QUEREMFLITS(p[2]) * 16);
5311         for ( ; i < nq; i++, p += 4, wr += 2)
5312                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5313                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5314                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5315                     G_QUEREMFLITS(p[2]) * 16);
5316
5317         rc = sbuf_finish(sb);
5318         sbuf_delete(sb);
5319
5320         return (rc);
5321 }
5322
5323 static int
5324 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5325 {
5326         struct adapter *sc = arg1;
5327         struct sbuf *sb;
5328         int rc;
5329         struct tp_cpl_stats stats;
5330
5331         rc = sysctl_wire_old_buffer(req, 0);
5332         if (rc != 0)
5333                 return (rc);
5334
5335         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5336         if (sb == NULL)
5337                 return (ENOMEM);
5338
5339         t4_tp_get_cpl_stats(sc, &stats);
5340
5341         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5342             "channel 3\n");
5343         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5344                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5345         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5346                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5347
5348         rc = sbuf_finish(sb);
5349         sbuf_delete(sb);
5350
5351         return (rc);
5352 }
5353
5354 static int
5355 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5356 {
5357         struct adapter *sc = arg1;
5358         struct sbuf *sb;
5359         int rc;
5360         struct tp_usm_stats stats;
5361
5362         rc = sysctl_wire_old_buffer(req, 0);
5363         if (rc != 0)
5364                 return(rc);
5365
5366         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5367         if (sb == NULL)
5368                 return (ENOMEM);
5369
5370         t4_get_usm_stats(sc, &stats);
5371
5372         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5373         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5374         sbuf_printf(sb, "Drops:  %u", stats.drops);
5375
5376         rc = sbuf_finish(sb);
5377         sbuf_delete(sb);
5378
5379         return (rc);
5380 }
5381
5382 const char *devlog_level_strings[] = {
5383         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5384         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5385         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5386         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5387         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5388         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5389 };
5390
5391 const char *devlog_facility_strings[] = {
5392         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5393         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5394         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5395         [FW_DEVLOG_FACILITY_RES]        = "RES",
5396         [FW_DEVLOG_FACILITY_HW]         = "HW",
5397         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5398         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5399         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5400         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5401         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5402         [FW_DEVLOG_FACILITY_VI]         = "VI",
5403         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5404         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5405         [FW_DEVLOG_FACILITY_TM]         = "TM",
5406         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5407         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5408         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5409         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5410         [FW_DEVLOG_FACILITY_RI]         = "RI",
5411         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5412         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5413         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5414         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5415 };
5416
5417 static int
5418 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5419 {
5420         struct adapter *sc = arg1;
5421         struct devlog_params *dparams = &sc->params.devlog;
5422         struct fw_devlog_e *buf, *e;
5423         int i, j, rc, nentries, first = 0, m;
5424         struct sbuf *sb;
5425         uint64_t ftstamp = UINT64_MAX;
5426
5427         if (dparams->start == 0) {
5428                 dparams->memtype = FW_MEMTYPE_EDC0;
5429                 dparams->start = 0x84000;
5430                 dparams->size = 32768;
5431         }
5432
5433         nentries = dparams->size / sizeof(struct fw_devlog_e);
5434
5435         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5436         if (buf == NULL)
5437                 return (ENOMEM);
5438
5439         m = fwmtype_to_hwmtype(dparams->memtype);
5440         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5441         if (rc != 0)
5442                 goto done;
5443
5444         for (i = 0; i < nentries; i++) {
5445                 e = &buf[i];
5446
5447                 if (e->timestamp == 0)
5448                         break;  /* end */
5449
5450                 e->timestamp = be64toh(e->timestamp);
5451                 e->seqno = be32toh(e->seqno);
5452                 for (j = 0; j < 8; j++)
5453                         e->params[j] = be32toh(e->params[j]);
5454
5455                 if (e->timestamp < ftstamp) {
5456                         ftstamp = e->timestamp;
5457                         first = i;
5458                 }
5459         }
5460
5461         if (buf[first].timestamp == 0)
5462                 goto done;      /* nothing in the log */
5463
5464         rc = sysctl_wire_old_buffer(req, 0);
5465         if (rc != 0)
5466                 goto done;
5467
5468         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5469         if (sb == NULL) {
5470                 rc = ENOMEM;
5471                 goto done;
5472         }
5473         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5474             "Seq#", "Tstamp", "Level", "Facility", "Message");
5475
5476         i = first;
5477         do {
5478                 e = &buf[i];
5479                 if (e->timestamp == 0)
5480                         break;  /* end */
5481
5482                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5483                     e->seqno, e->timestamp,
5484                     (e->level < nitems(devlog_level_strings) ?
5485                         devlog_level_strings[e->level] : "UNKNOWN"),
5486                     (e->facility < nitems(devlog_facility_strings) ?
5487                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5488                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5489                     e->params[2], e->params[3], e->params[4],
5490                     e->params[5], e->params[6], e->params[7]);
5491
5492                 if (++i == nentries)
5493                         i = 0;
5494         } while (i != first);
5495
5496         rc = sbuf_finish(sb);
5497         sbuf_delete(sb);
5498 done:
5499         free(buf, M_CXGBE);
5500         return (rc);
5501 }
5502
5503 static int
5504 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5505 {
5506         struct adapter *sc = arg1;
5507         struct sbuf *sb;
5508         int rc;
5509         struct tp_fcoe_stats stats[4];
5510
5511         rc = sysctl_wire_old_buffer(req, 0);
5512         if (rc != 0)
5513                 return (rc);
5514
5515         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5516         if (sb == NULL)
5517                 return (ENOMEM);
5518
5519         t4_get_fcoe_stats(sc, 0, &stats[0]);
5520         t4_get_fcoe_stats(sc, 1, &stats[1]);
5521         t4_get_fcoe_stats(sc, 2, &stats[2]);
5522         t4_get_fcoe_stats(sc, 3, &stats[3]);
5523
5524         sbuf_printf(sb, "                   channel 0        channel 1        "
5525             "channel 2        channel 3\n");
5526         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5527             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5528             stats[3].octetsDDP);
5529         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5530             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5531         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5532             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5533             stats[3].framesDrop);
5534
5535         rc = sbuf_finish(sb);
5536         sbuf_delete(sb);
5537
5538         return (rc);
5539 }
5540
5541 static int
5542 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5543 {
5544         struct adapter *sc = arg1;
5545         struct sbuf *sb;
5546         int rc, i;
5547         unsigned int map, kbps, ipg, mode;
5548         unsigned int pace_tab[NTX_SCHED];
5549
5550         rc = sysctl_wire_old_buffer(req, 0);
5551         if (rc != 0)
5552                 return (rc);
5553
5554         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5555         if (sb == NULL)
5556                 return (ENOMEM);
5557
5558         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5559         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5560         t4_read_pace_tbl(sc, pace_tab);
5561
5562         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5563             "Class IPG (0.1 ns)   Flow IPG (us)");
5564
5565         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5566                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5567                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5568                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5569                 if (kbps)
5570                         sbuf_printf(sb, "%9u     ", kbps);
5571                 else
5572                         sbuf_printf(sb, " disabled     ");
5573
5574                 if (ipg)
5575                         sbuf_printf(sb, "%13u        ", ipg);
5576                 else
5577                         sbuf_printf(sb, "     disabled        ");
5578
5579                 if (pace_tab[i])
5580                         sbuf_printf(sb, "%10u", pace_tab[i]);
5581                 else
5582                         sbuf_printf(sb, "  disabled");
5583         }
5584
5585         rc = sbuf_finish(sb);
5586         sbuf_delete(sb);
5587
5588         return (rc);
5589 }
5590
5591 static int
5592 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5593 {
5594         struct adapter *sc = arg1;
5595         struct sbuf *sb;
5596         int rc, i, j;
5597         uint64_t *p0, *p1;
5598         struct lb_port_stats s[2];
5599         static const char *stat_name[] = {
5600                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5601                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5602                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5603                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5604                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5605                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5606                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5607         };
5608
5609         rc = sysctl_wire_old_buffer(req, 0);
5610         if (rc != 0)
5611                 return (rc);
5612
5613         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5614         if (sb == NULL)
5615                 return (ENOMEM);
5616
5617         memset(s, 0, sizeof(s));
5618
5619         for (i = 0; i < 4; i += 2) {
5620                 t4_get_lb_stats(sc, i, &s[0]);
5621                 t4_get_lb_stats(sc, i + 1, &s[1]);
5622
5623                 p0 = &s[0].octets;
5624                 p1 = &s[1].octets;
5625                 sbuf_printf(sb, "%s                       Loopback %u"
5626                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5627
5628                 for (j = 0; j < nitems(stat_name); j++)
5629                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5630                                    *p0++, *p1++);
5631         }
5632
5633         rc = sbuf_finish(sb);
5634         sbuf_delete(sb);
5635
5636         return (rc);
5637 }
5638
5639 static int
5640 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5641 {
5642         int rc = 0;
5643         struct port_info *pi = arg1;
5644         struct sbuf *sb;
5645         static const char *linkdnreasons[] = {
5646                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5647                 "PHY overheated", "unknown", "rx los", "reserved7"
5648         };
5649
5650         rc = sysctl_wire_old_buffer(req, 0);
5651         if (rc != 0)
5652                 return(rc);
5653         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5654         if (sb == NULL)
5655                 return (ENOMEM);
5656
5657         if (pi->linkdnrc < 0)
5658                 sbuf_printf(sb, "n/a");
5659         else if (pi->linkdnrc < nitems(linkdnreasons))
5660                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5661         else
5662                 sbuf_printf(sb, "%d", pi->linkdnrc);
5663
5664         rc = sbuf_finish(sb);
5665         sbuf_delete(sb);
5666
5667         return (rc);
5668 }
5669
5670 struct mem_desc {
5671         unsigned int base;
5672         unsigned int limit;
5673         unsigned int idx;
5674 };
5675
5676 static int
5677 mem_desc_cmp(const void *a, const void *b)
5678 {
5679         return ((const struct mem_desc *)a)->base -
5680                ((const struct mem_desc *)b)->base;
5681 }
5682
5683 static void
5684 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5685     unsigned int to)
5686 {
5687         unsigned int size;
5688
5689         size = to - from + 1;
5690         if (size == 0)
5691                 return;
5692
5693         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5694         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5695 }
5696
5697 static int
5698 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5699 {
5700         struct adapter *sc = arg1;
5701         struct sbuf *sb;
5702         int rc, i, n;
5703         uint32_t lo, hi, used, alloc;
5704         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5705         static const char *region[] = {
5706                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5707                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5708                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5709                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5710                 "RQUDP region:", "PBL region:", "TXPBL region:",
5711                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5712                 "On-chip queues:"
5713         };
5714         struct mem_desc avail[4];
5715         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5716         struct mem_desc *md = mem;
5717
5718         rc = sysctl_wire_old_buffer(req, 0);
5719         if (rc != 0)
5720                 return (rc);
5721
5722         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5723         if (sb == NULL)
5724                 return (ENOMEM);
5725
5726         for (i = 0; i < nitems(mem); i++) {
5727                 mem[i].limit = 0;
5728                 mem[i].idx = i;
5729         }
5730
5731         /* Find and sort the populated memory ranges */
5732         i = 0;
5733         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5734         if (lo & F_EDRAM0_ENABLE) {
5735                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5736                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5737                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5738                 avail[i].idx = 0;
5739                 i++;
5740         }
5741         if (lo & F_EDRAM1_ENABLE) {
5742                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5743                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
5744                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5745                 avail[i].idx = 1;
5746                 i++;
5747         }
5748         if (lo & F_EXT_MEM_ENABLE) {
5749                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5750                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5751                 avail[i].limit = avail[i].base +
5752                     (G_EXT_MEM_SIZE(hi) << 20);
5753                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
5754                 i++;
5755         }
5756         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
5757                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
5758                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
5759                 avail[i].limit = avail[i].base +
5760                     (G_EXT_MEM1_SIZE(hi) << 20);
5761                 avail[i].idx = 4;
5762                 i++;
5763         }
5764         if (!i)                                    /* no memory available */
5765                 return 0;
5766         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
5767
5768         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
5769         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
5770         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
5771         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
5772         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
5773         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
5774         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
5775         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
5776         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
5777
5778         /* the next few have explicit upper bounds */
5779         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
5780         md->limit = md->base - 1 +
5781                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
5782                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
5783         md++;
5784
5785         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
5786         md->limit = md->base - 1 +
5787                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
5788                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
5789         md++;
5790
5791         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
5792                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
5793                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
5794                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
5795         } else {
5796                 md->base = 0;
5797                 md->idx = nitems(region);  /* hide it */
5798         }
5799         md++;
5800
5801 #define ulp_region(reg) \
5802         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
5803         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
5804
5805         ulp_region(RX_ISCSI);
5806         ulp_region(RX_TDDP);
5807         ulp_region(TX_TPT);
5808         ulp_region(RX_STAG);
5809         ulp_region(RX_RQ);
5810         ulp_region(RX_RQUDP);
5811         ulp_region(RX_PBL);
5812         ulp_region(TX_PBL);
5813 #undef ulp_region
5814
5815         md->base = 0;
5816         md->idx = nitems(region);
5817         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
5818                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
5819                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
5820                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
5821         }
5822         md++;
5823
5824         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
5825         md->limit = md->base + sc->tids.ntids - 1;
5826         md++;
5827         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
5828         md->limit = md->base + sc->tids.ntids - 1;
5829         md++;
5830
5831         md->base = sc->vres.ocq.start;
5832         if (sc->vres.ocq.size)
5833                 md->limit = md->base + sc->vres.ocq.size - 1;
5834         else
5835                 md->idx = nitems(region);  /* hide it */
5836         md++;
5837
5838         /* add any address-space holes, there can be up to 3 */
5839         for (n = 0; n < i - 1; n++)
5840                 if (avail[n].limit < avail[n + 1].base)
5841                         (md++)->base = avail[n].limit;
5842         if (avail[n].limit)
5843                 (md++)->base = avail[n].limit;
5844
5845         n = md - mem;
5846         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
5847
5848         for (lo = 0; lo < i; lo++)
5849                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
5850                                 avail[lo].limit - 1);
5851
5852         sbuf_printf(sb, "\n");
5853         for (i = 0; i < n; i++) {
5854                 if (mem[i].idx >= nitems(region))
5855                         continue;                        /* skip holes */
5856                 if (!mem[i].limit)
5857                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
5858                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
5859                                 mem[i].limit);
5860         }
5861
5862         sbuf_printf(sb, "\n");
5863         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
5864         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
5865         mem_region_show(sb, "uP RAM:", lo, hi);
5866
5867         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
5868         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
5869         mem_region_show(sb, "uP Extmem2:", lo, hi);
5870
5871         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
5872         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
5873                    G_PMRXMAXPAGE(lo),
5874                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
5875                    (lo & F_PMRXNUMCHN) ? 2 : 1);
5876
5877         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
5878         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
5879         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
5880                    G_PMTXMAXPAGE(lo),
5881                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
5882                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
5883         sbuf_printf(sb, "%u p-structs\n",
5884                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
5885
5886         for (i = 0; i < 4; i++) {
5887                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
5888                 if (is_t4(sc)) {
5889                         used = G_USED(lo);
5890                         alloc = G_ALLOC(lo);
5891                 } else {
5892                         used = G_T5_USED(lo);
5893                         alloc = G_T5_ALLOC(lo);
5894                 }
5895                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
5896                            i, used, alloc);
5897         }
5898         for (i = 0; i < 4; i++) {
5899                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
5900                 if (is_t4(sc)) {
5901                         used = G_USED(lo);
5902                         alloc = G_ALLOC(lo);
5903                 } else {
5904                         used = G_T5_USED(lo);
5905                         alloc = G_T5_ALLOC(lo);
5906                 }
5907                 sbuf_printf(sb,
5908                            "\nLoopback %d using %u pages out of %u allocated",
5909                            i, used, alloc);
5910         }
5911
5912         rc = sbuf_finish(sb);
5913         sbuf_delete(sb);
5914
5915         return (rc);
5916 }
5917
5918 static inline void
5919 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
5920 {
5921         *mask = x | y;
5922         y = htobe64(y);
5923         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
5924 }
5925
5926 static int
5927 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
5928 {
5929         struct adapter *sc = arg1;
5930         struct sbuf *sb;
5931         int rc, i, n;
5932
5933         rc = sysctl_wire_old_buffer(req, 0);
5934         if (rc != 0)
5935                 return (rc);
5936
5937         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5938         if (sb == NULL)
5939                 return (ENOMEM);
5940
5941         sbuf_printf(sb,
5942             "Idx  Ethernet address     Mask     Vld Ports PF"
5943             "  VF              Replication             P0 P1 P2 P3  ML");
5944         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
5945             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
5946         for (i = 0; i < n; i++) {
5947                 uint64_t tcamx, tcamy, mask;
5948                 uint32_t cls_lo, cls_hi;
5949                 uint8_t addr[ETHER_ADDR_LEN];
5950
5951                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
5952                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
5953                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
5954                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
5955
5956                 if (tcamx & tcamy)
5957                         continue;
5958
5959                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
5960                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
5961                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
5962                            addr[3], addr[4], addr[5], (uintmax_t)mask,
5963                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
5964                            G_PORTMAP(cls_hi), G_PF(cls_lo),
5965                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
5966
5967                 if (cls_lo & F_REPLICATE) {
5968                         struct fw_ldst_cmd ldst_cmd;
5969
5970                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
5971                         ldst_cmd.op_to_addrspace =
5972                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
5973                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
5974                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
5975                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
5976                         ldst_cmd.u.mps.fid_ctl =
5977                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
5978                                 V_FW_LDST_CMD_CTL(i));
5979
5980                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
5981                             "t4mps");
5982                         if (rc)
5983                                 break;
5984                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
5985                             sizeof(ldst_cmd), &ldst_cmd);
5986                         end_synchronized_op(sc, 0);
5987
5988                         if (rc != 0) {
5989                                 sbuf_printf(sb,
5990                                     " ------------ error %3u ------------", rc);
5991                                 rc = 0;
5992                         } else {
5993                                 sbuf_printf(sb, " %08x %08x %08x %08x",
5994                                     be32toh(ldst_cmd.u.mps.rplc127_96),
5995                                     be32toh(ldst_cmd.u.mps.rplc95_64),
5996                                     be32toh(ldst_cmd.u.mps.rplc63_32),
5997                                     be32toh(ldst_cmd.u.mps.rplc31_0));
5998                         }
5999                 } else
6000                         sbuf_printf(sb, "%36s", "");
6001
6002                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6003                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6004                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6005         }
6006
6007         if (rc)
6008                 (void) sbuf_finish(sb);
6009         else
6010                 rc = sbuf_finish(sb);
6011         sbuf_delete(sb);
6012
6013         return (rc);
6014 }
6015
6016 static int
6017 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6018 {
6019         struct adapter *sc = arg1;
6020         struct sbuf *sb;
6021         int rc;
6022         uint16_t mtus[NMTUS];
6023
6024         rc = sysctl_wire_old_buffer(req, 0);
6025         if (rc != 0)
6026                 return (rc);
6027
6028         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6029         if (sb == NULL)
6030                 return (ENOMEM);
6031
6032         t4_read_mtu_tbl(sc, mtus, NULL);
6033
6034         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6035             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6036             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6037             mtus[14], mtus[15]);
6038
6039         rc = sbuf_finish(sb);
6040         sbuf_delete(sb);
6041
6042         return (rc);
6043 }
6044
6045 static int
6046 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6047 {
6048         struct adapter *sc = arg1;
6049         struct sbuf *sb;
6050         int rc, i;
6051         uint32_t cnt[PM_NSTATS];
6052         uint64_t cyc[PM_NSTATS];
6053         static const char *rx_stats[] = {
6054                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6055         };
6056         static const char *tx_stats[] = {
6057                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6058         };
6059
6060         rc = sysctl_wire_old_buffer(req, 0);
6061         if (rc != 0)
6062                 return (rc);
6063
6064         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6065         if (sb == NULL)
6066                 return (ENOMEM);
6067
6068         t4_pmtx_get_stats(sc, cnt, cyc);
6069         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6070         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6071                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6072                     cyc[i]);
6073
6074         t4_pmrx_get_stats(sc, cnt, cyc);
6075         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6076         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6077                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6078                     cyc[i]);
6079
6080         rc = sbuf_finish(sb);
6081         sbuf_delete(sb);
6082
6083         return (rc);
6084 }
6085
6086 static int
6087 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6088 {
6089         struct adapter *sc = arg1;
6090         struct sbuf *sb;
6091         int rc;
6092         struct tp_rdma_stats stats;
6093
6094         rc = sysctl_wire_old_buffer(req, 0);
6095         if (rc != 0)
6096                 return (rc);
6097
6098         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6099         if (sb == NULL)
6100                 return (ENOMEM);
6101
6102         t4_tp_get_rdma_stats(sc, &stats);
6103         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6104         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6105
6106         rc = sbuf_finish(sb);
6107         sbuf_delete(sb);
6108
6109         return (rc);
6110 }
6111
6112 static int
6113 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6114 {
6115         struct adapter *sc = arg1;
6116         struct sbuf *sb;
6117         int rc;
6118         struct tp_tcp_stats v4, v6;
6119
6120         rc = sysctl_wire_old_buffer(req, 0);
6121         if (rc != 0)
6122                 return (rc);
6123
6124         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6125         if (sb == NULL)
6126                 return (ENOMEM);
6127
6128         t4_tp_get_tcp_stats(sc, &v4, &v6);
6129         sbuf_printf(sb,
6130             "                                IP                 IPv6\n");
6131         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6132             v4.tcpOutRsts, v6.tcpOutRsts);
6133         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6134             v4.tcpInSegs, v6.tcpInSegs);
6135         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6136             v4.tcpOutSegs, v6.tcpOutSegs);
6137         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6138             v4.tcpRetransSegs, v6.tcpRetransSegs);
6139
6140         rc = sbuf_finish(sb);
6141         sbuf_delete(sb);
6142
6143         return (rc);
6144 }
6145
6146 static int
6147 sysctl_tids(SYSCTL_HANDLER_ARGS)
6148 {
6149         struct adapter *sc = arg1;
6150         struct sbuf *sb;
6151         int rc;
6152         struct tid_info *t = &sc->tids;
6153
6154         rc = sysctl_wire_old_buffer(req, 0);
6155         if (rc != 0)
6156                 return (rc);
6157
6158         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6159         if (sb == NULL)
6160                 return (ENOMEM);
6161
6162         if (t->natids) {
6163                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6164                     t->atids_in_use);
6165         }
6166
6167         if (t->ntids) {
6168                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6169                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6170
6171                         if (b) {
6172                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6173                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6174                                     t->ntids - 1);
6175                         } else {
6176                                 sbuf_printf(sb, "TID range: %u-%u",
6177                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6178                                     t->ntids - 1);
6179                         }
6180                 } else
6181                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6182                 sbuf_printf(sb, ", in use: %u\n",
6183                     atomic_load_acq_int(&t->tids_in_use));
6184         }
6185
6186         if (t->nstids) {
6187                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6188                     t->stid_base + t->nstids - 1, t->stids_in_use);
6189         }
6190
6191         if (t->nftids) {
6192                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6193                     t->ftid_base + t->nftids - 1);
6194         }
6195
6196         if (t->netids) {
6197                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6198                     t->etid_base + t->netids - 1);
6199         }
6200
6201         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6202             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6203             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6204
6205         rc = sbuf_finish(sb);
6206         sbuf_delete(sb);
6207
6208         return (rc);
6209 }
6210
6211 static int
6212 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6213 {
6214         struct adapter *sc = arg1;
6215         struct sbuf *sb;
6216         int rc;
6217         struct tp_err_stats stats;
6218
6219         rc = sysctl_wire_old_buffer(req, 0);
6220         if (rc != 0)
6221                 return (rc);
6222
6223         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6224         if (sb == NULL)
6225                 return (ENOMEM);
6226
6227         t4_tp_get_err_stats(sc, &stats);
6228
6229         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6230                       "channel 3\n");
6231         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6232             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6233             stats.macInErrs[3]);
6234         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6235             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6236             stats.hdrInErrs[3]);
6237         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6238             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6239             stats.tcpInErrs[3]);
6240         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6241             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6242             stats.tcp6InErrs[3]);
6243         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6244             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6245             stats.tnlCongDrops[3]);
6246         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6247             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6248             stats.tnlTxDrops[3]);
6249         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6250             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6251             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6252         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6253             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6254             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6255         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6256             stats.ofldNoNeigh, stats.ofldCongDefer);
6257
6258         rc = sbuf_finish(sb);
6259         sbuf_delete(sb);
6260
6261         return (rc);
6262 }
6263
6264 struct field_desc {
6265         const char *name;
6266         u_int start;
6267         u_int width;
6268 };
6269
6270 static void
6271 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6272 {
6273         char buf[32];
6274         int line_size = 0;
6275
6276         while (f->name) {
6277                 uint64_t mask = (1ULL << f->width) - 1;
6278                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6279                     ((uintmax_t)v >> f->start) & mask);
6280
6281                 if (line_size + len >= 79) {
6282                         line_size = 8;
6283                         sbuf_printf(sb, "\n        ");
6284                 }
6285                 sbuf_printf(sb, "%s ", buf);
6286                 line_size += len + 1;
6287                 f++;
6288         }
6289         sbuf_printf(sb, "\n");
6290 }
6291
6292 static struct field_desc tp_la0[] = {
6293         { "RcfOpCodeOut", 60, 4 },
6294         { "State", 56, 4 },
6295         { "WcfState", 52, 4 },
6296         { "RcfOpcSrcOut", 50, 2 },
6297         { "CRxError", 49, 1 },
6298         { "ERxError", 48, 1 },
6299         { "SanityFailed", 47, 1 },
6300         { "SpuriousMsg", 46, 1 },
6301         { "FlushInputMsg", 45, 1 },
6302         { "FlushInputCpl", 44, 1 },
6303         { "RssUpBit", 43, 1 },
6304         { "RssFilterHit", 42, 1 },
6305         { "Tid", 32, 10 },
6306         { "InitTcb", 31, 1 },
6307         { "LineNumber", 24, 7 },
6308         { "Emsg", 23, 1 },
6309         { "EdataOut", 22, 1 },
6310         { "Cmsg", 21, 1 },
6311         { "CdataOut", 20, 1 },
6312         { "EreadPdu", 19, 1 },
6313         { "CreadPdu", 18, 1 },
6314         { "TunnelPkt", 17, 1 },
6315         { "RcfPeerFin", 16, 1 },
6316         { "RcfReasonOut", 12, 4 },
6317         { "TxCchannel", 10, 2 },
6318         { "RcfTxChannel", 8, 2 },
6319         { "RxEchannel", 6, 2 },
6320         { "RcfRxChannel", 5, 1 },
6321         { "RcfDataOutSrdy", 4, 1 },
6322         { "RxDvld", 3, 1 },
6323         { "RxOoDvld", 2, 1 },
6324         { "RxCongestion", 1, 1 },
6325         { "TxCongestion", 0, 1 },
6326         { NULL }
6327 };
6328
6329 static struct field_desc tp_la1[] = {
6330         { "CplCmdIn", 56, 8 },
6331         { "CplCmdOut", 48, 8 },
6332         { "ESynOut", 47, 1 },
6333         { "EAckOut", 46, 1 },
6334         { "EFinOut", 45, 1 },
6335         { "ERstOut", 44, 1 },
6336         { "SynIn", 43, 1 },
6337         { "AckIn", 42, 1 },
6338         { "FinIn", 41, 1 },
6339         { "RstIn", 40, 1 },
6340         { "DataIn", 39, 1 },
6341         { "DataInVld", 38, 1 },
6342         { "PadIn", 37, 1 },
6343         { "RxBufEmpty", 36, 1 },
6344         { "RxDdp", 35, 1 },
6345         { "RxFbCongestion", 34, 1 },
6346         { "TxFbCongestion", 33, 1 },
6347         { "TxPktSumSrdy", 32, 1 },
6348         { "RcfUlpType", 28, 4 },
6349         { "Eread", 27, 1 },
6350         { "Ebypass", 26, 1 },
6351         { "Esave", 25, 1 },
6352         { "Static0", 24, 1 },
6353         { "Cread", 23, 1 },
6354         { "Cbypass", 22, 1 },
6355         { "Csave", 21, 1 },
6356         { "CPktOut", 20, 1 },
6357         { "RxPagePoolFull", 18, 2 },
6358         { "RxLpbkPkt", 17, 1 },
6359         { "TxLpbkPkt", 16, 1 },
6360         { "RxVfValid", 15, 1 },
6361         { "SynLearned", 14, 1 },
6362         { "SetDelEntry", 13, 1 },
6363         { "SetInvEntry", 12, 1 },
6364         { "CpcmdDvld", 11, 1 },
6365         { "CpcmdSave", 10, 1 },
6366         { "RxPstructsFull", 8, 2 },
6367         { "EpcmdDvld", 7, 1 },
6368         { "EpcmdFlush", 6, 1 },
6369         { "EpcmdTrimPrefix", 5, 1 },
6370         { "EpcmdTrimPostfix", 4, 1 },
6371         { "ERssIp4Pkt", 3, 1 },
6372         { "ERssIp6Pkt", 2, 1 },
6373         { "ERssTcpUdpPkt", 1, 1 },
6374         { "ERssFceFipPkt", 0, 1 },
6375         { NULL }
6376 };
6377
6378 static struct field_desc tp_la2[] = {
6379         { "CplCmdIn", 56, 8 },
6380         { "MpsVfVld", 55, 1 },
6381         { "MpsPf", 52, 3 },
6382         { "MpsVf", 44, 8 },
6383         { "SynIn", 43, 1 },
6384         { "AckIn", 42, 1 },
6385         { "FinIn", 41, 1 },
6386         { "RstIn", 40, 1 },
6387         { "DataIn", 39, 1 },
6388         { "DataInVld", 38, 1 },
6389         { "PadIn", 37, 1 },
6390         { "RxBufEmpty", 36, 1 },
6391         { "RxDdp", 35, 1 },
6392         { "RxFbCongestion", 34, 1 },
6393         { "TxFbCongestion", 33, 1 },
6394         { "TxPktSumSrdy", 32, 1 },
6395         { "RcfUlpType", 28, 4 },
6396         { "Eread", 27, 1 },
6397         { "Ebypass", 26, 1 },
6398         { "Esave", 25, 1 },
6399         { "Static0", 24, 1 },
6400         { "Cread", 23, 1 },
6401         { "Cbypass", 22, 1 },
6402         { "Csave", 21, 1 },
6403         { "CPktOut", 20, 1 },
6404         { "RxPagePoolFull", 18, 2 },
6405         { "RxLpbkPkt", 17, 1 },
6406         { "TxLpbkPkt", 16, 1 },
6407         { "RxVfValid", 15, 1 },
6408         { "SynLearned", 14, 1 },
6409         { "SetDelEntry", 13, 1 },
6410         { "SetInvEntry", 12, 1 },
6411         { "CpcmdDvld", 11, 1 },
6412         { "CpcmdSave", 10, 1 },
6413         { "RxPstructsFull", 8, 2 },
6414         { "EpcmdDvld", 7, 1 },
6415         { "EpcmdFlush", 6, 1 },
6416         { "EpcmdTrimPrefix", 5, 1 },
6417         { "EpcmdTrimPostfix", 4, 1 },
6418         { "ERssIp4Pkt", 3, 1 },
6419         { "ERssIp6Pkt", 2, 1 },
6420         { "ERssTcpUdpPkt", 1, 1 },
6421         { "ERssFceFipPkt", 0, 1 },
6422         { NULL }
6423 };
6424
6425 static void
6426 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6427 {
6428
6429         field_desc_show(sb, *p, tp_la0);
6430 }
6431
6432 static void
6433 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6434 {
6435
6436         if (idx)
6437                 sbuf_printf(sb, "\n");
6438         field_desc_show(sb, p[0], tp_la0);
6439         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6440                 field_desc_show(sb, p[1], tp_la0);
6441 }
6442
6443 static void
6444 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6445 {
6446
6447         if (idx)
6448                 sbuf_printf(sb, "\n");
6449         field_desc_show(sb, p[0], tp_la0);
6450         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6451                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6452 }
6453
6454 static int
6455 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6456 {
6457         struct adapter *sc = arg1;
6458         struct sbuf *sb;
6459         uint64_t *buf, *p;
6460         int rc;
6461         u_int i, inc;
6462         void (*show_func)(struct sbuf *, uint64_t *, int);
6463
6464         rc = sysctl_wire_old_buffer(req, 0);
6465         if (rc != 0)
6466                 return (rc);
6467
6468         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6469         if (sb == NULL)
6470                 return (ENOMEM);
6471
6472         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6473
6474         t4_tp_read_la(sc, buf, NULL);
6475         p = buf;
6476
6477         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6478         case 2:
6479                 inc = 2;
6480                 show_func = tp_la_show2;
6481                 break;
6482         case 3:
6483                 inc = 2;
6484                 show_func = tp_la_show3;
6485                 break;
6486         default:
6487                 inc = 1;
6488                 show_func = tp_la_show;
6489         }
6490
6491         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6492                 (*show_func)(sb, p, i);
6493
6494         rc = sbuf_finish(sb);
6495         sbuf_delete(sb);
6496         free(buf, M_CXGBE);
6497         return (rc);
6498 }
6499
6500 static int
6501 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6502 {
6503         struct adapter *sc = arg1;
6504         struct sbuf *sb;
6505         int rc;
6506         u64 nrate[NCHAN], orate[NCHAN];
6507
6508         rc = sysctl_wire_old_buffer(req, 0);
6509         if (rc != 0)
6510                 return (rc);
6511
6512         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6513         if (sb == NULL)
6514                 return (ENOMEM);
6515
6516         t4_get_chan_txrate(sc, nrate, orate);
6517         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6518                  "channel 3\n");
6519         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6520             nrate[0], nrate[1], nrate[2], nrate[3]);
6521         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6522             orate[0], orate[1], orate[2], orate[3]);
6523
6524         rc = sbuf_finish(sb);
6525         sbuf_delete(sb);
6526
6527         return (rc);
6528 }
6529
6530 static int
6531 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6532 {
6533         struct adapter *sc = arg1;
6534         struct sbuf *sb;
6535         uint32_t *buf, *p;
6536         int rc, i;
6537
6538         rc = sysctl_wire_old_buffer(req, 0);
6539         if (rc != 0)
6540                 return (rc);
6541
6542         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6543         if (sb == NULL)
6544                 return (ENOMEM);
6545
6546         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6547             M_ZERO | M_WAITOK);
6548
6549         t4_ulprx_read_la(sc, buf);
6550         p = buf;
6551
6552         sbuf_printf(sb, "      Pcmd        Type   Message"
6553             "                Data");
6554         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6555                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6556                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6557         }
6558
6559         rc = sbuf_finish(sb);
6560         sbuf_delete(sb);
6561         free(buf, M_CXGBE);
6562         return (rc);
6563 }
6564
6565 static int
6566 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6567 {
6568         struct adapter *sc = arg1;
6569         struct sbuf *sb;
6570         int rc, v;
6571
6572         rc = sysctl_wire_old_buffer(req, 0);
6573         if (rc != 0)
6574                 return (rc);
6575
6576         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6577         if (sb == NULL)
6578                 return (ENOMEM);
6579
6580         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6581         if (G_STATSOURCE_T5(v) == 7) {
6582                 if (G_STATMODE(v) == 0) {
6583                         sbuf_printf(sb, "total %d, incomplete %d",
6584                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6585                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6586                 } else if (G_STATMODE(v) == 1) {
6587                         sbuf_printf(sb, "total %d, data overflow %d",
6588                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6589                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6590                 }
6591         }
6592         rc = sbuf_finish(sb);
6593         sbuf_delete(sb);
6594
6595         return (rc);
6596 }
6597 #endif
6598
6599 static inline void
6600 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6601 {
6602         struct buf_ring *br;
6603         struct mbuf *m;
6604
6605         TXQ_LOCK_ASSERT_OWNED(txq);
6606
6607         br = txq->br;
6608         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6609         if (m)
6610                 t4_eth_tx(ifp, txq, m);
6611 }
6612
6613 void
6614 t4_tx_callout(void *arg)
6615 {
6616         struct sge_eq *eq = arg;
6617         struct adapter *sc;
6618
6619         if (EQ_TRYLOCK(eq) == 0)
6620                 goto reschedule;
6621
6622         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6623                 EQ_UNLOCK(eq);
6624 reschedule:
6625                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6626                         callout_schedule(&eq->tx_callout, 1);
6627                 return;
6628         }
6629
6630         EQ_LOCK_ASSERT_OWNED(eq);
6631
6632         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6633
6634                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6635                         struct sge_txq *txq = arg;
6636                         struct port_info *pi = txq->ifp->if_softc;
6637
6638                         sc = pi->adapter;
6639                 } else {
6640                         struct sge_wrq *wrq = arg;
6641
6642                         sc = wrq->adapter;
6643                 }
6644
6645                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6646         }
6647
6648         EQ_UNLOCK(eq);
6649 }
6650
6651 void
6652 t4_tx_task(void *arg, int count)
6653 {
6654         struct sge_eq *eq = arg;
6655
6656         EQ_LOCK(eq);
6657         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6658                 struct sge_txq *txq = arg;
6659                 txq_start(txq->ifp, txq);
6660         } else {
6661                 struct sge_wrq *wrq = arg;
6662                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6663         }
6664         EQ_UNLOCK(eq);
6665 }
6666
6667 static uint32_t
6668 fconf_to_mode(uint32_t fconf)
6669 {
6670         uint32_t mode;
6671
6672         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6673             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6674
6675         if (fconf & F_FRAGMENTATION)
6676                 mode |= T4_FILTER_IP_FRAGMENT;
6677
6678         if (fconf & F_MPSHITTYPE)
6679                 mode |= T4_FILTER_MPS_HIT_TYPE;
6680
6681         if (fconf & F_MACMATCH)
6682                 mode |= T4_FILTER_MAC_IDX;
6683
6684         if (fconf & F_ETHERTYPE)
6685                 mode |= T4_FILTER_ETH_TYPE;
6686
6687         if (fconf & F_PROTOCOL)
6688                 mode |= T4_FILTER_IP_PROTO;
6689
6690         if (fconf & F_TOS)
6691                 mode |= T4_FILTER_IP_TOS;
6692
6693         if (fconf & F_VLAN)
6694                 mode |= T4_FILTER_VLAN;
6695
6696         if (fconf & F_VNIC_ID)
6697                 mode |= T4_FILTER_VNIC;
6698
6699         if (fconf & F_PORT)
6700                 mode |= T4_FILTER_PORT;
6701
6702         if (fconf & F_FCOE)
6703                 mode |= T4_FILTER_FCoE;
6704
6705         return (mode);
6706 }
6707
6708 static uint32_t
6709 mode_to_fconf(uint32_t mode)
6710 {
6711         uint32_t fconf = 0;
6712
6713         if (mode & T4_FILTER_IP_FRAGMENT)
6714                 fconf |= F_FRAGMENTATION;
6715
6716         if (mode & T4_FILTER_MPS_HIT_TYPE)
6717                 fconf |= F_MPSHITTYPE;
6718
6719         if (mode & T4_FILTER_MAC_IDX)
6720                 fconf |= F_MACMATCH;
6721
6722         if (mode & T4_FILTER_ETH_TYPE)
6723                 fconf |= F_ETHERTYPE;
6724
6725         if (mode & T4_FILTER_IP_PROTO)
6726                 fconf |= F_PROTOCOL;
6727
6728         if (mode & T4_FILTER_IP_TOS)
6729                 fconf |= F_TOS;
6730
6731         if (mode & T4_FILTER_VLAN)
6732                 fconf |= F_VLAN;
6733
6734         if (mode & T4_FILTER_VNIC)
6735                 fconf |= F_VNIC_ID;
6736
6737         if (mode & T4_FILTER_PORT)
6738                 fconf |= F_PORT;
6739
6740         if (mode & T4_FILTER_FCoE)
6741                 fconf |= F_FCOE;
6742
6743         return (fconf);
6744 }
6745
6746 static uint32_t
6747 fspec_to_fconf(struct t4_filter_specification *fs)
6748 {
6749         uint32_t fconf = 0;
6750
6751         if (fs->val.frag || fs->mask.frag)
6752                 fconf |= F_FRAGMENTATION;
6753
6754         if (fs->val.matchtype || fs->mask.matchtype)
6755                 fconf |= F_MPSHITTYPE;
6756
6757         if (fs->val.macidx || fs->mask.macidx)
6758                 fconf |= F_MACMATCH;
6759
6760         if (fs->val.ethtype || fs->mask.ethtype)
6761                 fconf |= F_ETHERTYPE;
6762
6763         if (fs->val.proto || fs->mask.proto)
6764                 fconf |= F_PROTOCOL;
6765
6766         if (fs->val.tos || fs->mask.tos)
6767                 fconf |= F_TOS;
6768
6769         if (fs->val.vlan_vld || fs->mask.vlan_vld)
6770                 fconf |= F_VLAN;
6771
6772         if (fs->val.vnic_vld || fs->mask.vnic_vld)
6773                 fconf |= F_VNIC_ID;
6774
6775         if (fs->val.iport || fs->mask.iport)
6776                 fconf |= F_PORT;
6777
6778         if (fs->val.fcoe || fs->mask.fcoe)
6779                 fconf |= F_FCOE;
6780
6781         return (fconf);
6782 }
6783
6784 static int
6785 get_filter_mode(struct adapter *sc, uint32_t *mode)
6786 {
6787         int rc;
6788         uint32_t fconf;
6789
6790         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6791             "t4getfm");
6792         if (rc)
6793                 return (rc);
6794
6795         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
6796             A_TP_VLAN_PRI_MAP);
6797
6798         if (sc->params.tp.vlan_pri_map != fconf) {
6799                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
6800                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
6801                     fconf);
6802                 sc->params.tp.vlan_pri_map = fconf;
6803         }
6804
6805         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
6806
6807         end_synchronized_op(sc, LOCK_HELD);
6808         return (0);
6809 }
6810
6811 static int
6812 set_filter_mode(struct adapter *sc, uint32_t mode)
6813 {
6814         uint32_t fconf;
6815         int rc;
6816
6817         fconf = mode_to_fconf(mode);
6818
6819         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6820             "t4setfm");
6821         if (rc)
6822                 return (rc);
6823
6824         if (sc->tids.ftids_in_use > 0) {
6825                 rc = EBUSY;
6826                 goto done;
6827         }
6828
6829 #ifdef TCP_OFFLOAD
6830         if (sc->offload_map) {
6831                 rc = EBUSY;
6832                 goto done;
6833         }
6834 #endif
6835
6836 #ifdef notyet
6837         rc = -t4_set_filter_mode(sc, fconf);
6838         if (rc == 0)
6839                 sc->filter_mode = fconf;
6840 #else
6841         rc = ENOTSUP;
6842 #endif
6843
6844 done:
6845         end_synchronized_op(sc, LOCK_HELD);
6846         return (rc);
6847 }
6848
6849 static inline uint64_t
6850 get_filter_hits(struct adapter *sc, uint32_t fid)
6851 {
6852         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6853         uint64_t hits;
6854
6855         memwin_info(sc, 0, &mw_base, NULL);
6856         off = position_memwin(sc, 0,
6857             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
6858         if (is_t4(sc)) {
6859                 hits = t4_read_reg64(sc, mw_base + off + 16);
6860                 hits = be64toh(hits);
6861         } else {
6862                 hits = t4_read_reg(sc, mw_base + off + 24);
6863                 hits = be32toh(hits);
6864         }
6865
6866         return (hits);
6867 }
6868
6869 static int
6870 get_filter(struct adapter *sc, struct t4_filter *t)
6871 {
6872         int i, rc, nfilters = sc->tids.nftids;
6873         struct filter_entry *f;
6874
6875         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6876             "t4getf");
6877         if (rc)
6878                 return (rc);
6879
6880         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
6881             t->idx >= nfilters) {
6882                 t->idx = 0xffffffff;
6883                 goto done;
6884         }
6885
6886         f = &sc->tids.ftid_tab[t->idx];
6887         for (i = t->idx; i < nfilters; i++, f++) {
6888                 if (f->valid) {
6889                         t->idx = i;
6890                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
6891                         t->smtidx = f->smtidx;
6892                         if (f->fs.hitcnts)
6893                                 t->hits = get_filter_hits(sc, t->idx);
6894                         else
6895                                 t->hits = UINT64_MAX;
6896                         t->fs = f->fs;
6897
6898                         goto done;
6899                 }
6900         }
6901
6902         t->idx = 0xffffffff;
6903 done:
6904         end_synchronized_op(sc, LOCK_HELD);
6905         return (0);
6906 }
6907
6908 static int
6909 set_filter(struct adapter *sc, struct t4_filter *t)
6910 {
6911         unsigned int nfilters, nports;
6912         struct filter_entry *f;
6913         int i, rc;
6914
6915         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
6916         if (rc)
6917                 return (rc);
6918
6919         nfilters = sc->tids.nftids;
6920         nports = sc->params.nports;
6921
6922         if (nfilters == 0) {
6923                 rc = ENOTSUP;
6924                 goto done;
6925         }
6926
6927         if (!(sc->flags & FULL_INIT_DONE)) {
6928                 rc = EAGAIN;
6929                 goto done;
6930         }
6931
6932         if (t->idx >= nfilters) {
6933                 rc = EINVAL;
6934                 goto done;
6935         }
6936
6937         /* Validate against the global filter mode */
6938         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
6939             sc->params.tp.vlan_pri_map) {
6940                 rc = E2BIG;
6941                 goto done;
6942         }
6943
6944         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
6945                 rc = EINVAL;
6946                 goto done;
6947         }
6948
6949         if (t->fs.val.iport >= nports) {
6950                 rc = EINVAL;
6951                 goto done;
6952         }
6953
6954         /* Can't specify an iq if not steering to it */
6955         if (!t->fs.dirsteer && t->fs.iq) {
6956                 rc = EINVAL;
6957                 goto done;
6958         }
6959
6960         /* IPv6 filter idx must be 4 aligned */
6961         if (t->fs.type == 1 &&
6962             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
6963                 rc = EINVAL;
6964                 goto done;
6965         }
6966
6967         if (sc->tids.ftid_tab == NULL) {
6968                 KASSERT(sc->tids.ftids_in_use == 0,
6969                     ("%s: no memory allocated but filters_in_use > 0",
6970                     __func__));
6971
6972                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
6973                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
6974                 if (sc->tids.ftid_tab == NULL) {
6975                         rc = ENOMEM;
6976                         goto done;
6977                 }
6978                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
6979         }
6980
6981         for (i = 0; i < 4; i++) {
6982                 f = &sc->tids.ftid_tab[t->idx + i];
6983
6984                 if (f->pending || f->valid) {
6985                         rc = EBUSY;
6986                         goto done;
6987                 }
6988                 if (f->locked) {
6989                         rc = EPERM;
6990                         goto done;
6991                 }
6992
6993                 if (t->fs.type == 0)
6994                         break;
6995         }
6996
6997         f = &sc->tids.ftid_tab[t->idx];
6998         f->fs = t->fs;
6999
7000         rc = set_filter_wr(sc, t->idx);
7001 done:
7002         end_synchronized_op(sc, 0);
7003
7004         if (rc == 0) {
7005                 mtx_lock(&sc->tids.ftid_lock);
7006                 for (;;) {
7007                         if (f->pending == 0) {
7008                                 rc = f->valid ? 0 : EIO;
7009                                 break;
7010                         }
7011
7012                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7013                             PCATCH, "t4setfw", 0)) {
7014                                 rc = EINPROGRESS;
7015                                 break;
7016                         }
7017                 }
7018                 mtx_unlock(&sc->tids.ftid_lock);
7019         }
7020         return (rc);
7021 }
7022
7023 static int
7024 del_filter(struct adapter *sc, struct t4_filter *t)
7025 {
7026         unsigned int nfilters;
7027         struct filter_entry *f;
7028         int rc;
7029
7030         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7031         if (rc)
7032                 return (rc);
7033
7034         nfilters = sc->tids.nftids;
7035
7036         if (nfilters == 0) {
7037                 rc = ENOTSUP;
7038                 goto done;
7039         }
7040
7041         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7042             t->idx >= nfilters) {
7043                 rc = EINVAL;
7044                 goto done;
7045         }
7046
7047         if (!(sc->flags & FULL_INIT_DONE)) {
7048                 rc = EAGAIN;
7049                 goto done;
7050         }
7051
7052         f = &sc->tids.ftid_tab[t->idx];
7053
7054         if (f->pending) {
7055                 rc = EBUSY;
7056                 goto done;
7057         }
7058         if (f->locked) {
7059                 rc = EPERM;
7060                 goto done;
7061         }
7062
7063         if (f->valid) {
7064                 t->fs = f->fs;  /* extra info for the caller */
7065                 rc = del_filter_wr(sc, t->idx);
7066         }
7067
7068 done:
7069         end_synchronized_op(sc, 0);
7070
7071         if (rc == 0) {
7072                 mtx_lock(&sc->tids.ftid_lock);
7073                 for (;;) {
7074                         if (f->pending == 0) {
7075                                 rc = f->valid ? EIO : 0;
7076                                 break;
7077                         }
7078
7079                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7080                             PCATCH, "t4delfw", 0)) {
7081                                 rc = EINPROGRESS;
7082                                 break;
7083                         }
7084                 }
7085                 mtx_unlock(&sc->tids.ftid_lock);
7086         }
7087
7088         return (rc);
7089 }
7090
7091 static void
7092 clear_filter(struct filter_entry *f)
7093 {
7094         if (f->l2t)
7095                 t4_l2t_release(f->l2t);
7096
7097         bzero(f, sizeof (*f));
7098 }
7099
7100 static int
7101 set_filter_wr(struct adapter *sc, int fidx)
7102 {
7103         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7104         struct wrqe *wr;
7105         struct fw_filter_wr *fwr;
7106         unsigned int ftid;
7107
7108         ASSERT_SYNCHRONIZED_OP(sc);
7109
7110         if (f->fs.newdmac || f->fs.newvlan) {
7111                 /* This filter needs an L2T entry; allocate one. */
7112                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7113                 if (f->l2t == NULL)
7114                         return (EAGAIN);
7115                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7116                     f->fs.dmac)) {
7117                         t4_l2t_release(f->l2t);
7118                         f->l2t = NULL;
7119                         return (ENOMEM);
7120                 }
7121         }
7122
7123         ftid = sc->tids.ftid_base + fidx;
7124
7125         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7126         if (wr == NULL)
7127                 return (ENOMEM);
7128
7129         fwr = wrtod(wr);
7130         bzero(fwr, sizeof (*fwr));
7131
7132         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7133         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7134         fwr->tid_to_iq =
7135             htobe32(V_FW_FILTER_WR_TID(ftid) |
7136                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7137                 V_FW_FILTER_WR_NOREPLY(0) |
7138                 V_FW_FILTER_WR_IQ(f->fs.iq));
7139         fwr->del_filter_to_l2tix =
7140             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7141                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7142                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7143                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7144                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7145                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7146                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7147                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7148                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7149                     f->fs.newvlan == VLAN_REWRITE) |
7150                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7151                     f->fs.newvlan == VLAN_REWRITE) |
7152                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7153                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7154                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7155                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7156         fwr->ethtype = htobe16(f->fs.val.ethtype);
7157         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7158         fwr->frag_to_ovlan_vldm =
7159             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7160                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7161                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7162                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7163                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7164                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7165         fwr->smac_sel = 0;
7166         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7167             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7168         fwr->maci_to_matchtypem =
7169             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7170                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7171                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7172                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7173                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7174                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7175                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7176                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7177         fwr->ptcl = f->fs.val.proto;
7178         fwr->ptclm = f->fs.mask.proto;
7179         fwr->ttyp = f->fs.val.tos;
7180         fwr->ttypm = f->fs.mask.tos;
7181         fwr->ivlan = htobe16(f->fs.val.vlan);
7182         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7183         fwr->ovlan = htobe16(f->fs.val.vnic);
7184         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7185         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7186         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7187         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7188         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7189         fwr->lp = htobe16(f->fs.val.dport);
7190         fwr->lpm = htobe16(f->fs.mask.dport);
7191         fwr->fp = htobe16(f->fs.val.sport);
7192         fwr->fpm = htobe16(f->fs.mask.sport);
7193         if (f->fs.newsmac)
7194                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7195
7196         f->pending = 1;
7197         sc->tids.ftids_in_use++;
7198
7199         t4_wrq_tx(sc, wr);
7200         return (0);
7201 }
7202
7203 static int
7204 del_filter_wr(struct adapter *sc, int fidx)
7205 {
7206         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7207         struct wrqe *wr;
7208         struct fw_filter_wr *fwr;
7209         unsigned int ftid;
7210
7211         ftid = sc->tids.ftid_base + fidx;
7212
7213         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7214         if (wr == NULL)
7215                 return (ENOMEM);
7216         fwr = wrtod(wr);
7217         bzero(fwr, sizeof (*fwr));
7218
7219         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7220
7221         f->pending = 1;
7222         t4_wrq_tx(sc, wr);
7223         return (0);
7224 }
7225
7226 int
7227 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7228 {
7229         struct adapter *sc = iq->adapter;
7230         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7231         unsigned int idx = GET_TID(rpl);
7232         unsigned int rc;
7233         struct filter_entry *f;
7234
7235         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7236             rss->opcode));
7237
7238         if (is_ftid(sc, idx)) {
7239
7240                 idx -= sc->tids.ftid_base;
7241                 f = &sc->tids.ftid_tab[idx];
7242                 rc = G_COOKIE(rpl->cookie);
7243
7244                 mtx_lock(&sc->tids.ftid_lock);
7245                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7246                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7247                             __func__, idx));
7248                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7249                         f->pending = 0;  /* asynchronous setup completed */
7250                         f->valid = 1;
7251                 } else {
7252                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7253                                 /* Add or delete failed, display an error */
7254                                 log(LOG_ERR,
7255                                     "filter %u setup failed with error %u\n",
7256                                     idx, rc);
7257                         }
7258
7259                         clear_filter(f);
7260                         sc->tids.ftids_in_use--;
7261                 }
7262                 wakeup(&sc->tids.ftid_tab);
7263                 mtx_unlock(&sc->tids.ftid_lock);
7264         }
7265
7266         return (0);
7267 }
7268
7269 static int
7270 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7271 {
7272         int rc;
7273
7274         if (cntxt->cid > M_CTXTQID)
7275                 return (EINVAL);
7276
7277         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7278             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7279                 return (EINVAL);
7280
7281         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7282         if (rc)
7283                 return (rc);
7284
7285         if (sc->flags & FW_OK) {
7286                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7287                     &cntxt->data[0]);
7288                 if (rc == 0)
7289                         goto done;
7290         }
7291
7292         /*
7293          * Read via firmware failed or wasn't even attempted.  Read directly via
7294          * the backdoor.
7295          */
7296         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7297 done:
7298         end_synchronized_op(sc, 0);
7299         return (rc);
7300 }
7301
7302 static int
7303 load_fw(struct adapter *sc, struct t4_data *fw)
7304 {
7305         int rc;
7306         uint8_t *fw_data;
7307
7308         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7309         if (rc)
7310                 return (rc);
7311
7312         if (sc->flags & FULL_INIT_DONE) {
7313                 rc = EBUSY;
7314                 goto done;
7315         }
7316
7317         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7318         if (fw_data == NULL) {
7319                 rc = ENOMEM;
7320                 goto done;
7321         }
7322
7323         rc = copyin(fw->data, fw_data, fw->len);
7324         if (rc == 0)
7325                 rc = -t4_load_fw(sc, fw_data, fw->len);
7326
7327         free(fw_data, M_CXGBE);
7328 done:
7329         end_synchronized_op(sc, 0);
7330         return (rc);
7331 }
7332
7333 static int
7334 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7335 {
7336         uint32_t addr, off, remaining, i, n;
7337         uint32_t *buf, *b;
7338         uint32_t mw_base, mw_aperture;
7339         int rc;
7340         uint8_t *dst;
7341
7342         rc = validate_mem_range(sc, mr->addr, mr->len);
7343         if (rc != 0)
7344                 return (rc);
7345
7346         memwin_info(sc, win, &mw_base, &mw_aperture);
7347         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7348         addr = mr->addr;
7349         remaining = mr->len;
7350         dst = (void *)mr->data;
7351
7352         while (remaining) {
7353                 off = position_memwin(sc, win, addr);
7354
7355                 /* number of bytes that we'll copy in the inner loop */
7356                 n = min(remaining, mw_aperture - off);
7357                 for (i = 0; i < n; i += 4)
7358                         *b++ = t4_read_reg(sc, mw_base + off + i);
7359
7360                 rc = copyout(buf, dst, n);
7361                 if (rc != 0)
7362                         break;
7363
7364                 b = buf;
7365                 dst += n;
7366                 remaining -= n;
7367                 addr += n;
7368         }
7369
7370         free(buf, M_CXGBE);
7371         return (rc);
7372 }
7373
7374 static int
7375 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7376 {
7377         int rc;
7378
7379         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7380                 return (EINVAL);
7381
7382         if (i2cd->len > 1) {
7383                 /* XXX: need fw support for longer reads in one go */
7384                 return (ENOTSUP);
7385         }
7386
7387         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7388         if (rc)
7389                 return (rc);
7390         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7391             i2cd->offset, &i2cd->data[0]);
7392         end_synchronized_op(sc, 0);
7393
7394         return (rc);
7395 }
7396
7397 static int
7398 in_range(int val, int lo, int hi)
7399 {
7400
7401         return (val < 0 || (val <= hi && val >= lo));
7402 }
7403
7404 static int
7405 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7406 {
7407         int fw_subcmd, fw_type, rc;
7408
7409         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7410         if (rc)
7411                 return (rc);
7412
7413         if (!(sc->flags & FULL_INIT_DONE)) {
7414                 rc = EAGAIN;
7415                 goto done;
7416         }
7417
7418         /*
7419          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7420          * sub-command and type are in common locations.)
7421          */
7422         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7423                 fw_subcmd = FW_SCHED_SC_CONFIG;
7424         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7425                 fw_subcmd = FW_SCHED_SC_PARAMS;
7426         else {
7427                 rc = EINVAL;
7428                 goto done;
7429         }
7430         if (p->type == SCHED_CLASS_TYPE_PACKET)
7431                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7432         else {
7433                 rc = EINVAL;
7434                 goto done;
7435         }
7436
7437         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7438                 /* Vet our parameters ..*/
7439                 if (p->u.config.minmax < 0) {
7440                         rc = EINVAL;
7441                         goto done;
7442                 }
7443
7444                 /* And pass the request to the firmware ...*/
7445                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax);
7446                 goto done;
7447         }
7448
7449         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7450                 int fw_level;
7451                 int fw_mode;
7452                 int fw_rateunit;
7453                 int fw_ratemode;
7454
7455                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7456                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7457                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7458                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7459                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7460                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7461                 else {
7462                         rc = EINVAL;
7463                         goto done;
7464                 }
7465
7466                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7467                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7468                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7469                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7470                 else {
7471                         rc = EINVAL;
7472                         goto done;
7473                 }
7474
7475                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7476                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7477                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7478                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7479                 else {
7480                         rc = EINVAL;
7481                         goto done;
7482                 }
7483
7484                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7485                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7486                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7487                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7488                 else {
7489                         rc = EINVAL;
7490                         goto done;
7491                 }
7492
7493                 /* Vet our parameters ... */
7494                 if (!in_range(p->u.params.channel, 0, 3) ||
7495                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7496                     !in_range(p->u.params.minrate, 0, 10000000) ||
7497                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7498                     !in_range(p->u.params.weight, 0, 100)) {
7499                         rc = ERANGE;
7500                         goto done;
7501                 }
7502
7503                 /*
7504                  * Translate any unset parameters into the firmware's
7505                  * nomenclature and/or fail the call if the parameters
7506                  * are required ...
7507                  */
7508                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7509                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7510                         rc = EINVAL;
7511                         goto done;
7512                 }
7513                 if (p->u.params.minrate < 0)
7514                         p->u.params.minrate = 0;
7515                 if (p->u.params.maxrate < 0) {
7516                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7517                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7518                                 rc = EINVAL;
7519                                 goto done;
7520                         } else
7521                                 p->u.params.maxrate = 0;
7522                 }
7523                 if (p->u.params.weight < 0) {
7524                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7525                                 rc = EINVAL;
7526                                 goto done;
7527                         } else
7528                                 p->u.params.weight = 0;
7529                 }
7530                 if (p->u.params.pktsize < 0) {
7531                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7532                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7533                                 rc = EINVAL;
7534                                 goto done;
7535                         } else
7536                                 p->u.params.pktsize = 0;
7537                 }
7538
7539                 /* See what the firmware thinks of the request ... */
7540                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7541                     fw_rateunit, fw_ratemode, p->u.params.channel,
7542                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7543                     p->u.params.weight, p->u.params.pktsize);
7544                 goto done;
7545         }
7546
7547         rc = EINVAL;
7548 done:
7549         end_synchronized_op(sc, 0);
7550         return (rc);
7551 }
7552
7553 static int
7554 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7555 {
7556         struct port_info *pi = NULL;
7557         struct sge_txq *txq;
7558         uint32_t fw_mnem, fw_queue, fw_class;
7559         int i, rc;
7560
7561         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7562         if (rc)
7563                 return (rc);
7564
7565         if (!(sc->flags & FULL_INIT_DONE)) {
7566                 rc = EAGAIN;
7567                 goto done;
7568         }
7569
7570         if (p->port >= sc->params.nports) {
7571                 rc = EINVAL;
7572                 goto done;
7573         }
7574
7575         pi = sc->port[p->port];
7576         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7577                 rc = EINVAL;
7578                 goto done;
7579         }
7580
7581         /*
7582          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7583          * Scheduling Class in this case).
7584          */
7585         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7586             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7587         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7588
7589         /*
7590          * If op.queue is non-negative, then we're only changing the scheduling
7591          * on a single specified TX queue.
7592          */
7593         if (p->queue >= 0) {
7594                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7595                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7596                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7597                     &fw_class);
7598                 goto done;
7599         }
7600
7601         /*
7602          * Change the scheduling on all the TX queues for the
7603          * interface.
7604          */
7605         for_each_txq(pi, i, txq) {
7606                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7607                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7608                     &fw_class);
7609                 if (rc)
7610                         goto done;
7611         }
7612
7613         rc = 0;
7614 done:
7615         end_synchronized_op(sc, 0);
7616         return (rc);
7617 }
7618
7619 int
7620 t4_os_find_pci_capability(struct adapter *sc, int cap)
7621 {
7622         int i;
7623
7624         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7625 }
7626
7627 int
7628 t4_os_pci_save_state(struct adapter *sc)
7629 {
7630         device_t dev;
7631         struct pci_devinfo *dinfo;
7632
7633         dev = sc->dev;
7634         dinfo = device_get_ivars(dev);
7635
7636         pci_cfg_save(dev, dinfo, 0);
7637         return (0);
7638 }
7639
7640 int
7641 t4_os_pci_restore_state(struct adapter *sc)
7642 {
7643         device_t dev;
7644         struct pci_devinfo *dinfo;
7645
7646         dev = sc->dev;
7647         dinfo = device_get_ivars(dev);
7648
7649         pci_cfg_restore(dev, dinfo);
7650         return (0);
7651 }
7652
7653 void
7654 t4_os_portmod_changed(const struct adapter *sc, int idx)
7655 {
7656         struct port_info *pi = sc->port[idx];
7657         static const char *mod_str[] = {
7658                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7659         };
7660
7661         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7662                 if_printf(pi->ifp, "transceiver unplugged.\n");
7663         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7664                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7665         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7666                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7667         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7668                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7669                     mod_str[pi->mod_type]);
7670         } else {
7671                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7672                     pi->mod_type);
7673         }
7674 }
7675
7676 void
7677 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7678 {
7679         struct port_info *pi = sc->port[idx];
7680         struct ifnet *ifp = pi->ifp;
7681
7682         if (link_stat) {
7683                 pi->linkdnrc = -1;
7684                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7685                 if_link_state_change(ifp, LINK_STATE_UP);
7686         } else {
7687                 if (reason >= 0)
7688                         pi->linkdnrc = reason;
7689                 if_link_state_change(ifp, LINK_STATE_DOWN);
7690         }
7691 }
7692
7693 void
7694 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7695 {
7696         struct adapter *sc;
7697
7698         sx_slock(&t4_list_lock);
7699         SLIST_FOREACH(sc, &t4_list, link) {
7700                 /*
7701                  * func should not make any assumptions about what state sc is
7702                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7703                  */
7704                 func(sc, arg);
7705         }
7706         sx_sunlock(&t4_list_lock);
7707 }
7708
7709 static int
7710 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7711 {
7712        return (0);
7713 }
7714
7715 static int
7716 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7717 {
7718        return (0);
7719 }
7720
7721 static int
7722 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7723     struct thread *td)
7724 {
7725         int rc;
7726         struct adapter *sc = dev->si_drv1;
7727
7728         rc = priv_check(td, PRIV_DRIVER);
7729         if (rc != 0)
7730                 return (rc);
7731
7732         switch (cmd) {
7733         case CHELSIO_T4_GETREG: {
7734                 struct t4_reg *edata = (struct t4_reg *)data;
7735
7736                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7737                         return (EFAULT);
7738
7739                 if (edata->size == 4)
7740                         edata->val = t4_read_reg(sc, edata->addr);
7741                 else if (edata->size == 8)
7742                         edata->val = t4_read_reg64(sc, edata->addr);
7743                 else
7744                         return (EINVAL);
7745
7746                 break;
7747         }
7748         case CHELSIO_T4_SETREG: {
7749                 struct t4_reg *edata = (struct t4_reg *)data;
7750
7751                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7752                         return (EFAULT);
7753
7754                 if (edata->size == 4) {
7755                         if (edata->val & 0xffffffff00000000)
7756                                 return (EINVAL);
7757                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7758                 } else if (edata->size == 8)
7759                         t4_write_reg64(sc, edata->addr, edata->val);
7760                 else
7761                         return (EINVAL);
7762                 break;
7763         }
7764         case CHELSIO_T4_REGDUMP: {
7765                 struct t4_regdump *regs = (struct t4_regdump *)data;
7766                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7767                 uint8_t *buf;
7768
7769                 if (regs->len < reglen) {
7770                         regs->len = reglen; /* hint to the caller */
7771                         return (ENOBUFS);
7772                 }
7773
7774                 regs->len = reglen;
7775                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
7776                 t4_get_regs(sc, regs, buf);
7777                 rc = copyout(buf, regs->data, reglen);
7778                 free(buf, M_CXGBE);
7779                 break;
7780         }
7781         case CHELSIO_T4_GET_FILTER_MODE:
7782                 rc = get_filter_mode(sc, (uint32_t *)data);
7783                 break;
7784         case CHELSIO_T4_SET_FILTER_MODE:
7785                 rc = set_filter_mode(sc, *(uint32_t *)data);
7786                 break;
7787         case CHELSIO_T4_GET_FILTER:
7788                 rc = get_filter(sc, (struct t4_filter *)data);
7789                 break;
7790         case CHELSIO_T4_SET_FILTER:
7791                 rc = set_filter(sc, (struct t4_filter *)data);
7792                 break;
7793         case CHELSIO_T4_DEL_FILTER:
7794                 rc = del_filter(sc, (struct t4_filter *)data);
7795                 break;
7796         case CHELSIO_T4_GET_SGE_CONTEXT:
7797                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
7798                 break;
7799         case CHELSIO_T4_LOAD_FW:
7800                 rc = load_fw(sc, (struct t4_data *)data);
7801                 break;
7802         case CHELSIO_T4_GET_MEM:
7803                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
7804                 break;
7805         case CHELSIO_T4_GET_I2C:
7806                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
7807                 break;
7808         case CHELSIO_T4_CLEAR_STATS: {
7809                 int i;
7810                 u_int port_id = *(uint32_t *)data;
7811                 struct port_info *pi;
7812
7813                 if (port_id >= sc->params.nports)
7814                         return (EINVAL);
7815                 pi = sc->port[port_id];
7816
7817                 /* MAC stats */
7818                 t4_clr_port_stats(sc, pi->tx_chan);
7819
7820                 if (pi->flags & PORT_INIT_DONE) {
7821                         struct sge_rxq *rxq;
7822                         struct sge_txq *txq;
7823                         struct sge_wrq *wrq;
7824
7825                         for_each_rxq(pi, i, rxq) {
7826 #if defined(INET) || defined(INET6)
7827                                 rxq->lro.lro_queued = 0;
7828                                 rxq->lro.lro_flushed = 0;
7829 #endif
7830                                 rxq->rxcsum = 0;
7831                                 rxq->vlan_extraction = 0;
7832                         }
7833
7834                         for_each_txq(pi, i, txq) {
7835                                 txq->txcsum = 0;
7836                                 txq->tso_wrs = 0;
7837                                 txq->vlan_insertion = 0;
7838                                 txq->imm_wrs = 0;
7839                                 txq->sgl_wrs = 0;
7840                                 txq->txpkt_wrs = 0;
7841                                 txq->txpkts_wrs = 0;
7842                                 txq->txpkts_pkts = 0;
7843                                 txq->br->br_drops = 0;
7844                                 txq->no_dmamap = 0;
7845                                 txq->no_desc = 0;
7846                         }
7847
7848 #ifdef TCP_OFFLOAD
7849                         /* nothing to clear for each ofld_rxq */
7850
7851                         for_each_ofld_txq(pi, i, wrq) {
7852                                 wrq->tx_wrs = 0;
7853                                 wrq->no_desc = 0;
7854                         }
7855 #endif
7856                         wrq = &sc->sge.ctrlq[pi->port_id];
7857                         wrq->tx_wrs = 0;
7858                         wrq->no_desc = 0;
7859                 }
7860                 break;
7861         }
7862         case CHELSIO_T4_SCHED_CLASS:
7863                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
7864                 break;
7865         case CHELSIO_T4_SCHED_QUEUE:
7866                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
7867                 break;
7868         case CHELSIO_T4_GET_TRACER:
7869                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
7870                 break;
7871         case CHELSIO_T4_SET_TRACER:
7872                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
7873                 break;
7874         default:
7875                 rc = EINVAL;
7876         }
7877
7878         return (rc);
7879 }
7880
7881 #ifdef TCP_OFFLOAD
7882 static int
7883 toe_capability(struct port_info *pi, int enable)
7884 {
7885         int rc;
7886         struct adapter *sc = pi->adapter;
7887
7888         ASSERT_SYNCHRONIZED_OP(sc);
7889
7890         if (!is_offload(sc))
7891                 return (ENODEV);
7892
7893         if (enable) {
7894                 if (!(sc->flags & FULL_INIT_DONE)) {
7895                         rc = cxgbe_init_synchronized(pi);
7896                         if (rc)
7897                                 return (rc);
7898                 }
7899
7900                 if (isset(&sc->offload_map, pi->port_id))
7901                         return (0);
7902
7903                 if (!(sc->flags & TOM_INIT_DONE)) {
7904                         rc = t4_activate_uld(sc, ULD_TOM);
7905                         if (rc == EAGAIN) {
7906                                 log(LOG_WARNING,
7907                                     "You must kldload t4_tom.ko before trying "
7908                                     "to enable TOE on a cxgbe interface.\n");
7909                         }
7910                         if (rc != 0)
7911                                 return (rc);
7912                         KASSERT(sc->tom_softc != NULL,
7913                             ("%s: TOM activated but softc NULL", __func__));
7914                         KASSERT(sc->flags & TOM_INIT_DONE,
7915                             ("%s: TOM activated but flag not set", __func__));
7916                 }
7917
7918                 setbit(&sc->offload_map, pi->port_id);
7919         } else {
7920                 if (!isset(&sc->offload_map, pi->port_id))
7921                         return (0);
7922
7923                 KASSERT(sc->flags & TOM_INIT_DONE,
7924                     ("%s: TOM never initialized?", __func__));
7925                 clrbit(&sc->offload_map, pi->port_id);
7926         }
7927
7928         return (0);
7929 }
7930
7931 /*
7932  * Add an upper layer driver to the global list.
7933  */
7934 int
7935 t4_register_uld(struct uld_info *ui)
7936 {
7937         int rc = 0;
7938         struct uld_info *u;
7939
7940         sx_xlock(&t4_uld_list_lock);
7941         SLIST_FOREACH(u, &t4_uld_list, link) {
7942             if (u->uld_id == ui->uld_id) {
7943                     rc = EEXIST;
7944                     goto done;
7945             }
7946         }
7947
7948         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
7949         ui->refcount = 0;
7950 done:
7951         sx_xunlock(&t4_uld_list_lock);
7952         return (rc);
7953 }
7954
7955 int
7956 t4_unregister_uld(struct uld_info *ui)
7957 {
7958         int rc = EINVAL;
7959         struct uld_info *u;
7960
7961         sx_xlock(&t4_uld_list_lock);
7962
7963         SLIST_FOREACH(u, &t4_uld_list, link) {
7964             if (u == ui) {
7965                     if (ui->refcount > 0) {
7966                             rc = EBUSY;
7967                             goto done;
7968                     }
7969
7970                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
7971                     rc = 0;
7972                     goto done;
7973             }
7974         }
7975 done:
7976         sx_xunlock(&t4_uld_list_lock);
7977         return (rc);
7978 }
7979
7980 int
7981 t4_activate_uld(struct adapter *sc, int id)
7982 {
7983         int rc = EAGAIN;
7984         struct uld_info *ui;
7985
7986         ASSERT_SYNCHRONIZED_OP(sc);
7987
7988         sx_slock(&t4_uld_list_lock);
7989
7990         SLIST_FOREACH(ui, &t4_uld_list, link) {
7991                 if (ui->uld_id == id) {
7992                         rc = ui->activate(sc);
7993                         if (rc == 0)
7994                                 ui->refcount++;
7995                         goto done;
7996                 }
7997         }
7998 done:
7999         sx_sunlock(&t4_uld_list_lock);
8000
8001         return (rc);
8002 }
8003
8004 int
8005 t4_deactivate_uld(struct adapter *sc, int id)
8006 {
8007         int rc = EINVAL;
8008         struct uld_info *ui;
8009
8010         ASSERT_SYNCHRONIZED_OP(sc);
8011
8012         sx_slock(&t4_uld_list_lock);
8013
8014         SLIST_FOREACH(ui, &t4_uld_list, link) {
8015                 if (ui->uld_id == id) {
8016                         rc = ui->deactivate(sc);
8017                         if (rc == 0)
8018                                 ui->refcount--;
8019                         goto done;
8020                 }
8021         }
8022 done:
8023         sx_sunlock(&t4_uld_list_lock);
8024
8025         return (rc);
8026 }
8027 #endif
8028
8029 /*
8030  * Come up with reasonable defaults for some of the tunables, provided they're
8031  * not set by the user (in which case we'll use the values as is).
8032  */
8033 static void
8034 tweak_tunables(void)
8035 {
8036         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8037
8038         if (t4_ntxq10g < 1)
8039                 t4_ntxq10g = min(nc, NTXQ_10G);
8040
8041         if (t4_ntxq1g < 1)
8042                 t4_ntxq1g = min(nc, NTXQ_1G);
8043
8044         if (t4_nrxq10g < 1)
8045                 t4_nrxq10g = min(nc, NRXQ_10G);
8046
8047         if (t4_nrxq1g < 1)
8048                 t4_nrxq1g = min(nc, NRXQ_1G);
8049
8050 #ifdef TCP_OFFLOAD
8051         if (t4_nofldtxq10g < 1)
8052                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8053
8054         if (t4_nofldtxq1g < 1)
8055                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8056
8057         if (t4_nofldrxq10g < 1)
8058                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8059
8060         if (t4_nofldrxq1g < 1)
8061                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8062
8063         if (t4_toecaps_allowed == -1)
8064                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8065 #else
8066         if (t4_toecaps_allowed == -1)
8067                 t4_toecaps_allowed = 0;
8068 #endif
8069
8070         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8071                 t4_tmr_idx_10g = TMR_IDX_10G;
8072
8073         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8074                 t4_pktc_idx_10g = PKTC_IDX_10G;
8075
8076         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8077                 t4_tmr_idx_1g = TMR_IDX_1G;
8078
8079         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8080                 t4_pktc_idx_1g = PKTC_IDX_1G;
8081
8082         if (t4_qsize_txq < 128)
8083                 t4_qsize_txq = 128;
8084
8085         if (t4_qsize_rxq < 128)
8086                 t4_qsize_rxq = 128;
8087         while (t4_qsize_rxq & 7)
8088                 t4_qsize_rxq++;
8089
8090         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8091 }
8092
8093 static int
8094 mod_event(module_t mod, int cmd, void *arg)
8095 {
8096         int rc = 0;
8097         static int loaded = 0;
8098
8099         switch (cmd) {
8100         case MOD_LOAD:
8101                 if (atomic_fetchadd_int(&loaded, 1))
8102                         break;
8103                 t4_sge_modload();
8104                 sx_init(&t4_list_lock, "T4/T5 adapters");
8105                 SLIST_INIT(&t4_list);
8106 #ifdef TCP_OFFLOAD
8107                 sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8108                 SLIST_INIT(&t4_uld_list);
8109 #endif
8110                 t4_tracer_modload();
8111                 tweak_tunables();
8112                 break;
8113
8114         case MOD_UNLOAD:
8115                 if (atomic_fetchadd_int(&loaded, -1) > 1)
8116                         break;
8117                 t4_tracer_modunload();
8118 #ifdef TCP_OFFLOAD
8119                 sx_slock(&t4_uld_list_lock);
8120                 if (!SLIST_EMPTY(&t4_uld_list)) {
8121                         rc = EBUSY;
8122                         sx_sunlock(&t4_uld_list_lock);
8123                         break;
8124                 }
8125                 sx_sunlock(&t4_uld_list_lock);
8126                 sx_destroy(&t4_uld_list_lock);
8127 #endif
8128                 sx_slock(&t4_list_lock);
8129                 if (!SLIST_EMPTY(&t4_list)) {
8130                         rc = EBUSY;
8131                         sx_sunlock(&t4_list_lock);
8132                         break;
8133                 }
8134                 sx_sunlock(&t4_list_lock);
8135                 sx_destroy(&t4_list_lock);
8136                 break;
8137         }
8138
8139         return (rc);
8140 }
8141
8142 static devclass_t t4_devclass, t5_devclass;
8143 static devclass_t cxgbe_devclass, cxl_devclass;
8144
8145 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8146 MODULE_VERSION(t4nex, 1);
8147 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8148
8149 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8150 MODULE_VERSION(t5nex, 1);
8151 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8152
8153 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8154 MODULE_VERSION(cxgbe, 1);
8155
8156 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8157 MODULE_VERSION(cxl, 1);