sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2011 Chelsio Communications, Inc.
   5  * All rights reserved.
   6  * Written by: Navdeep Parhar <np@FreeBSD.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_ddb.h"
  34 #include "opt_inet.h"
  35 #include "opt_inet6.h"
  36 #include "opt_ratelimit.h"
  37 #include "opt_rss.h"
  38
  39 #include <sys/param.h>
  40 #include <sys/conf.h>
  41 #include <sys/priv.h>
  42 #include <sys/kernel.h>
  43 #include <sys/bus.h>
  44 #include <sys/module.h>
  45 #include <sys/malloc.h>
  46 #include <sys/queue.h>
  47 #include <sys/taskqueue.h>
  48 #include <sys/pciio.h>
  49 #include <dev/pci/pcireg.h>
  50 #include <dev/pci/pcivar.h>
  51 #include <dev/pci/pci_private.h>
  52 #include <sys/firmware.h>
  53 #include <sys/sbuf.h>
  54 #include <sys/smp.h>
  55 #include <sys/socket.h>
  56 #include <sys/sockio.h>
  57 #include <sys/sysctl.h>
  58 #include <net/ethernet.h>
  59 #include <net/if.h>
  60 #include <net/if_types.h>
  61 #include <net/if_dl.h>
  62 #include <net/if_vlan_var.h>
  63 #ifdef RSS
  64 #include <net/rss_config.h>
  65 #endif
  66 #include <netinet/in.h>
  67 #include <netinet/ip.h>
  68 #if defined(__i386__) || defined(__amd64__)
  69 #include <machine/md_var.h>
  70 #include <machine/cputypes.h>
  71 #include <vm/vm.h>
  72 #include <vm/pmap.h>
  73 #endif
  74 #include <crypto/rijndael/rijndael.h>
  75 #ifdef DDB
  76 #include <ddb/ddb.h>
  77 #include <ddb/db_lex.h>
  78 #endif
  79
  80 #include "common/common.h"
  81 #include "common/t4_msg.h"
  82 #include "common/t4_regs.h"
  83 #include "common/t4_regs_values.h"
  84 #include "cudbg/cudbg.h"
  85 #include "t4_ioctl.h"
  86 #include "t4_l2t.h"
  87 #include "t4_mp_ring.h"
  88 #include "t4_if.h"
  89 #include "t4_smt.h"
  90
  91 /* T4 bus driver interface */
  92 static int t4_probe(device_t);
  93 static int t4_attach(device_t);
  94 static int t4_detach(device_t);
  95 static int t4_child_location_str(device_t, device_t, char *, size_t);
  96 static int t4_ready(device_t);
  97 static int t4_read_port_device(device_t, int, device_t *);
  98 static device_method_t t4_methods[] = {
  99         DEVMETHOD(device_probe,         t4_probe),
 100         DEVMETHOD(device_attach,        t4_attach),
 101         DEVMETHOD(device_detach,        t4_detach),
 102
 103         DEVMETHOD(bus_child_location_str, t4_child_location_str),
 104
 105         DEVMETHOD(t4_is_main_ready,     t4_ready),
 106         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
 107
 108         DEVMETHOD_END
 109 };
 110 static driver_t t4_driver = {
 111         "t4nex",
 112         t4_methods,
 113         sizeof(struct adapter)
 114 };
 115
 116
 117 /* T4 port (cxgbe) interface */
 118 static int cxgbe_probe(device_t);
 119 static int cxgbe_attach(device_t);
 120 static int cxgbe_detach(device_t);
 121 device_method_t cxgbe_methods[] = {
 122         DEVMETHOD(device_probe,         cxgbe_probe),
 123         DEVMETHOD(device_attach,        cxgbe_attach),
 124         DEVMETHOD(device_detach,        cxgbe_detach),
 125         { 0, 0 }
 126 };
 127 static driver_t cxgbe_driver = {
 128         "cxgbe",
 129         cxgbe_methods,
 130         sizeof(struct port_info)
 131 };
 132
 133 /* T4 VI (vcxgbe) interface */
 134 static int vcxgbe_probe(device_t);
 135 static int vcxgbe_attach(device_t);
 136 static int vcxgbe_detach(device_t);
 137 static device_method_t vcxgbe_methods[] = {
 138         DEVMETHOD(device_probe,         vcxgbe_probe),
 139         DEVMETHOD(device_attach,        vcxgbe_attach),
 140         DEVMETHOD(device_detach,        vcxgbe_detach),
 141         { 0, 0 }
 142 };
 143 static driver_t vcxgbe_driver = {
 144         "vcxgbe",
 145         vcxgbe_methods,
 146         sizeof(struct vi_info)
 147 };
 148
 149 static d_ioctl_t t4_ioctl;
 150
 151 static struct cdevsw t4_cdevsw = {
 152        .d_version = D_VERSION,
 153        .d_ioctl = t4_ioctl,
 154        .d_name = "t4nex",
 155 };
 156
 157 /* T5 bus driver interface */
 158 static int t5_probe(device_t);
 159 static device_method_t t5_methods[] = {
 160         DEVMETHOD(device_probe,         t5_probe),
 161         DEVMETHOD(device_attach,        t4_attach),
 162         DEVMETHOD(device_detach,        t4_detach),
 163
 164         DEVMETHOD(bus_child_location_str, t4_child_location_str),
 165
 166         DEVMETHOD(t4_is_main_ready,     t4_ready),
 167         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
 168
 169         DEVMETHOD_END
 170 };
 171 static driver_t t5_driver = {
 172         "t5nex",
 173         t5_methods,
 174         sizeof(struct adapter)
 175 };
 176
 177
 178 /* T5 port (cxl) interface */
 179 static driver_t cxl_driver = {
 180         "cxl",
 181         cxgbe_methods,
 182         sizeof(struct port_info)
 183 };
 184
 185 /* T5 VI (vcxl) interface */
 186 static driver_t vcxl_driver = {
 187         "vcxl",
 188         vcxgbe_methods,
 189         sizeof(struct vi_info)
 190 };
 191
 192 /* T6 bus driver interface */
 193 static int t6_probe(device_t);
 194 static device_method_t t6_methods[] = {
 195         DEVMETHOD(device_probe,         t6_probe),
 196         DEVMETHOD(device_attach,        t4_attach),
 197         DEVMETHOD(device_detach,        t4_detach),
 198
 199         DEVMETHOD(bus_child_location_str, t4_child_location_str),
 200
 201         DEVMETHOD(t4_is_main_ready,     t4_ready),
 202         DEVMETHOD(t4_read_port_device,  t4_read_port_device),
 203
 204         DEVMETHOD_END
 205 };
 206 static driver_t t6_driver = {
 207         "t6nex",
 208         t6_methods,
 209         sizeof(struct adapter)
 210 };
 211
 212
 213 /* T6 port (cc) interface */
 214 static driver_t cc_driver = {
 215         "cc",
 216         cxgbe_methods,
 217         sizeof(struct port_info)
 218 };
 219
 220 /* T6 VI (vcc) interface */
 221 static driver_t vcc_driver = {
 222         "vcc",
 223         vcxgbe_methods,
 224         sizeof(struct vi_info)
 225 };
 226
 227 /* ifnet interface */
 228 static void cxgbe_init(void *);
 229 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 230 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 231 static void cxgbe_qflush(struct ifnet *);
 232
 233 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 234
 235 /*
 236  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 237  * then ADAPTER_LOCK, then t4_uld_list_lock.
 238  */
 239 static struct sx t4_list_lock;
 240 SLIST_HEAD(, adapter) t4_list;
 241 #ifdef TCP_OFFLOAD
 242 static struct sx t4_uld_list_lock;
 243 SLIST_HEAD(, uld_info) t4_uld_list;
 244 #endif
 245
 246 /*
 247  * Tunables.  See tweak_tunables() too.
 248  *
 249  * Each tunable is set to a default value here if it's known at compile-time.
 250  * Otherwise it is set to -n as an indication to tweak_tunables() that it should
 251  * provide a reasonable default (upto n) when the driver is loaded.
 252  *
 253  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 254  * T5 are under hw.cxl.
 255  */
 256
 257 /*
 258  * Number of queues for tx and rx, NIC and offload.
 259  */
 260 #define NTXQ 16
 261 int t4_ntxq = -NTXQ;
 262 TUNABLE_INT("hw.cxgbe.ntxq", &t4_ntxq);
 263 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq);      /* Old name, undocumented */
 264
 265 #define NRXQ 8
 266 int t4_nrxq = -NRXQ;
 267 TUNABLE_INT("hw.cxgbe.nrxq", &t4_nrxq);
 268 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq);      /* Old name, undocumented */
 269
 270 #define NTXQ_VI 1
 271 static int t4_ntxq_vi = -NTXQ_VI;
 272 TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi);
 273
 274 #define NRXQ_VI 1
 275 static int t4_nrxq_vi = -NRXQ_VI;
 276 TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi);
 277
 278 static int t4_rsrv_noflowq = 0;
 279 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 280
 281 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
 282 #define NOFLDTXQ 8
 283 static int t4_nofldtxq = -NOFLDTXQ;
 284 TUNABLE_INT("hw.cxgbe.nofldtxq", &t4_nofldtxq);
 285
 286 #define NOFLDRXQ 2
 287 static int t4_nofldrxq = -NOFLDRXQ;
 288 TUNABLE_INT("hw.cxgbe.nofldrxq", &t4_nofldrxq);
 289
 290 #define NOFLDTXQ_VI 1
 291 static int t4_nofldtxq_vi = -NOFLDTXQ_VI;
 292 TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi);
 293
 294 #define NOFLDRXQ_VI 1
 295 static int t4_nofldrxq_vi = -NOFLDRXQ_VI;
 296 TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi);
 297
 298 #define TMR_IDX_OFLD 1
 299 int t4_tmr_idx_ofld = TMR_IDX_OFLD;
 300 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_ofld", &t4_tmr_idx_ofld);
 301
 302 #define PKTC_IDX_OFLD (-1)
 303 int t4_pktc_idx_ofld = PKTC_IDX_OFLD;
 304 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_ofld", &t4_pktc_idx_ofld);
 305
 306 /* 0 means chip/fw default, non-zero number is value in microseconds */
 307 static u_long t4_toe_keepalive_idle = 0;
 308 TUNABLE_ULONG("hw.cxgbe.toe.keepalive_idle", &t4_toe_keepalive_idle);
 309
 310 /* 0 means chip/fw default, non-zero number is value in microseconds */
 311 static u_long t4_toe_keepalive_interval = 0;
 312 TUNABLE_ULONG("hw.cxgbe.toe.keepalive_interval", &t4_toe_keepalive_interval);
 313
 314 /* 0 means chip/fw default, non-zero number is # of keepalives before abort */
 315 static int t4_toe_keepalive_count = 0;
 316 TUNABLE_INT("hw.cxgbe.toe.keepalive_count", &t4_toe_keepalive_count);
 317
 318 /* 0 means chip/fw default, non-zero number is value in microseconds */
 319 static u_long t4_toe_rexmt_min = 0;
 320 TUNABLE_ULONG("hw.cxgbe.toe.rexmt_min", &t4_toe_rexmt_min);
 321
 322 /* 0 means chip/fw default, non-zero number is value in microseconds */
 323 static u_long t4_toe_rexmt_max = 0;
 324 TUNABLE_ULONG("hw.cxgbe.toe.rexmt_max", &t4_toe_rexmt_max);
 325
 326 /* 0 means chip/fw default, non-zero number is # of rexmt before abort */
 327 static int t4_toe_rexmt_count = 0;
 328 TUNABLE_INT("hw.cxgbe.toe.rexmt_count", &t4_toe_rexmt_count);
 329
 330 /* -1 means chip/fw default, other values are raw backoff values to use */
 331 static int t4_toe_rexmt_backoff[16] = {
 332         -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
 333 };
 334 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.0", &t4_toe_rexmt_backoff[0]);
 335 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.1", &t4_toe_rexmt_backoff[1]);
 336 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.2", &t4_toe_rexmt_backoff[2]);
 337 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.3", &t4_toe_rexmt_backoff[3]);
 338 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.4", &t4_toe_rexmt_backoff[4]);
 339 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.5", &t4_toe_rexmt_backoff[5]);
 340 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.6", &t4_toe_rexmt_backoff[6]);
 341 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.7", &t4_toe_rexmt_backoff[7]);
 342 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.8", &t4_toe_rexmt_backoff[8]);
 343 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.9", &t4_toe_rexmt_backoff[9]);
 344 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.10", &t4_toe_rexmt_backoff[10]);
 345 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.11", &t4_toe_rexmt_backoff[11]);
 346 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.12", &t4_toe_rexmt_backoff[12]);
 347 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.13", &t4_toe_rexmt_backoff[13]);
 348 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.14", &t4_toe_rexmt_backoff[14]);
 349 TUNABLE_INT("hw.cxgbe.toe.rexmt_backoff.15", &t4_toe_rexmt_backoff[15]);
 350 #endif
 351
 352 #ifdef DEV_NETMAP
 353 #define NNMTXQ_VI 2
 354 static int t4_nnmtxq_vi = -NNMTXQ_VI;
 355 TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi);
 356
 357 #define NNMRXQ_VI 2
 358 static int t4_nnmrxq_vi = -NNMRXQ_VI;
 359 TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi);
 360 #endif
 361
 362 /*
 363  * Holdoff parameters for ports.
 364  */
 365 #define TMR_IDX 1
 366 int t4_tmr_idx = TMR_IDX;
 367 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx", &t4_tmr_idx);
 368 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx);     /* Old name */
 369
 370 #define PKTC_IDX (-1)
 371 int t4_pktc_idx = PKTC_IDX;
 372 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx", &t4_pktc_idx);
 373 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx);     /* Old name */
 374
 375 /*
 376  * Size (# of entries) of each tx and rx queue.
 377  */
 378 unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 379 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 380
 381 unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 382 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 383
 384 /*
 385  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 386  */
 387 int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 388 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 389
 390 /*
 391  * Configuration file.  All the _CF names here are special.
 392  */
 393 #define DEFAULT_CF      "default"
 394 #define BUILTIN_CF      "built-in"
 395 #define FLASH_CF        "flash"
 396 #define UWIRE_CF        "uwire"
 397 #define FPGA_CF         "fpga"
 398 static char t4_cfg_file[32] = DEFAULT_CF;
 399 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 400
 401 /*
 402  * PAUSE settings (bit 0, 1, 2 = rx_pause, tx_pause, pause_autoneg respectively).
 403  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 404  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 405  *            mark or when signalled to do so, 0 to never emit PAUSE.
 406  * pause_autoneg = 1 means PAUSE will be negotiated if possible and the
 407  *                 negotiated settings will override rx_pause/tx_pause.
 408  *                 Otherwise rx_pause/tx_pause are applied forcibly.
 409  */
 410 static int t4_pause_settings = PAUSE_RX | PAUSE_TX | PAUSE_AUTONEG;
 411 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 412
 413 /*
 414  * Forward Error Correction settings (bit 0, 1 = RS, BASER respectively).
 415  * -1 to run with the firmware default.  Same as FEC_AUTO (bit 5)
 416  *  0 to disable FEC.
 417  */
 418 static int t4_fec = -1;
 419 TUNABLE_INT("hw.cxgbe.fec", &t4_fec);
 420
 421 /*
 422  * Link autonegotiation.
 423  * -1 to run with the firmware default.
 424  *  0 to disable.
 425  *  1 to enable.
 426  */
 427 static int t4_autoneg = -1;
 428 TUNABLE_INT("hw.cxgbe.autoneg", &t4_autoneg);
 429
 430 /*
 431  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 432  * encouraged respectively).
 433  */
 434 static unsigned int t4_fw_install = 1;
 435 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 436
 437 /*
 438  * ASIC features that will be used.  Disable the ones you don't want so that the
 439  * chip resources aren't wasted on features that will not be used.
 440  */
 441 static int t4_nbmcaps_allowed = 0;
 442 TUNABLE_INT("hw.cxgbe.nbmcaps_allowed", &t4_nbmcaps_allowed);
 443
 444 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 445 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 446
 447 static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS |
 448     FW_CAPS_CONFIG_SWITCH_EGRESS;
 449 TUNABLE_INT("hw.cxgbe.switchcaps_allowed", &t4_switchcaps_allowed);
 450
 451 #ifdef RATELIMIT
 452 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC |
 453         FW_CAPS_CONFIG_NIC_HASHFILTER | FW_CAPS_CONFIG_NIC_ETHOFLD;
 454 #else
 455 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC |
 456         FW_CAPS_CONFIG_NIC_HASHFILTER;
 457 #endif
 458 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 459
 460 static int t4_toecaps_allowed = -1;
 461 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 462
 463 static int t4_rdmacaps_allowed = -1;
 464 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 465
 466 static int t4_cryptocaps_allowed = -1;
 467 TUNABLE_INT("hw.cxgbe.cryptocaps_allowed", &t4_cryptocaps_allowed);
 468
 469 static int t4_iscsicaps_allowed = -1;
 470 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 471
 472 static int t4_fcoecaps_allowed = 0;
 473 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 474
 475 static int t5_write_combine = 0;
 476 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 477
 478 static int t4_num_vis = 1;
 479 TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis);
 480 /*
 481  * PCIe Relaxed Ordering.
 482  * -1: driver should figure out a good value.
 483  * 0: disable RO.
 484  * 1: enable RO.
 485  * 2: leave RO alone.
 486  */
 487 static int pcie_relaxed_ordering = -1;
 488 TUNABLE_INT("hw.cxgbe.pcie_relaxed_ordering", &pcie_relaxed_ordering);
 489
 490 static int t4_panic_on_fatal_err = 0;
 491 TUNABLE_INT("hw.cxgbe.panic_on_fatal_err", &t4_panic_on_fatal_err);
 492
 493 #ifdef TCP_OFFLOAD
 494 /*
 495  * TOE tunables.
 496  */
 497 static int t4_cop_managed_offloading = 0;
 498 TUNABLE_INT("hw.cxgbe.cop_managed_offloading", &t4_cop_managed_offloading);
 499 #endif
 500
 501 /* Functions used by VIs to obtain unique MAC addresses for each VI. */
 502 static int vi_mac_funcs[] = {
 503         FW_VI_FUNC_ETH,
 504         FW_VI_FUNC_OFLD,
 505         FW_VI_FUNC_IWARP,
 506         FW_VI_FUNC_OPENISCSI,
 507         FW_VI_FUNC_OPENFCOE,
 508         FW_VI_FUNC_FOISCSI,
 509         FW_VI_FUNC_FOFCOE,
 510 };
 511
 512 struct intrs_and_queues {
 513         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 514         uint16_t num_vis;       /* number of VIs for each port */
 515         uint16_t nirq;          /* Total # of vectors */
 516         uint16_t ntxq;          /* # of NIC txq's for each port */
 517         uint16_t nrxq;          /* # of NIC rxq's for each port */
 518         uint16_t nofldtxq;      /* # of TOE/ETHOFLD txq's for each port */
 519         uint16_t nofldrxq;      /* # of TOE rxq's for each port */
 520
 521         /* The vcxgbe/vcxl interfaces use these and not the ones above. */
 522         uint16_t ntxq_vi;       /* # of NIC txq's */
 523         uint16_t nrxq_vi;       /* # of NIC rxq's */
 524         uint16_t nofldtxq_vi;   /* # of TOE txq's */
 525         uint16_t nofldrxq_vi;   /* # of TOE rxq's */
 526         uint16_t nnmtxq_vi;     /* # of netmap txq's */
 527         uint16_t nnmrxq_vi;     /* # of netmap rxq's */
 528 };
 529
 530 static void setup_memwin(struct adapter *);
 531 static void position_memwin(struct adapter *, int, uint32_t);
 532 static int validate_mem_range(struct adapter *, uint32_t, int);
 533 static int fwmtype_to_hwmtype(int);
 534 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 535     uint32_t *);
 536 static int fixup_devlog_params(struct adapter *);
 537 static int cfg_itype_and_nqueues(struct adapter *, struct intrs_and_queues *);
 538 static int prep_firmware(struct adapter *);
 539 static int partition_resources(struct adapter *, const struct firmware *,
 540     const char *);
 541 static int get_params__pre_init(struct adapter *);
 542 static int get_params__post_init(struct adapter *);
 543 static int set_params__post_init(struct adapter *);
 544 static void t4_set_desc(struct adapter *);
 545 static bool fixed_ifmedia(struct port_info *);
 546 static void build_medialist(struct port_info *);
 547 static void init_link_config(struct port_info *);
 548 static int fixup_link_config(struct port_info *);
 549 static int apply_link_config(struct port_info *);
 550 static int cxgbe_init_synchronized(struct vi_info *);
 551 static int cxgbe_uninit_synchronized(struct vi_info *);
 552 static void quiesce_txq(struct adapter *, struct sge_txq *);
 553 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 554 static void quiesce_iq(struct adapter *, struct sge_iq *);
 555 static void quiesce_fl(struct adapter *, struct sge_fl *);
 556 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 557     driver_intr_t *, void *, char *);
 558 static int t4_free_irq(struct adapter *, struct irq *);
 559 static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 560 static void vi_refresh_stats(struct adapter *, struct vi_info *);
 561 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 562 static void cxgbe_tick(void *);
 563 static void cxgbe_sysctls(struct port_info *);
 564 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 565 static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS);
 566 static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS);
 567 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 568 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 569 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 570 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 571 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 572 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 573 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 574 static int sysctl_fec(SYSCTL_HANDLER_ARGS);
 575 static int sysctl_autoneg(SYSCTL_HANDLER_ARGS);
 576 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 577 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 578 static int sysctl_loadavg(SYSCTL_HANDLER_ARGS);
 579 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 580 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 581 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 582 static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS);
 583 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 584 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 585 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 586 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 587 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 588 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 589 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 590 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 591 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 592 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 593 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 594 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 595 static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS);
 596 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 597 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 598 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 599 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 600 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 601 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 602 static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS);
 603 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 604 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 605 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 606 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 607 static int sysctl_cpus(SYSCTL_HANDLER_ARGS);
 608 #ifdef TCP_OFFLOAD
 609 static int sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS);
 610 static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
 611 static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
 612 static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
 613 static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS);
 614 static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS);
 615 static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS);
 616 static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS);
 617 #endif
 618 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 619 static int load_fw(struct adapter *, struct t4_data *);
 620 static int load_cfg(struct adapter *, struct t4_data *);
 621 static int load_boot(struct adapter *, struct t4_bootrom *);
 622 static int load_bootcfg(struct adapter *, struct t4_data *);
 623 static int cudbg_dump(struct adapter *, struct t4_cudbg_dump *);
 624 static void free_offload_policy(struct t4_offload_policy *);
 625 static int set_offload_policy(struct adapter *, struct t4_offload_policy *);
 626 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 627 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 628 #ifdef TCP_OFFLOAD
 629 static int toe_capability(struct vi_info *, int);
 630 #endif
 631 static int mod_event(module_t, int, void *);
 632 static int notify_siblings(device_t, int);
 633
 634 struct {
 635         uint16_t device;
 636         char *desc;
 637 } t4_pciids[] = {
 638         {0xa000, "Chelsio Terminator 4 FPGA"},
 639         {0x4400, "Chelsio T440-dbg"},
 640         {0x4401, "Chelsio T420-CR"},
 641         {0x4402, "Chelsio T422-CR"},
 642         {0x4403, "Chelsio T440-CR"},
 643         {0x4404, "Chelsio T420-BCH"},
 644         {0x4405, "Chelsio T440-BCH"},
 645         {0x4406, "Chelsio T440-CH"},
 646         {0x4407, "Chelsio T420-SO"},
 647         {0x4408, "Chelsio T420-CX"},
 648         {0x4409, "Chelsio T420-BT"},
 649         {0x440a, "Chelsio T404-BT"},
 650         {0x440e, "Chelsio T440-LP-CR"},
 651 }, t5_pciids[] = {
 652         {0xb000, "Chelsio Terminator 5 FPGA"},
 653         {0x5400, "Chelsio T580-dbg"},
 654         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 655         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 656         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 657         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 658         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 659         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 660         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 661         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 662         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 663         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 664         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 665         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 666         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 667         {0x5418,  "Chelsio T540-BT"},           /* 4 x 10GBaseT */
 668         {0x5419,  "Chelsio T540-LP-BT"},        /* 4 x 10GBaseT */
 669         {0x541a,  "Chelsio T540-SO-BT"},        /* 4 x 10GBaseT, nomem */
 670         {0x541b,  "Chelsio T540-SO-CR"},        /* 4 x 10G, nomem */
 671 }, t6_pciids[] = {
 672         {0xc006, "Chelsio Terminator 6 FPGA"},  /* T6 PE10K6 FPGA (PF0) */
 673         {0x6400, "Chelsio T6-DBG-25"},          /* 2 x 10/25G, debug */
 674         {0x6401, "Chelsio T6225-CR"},           /* 2 x 10/25G */
 675         {0x6402, "Chelsio T6225-SO-CR"},        /* 2 x 10/25G, nomem */
 676         {0x6403, "Chelsio T6425-CR"},           /* 4 x 10/25G */
 677         {0x6404, "Chelsio T6425-SO-CR"},        /* 4 x 10/25G, nomem */
 678         {0x6405, "Chelsio T6225-OCP-SO"},       /* 2 x 10/25G, nomem */
 679         {0x6406, "Chelsio T62100-OCP-SO"},      /* 2 x 40/50/100G, nomem */
 680         {0x6407, "Chelsio T62100-LP-CR"},       /* 2 x 40/50/100G */
 681         {0x6408, "Chelsio T62100-SO-CR"},       /* 2 x 40/50/100G, nomem */
 682         {0x6409, "Chelsio T6210-BT"},           /* 2 x 10GBASE-T */
 683         {0x640d, "Chelsio T62100-CR"},          /* 2 x 40/50/100G */
 684         {0x6410, "Chelsio T6-DBG-100"},         /* 2 x 40/50/100G, debug */
 685         {0x6411, "Chelsio T6225-LL-CR"},        /* 2 x 10/25G */
 686         {0x6414, "Chelsio T61100-OCP-SO"},      /* 1 x 40/50/100G, nomem */
 687         {0x6415, "Chelsio T6201-BT"},           /* 2 x 1000BASE-T */
 688
 689         /* Custom */
 690         {0x6480, "Custom T6225-CR"},
 691         {0x6481, "Custom T62100-CR"},
 692         {0x6482, "Custom T6225-CR"},
 693         {0x6483, "Custom T62100-CR"},
 694         {0x6484, "Custom T64100-CR"},
 695         {0x6485, "Custom T6240-SO"},
 696         {0x6486, "Custom T6225-SO-CR"},
 697         {0x6487, "Custom T6225-CR"},
 698 };
 699
 700 #ifdef TCP_OFFLOAD
 701 /*
 702  * service_iq_fl() has an iq and needs the fl.  Offset of fl from the iq should
 703  * be exactly the same for both rxq and ofld_rxq.
 704  */
 705 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 706 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 707 #endif
 708 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 709
 710 static int
 711 t4_probe(device_t dev)
 712 {
 713         int i;
 714         uint16_t v = pci_get_vendor(dev);
 715         uint16_t d = pci_get_device(dev);
 716         uint8_t f = pci_get_function(dev);
 717
 718         if (v != PCI_VENDOR_ID_CHELSIO)
 719                 return (ENXIO);
 720
 721         /* Attach only to PF0 of the FPGA */
 722         if (d == 0xa000 && f != 0)
 723                 return (ENXIO);
 724
 725         for (i = 0; i < nitems(t4_pciids); i++) {
 726                 if (d == t4_pciids[i].device) {
 727                         device_set_desc(dev, t4_pciids[i].desc);
 728                         return (BUS_PROBE_DEFAULT);
 729                 }
 730         }
 731
 732         return (ENXIO);
 733 }
 734
 735 static int
 736 t5_probe(device_t dev)
 737 {
 738         int i;
 739         uint16_t v = pci_get_vendor(dev);
 740         uint16_t d = pci_get_device(dev);
 741         uint8_t f = pci_get_function(dev);
 742
 743         if (v != PCI_VENDOR_ID_CHELSIO)
 744                 return (ENXIO);
 745
 746         /* Attach only to PF0 of the FPGA */
 747         if (d == 0xb000 && f != 0)
 748                 return (ENXIO);
 749
 750         for (i = 0; i < nitems(t5_pciids); i++) {
 751                 if (d == t5_pciids[i].device) {
 752                         device_set_desc(dev, t5_pciids[i].desc);
 753                         return (BUS_PROBE_DEFAULT);
 754                 }
 755         }
 756
 757         return (ENXIO);
 758 }
 759
 760 static int
 761 t6_probe(device_t dev)
 762 {
 763         int i;
 764         uint16_t v = pci_get_vendor(dev);
 765         uint16_t d = pci_get_device(dev);
 766
 767         if (v != PCI_VENDOR_ID_CHELSIO)
 768                 return (ENXIO);
 769
 770         for (i = 0; i < nitems(t6_pciids); i++) {
 771                 if (d == t6_pciids[i].device) {
 772                         device_set_desc(dev, t6_pciids[i].desc);
 773                         return (BUS_PROBE_DEFAULT);
 774                 }
 775         }
 776
 777         return (ENXIO);
 778 }
 779
 780 static void
 781 t5_attribute_workaround(device_t dev)
 782 {
 783         device_t root_port;
 784         uint32_t v;
 785
 786         /*
 787          * The T5 chips do not properly echo the No Snoop and Relaxed
 788          * Ordering attributes when replying to a TLP from a Root
 789          * Port.  As a workaround, find the parent Root Port and
 790          * disable No Snoop and Relaxed Ordering.  Note that this
 791          * affects all devices under this root port.
 792          */
 793         root_port = pci_find_pcie_root_port(dev);
 794         if (root_port == NULL) {
 795                 device_printf(dev, "Unable to find parent root port\n");
 796                 return;
 797         }
 798
 799         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 800             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 801         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 802             0)
 803                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 804                     device_get_nameunit(root_port));
 805 }
 806
 807 static const struct devnames devnames[] = {
 808         {
 809                 .nexus_name = "t4nex",
 810                 .ifnet_name = "cxgbe",
 811                 .vi_ifnet_name = "vcxgbe",
 812                 .pf03_drv_name = "t4iov",
 813                 .vf_nexus_name = "t4vf",
 814                 .vf_ifnet_name = "cxgbev"
 815         }, {
 816                 .nexus_name = "t5nex",
 817                 .ifnet_name = "cxl",
 818                 .vi_ifnet_name = "vcxl",
 819                 .pf03_drv_name = "t5iov",
 820                 .vf_nexus_name = "t5vf",
 821                 .vf_ifnet_name = "cxlv"
 822         }, {
 823                 .nexus_name = "t6nex",
 824                 .ifnet_name = "cc",
 825                 .vi_ifnet_name = "vcc",
 826                 .pf03_drv_name = "t6iov",
 827                 .vf_nexus_name = "t6vf",
 828                 .vf_ifnet_name = "ccv"
 829         }
 830 };
 831
 832 void
 833 t4_init_devnames(struct adapter *sc)
 834 {
 835         int id;
 836
 837         id = chip_id(sc);
 838         if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames))
 839                 sc->names = &devnames[id - CHELSIO_T4];
 840         else {
 841                 device_printf(sc->dev, "chip id %d is not supported.\n", id);
 842                 sc->names = NULL;
 843         }
 844 }
 845
 846 static int
 847 t4_ifnet_unit(struct adapter *sc, struct port_info *pi)
 848 {
 849         const char *parent, *name;
 850         long value;
 851         int line, unit;
 852
 853         line = 0;
 854         parent = device_get_nameunit(sc->dev);
 855         name = sc->names->ifnet_name;
 856         while (resource_find_dev(&line, name, &unit, "at", parent) == 0) {
 857                 if (resource_long_value(name, unit, "port", &value) == 0 &&
 858                     value == pi->port_id)
 859                         return (unit);
 860         }
 861         return (-1);
 862 }
 863
 864 static int
 865 t4_attach(device_t dev)
 866 {
 867         struct adapter *sc;
 868         int rc = 0, i, j, rqidx, tqidx, nports;
 869         struct make_dev_args mda;
 870         struct intrs_and_queues iaq;
 871         struct sge *s;
 872         uint32_t *buf;
 873 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
 874         int ofld_tqidx;
 875 #endif
 876 #ifdef TCP_OFFLOAD
 877         int ofld_rqidx;
 878 #endif
 879 #ifdef DEV_NETMAP
 880         int nm_rqidx, nm_tqidx;
 881 #endif
 882         int num_vis;
 883
 884         sc = device_get_softc(dev);
 885         sc->dev = dev;
 886         TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
 887
 888         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 889                 t5_attribute_workaround(dev);
 890         pci_enable_busmaster(dev);
 891         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 892                 uint32_t v;
 893
 894                 pci_set_max_read_req(dev, 4096);
 895                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 896                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 897                 if (pcie_relaxed_ordering == 0 &&
 898                     (v & PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) {
 899                         v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE;
 900                         pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 901                 } else if (pcie_relaxed_ordering == 1 &&
 902                     (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) {
 903                         v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 904                         pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 905                 }
 906         }
 907
 908         sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
 909         sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
 910         sc->traceq = -1;
 911         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 912         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 913             device_get_nameunit(dev));
 914
 915         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 916             device_get_nameunit(dev));
 917         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 918         t4_add_adapter(sc);
 919
 920         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 921         TAILQ_INIT(&sc->sfl);
 922         callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
 923
 924         mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
 925
 926         sc->policy = NULL;
 927         rw_init(&sc->policy_lock, "connection offload policy");
 928
 929         rc = t4_map_bars_0_and_4(sc);
 930         if (rc != 0)
 931                 goto done; /* error message displayed already */
 932
 933         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 934
 935         /* Prepare the adapter for operation. */
 936         buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK);
 937         rc = -t4_prep_adapter(sc, buf);
 938         free(buf, M_CXGBE);
 939         if (rc != 0) {
 940                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 941                 goto done;
 942         }
 943
 944         /*
 945          * This is the real PF# to which we're attaching.  Works from within PCI
 946          * passthrough environments too, where pci_get_function() could return a
 947          * different PF# depending on the passthrough configuration.  We need to
 948          * use the real PF# in all our communication with the firmware.
 949          */
 950         j = t4_read_reg(sc, A_PL_WHOAMI);
 951         sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
 952         sc->mbox = sc->pf;
 953
 954         t4_init_devnames(sc);
 955         if (sc->names == NULL) {
 956                 rc = ENOTSUP;
 957                 goto done; /* error message displayed already */
 958         }
 959
 960         /*
 961          * Do this really early, with the memory windows set up even before the
 962          * character device.  The userland tool's register i/o and mem read
 963          * will work even in "recovery mode".
 964          */
 965         setup_memwin(sc);
 966         if (t4_init_devlog_params(sc, 0) == 0)
 967                 fixup_devlog_params(sc);
 968         make_dev_args_init(&mda);
 969         mda.mda_devsw = &t4_cdevsw;
 970         mda.mda_uid = UID_ROOT;
 971         mda.mda_gid = GID_WHEEL;
 972         mda.mda_mode = 0600;
 973         mda.mda_si_drv1 = sc;
 974         rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
 975         if (rc != 0)
 976                 device_printf(dev, "failed to create nexus char device: %d.\n",
 977                     rc);
 978
 979         /* Go no further if recovery mode has been requested. */
 980         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 981                 device_printf(dev, "recovery mode.\n");
 982                 goto done;
 983         }
 984
 985 #if defined(__i386__)
 986         if ((cpu_feature & CPUID_CX8) == 0) {
 987                 device_printf(dev, "64 bit atomics not available.\n");
 988                 rc = ENOTSUP;
 989                 goto done;
 990         }
 991 #endif
 992
 993         /* Prepare the firmware for operation */
 994         rc = prep_firmware(sc);
 995         if (rc != 0)
 996                 goto done; /* error message displayed already */
 997
 998         rc = get_params__post_init(sc);
 999         if (rc != 0)
1000                 goto done; /* error message displayed already */
1001
1002         rc = set_params__post_init(sc);
1003         if (rc != 0)
1004                 goto done; /* error message displayed already */
1005
1006         rc = t4_map_bar_2(sc);
1007         if (rc != 0)
1008                 goto done; /* error message displayed already */
1009
1010         rc = t4_create_dma_tag(sc);
1011         if (rc != 0)
1012                 goto done; /* error message displayed already */
1013
1014         /*
1015          * First pass over all the ports - allocate VIs and initialize some
1016          * basic parameters like mac address, port type, etc.
1017          */
1018         for_each_port(sc, i) {
1019                 struct port_info *pi;
1020
1021                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
1022                 sc->port[i] = pi;
1023
1024                 /* These must be set before t4_port_init */
1025                 pi->adapter = sc;
1026                 pi->port_id = i;
1027                 /*
1028                  * XXX: vi[0] is special so we can't delay this allocation until
1029                  * pi->nvi's final value is known.
1030                  */
1031                 pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE,
1032                     M_ZERO | M_WAITOK);
1033
1034                 /*
1035                  * Allocate the "main" VI and initialize parameters
1036                  * like mac addr.
1037                  */
1038                 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
1039                 if (rc != 0) {
1040                         device_printf(dev, "unable to initialize port %d: %d\n",
1041                             i, rc);
1042                         free(pi->vi, M_CXGBE);
1043                         free(pi, M_CXGBE);
1044                         sc->port[i] = NULL;
1045                         goto done;
1046                 }
1047
1048                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
1049                     device_get_nameunit(dev), i);
1050                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
1051                 sc->chan_map[pi->tx_chan] = i;
1052
1053                 /* All VIs on this port share this media. */
1054                 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1055                     cxgbe_media_status);
1056
1057                 PORT_LOCK(pi);
1058                 init_link_config(pi);
1059                 fixup_link_config(pi);
1060                 build_medialist(pi);
1061                 if (fixed_ifmedia(pi))
1062                         pi->flags |= FIXED_IFMEDIA;
1063                 PORT_UNLOCK(pi);
1064
1065                 pi->dev = device_add_child(dev, sc->names->ifnet_name,
1066                     t4_ifnet_unit(sc, pi));
1067                 if (pi->dev == NULL) {
1068                         device_printf(dev,
1069                             "failed to add device for port %d.\n", i);
1070                         rc = ENXIO;
1071                         goto done;
1072                 }
1073                 pi->vi[0].dev = pi->dev;
1074                 device_set_softc(pi->dev, pi);
1075         }
1076
1077         /*
1078          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
1079          */
1080         nports = sc->params.nports;
1081         rc = cfg_itype_and_nqueues(sc, &iaq);
1082         if (rc != 0)
1083                 goto done; /* error message displayed already */
1084
1085         num_vis = iaq.num_vis;
1086         sc->intr_type = iaq.intr_type;
1087         sc->intr_count = iaq.nirq;
1088
1089         s = &sc->sge;
1090         s->nrxq = nports * iaq.nrxq;
1091         s->ntxq = nports * iaq.ntxq;
1092         if (num_vis > 1) {
1093                 s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi;
1094                 s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi;
1095         }
1096         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
1097         s->neq += nports;               /* ctrl queues: 1 per port */
1098         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
1099 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1100         if (is_offload(sc) || is_ethoffload(sc)) {
1101                 s->nofldtxq = nports * iaq.nofldtxq;
1102                 if (num_vis > 1)
1103                         s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi;
1104                 s->neq += s->nofldtxq;
1105
1106                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
1107                     M_CXGBE, M_ZERO | M_WAITOK);
1108         }
1109 #endif
1110 #ifdef TCP_OFFLOAD
1111         if (is_offload(sc)) {
1112                 s->nofldrxq = nports * iaq.nofldrxq;
1113                 if (num_vis > 1)
1114                         s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi;
1115                 s->neq += s->nofldrxq;  /* free list */
1116                 s->niq += s->nofldrxq;
1117
1118                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
1119                     M_CXGBE, M_ZERO | M_WAITOK);
1120         }
1121 #endif
1122 #ifdef DEV_NETMAP
1123         if (num_vis > 1) {
1124                 s->nnmrxq = nports * (num_vis - 1) * iaq.nnmrxq_vi;
1125                 s->nnmtxq = nports * (num_vis - 1) * iaq.nnmtxq_vi;
1126         }
1127         s->neq += s->nnmtxq + s->nnmrxq;
1128         s->niq += s->nnmrxq;
1129
1130         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
1131             M_CXGBE, M_ZERO | M_WAITOK);
1132         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
1133             M_CXGBE, M_ZERO | M_WAITOK);
1134 #endif
1135
1136         s->ctrlq = malloc(nports * sizeof(struct sge_wrq), M_CXGBE,
1137             M_ZERO | M_WAITOK);
1138         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
1139             M_ZERO | M_WAITOK);
1140         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
1141             M_ZERO | M_WAITOK);
1142         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
1143             M_ZERO | M_WAITOK);
1144         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
1145             M_ZERO | M_WAITOK);
1146
1147         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
1148             M_ZERO | M_WAITOK);
1149
1150         t4_init_l2t(sc, M_WAITOK);
1151         t4_init_smt(sc, M_WAITOK);
1152         t4_init_tx_sched(sc);
1153 #ifdef RATELIMIT
1154         t4_init_etid_table(sc);
1155 #endif
1156
1157         /*
1158          * Second pass over the ports.  This time we know the number of rx and
1159          * tx queues that each port should get.
1160          */
1161         rqidx = tqidx = 0;
1162 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1163         ofld_tqidx = 0;
1164 #endif
1165 #ifdef TCP_OFFLOAD
1166         ofld_rqidx = 0;
1167 #endif
1168 #ifdef DEV_NETMAP
1169         nm_rqidx = nm_tqidx = 0;
1170 #endif
1171         for_each_port(sc, i) {
1172                 struct port_info *pi = sc->port[i];
1173                 struct vi_info *vi;
1174
1175                 if (pi == NULL)
1176                         continue;
1177
1178                 pi->nvi = num_vis;
1179                 for_each_vi(pi, j, vi) {
1180                         vi->pi = pi;
1181                         vi->qsize_rxq = t4_qsize_rxq;
1182                         vi->qsize_txq = t4_qsize_txq;
1183
1184                         vi->first_rxq = rqidx;
1185                         vi->first_txq = tqidx;
1186                         vi->tmr_idx = t4_tmr_idx;
1187                         vi->pktc_idx = t4_pktc_idx;
1188                         vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi;
1189                         vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi;
1190
1191                         rqidx += vi->nrxq;
1192                         tqidx += vi->ntxq;
1193
1194                         if (j == 0 && vi->ntxq > 1)
1195                                 vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0;
1196                         else
1197                                 vi->rsrv_noflowq = 0;
1198
1199 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1200                         vi->first_ofld_txq = ofld_tqidx;
1201                         vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi;
1202                         ofld_tqidx += vi->nofldtxq;
1203 #endif
1204 #ifdef TCP_OFFLOAD
1205                         vi->ofld_tmr_idx = t4_tmr_idx_ofld;
1206                         vi->ofld_pktc_idx = t4_pktc_idx_ofld;
1207                         vi->first_ofld_rxq = ofld_rqidx;
1208                         vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi;
1209
1210                         ofld_rqidx += vi->nofldrxq;
1211 #endif
1212 #ifdef DEV_NETMAP
1213                         if (j > 0) {
1214                                 vi->first_nm_rxq = nm_rqidx;
1215                                 vi->first_nm_txq = nm_tqidx;
1216                                 vi->nnmrxq = iaq.nnmrxq_vi;
1217                                 vi->nnmtxq = iaq.nnmtxq_vi;
1218                                 nm_rqidx += vi->nnmrxq;
1219                                 nm_tqidx += vi->nnmtxq;
1220                         }
1221 #endif
1222                 }
1223         }
1224
1225         rc = t4_setup_intr_handlers(sc);
1226         if (rc != 0) {
1227                 device_printf(dev,
1228                     "failed to setup interrupt handlers: %d\n", rc);
1229                 goto done;
1230         }
1231
1232         rc = bus_generic_probe(dev);
1233         if (rc != 0) {
1234                 device_printf(dev, "failed to probe child drivers: %d\n", rc);
1235                 goto done;
1236         }
1237
1238         /*
1239          * Ensure thread-safe mailbox access (in debug builds).
1240          *
1241          * So far this was the only thread accessing the mailbox but various
1242          * ifnets and sysctls are about to be created and their handlers/ioctls
1243          * will access the mailbox from different threads.
1244          */
1245         sc->flags |= CHK_MBOX_ACCESS;
1246
1247         rc = bus_generic_attach(dev);
1248         if (rc != 0) {
1249                 device_printf(dev,
1250                     "failed to attach all child ports: %d\n", rc);
1251                 goto done;
1252         }
1253
1254         device_printf(dev,
1255             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
1256             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1257             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1258             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1259             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1260
1261         t4_set_desc(sc);
1262
1263         notify_siblings(dev, 0);
1264
1265 done:
1266         if (rc != 0 && sc->cdev) {
1267                 /* cdev was created and so cxgbetool works; recover that way. */
1268                 device_printf(dev,
1269                     "error during attach, adapter is now in recovery mode.\n");
1270                 rc = 0;
1271         }
1272
1273         if (rc != 0)
1274                 t4_detach_common(dev);
1275         else
1276                 t4_sysctls(sc);
1277
1278         return (rc);
1279 }
1280
1281 static int
1282 t4_child_location_str(device_t bus, device_t dev, char *buf, size_t buflen)
1283 {
1284         struct port_info *pi;
1285
1286         pi = device_get_softc(dev);
1287         snprintf(buf, buflen, "port=%d", pi->port_id);
1288         return (0);
1289 }
1290
1291 static int
1292 t4_ready(device_t dev)
1293 {
1294         struct adapter *sc;
1295
1296         sc = device_get_softc(dev);
1297         if (sc->flags & FW_OK)
1298                 return (0);
1299         return (ENXIO);
1300 }
1301
1302 static int
1303 t4_read_port_device(device_t dev, int port, device_t *child)
1304 {
1305         struct adapter *sc;
1306         struct port_info *pi;
1307
1308         sc = device_get_softc(dev);
1309         if (port < 0 || port >= MAX_NPORTS)
1310                 return (EINVAL);
1311         pi = sc->port[port];
1312         if (pi == NULL || pi->dev == NULL)
1313                 return (ENXIO);
1314         *child = pi->dev;
1315         return (0);
1316 }
1317
1318 static int
1319 notify_siblings(device_t dev, int detaching)
1320 {
1321         device_t sibling;
1322         int error, i;
1323
1324         error = 0;
1325         for (i = 0; i < PCI_FUNCMAX; i++) {
1326                 if (i == pci_get_function(dev))
1327                         continue;
1328                 sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
1329                     pci_get_slot(dev), i);
1330                 if (sibling == NULL || !device_is_attached(sibling))
1331                         continue;
1332                 if (detaching)
1333                         error = T4_DETACH_CHILD(sibling);
1334                 else
1335                         (void)T4_ATTACH_CHILD(sibling);
1336                 if (error)
1337                         break;
1338         }
1339         return (error);
1340 }
1341
1342 /*
1343  * Idempotent
1344  */
1345 static int
1346 t4_detach(device_t dev)
1347 {
1348         struct adapter *sc;
1349         int rc;
1350
1351         sc = device_get_softc(dev);
1352
1353         rc = notify_siblings(dev, 1);
1354         if (rc) {
1355                 device_printf(dev,
1356                     "failed to detach sibling devices: %d\n", rc);
1357                 return (rc);
1358         }
1359
1360         return (t4_detach_common(dev));
1361 }
1362
1363 int
1364 t4_detach_common(device_t dev)
1365 {
1366         struct adapter *sc;
1367         struct port_info *pi;
1368         int i, rc;
1369
1370         sc = device_get_softc(dev);
1371
1372         if (sc->cdev) {
1373                 destroy_dev(sc->cdev);
1374                 sc->cdev = NULL;
1375         }
1376
1377         sc->flags &= ~CHK_MBOX_ACCESS;
1378         if (sc->flags & FULL_INIT_DONE) {
1379                 if (!(sc->flags & IS_VF))
1380                         t4_intr_disable(sc);
1381         }
1382
1383         if (device_is_attached(dev)) {
1384                 rc = bus_generic_detach(dev);
1385                 if (rc) {
1386                         device_printf(dev,
1387                             "failed to detach child devices: %d\n", rc);
1388                         return (rc);
1389                 }
1390         }
1391
1392         for (i = 0; i < sc->intr_count; i++)
1393                 t4_free_irq(sc, &sc->irq[i]);
1394
1395         if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1396                 t4_free_tx_sched(sc);
1397
1398         for (i = 0; i < MAX_NPORTS; i++) {
1399                 pi = sc->port[i];
1400                 if (pi) {
1401                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1402                         if (pi->dev)
1403                                 device_delete_child(dev, pi->dev);
1404
1405                         mtx_destroy(&pi->pi_lock);
1406                         free(pi->vi, M_CXGBE);
1407                         free(pi, M_CXGBE);
1408                 }
1409         }
1410
1411         device_delete_children(dev);
1412
1413         if (sc->flags & FULL_INIT_DONE)
1414                 adapter_full_uninit(sc);
1415
1416         if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1417                 t4_fw_bye(sc, sc->mbox);
1418
1419         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1420                 pci_release_msi(dev);
1421
1422         if (sc->regs_res)
1423                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1424                     sc->regs_res);
1425
1426         if (sc->udbs_res)
1427                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1428                     sc->udbs_res);
1429
1430         if (sc->msix_res)
1431                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1432                     sc->msix_res);
1433
1434         if (sc->l2t)
1435                 t4_free_l2t(sc->l2t);
1436         if (sc->smt)
1437                 t4_free_smt(sc->smt);
1438 #ifdef RATELIMIT
1439         t4_free_etid_table(sc);
1440 #endif
1441
1442 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1443         free(sc->sge.ofld_txq, M_CXGBE);
1444 #endif
1445 #ifdef TCP_OFFLOAD
1446         free(sc->sge.ofld_rxq, M_CXGBE);
1447 #endif
1448 #ifdef DEV_NETMAP
1449         free(sc->sge.nm_rxq, M_CXGBE);
1450         free(sc->sge.nm_txq, M_CXGBE);
1451 #endif
1452         free(sc->irq, M_CXGBE);
1453         free(sc->sge.rxq, M_CXGBE);
1454         free(sc->sge.txq, M_CXGBE);
1455         free(sc->sge.ctrlq, M_CXGBE);
1456         free(sc->sge.iqmap, M_CXGBE);
1457         free(sc->sge.eqmap, M_CXGBE);
1458         free(sc->tids.ftid_tab, M_CXGBE);
1459         free(sc->tids.hpftid_tab, M_CXGBE);
1460         free_hftid_hash(&sc->tids);
1461         free(sc->tids.atid_tab, M_CXGBE);
1462         free(sc->tids.tid_tab, M_CXGBE);
1463         free(sc->tt.tls_rx_ports, M_CXGBE);
1464         t4_destroy_dma_tag(sc);
1465         if (mtx_initialized(&sc->sc_lock)) {
1466                 sx_xlock(&t4_list_lock);
1467                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1468                 sx_xunlock(&t4_list_lock);
1469                 mtx_destroy(&sc->sc_lock);
1470         }
1471
1472         callout_drain(&sc->sfl_callout);
1473         if (mtx_initialized(&sc->tids.ftid_lock)) {
1474                 mtx_destroy(&sc->tids.ftid_lock);
1475                 cv_destroy(&sc->tids.ftid_cv);
1476         }
1477         if (mtx_initialized(&sc->tids.atid_lock))
1478                 mtx_destroy(&sc->tids.atid_lock);
1479         if (mtx_initialized(&sc->sfl_lock))
1480                 mtx_destroy(&sc->sfl_lock);
1481         if (mtx_initialized(&sc->ifp_lock))
1482                 mtx_destroy(&sc->ifp_lock);
1483         if (mtx_initialized(&sc->reg_lock))
1484                 mtx_destroy(&sc->reg_lock);
1485
1486         if (rw_initialized(&sc->policy_lock)) {
1487                 rw_destroy(&sc->policy_lock);
1488 #ifdef TCP_OFFLOAD
1489                 if (sc->policy != NULL)
1490                         free_offload_policy(sc->policy);
1491 #endif
1492         }
1493
1494         for (i = 0; i < NUM_MEMWIN; i++) {
1495                 struct memwin *mw = &sc->memwin[i];
1496
1497                 if (rw_initialized(&mw->mw_lock))
1498                         rw_destroy(&mw->mw_lock);
1499         }
1500
1501         bzero(sc, sizeof(*sc));
1502
1503         return (0);
1504 }
1505
1506 static int
1507 cxgbe_probe(device_t dev)
1508 {
1509         char buf[128];
1510         struct port_info *pi = device_get_softc(dev);
1511
1512         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1513         device_set_desc_copy(dev, buf);
1514
1515         return (BUS_PROBE_DEFAULT);
1516 }
1517
1518 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1519     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1520     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS | \
1521     IFCAP_HWRXTSTMP)
1522 #define T4_CAP_ENABLE (T4_CAP)
1523
1524 static int
1525 cxgbe_vi_attach(device_t dev, struct vi_info *vi)
1526 {
1527         struct ifnet *ifp;
1528         struct sbuf *sb;
1529
1530         vi->xact_addr_filt = -1;
1531         callout_init(&vi->tick, 1);
1532
1533         /* Allocate an ifnet and set it up */
1534         ifp = if_alloc(IFT_ETHER);
1535         if (ifp == NULL) {
1536                 device_printf(dev, "Cannot allocate ifnet\n");
1537                 return (ENOMEM);
1538         }
1539         vi->ifp = ifp;
1540         ifp->if_softc = vi;
1541
1542         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1543         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1544
1545         ifp->if_init = cxgbe_init;
1546         ifp->if_ioctl = cxgbe_ioctl;
1547         ifp->if_transmit = cxgbe_transmit;
1548         ifp->if_qflush = cxgbe_qflush;
1549         ifp->if_get_counter = cxgbe_get_counter;
1550 #ifdef RATELIMIT
1551         ifp->if_snd_tag_alloc = cxgbe_snd_tag_alloc;
1552         ifp->if_snd_tag_modify = cxgbe_snd_tag_modify;
1553         ifp->if_snd_tag_query = cxgbe_snd_tag_query;
1554         ifp->if_snd_tag_free = cxgbe_snd_tag_free;
1555 #endif
1556
1557         ifp->if_capabilities = T4_CAP;
1558         ifp->if_capenable = T4_CAP_ENABLE;
1559 #ifdef TCP_OFFLOAD
1560         if (vi->nofldrxq != 0)
1561                 ifp->if_capabilities |= IFCAP_TOE;
1562 #endif
1563 #ifdef DEV_NETMAP
1564         if (vi->nnmrxq != 0)
1565                 ifp->if_capabilities |= IFCAP_NETMAP;
1566 #endif
1567 #ifdef RATELIMIT
1568         if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) {
1569                 ifp->if_capabilities |= IFCAP_TXRTLMT;
1570                 ifp->if_capenable |= IFCAP_TXRTLMT;
1571         }
1572 #endif
1573         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1574             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1575
1576         ifp->if_hw_tsomax = IP_MAXPACKET;
1577         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_TSO;
1578 #ifdef RATELIMIT
1579         if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0)
1580                 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_EO_TSO;
1581 #endif
1582         ifp->if_hw_tsomaxsegsize = 65536;
1583
1584         ether_ifattach(ifp, vi->hw_addr);
1585 #ifdef DEV_NETMAP
1586         if (ifp->if_capabilities & IFCAP_NETMAP)
1587                 cxgbe_nm_attach(vi);
1588 #endif
1589         sb = sbuf_new_auto();
1590         sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
1591 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1592         switch (ifp->if_capabilities & (IFCAP_TOE | IFCAP_TXRTLMT)) {
1593         case IFCAP_TOE:
1594                 sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq);
1595                 break;
1596         case IFCAP_TOE | IFCAP_TXRTLMT:
1597                 sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq);
1598                 break;
1599         case IFCAP_TXRTLMT:
1600                 sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq);
1601                 break;
1602         }
1603 #endif
1604 #ifdef TCP_OFFLOAD
1605         if (ifp->if_capabilities & IFCAP_TOE)
1606                 sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq);
1607 #endif
1608 #ifdef DEV_NETMAP
1609         if (ifp->if_capabilities & IFCAP_NETMAP)
1610                 sbuf_printf(sb, "; %d txq, %d rxq (netmap)",
1611                     vi->nnmtxq, vi->nnmrxq);
1612 #endif
1613         sbuf_finish(sb);
1614         device_printf(dev, "%s\n", sbuf_data(sb));
1615         sbuf_delete(sb);
1616
1617         vi_sysctls(vi);
1618
1619         return (0);
1620 }
1621
1622 static int
1623 cxgbe_attach(device_t dev)
1624 {
1625         struct port_info *pi = device_get_softc(dev);
1626         struct adapter *sc = pi->adapter;
1627         struct vi_info *vi;
1628         int i, rc;
1629
1630         callout_init_mtx(&pi->tick, &pi->pi_lock, 0);
1631
1632         rc = cxgbe_vi_attach(dev, &pi->vi[0]);
1633         if (rc)
1634                 return (rc);
1635
1636         for_each_vi(pi, i, vi) {
1637                 if (i == 0)
1638                         continue;
1639                 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1);
1640                 if (vi->dev == NULL) {
1641                         device_printf(dev, "failed to add VI %d\n", i);
1642                         continue;
1643                 }
1644                 device_set_softc(vi->dev, vi);
1645         }
1646
1647         cxgbe_sysctls(pi);
1648
1649         bus_generic_attach(dev);
1650
1651         return (0);
1652 }
1653
1654 static void
1655 cxgbe_vi_detach(struct vi_info *vi)
1656 {
1657         struct ifnet *ifp = vi->ifp;
1658
1659         ether_ifdetach(ifp);
1660
1661         /* Let detach proceed even if these fail. */
1662 #ifdef DEV_NETMAP
1663         if (ifp->if_capabilities & IFCAP_NETMAP)
1664                 cxgbe_nm_detach(vi);
1665 #endif
1666         cxgbe_uninit_synchronized(vi);
1667         callout_drain(&vi->tick);
1668         vi_full_uninit(vi);
1669
1670         if_free(vi->ifp);
1671         vi->ifp = NULL;
1672 }
1673
1674 static int
1675 cxgbe_detach(device_t dev)
1676 {
1677         struct port_info *pi = device_get_softc(dev);
1678         struct adapter *sc = pi->adapter;
1679         int rc;
1680
1681         /* Detach the extra VIs first. */
1682         rc = bus_generic_detach(dev);
1683         if (rc)
1684                 return (rc);
1685         device_delete_children(dev);
1686
1687         doom_vi(sc, &pi->vi[0]);
1688
1689         if (pi->flags & HAS_TRACEQ) {
1690                 sc->traceq = -1;        /* cloner should not create ifnet */
1691                 t4_tracer_port_detach(sc);
1692         }
1693
1694         cxgbe_vi_detach(&pi->vi[0]);
1695         callout_drain(&pi->tick);
1696         ifmedia_removeall(&pi->media);
1697
1698         end_synchronized_op(sc, 0);
1699
1700         return (0);
1701 }
1702
1703 static void
1704 cxgbe_init(void *arg)
1705 {
1706         struct vi_info *vi = arg;
1707         struct adapter *sc = vi->pi->adapter;
1708
1709         if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
1710                 return;
1711         cxgbe_init_synchronized(vi);
1712         end_synchronized_op(sc, 0);
1713 }
1714
1715 static int
1716 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1717 {
1718         int rc = 0, mtu, flags;
1719         struct vi_info *vi = ifp->if_softc;
1720         struct port_info *pi = vi->pi;
1721         struct adapter *sc = pi->adapter;
1722         struct ifreq *ifr = (struct ifreq *)data;
1723         uint32_t mask;
1724
1725         switch (cmd) {
1726         case SIOCSIFMTU:
1727                 mtu = ifr->ifr_mtu;
1728                 if (mtu < ETHERMIN || mtu > MAX_MTU)
1729                         return (EINVAL);
1730
1731                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
1732                 if (rc)
1733                         return (rc);
1734                 ifp->if_mtu = mtu;
1735                 if (vi->flags & VI_INIT_DONE) {
1736                         t4_update_fl_bufsize(ifp);
1737                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1738                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1739                 }
1740                 end_synchronized_op(sc, 0);
1741                 break;
1742
1743         case SIOCSIFFLAGS:
1744                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4flg");
1745                 if (rc)
1746                         return (rc);
1747
1748                 if (ifp->if_flags & IFF_UP) {
1749                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1750                                 flags = vi->if_flags;
1751                                 if ((ifp->if_flags ^ flags) &
1752                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1753                                         rc = update_mac_settings(ifp,
1754                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1755                                 }
1756                         } else {
1757                                 rc = cxgbe_init_synchronized(vi);
1758                         }
1759                         vi->if_flags = ifp->if_flags;
1760                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1761                         rc = cxgbe_uninit_synchronized(vi);
1762                 }
1763                 end_synchronized_op(sc, 0);
1764                 break;
1765
1766         case SIOCADDMULTI:
1767         case SIOCDELMULTI:
1768                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4multi");
1769                 if (rc)
1770                         return (rc);
1771                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1772                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1773                 end_synchronized_op(sc, 0);
1774                 break;
1775
1776         case SIOCSIFCAP:
1777                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
1778                 if (rc)
1779                         return (rc);
1780
1781                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1782                 if (mask & IFCAP_TXCSUM) {
1783                         ifp->if_capenable ^= IFCAP_TXCSUM;
1784                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1785
1786                         if (IFCAP_TSO4 & ifp->if_capenable &&
1787                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1788                                 ifp->if_capenable &= ~IFCAP_TSO4;
1789                                 if_printf(ifp,
1790                                     "tso4 disabled due to -txcsum.\n");
1791                         }
1792                 }
1793                 if (mask & IFCAP_TXCSUM_IPV6) {
1794                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1795                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1796
1797                         if (IFCAP_TSO6 & ifp->if_capenable &&
1798                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1799                                 ifp->if_capenable &= ~IFCAP_TSO6;
1800                                 if_printf(ifp,
1801                                     "tso6 disabled due to -txcsum6.\n");
1802                         }
1803                 }
1804                 if (mask & IFCAP_RXCSUM)
1805                         ifp->if_capenable ^= IFCAP_RXCSUM;
1806                 if (mask & IFCAP_RXCSUM_IPV6)
1807                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1808
1809                 /*
1810                  * Note that we leave CSUM_TSO alone (it is always set).  The
1811                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1812                  * sending a TSO request our way, so it's sufficient to toggle
1813                  * IFCAP_TSOx only.
1814                  */
1815                 if (mask & IFCAP_TSO4) {
1816                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1817                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1818                                 if_printf(ifp, "enable txcsum first.\n");
1819                                 rc = EAGAIN;
1820                                 goto fail;
1821                         }
1822                         ifp->if_capenable ^= IFCAP_TSO4;
1823                 }
1824                 if (mask & IFCAP_TSO6) {
1825                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1826                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1827                                 if_printf(ifp, "enable txcsum6 first.\n");
1828                                 rc = EAGAIN;
1829                                 goto fail;
1830                         }
1831                         ifp->if_capenable ^= IFCAP_TSO6;
1832                 }
1833                 if (mask & IFCAP_LRO) {
1834 #if defined(INET) || defined(INET6)
1835                         int i;
1836                         struct sge_rxq *rxq;
1837
1838                         ifp->if_capenable ^= IFCAP_LRO;
1839                         for_each_rxq(vi, i, rxq) {
1840                                 if (ifp->if_capenable & IFCAP_LRO)
1841                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1842                                 else
1843                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1844                         }
1845 #endif
1846                 }
1847 #ifdef TCP_OFFLOAD
1848                 if (mask & IFCAP_TOE) {
1849                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1850
1851                         rc = toe_capability(vi, enable);
1852                         if (rc != 0)
1853                                 goto fail;
1854
1855                         ifp->if_capenable ^= mask;
1856                 }
1857 #endif
1858                 if (mask & IFCAP_VLAN_HWTAGGING) {
1859                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1860                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1861                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1862                 }
1863                 if (mask & IFCAP_VLAN_MTU) {
1864                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1865
1866                         /* Need to find out how to disable auto-mtu-inflation */
1867                 }
1868                 if (mask & IFCAP_VLAN_HWTSO)
1869                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1870                 if (mask & IFCAP_VLAN_HWCSUM)
1871                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1872 #ifdef RATELIMIT
1873                 if (mask & IFCAP_TXRTLMT)
1874                         ifp->if_capenable ^= IFCAP_TXRTLMT;
1875 #endif
1876                 if (mask & IFCAP_HWRXTSTMP) {
1877                         int i;
1878                         struct sge_rxq *rxq;
1879
1880                         ifp->if_capenable ^= IFCAP_HWRXTSTMP;
1881                         for_each_rxq(vi, i, rxq) {
1882                                 if (ifp->if_capenable & IFCAP_HWRXTSTMP)
1883                                         rxq->iq.flags |= IQ_RX_TIMESTAMP;
1884                                 else
1885                                         rxq->iq.flags &= ~IQ_RX_TIMESTAMP;
1886                         }
1887                 }
1888
1889 #ifdef VLAN_CAPABILITIES
1890                 VLAN_CAPABILITIES(ifp);
1891 #endif
1892 fail:
1893                 end_synchronized_op(sc, 0);
1894                 break;
1895
1896         case SIOCSIFMEDIA:
1897         case SIOCGIFMEDIA:
1898         case SIOCGIFXMEDIA:
1899                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1900                 break;
1901
1902         case SIOCGI2C: {
1903                 struct ifi2creq i2c;
1904
1905                 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
1906                 if (rc != 0)
1907                         break;
1908                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1909                         rc = EPERM;
1910                         break;
1911                 }
1912                 if (i2c.len > sizeof(i2c.data)) {
1913                         rc = EINVAL;
1914                         break;
1915                 }
1916                 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
1917                 if (rc)
1918                         return (rc);
1919                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1920                     i2c.offset, i2c.len, &i2c.data[0]);
1921                 end_synchronized_op(sc, 0);
1922                 if (rc == 0)
1923                         rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
1924                 break;
1925         }
1926
1927         default:
1928                 rc = ether_ioctl(ifp, cmd, data);
1929         }
1930
1931         return (rc);
1932 }
1933
1934 static int
1935 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1936 {
1937         struct vi_info *vi = ifp->if_softc;
1938         struct port_info *pi = vi->pi;
1939         struct adapter *sc = pi->adapter;
1940         struct sge_txq *txq;
1941         void *items[1];
1942         int rc;
1943
1944         M_ASSERTPKTHDR(m);
1945         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1946
1947         if (__predict_false(pi->link_cfg.link_ok == false)) {
1948                 m_freem(m);
1949                 return (ENETDOWN);
1950         }
1951
1952         rc = parse_pkt(sc, &m);
1953         if (__predict_false(rc != 0)) {
1954                 MPASS(m == NULL);                       /* was freed already */
1955                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1956                 return (rc);
1957         }
1958 #ifdef RATELIMIT
1959         if (m->m_pkthdr.snd_tag != NULL) {
1960                 /* EAGAIN tells the stack we are not the correct interface. */
1961                 if (__predict_false(ifp != m->m_pkthdr.snd_tag->ifp)) {
1962                         m_freem(m);
1963                         return (EAGAIN);
1964                 }
1965
1966                 return (ethofld_transmit(ifp, m));
1967         }
1968 #endif
1969
1970         /* Select a txq. */
1971         txq = &sc->sge.txq[vi->first_txq];
1972         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1973                 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
1974                     vi->rsrv_noflowq);
1975
1976         items[0] = m;
1977         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1978         if (__predict_false(rc != 0))
1979                 m_freem(m);
1980
1981         return (rc);
1982 }
1983
1984 static void
1985 cxgbe_qflush(struct ifnet *ifp)
1986 {
1987         struct vi_info *vi = ifp->if_softc;
1988         struct sge_txq *txq;
1989         int i;
1990
1991         /* queues do not exist if !VI_INIT_DONE. */
1992         if (vi->flags & VI_INIT_DONE) {
1993                 for_each_txq(vi, i, txq) {
1994                         TXQ_LOCK(txq);
1995                         txq->eq.flags |= EQ_QFLUSH;
1996                         TXQ_UNLOCK(txq);
1997                         while (!mp_ring_is_idle(txq->r)) {
1998                                 mp_ring_check_drainage(txq->r, 0);
1999                                 pause("qflush", 1);
2000                         }
2001                         TXQ_LOCK(txq);
2002                         txq->eq.flags &= ~EQ_QFLUSH;
2003                         TXQ_UNLOCK(txq);
2004                 }
2005         }
2006         if_qflush(ifp);
2007 }
2008
2009 static uint64_t
2010 vi_get_counter(struct ifnet *ifp, ift_counter c)
2011 {
2012         struct vi_info *vi = ifp->if_softc;
2013         struct fw_vi_stats_vf *s = &vi->stats;
2014
2015         vi_refresh_stats(vi->pi->adapter, vi);
2016
2017         switch (c) {
2018         case IFCOUNTER_IPACKETS:
2019                 return (s->rx_bcast_frames + s->rx_mcast_frames +
2020                     s->rx_ucast_frames);
2021         case IFCOUNTER_IERRORS:
2022                 return (s->rx_err_frames);
2023         case IFCOUNTER_OPACKETS:
2024                 return (s->tx_bcast_frames + s->tx_mcast_frames +
2025                     s->tx_ucast_frames + s->tx_offload_frames);
2026         case IFCOUNTER_OERRORS:
2027                 return (s->tx_drop_frames);
2028         case IFCOUNTER_IBYTES:
2029                 return (s->rx_bcast_bytes + s->rx_mcast_bytes +
2030                     s->rx_ucast_bytes);
2031         case IFCOUNTER_OBYTES:
2032                 return (s->tx_bcast_bytes + s->tx_mcast_bytes +
2033                     s->tx_ucast_bytes + s->tx_offload_bytes);
2034         case IFCOUNTER_IMCASTS:
2035                 return (s->rx_mcast_frames);
2036         case IFCOUNTER_OMCASTS:
2037                 return (s->tx_mcast_frames);
2038         case IFCOUNTER_OQDROPS: {
2039                 uint64_t drops;
2040
2041                 drops = 0;
2042                 if (vi->flags & VI_INIT_DONE) {
2043                         int i;
2044                         struct sge_txq *txq;
2045
2046                         for_each_txq(vi, i, txq)
2047                                 drops += counter_u64_fetch(txq->r->drops);
2048                 }
2049
2050                 return (drops);
2051
2052         }
2053
2054         default:
2055                 return (if_get_counter_default(ifp, c));
2056         }
2057 }
2058
2059 uint64_t
2060 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
2061 {
2062         struct vi_info *vi = ifp->if_softc;
2063         struct port_info *pi = vi->pi;
2064         struct adapter *sc = pi->adapter;
2065         struct port_stats *s = &pi->stats;
2066
2067         if (pi->nvi > 1 || sc->flags & IS_VF)
2068                 return (vi_get_counter(ifp, c));
2069
2070         cxgbe_refresh_stats(sc, pi);
2071
2072         switch (c) {
2073         case IFCOUNTER_IPACKETS:
2074                 return (s->rx_frames);
2075
2076         case IFCOUNTER_IERRORS:
2077                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
2078                     s->rx_fcs_err + s->rx_len_err);
2079
2080         case IFCOUNTER_OPACKETS:
2081                 return (s->tx_frames);
2082
2083         case IFCOUNTER_OERRORS:
2084                 return (s->tx_error_frames);
2085
2086         case IFCOUNTER_IBYTES:
2087                 return (s->rx_octets);
2088
2089         case IFCOUNTER_OBYTES:
2090                 return (s->tx_octets);
2091
2092         case IFCOUNTER_IMCASTS:
2093                 return (s->rx_mcast_frames);
2094
2095         case IFCOUNTER_OMCASTS:
2096                 return (s->tx_mcast_frames);
2097
2098         case IFCOUNTER_IQDROPS:
2099                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
2100                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
2101                     s->rx_trunc3 + pi->tnl_cong_drops);
2102
2103         case IFCOUNTER_OQDROPS: {
2104                 uint64_t drops;
2105
2106                 drops = s->tx_drop;
2107                 if (vi->flags & VI_INIT_DONE) {
2108                         int i;
2109                         struct sge_txq *txq;
2110
2111                         for_each_txq(vi, i, txq)
2112                                 drops += counter_u64_fetch(txq->r->drops);
2113                 }
2114
2115                 return (drops);
2116
2117         }
2118
2119         default:
2120                 return (if_get_counter_default(ifp, c));
2121         }
2122 }
2123
2124 /*
2125  * The kernel picks a media from the list we had provided but we still validate
2126  * the requeste.
2127  */
2128 int
2129 cxgbe_media_change(struct ifnet *ifp)
2130 {
2131         struct vi_info *vi = ifp->if_softc;
2132         struct port_info *pi = vi->pi;
2133         struct ifmedia *ifm = &pi->media;
2134         struct link_config *lc = &pi->link_cfg;
2135         struct adapter *sc = pi->adapter;
2136         int rc;
2137
2138         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mec");
2139         if (rc != 0)
2140                 return (rc);
2141         PORT_LOCK(pi);
2142         if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) {
2143                 /* ifconfig .. media autoselect */
2144                 if (!(lc->supported & FW_PORT_CAP32_ANEG)) {
2145                         rc = ENOTSUP; /* AN not supported by transceiver */
2146                         goto done;
2147                 }
2148                 lc->requested_aneg = AUTONEG_ENABLE;
2149                 lc->requested_speed = 0;
2150                 lc->requested_fc |= PAUSE_AUTONEG;
2151         } else {
2152                 lc->requested_aneg = AUTONEG_DISABLE;
2153                 lc->requested_speed =
2154                     ifmedia_baudrate(ifm->ifm_media) / 1000000;
2155                 lc->requested_fc = 0;
2156                 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE)
2157                         lc->requested_fc |= PAUSE_RX;
2158                 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)
2159                         lc->requested_fc |= PAUSE_TX;
2160         }
2161         if (pi->up_vis > 0) {
2162                 fixup_link_config(pi);
2163                 rc = apply_link_config(pi);
2164         }
2165 done:
2166         PORT_UNLOCK(pi);
2167         end_synchronized_op(sc, 0);
2168         return (rc);
2169 }
2170
2171 /*
2172  * Base media word (without ETHER, pause, link active, etc.) for the port at the
2173  * given speed.
2174  */
2175 static int
2176 port_mword(struct port_info *pi, uint32_t speed)
2177 {
2178
2179         MPASS(speed & M_FW_PORT_CAP32_SPEED);
2180         MPASS(powerof2(speed));
2181
2182         switch(pi->port_type) {
2183         case FW_PORT_TYPE_BT_SGMII:
2184         case FW_PORT_TYPE_BT_XFI:
2185         case FW_PORT_TYPE_BT_XAUI:
2186                 /* BaseT */
2187                 switch (speed) {
2188                 case FW_PORT_CAP32_SPEED_100M:
2189                         return (IFM_100_T);
2190                 case FW_PORT_CAP32_SPEED_1G:
2191                         return (IFM_1000_T);
2192                 case FW_PORT_CAP32_SPEED_10G:
2193                         return (IFM_10G_T);
2194                 }
2195                 break;
2196         case FW_PORT_TYPE_KX4:
2197                 if (speed == FW_PORT_CAP32_SPEED_10G)
2198                         return (IFM_10G_KX4);
2199                 break;
2200         case FW_PORT_TYPE_CX4:
2201                 if (speed == FW_PORT_CAP32_SPEED_10G)
2202                         return (IFM_10G_CX4);
2203                 break;
2204         case FW_PORT_TYPE_KX:
2205                 if (speed == FW_PORT_CAP32_SPEED_1G)
2206                         return (IFM_1000_KX);
2207                 break;
2208         case FW_PORT_TYPE_KR:
2209         case FW_PORT_TYPE_BP_AP:
2210         case FW_PORT_TYPE_BP4_AP:
2211         case FW_PORT_TYPE_BP40_BA:
2212         case FW_PORT_TYPE_KR4_100G:
2213         case FW_PORT_TYPE_KR_SFP28:
2214         case FW_PORT_TYPE_KR_XLAUI:
2215                 switch (speed) {
2216                 case FW_PORT_CAP32_SPEED_1G:
2217                         return (IFM_1000_KX);
2218                 case FW_PORT_CAP32_SPEED_10G:
2219                         return (IFM_10G_KR);
2220                 case FW_PORT_CAP32_SPEED_25G:
2221                         return (IFM_25G_KR);
2222                 case FW_PORT_CAP32_SPEED_40G:
2223                         return (IFM_40G_KR4);
2224                 case FW_PORT_CAP32_SPEED_50G:
2225                         return (IFM_50G_KR2);
2226                 case FW_PORT_CAP32_SPEED_100G:
2227                         return (IFM_100G_KR4);
2228                 }
2229                 break;
2230         case FW_PORT_TYPE_FIBER_XFI:
2231         case FW_PORT_TYPE_FIBER_XAUI:
2232         case FW_PORT_TYPE_SFP:
2233         case FW_PORT_TYPE_QSFP_10G:
2234         case FW_PORT_TYPE_QSA:
2235         case FW_PORT_TYPE_QSFP:
2236         case FW_PORT_TYPE_CR4_QSFP:
2237         case FW_PORT_TYPE_CR_QSFP:
2238         case FW_PORT_TYPE_CR2_QSFP:
2239         case FW_PORT_TYPE_SFP28:
2240                 /* Pluggable transceiver */
2241                 switch (pi->mod_type) {
2242                 case FW_PORT_MOD_TYPE_LR:
2243                         switch (speed) {
2244                         case FW_PORT_CAP32_SPEED_1G:
2245                                 return (IFM_1000_LX);
2246                         case FW_PORT_CAP32_SPEED_10G:
2247                                 return (IFM_10G_LR);
2248                         case FW_PORT_CAP32_SPEED_25G:
2249                                 return (IFM_25G_LR);
2250                         case FW_PORT_CAP32_SPEED_40G:
2251                                 return (IFM_40G_LR4);
2252                         case FW_PORT_CAP32_SPEED_50G:
2253                                 return (IFM_50G_LR2);
2254                         case FW_PORT_CAP32_SPEED_100G:
2255                                 return (IFM_100G_LR4);
2256                         }
2257                         break;
2258                 case FW_PORT_MOD_TYPE_SR:
2259                         switch (speed) {
2260                         case FW_PORT_CAP32_SPEED_1G:
2261                                 return (IFM_1000_SX);
2262                         case FW_PORT_CAP32_SPEED_10G:
2263                                 return (IFM_10G_SR);
2264                         case FW_PORT_CAP32_SPEED_25G:
2265                                 return (IFM_25G_SR);
2266                         case FW_PORT_CAP32_SPEED_40G:
2267                                 return (IFM_40G_SR4);
2268                         case FW_PORT_CAP32_SPEED_50G:
2269                                 return (IFM_50G_SR2);
2270                         case FW_PORT_CAP32_SPEED_100G:
2271                                 return (IFM_100G_SR4);
2272                         }
2273                         break;
2274                 case FW_PORT_MOD_TYPE_ER:
2275                         if (speed == FW_PORT_CAP32_SPEED_10G)
2276                                 return (IFM_10G_ER);
2277                         break;
2278                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2279                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2280                         switch (speed) {
2281                         case FW_PORT_CAP32_SPEED_1G:
2282                                 return (IFM_1000_CX);
2283                         case FW_PORT_CAP32_SPEED_10G:
2284                                 return (IFM_10G_TWINAX);
2285                         case FW_PORT_CAP32_SPEED_25G:
2286                                 return (IFM_25G_CR);
2287                         case FW_PORT_CAP32_SPEED_40G:
2288                                 return (IFM_40G_CR4);
2289                         case FW_PORT_CAP32_SPEED_50G:
2290                                 return (IFM_50G_CR2);
2291                         case FW_PORT_CAP32_SPEED_100G:
2292                                 return (IFM_100G_CR4);
2293                         }
2294                         break;
2295                 case FW_PORT_MOD_TYPE_LRM:
2296                         if (speed == FW_PORT_CAP32_SPEED_10G)
2297                                 return (IFM_10G_LRM);
2298                         break;
2299                 case FW_PORT_MOD_TYPE_NA:
2300                         MPASS(0);       /* Not pluggable? */
2301                         /* fall throough */
2302                 case FW_PORT_MOD_TYPE_ERROR:
2303                 case FW_PORT_MOD_TYPE_UNKNOWN:
2304                 case FW_PORT_MOD_TYPE_NOTSUPPORTED:
2305                         break;
2306                 case FW_PORT_MOD_TYPE_NONE:
2307                         return (IFM_NONE);
2308                 }
2309                 break;
2310         case FW_PORT_TYPE_NONE:
2311                 return (IFM_NONE);
2312         }
2313
2314         return (IFM_UNKNOWN);
2315 }
2316
2317 void
2318 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2319 {
2320         struct vi_info *vi = ifp->if_softc;
2321         struct port_info *pi = vi->pi;
2322         struct adapter *sc = pi->adapter;
2323         struct link_config *lc = &pi->link_cfg;
2324
2325         if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4med") != 0)
2326                 return;
2327         PORT_LOCK(pi);
2328
2329         if (pi->up_vis == 0) {
2330                 /*
2331                  * If all the interfaces are administratively down the firmware
2332                  * does not report transceiver changes.  Refresh port info here
2333                  * so that ifconfig displays accurate ifmedia at all times.
2334                  * This is the only reason we have a synchronized op in this
2335                  * function.  Just PORT_LOCK would have been enough otherwise.
2336                  */
2337                 t4_update_port_info(pi);
2338                 build_medialist(pi);
2339         }
2340
2341         /* ifm_status */
2342         ifmr->ifm_status = IFM_AVALID;
2343         if (lc->link_ok == false)
2344                 goto done;
2345         ifmr->ifm_status |= IFM_ACTIVE;
2346
2347         /* ifm_active */
2348         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2349         ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE);
2350         if (lc->fc & PAUSE_RX)
2351                 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
2352         if (lc->fc & PAUSE_TX)
2353                 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
2354         ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed));
2355 done:
2356         PORT_UNLOCK(pi);
2357         end_synchronized_op(sc, 0);
2358 }
2359
2360 static int
2361 vcxgbe_probe(device_t dev)
2362 {
2363         char buf[128];
2364         struct vi_info *vi = device_get_softc(dev);
2365
2366         snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
2367             vi - vi->pi->vi);
2368         device_set_desc_copy(dev, buf);
2369
2370         return (BUS_PROBE_DEFAULT);
2371 }
2372
2373 static int
2374 alloc_extra_vi(struct adapter *sc, struct port_info *pi, struct vi_info *vi)
2375 {
2376         int func, index, rc;
2377         uint32_t param, val;
2378
2379         ASSERT_SYNCHRONIZED_OP(sc);
2380
2381         index = vi - pi->vi;
2382         MPASS(index > 0);       /* This function deals with _extra_ VIs only */
2383         KASSERT(index < nitems(vi_mac_funcs),
2384             ("%s: VI %s doesn't have a MAC func", __func__,
2385             device_get_nameunit(vi->dev)));
2386         func = vi_mac_funcs[index];
2387         rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
2388             vi->hw_addr, &vi->rss_size, func, 0);
2389         if (rc < 0) {
2390                 device_printf(vi->dev, "failed to allocate virtual interface %d"
2391                     "for port %d: %d\n", index, pi->port_id, -rc);
2392                 return (-rc);
2393         }
2394         vi->viid = rc;
2395         if (chip_id(sc) <= CHELSIO_T5)
2396                 vi->smt_idx = (rc & 0x7f) << 1;
2397         else
2398                 vi->smt_idx = (rc & 0x7f);
2399
2400         if (vi->rss_size == 1) {
2401                 /*
2402                  * This VI didn't get a slice of the RSS table.  Reduce the
2403                  * number of VIs being created (hw.cxgbe.num_vis) or modify the
2404                  * configuration file (nvi, rssnvi for this PF) if this is a
2405                  * problem.
2406                  */
2407                 device_printf(vi->dev, "RSS table not available.\n");
2408                 vi->rss_base = 0xffff;
2409
2410                 return (0);
2411         }
2412
2413         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
2414             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
2415             V_FW_PARAMS_PARAM_YZ(vi->viid);
2416         rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2417         if (rc)
2418                 vi->rss_base = 0xffff;
2419         else {
2420                 MPASS((val >> 16) == vi->rss_size);
2421                 vi->rss_base = val & 0xffff;
2422         }
2423
2424         return (0);
2425 }
2426
2427 static int
2428 vcxgbe_attach(device_t dev)
2429 {
2430         struct vi_info *vi;
2431         struct port_info *pi;
2432         struct adapter *sc;
2433         int rc;
2434
2435         vi = device_get_softc(dev);
2436         pi = vi->pi;
2437         sc = pi->adapter;
2438
2439         rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4via");
2440         if (rc)
2441                 return (rc);
2442         rc = alloc_extra_vi(sc, pi, vi);
2443         end_synchronized_op(sc, 0);
2444         if (rc)
2445                 return (rc);
2446
2447         rc = cxgbe_vi_attach(dev, vi);
2448         if (rc) {
2449                 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
2450                 return (rc);
2451         }
2452         return (0);
2453 }
2454
2455 static int
2456 vcxgbe_detach(device_t dev)
2457 {
2458         struct vi_info *vi;
2459         struct adapter *sc;
2460
2461         vi = device_get_softc(dev);
2462         sc = vi->pi->adapter;
2463
2464         doom_vi(sc, vi);
2465
2466         cxgbe_vi_detach(vi);
2467         t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
2468
2469         end_synchronized_op(sc, 0);
2470
2471         return (0);
2472 }
2473
2474 void
2475 t4_fatal_err(struct adapter *sc)
2476 {
2477         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
2478         t4_intr_disable(sc);
2479         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
2480             device_get_nameunit(sc->dev));
2481         if (t4_panic_on_fatal_err)
2482                 panic("panic requested on fatal error");
2483 }
2484
2485 void
2486 t4_add_adapter(struct adapter *sc)
2487 {
2488         sx_xlock(&t4_list_lock);
2489         SLIST_INSERT_HEAD(&t4_list, sc, link);
2490         sx_xunlock(&t4_list_lock);
2491 }
2492
2493 int
2494 t4_map_bars_0_and_4(struct adapter *sc)
2495 {
2496         sc->regs_rid = PCIR_BAR(0);
2497         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2498             &sc->regs_rid, RF_ACTIVE);
2499         if (sc->regs_res == NULL) {
2500                 device_printf(sc->dev, "cannot map registers.\n");
2501                 return (ENXIO);
2502         }
2503         sc->bt = rman_get_bustag(sc->regs_res);
2504         sc->bh = rman_get_bushandle(sc->regs_res);
2505         sc->mmio_len = rman_get_size(sc->regs_res);
2506         setbit(&sc->doorbells, DOORBELL_KDB);
2507
2508         sc->msix_rid = PCIR_BAR(4);
2509         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2510             &sc->msix_rid, RF_ACTIVE);
2511         if (sc->msix_res == NULL) {
2512                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
2513                 return (ENXIO);
2514         }
2515
2516         return (0);
2517 }
2518
2519 int
2520 t4_map_bar_2(struct adapter *sc)
2521 {
2522
2523         /*
2524          * T4: only iWARP driver uses the userspace doorbells.  There is no need
2525          * to map it if RDMA is disabled.
2526          */
2527         if (is_t4(sc) && sc->rdmacaps == 0)
2528                 return (0);
2529
2530         sc->udbs_rid = PCIR_BAR(2);
2531         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
2532             &sc->udbs_rid, RF_ACTIVE);
2533         if (sc->udbs_res == NULL) {
2534                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
2535                 return (ENXIO);
2536         }
2537         sc->udbs_base = rman_get_virtual(sc->udbs_res);
2538
2539         if (chip_id(sc) >= CHELSIO_T5) {
2540                 setbit(&sc->doorbells, DOORBELL_UDB);
2541 #if defined(__i386__) || defined(__amd64__)
2542                 if (t5_write_combine) {
2543                         int rc, mode;
2544
2545                         /*
2546                          * Enable write combining on BAR2.  This is the
2547                          * userspace doorbell BAR and is split into 128B
2548                          * (UDBS_SEG_SIZE) doorbell regions, each associated
2549                          * with an egress queue.  The first 64B has the doorbell
2550                          * and the second 64B can be used to submit a tx work
2551                          * request with an implicit doorbell.
2552                          */
2553
2554                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
2555                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
2556                         if (rc == 0) {
2557                                 clrbit(&sc->doorbells, DOORBELL_UDB);
2558                                 setbit(&sc->doorbells, DOORBELL_WCWR);
2559                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
2560                         } else {
2561                                 device_printf(sc->dev,
2562                                     "couldn't enable write combining: %d\n",
2563                                     rc);
2564                         }
2565
2566                         mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0);
2567                         t4_write_reg(sc, A_SGE_STAT_CFG,
2568                             V_STATSOURCE_T5(7) | mode);
2569                 }
2570 #endif
2571         }
2572         sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0;
2573
2574         return (0);
2575 }
2576
2577 struct memwin_init {
2578         uint32_t base;
2579         uint32_t aperture;
2580 };
2581
2582 static const struct memwin_init t4_memwin[NUM_MEMWIN] = {
2583         { MEMWIN0_BASE, MEMWIN0_APERTURE },
2584         { MEMWIN1_BASE, MEMWIN1_APERTURE },
2585         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
2586 };
2587
2588 static const struct memwin_init t5_memwin[NUM_MEMWIN] = {
2589         { MEMWIN0_BASE, MEMWIN0_APERTURE },
2590         { MEMWIN1_BASE, MEMWIN1_APERTURE },
2591         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
2592 };
2593
2594 static void
2595 setup_memwin(struct adapter *sc)
2596 {
2597         const struct memwin_init *mw_init;
2598         struct memwin *mw;
2599         int i;
2600         uint32_t bar0;
2601
2602         if (is_t4(sc)) {
2603                 /*
2604                  * Read low 32b of bar0 indirectly via the hardware backdoor
2605                  * mechanism.  Works from within PCI passthrough environments
2606                  * too, where rman_get_start() can return a different value.  We
2607                  * need to program the T4 memory window decoders with the actual
2608                  * addresses that will be coming across the PCIe link.
2609                  */
2610                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
2611                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
2612
2613                 mw_init = &t4_memwin[0];
2614         } else {
2615                 /* T5+ use the relative offset inside the PCIe BAR */
2616                 bar0 = 0;
2617
2618                 mw_init = &t5_memwin[0];
2619         }
2620
2621         for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
2622                 rw_init(&mw->mw_lock, "memory window access");
2623                 mw->mw_base = mw_init->base;
2624                 mw->mw_aperture = mw_init->aperture;
2625                 mw->mw_curpos = 0;
2626                 t4_write_reg(sc,
2627                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
2628                     (mw->mw_base + bar0) | V_BIR(0) |
2629                     V_WINDOW(ilog2(mw->mw_aperture) - 10));
2630                 rw_wlock(&mw->mw_lock);
2631                 position_memwin(sc, i, 0);
2632                 rw_wunlock(&mw->mw_lock);
2633         }
2634
2635         /* flush */
2636         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
2637 }
2638
2639 /*
2640  * Positions the memory window at the given address in the card's address space.
2641  * There are some alignment requirements and the actual position may be at an
2642  * address prior to the requested address.  mw->mw_curpos always has the actual
2643  * position of the window.
2644  */
2645 static void
2646 position_memwin(struct adapter *sc, int idx, uint32_t addr)
2647 {
2648         struct memwin *mw;
2649         uint32_t pf;
2650         uint32_t reg;
2651
2652         MPASS(idx >= 0 && idx < NUM_MEMWIN);
2653         mw = &sc->memwin[idx];
2654         rw_assert(&mw->mw_lock, RA_WLOCKED);
2655
2656         if (is_t4(sc)) {
2657                 pf = 0;
2658                 mw->mw_curpos = addr & ~0xf;    /* start must be 16B aligned */
2659         } else {
2660                 pf = V_PFNUM(sc->pf);
2661                 mw->mw_curpos = addr & ~0x7f;   /* start must be 128B aligned */
2662         }
2663         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx);
2664         t4_write_reg(sc, reg, mw->mw_curpos | pf);
2665         t4_read_reg(sc, reg);   /* flush */
2666 }
2667
2668 int
2669 rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
2670     int len, int rw)
2671 {
2672         struct memwin *mw;
2673         uint32_t mw_end, v;
2674
2675         MPASS(idx >= 0 && idx < NUM_MEMWIN);
2676
2677         /* Memory can only be accessed in naturally aligned 4 byte units */
2678         if (addr & 3 || len & 3 || len <= 0)
2679                 return (EINVAL);
2680
2681         mw = &sc->memwin[idx];
2682         while (len > 0) {
2683                 rw_rlock(&mw->mw_lock);
2684                 mw_end = mw->mw_curpos + mw->mw_aperture;
2685                 if (addr >= mw_end || addr < mw->mw_curpos) {
2686                         /* Will need to reposition the window */
2687                         if (!rw_try_upgrade(&mw->mw_lock)) {
2688                                 rw_runlock(&mw->mw_lock);
2689                                 rw_wlock(&mw->mw_lock);
2690                         }
2691                         rw_assert(&mw->mw_lock, RA_WLOCKED);
2692                         position_memwin(sc, idx, addr);
2693                         rw_downgrade(&mw->mw_lock);
2694                         mw_end = mw->mw_curpos + mw->mw_aperture;
2695                 }
2696                 rw_assert(&mw->mw_lock, RA_RLOCKED);
2697                 while (addr < mw_end && len > 0) {
2698                         if (rw == 0) {
2699                                 v = t4_read_reg(sc, mw->mw_base + addr -
2700                                     mw->mw_curpos);
2701                                 *val++ = le32toh(v);
2702                         } else {
2703                                 v = *val++;
2704                                 t4_write_reg(sc, mw->mw_base + addr -
2705                                     mw->mw_curpos, htole32(v));
2706                         }
2707                         addr += 4;
2708                         len -= 4;
2709                 }
2710                 rw_runlock(&mw->mw_lock);
2711         }
2712
2713         return (0);
2714 }
2715
2716 int
2717 alloc_atid_tab(struct tid_info *t, int flags)
2718 {
2719         int i;
2720
2721         MPASS(t->natids > 0);
2722         MPASS(t->atid_tab == NULL);
2723
2724         t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE,
2725             M_ZERO | flags);
2726         if (t->atid_tab == NULL)
2727                 return (ENOMEM);
2728         mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF);
2729         t->afree = t->atid_tab;
2730         t->atids_in_use = 0;
2731         for (i = 1; i < t->natids; i++)
2732                 t->atid_tab[i - 1].next = &t->atid_tab[i];
2733         t->atid_tab[t->natids - 1].next = NULL;
2734
2735         return (0);
2736 }
2737
2738 void
2739 free_atid_tab(struct tid_info *t)
2740 {
2741
2742         KASSERT(t->atids_in_use == 0,
2743             ("%s: %d atids still in use.", __func__, t->atids_in_use));
2744
2745         if (mtx_initialized(&t->atid_lock))
2746                 mtx_destroy(&t->atid_lock);
2747         free(t->atid_tab, M_CXGBE);
2748         t->atid_tab = NULL;
2749 }
2750
2751 int
2752 alloc_atid(struct adapter *sc, void *ctx)
2753 {
2754         struct tid_info *t = &sc->tids;
2755         int atid = -1;
2756
2757         mtx_lock(&t->atid_lock);
2758         if (t->afree) {
2759                 union aopen_entry *p = t->afree;
2760
2761                 atid = p - t->atid_tab;
2762                 MPASS(atid <= M_TID_TID);
2763                 t->afree = p->next;
2764                 p->data = ctx;
2765                 t->atids_in_use++;
2766         }
2767         mtx_unlock(&t->atid_lock);
2768         return (atid);
2769 }
2770
2771 void *
2772 lookup_atid(struct adapter *sc, int atid)
2773 {
2774         struct tid_info *t = &sc->tids;
2775
2776         return (t->atid_tab[atid].data);
2777 }
2778
2779 void
2780 free_atid(struct adapter *sc, int atid)
2781 {
2782         struct tid_info *t = &sc->tids;
2783         union aopen_entry *p = &t->atid_tab[atid];
2784
2785         mtx_lock(&t->atid_lock);
2786         p->next = t->afree;
2787         t->afree = p;
2788         t->atids_in_use--;
2789         mtx_unlock(&t->atid_lock);
2790 }
2791
2792 static void
2793 queue_tid_release(struct adapter *sc, int tid)
2794 {
2795
2796         CXGBE_UNIMPLEMENTED("deferred tid release");
2797 }
2798
2799 void
2800 release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq)
2801 {
2802         struct wrqe *wr;
2803         struct cpl_tid_release *req;
2804
2805         wr = alloc_wrqe(sizeof(*req), ctrlq);
2806         if (wr == NULL) {
2807                 queue_tid_release(sc, tid);     /* defer */
2808                 return;
2809         }
2810         req = wrtod(wr);
2811
2812         INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid);
2813
2814         t4_wrq_tx(sc, wr);
2815 }
2816
2817 static int
2818 t4_range_cmp(const void *a, const void *b)
2819 {
2820         return ((const struct t4_range *)a)->start -
2821                ((const struct t4_range *)b)->start;
2822 }
2823
2824 /*
2825  * Verify that the memory range specified by the addr/len pair is valid within
2826  * the card's address space.
2827  */
2828 static int
2829 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
2830 {
2831         struct t4_range mem_ranges[4], *r, *next;
2832         uint32_t em, addr_len;
2833         int i, n, remaining;
2834
2835         /* Memory can only be accessed in naturally aligned 4 byte units */
2836         if (addr & 3 || len & 3 || len <= 0)
2837                 return (EINVAL);
2838
2839         /* Enabled memories */
2840         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2841
2842         r = &mem_ranges[0];
2843         n = 0;
2844         bzero(r, sizeof(mem_ranges));
2845         if (em & F_EDRAM0_ENABLE) {
2846                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2847                 r->size = G_EDRAM0_SIZE(addr_len) << 20;
2848                 if (r->size > 0) {
2849                         r->start = G_EDRAM0_BASE(addr_len) << 20;
2850                         if (addr >= r->start &&
2851                             addr + len <= r->start + r->size)
2852                                 return (0);
2853                         r++;
2854                         n++;
2855                 }
2856         }
2857         if (em & F_EDRAM1_ENABLE) {
2858                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2859                 r->size = G_EDRAM1_SIZE(addr_len) << 20;
2860                 if (r->size > 0) {
2861                         r->start = G_EDRAM1_BASE(addr_len) << 20;
2862                         if (addr >= r->start &&
2863                             addr + len <= r->start + r->size)
2864                                 return (0);
2865                         r++;
2866                         n++;
2867                 }
2868         }
2869         if (em & F_EXT_MEM_ENABLE) {
2870                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
2871                 r->size = G_EXT_MEM_SIZE(addr_len) << 20;
2872                 if (r->size > 0) {
2873                         r->start = G_EXT_MEM_BASE(addr_len) << 20;
2874                         if (addr >= r->start &&
2875                             addr + len <= r->start + r->size)
2876                                 return (0);
2877                         r++;
2878                         n++;
2879                 }
2880         }
2881         if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) {
2882                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
2883                 r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
2884                 if (r->size > 0) {
2885                         r->start = G_EXT_MEM1_BASE(addr_len) << 20;
2886                         if (addr >= r->start &&
2887                             addr + len <= r->start + r->size)
2888                                 return (0);
2889                         r++;
2890                         n++;
2891                 }
2892         }
2893         MPASS(n <= nitems(mem_ranges));
2894
2895         if (n > 1) {
2896                 /* Sort and merge the ranges. */
2897                 qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp);
2898
2899                 /* Start from index 0 and examine the next n - 1 entries. */
2900                 r = &mem_ranges[0];
2901                 for (remaining = n - 1; remaining > 0; remaining--, r++) {
2902
2903                         MPASS(r->size > 0);     /* r is a valid entry. */
2904                         next = r + 1;
2905                         MPASS(next->size > 0);  /* and so is the next one. */
2906
2907                         while (r->start + r->size >= next->start) {
2908                                 /* Merge the next one into the current entry. */
2909                                 r->size = max(r->start + r->size,
2910                                     next->start + next->size) - r->start;
2911                                 n--;    /* One fewer entry in total. */
2912                                 if (--remaining == 0)
2913                                         goto done;      /* short circuit */
2914                                 next++;
2915                         }
2916                         if (next != r + 1) {
2917                                 /*
2918                                  * Some entries were merged into r and next
2919                                  * points to the first valid entry that couldn't
2920                                  * be merged.
2921                                  */
2922                                 MPASS(next->size > 0);  /* must be valid */
2923                                 memcpy(r + 1, next, remaining * sizeof(*r));
2924 #ifdef INVARIANTS
2925                                 /*
2926                                  * This so that the foo->size assertion in the
2927                                  * next iteration of the loop do the right
2928                                  * thing for entries that were pulled up and are
2929                                  * no longer valid.
2930                                  */
2931                                 MPASS(n < nitems(mem_ranges));
2932                                 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
2933                                     sizeof(struct t4_range));
2934 #endif
2935                         }
2936                 }
2937 done:
2938                 /* Done merging the ranges. */
2939                 MPASS(n > 0);
2940                 r = &mem_ranges[0];
2941                 for (i = 0; i < n; i++, r++) {
2942                         if (addr >= r->start &&
2943                             addr + len <= r->start + r->size)
2944                                 return (0);
2945                 }
2946         }
2947
2948         return (EFAULT);
2949 }
2950
2951 static int
2952 fwmtype_to_hwmtype(int mtype)
2953 {
2954
2955         switch (mtype) {
2956         case FW_MEMTYPE_EDC0:
2957                 return (MEM_EDC0);
2958         case FW_MEMTYPE_EDC1:
2959                 return (MEM_EDC1);
2960         case FW_MEMTYPE_EXTMEM:
2961                 return (MEM_MC0);
2962         case FW_MEMTYPE_EXTMEM1:
2963                 return (MEM_MC1);
2964         default:
2965                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
2966         }
2967 }
2968
2969 /*
2970  * Verify that the memory range specified by the memtype/offset/len pair is
2971  * valid and lies entirely within the memtype specified.  The global address of
2972  * the start of the range is returned in addr.
2973  */
2974 static int
2975 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
2976     uint32_t *addr)
2977 {
2978         uint32_t em, addr_len, maddr;
2979
2980         /* Memory can only be accessed in naturally aligned 4 byte units */
2981         if (off & 3 || len & 3 || len == 0)
2982                 return (EINVAL);
2983
2984         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
2985         switch (fwmtype_to_hwmtype(mtype)) {
2986         case MEM_EDC0:
2987                 if (!(em & F_EDRAM0_ENABLE))
2988                         return (EINVAL);
2989                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
2990                 maddr = G_EDRAM0_BASE(addr_len) << 20;
2991                 break;
2992         case MEM_EDC1:
2993                 if (!(em & F_EDRAM1_ENABLE))
2994                         return (EINVAL);
2995                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
2996                 maddr = G_EDRAM1_BASE(addr_len) << 20;
2997                 break;
2998         case MEM_MC:
2999                 if (!(em & F_EXT_MEM_ENABLE))
3000                         return (EINVAL);
3001                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
3002                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
3003                 break;
3004         case MEM_MC1:
3005                 if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE))
3006                         return (EINVAL);
3007                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
3008                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
3009                 break;
3010         default:
3011                 return (EINVAL);
3012         }
3013
3014         *addr = maddr + off;    /* global address */
3015         return (validate_mem_range(sc, *addr, len));
3016 }
3017
3018 static int
3019 fixup_devlog_params(struct adapter *sc)
3020 {
3021         struct devlog_params *dparams = &sc->params.devlog;
3022         int rc;
3023
3024         rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
3025             dparams->size, &dparams->addr);
3026
3027         return (rc);
3028 }
3029
3030 static void
3031 update_nirq(struct intrs_and_queues *iaq, int nports)
3032 {
3033         int extra = T4_EXTRA_INTR;
3034
3035         iaq->nirq = extra;
3036         iaq->nirq += nports * (iaq->nrxq + iaq->nofldrxq);
3037         iaq->nirq += nports * (iaq->num_vis - 1) *
3038             max(iaq->nrxq_vi, iaq->nnmrxq_vi);
3039         iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi;
3040 }
3041
3042 /*
3043  * Adjust requirements to fit the number of interrupts available.
3044  */
3045 static void
3046 calculate_iaq(struct adapter *sc, struct intrs_and_queues *iaq, int itype,
3047     int navail)
3048 {
3049         int old_nirq;
3050         const int nports = sc->params.nports;
3051
3052         MPASS(nports > 0);
3053         MPASS(navail > 0);
3054
3055         bzero(iaq, sizeof(*iaq));
3056         iaq->intr_type = itype;
3057         iaq->num_vis = t4_num_vis;
3058         iaq->ntxq = t4_ntxq;
3059         iaq->ntxq_vi = t4_ntxq_vi;
3060         iaq->nrxq = t4_nrxq;
3061         iaq->nrxq_vi = t4_nrxq_vi;
3062 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
3063         if (is_offload(sc) || is_ethoffload(sc)) {
3064                 iaq->nofldtxq = t4_nofldtxq;
3065                 iaq->nofldtxq_vi = t4_nofldtxq_vi;
3066         }
3067 #endif
3068 #ifdef TCP_OFFLOAD
3069         if (is_offload(sc)) {
3070                 iaq->nofldrxq = t4_nofldrxq;
3071                 iaq->nofldrxq_vi = t4_nofldrxq_vi;
3072         }
3073 #endif
3074 #ifdef DEV_NETMAP
3075         iaq->nnmtxq_vi = t4_nnmtxq_vi;
3076         iaq->nnmrxq_vi = t4_nnmrxq_vi;
3077 #endif
3078
3079         update_nirq(iaq, nports);
3080         if (iaq->nirq <= navail &&
3081             (itype != INTR_MSI || powerof2(iaq->nirq))) {
3082                 /*
3083                  * This is the normal case -- there are enough interrupts for
3084                  * everything.
3085                  */
3086                 goto done;
3087         }
3088
3089         /*
3090          * If extra VIs have been configured try reducing their count and see if
3091          * that works.
3092          */
3093         while (iaq->num_vis > 1) {
3094                 iaq->num_vis--;
3095                 update_nirq(iaq, nports);
3096                 if (iaq->nirq <= navail &&
3097                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
3098                         device_printf(sc->dev, "virtual interfaces per port "
3099                             "reduced to %d from %d.  nrxq=%u, nofldrxq=%u, "
3100                             "nrxq_vi=%u nofldrxq_vi=%u, nnmrxq_vi=%u.  "
3101                             "itype %d, navail %u, nirq %d.\n",
3102                             iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq,
3103                             iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi,
3104                             itype, navail, iaq->nirq);
3105                         goto done;
3106                 }
3107         }
3108
3109         /*
3110          * Extra VIs will not be created.  Log a message if they were requested.
3111          */
3112         MPASS(iaq->num_vis == 1);
3113         iaq->ntxq_vi = iaq->nrxq_vi = 0;
3114         iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
3115         iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
3116         if (iaq->num_vis != t4_num_vis) {
3117                 device_printf(sc->dev, "extra virtual interfaces disabled.  "
3118                     "nrxq=%u, nofldrxq=%u, nrxq_vi=%u nofldrxq_vi=%u, "
3119                     "nnmrxq_vi=%u.  itype %d, navail %u, nirq %d.\n",
3120                     iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi,
3121                     iaq->nnmrxq_vi, itype, navail, iaq->nirq);
3122         }
3123
3124         /*
3125          * Keep reducing the number of NIC rx queues to the next lower power of
3126          * 2 (for even RSS distribution) and halving the TOE rx queues and see
3127          * if that works.
3128          */
3129         do {
3130                 if (iaq->nrxq > 1) {
3131                         do {
3132                                 iaq->nrxq--;
3133                         } while (!powerof2(iaq->nrxq));
3134                 }
3135                 if (iaq->nofldrxq > 1)
3136                         iaq->nofldrxq >>= 1;
3137
3138                 old_nirq = iaq->nirq;
3139                 update_nirq(iaq, nports);
3140                 if (iaq->nirq <= navail &&
3141                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
3142                         device_printf(sc->dev, "running with reduced number of "
3143                             "rx queues because of shortage of interrupts.  "
3144                             "nrxq=%u, nofldrxq=%u.  "
3145                             "itype %d, navail %u, nirq %d.\n", iaq->nrxq,
3146                             iaq->nofldrxq, itype, navail, iaq->nirq);
3147                         goto done;
3148                 }
3149         } while (old_nirq != iaq->nirq);
3150
3151         /* One interrupt for everything.  Ugh. */
3152         device_printf(sc->dev, "running with minimal number of queues.  "
3153             "itype %d, navail %u.\n", itype, navail);
3154         iaq->nirq = 1;
3155         MPASS(iaq->nrxq == 1);
3156         iaq->ntxq = 1;
3157         if (iaq->nofldrxq > 1)
3158                 iaq->nofldtxq = 1;
3159 done:
3160         MPASS(iaq->num_vis > 0);
3161         if (iaq->num_vis > 1) {
3162                 MPASS(iaq->nrxq_vi > 0);
3163                 MPASS(iaq->ntxq_vi > 0);
3164         }
3165         MPASS(iaq->nirq > 0);
3166         MPASS(iaq->nrxq > 0);
3167         MPASS(iaq->ntxq > 0);
3168         if (itype == INTR_MSI) {
3169                 MPASS(powerof2(iaq->nirq));
3170         }
3171 }
3172
3173 static int
3174 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq)
3175 {
3176         int rc, itype, navail, nalloc;
3177
3178         for (itype = INTR_MSIX; itype; itype >>= 1) {
3179
3180                 if ((itype & t4_intr_types) == 0)
3181                         continue;       /* not allowed */
3182
3183                 if (itype == INTR_MSIX)
3184                         navail = pci_msix_count(sc->dev);
3185                 else if (itype == INTR_MSI)
3186                         navail = pci_msi_count(sc->dev);
3187                 else
3188                         navail = 1;
3189 restart:
3190                 if (navail == 0)
3191                         continue;
3192
3193                 calculate_iaq(sc, iaq, itype, navail);
3194                 nalloc = iaq->nirq;
3195                 rc = 0;
3196                 if (itype == INTR_MSIX)
3197                         rc = pci_alloc_msix(sc->dev, &nalloc);
3198                 else if (itype == INTR_MSI)
3199                         rc = pci_alloc_msi(sc->dev, &nalloc);
3200
3201                 if (rc == 0 && nalloc > 0) {
3202                         if (nalloc == iaq->nirq)
3203                                 return (0);
3204
3205                         /*
3206                          * Didn't get the number requested.  Use whatever number
3207                          * the kernel is willing to allocate.
3208                          */
3209                         device_printf(sc->dev, "fewer vectors than requested, "
3210                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
3211                             itype, iaq->nirq, nalloc);
3212                         pci_release_msi(sc->dev);
3213                         navail = nalloc;
3214                         goto restart;
3215                 }
3216
3217                 device_printf(sc->dev,
3218                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
3219                     itype, rc, iaq->nirq, nalloc);
3220         }
3221
3222         device_printf(sc->dev,
3223             "failed to find a usable interrupt type.  "
3224             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
3225             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
3226
3227         return (ENXIO);
3228 }
3229
3230 #define FW_VERSION(chip) ( \
3231     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
3232     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
3233     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
3234     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
3235 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
3236
3237 struct fw_info {
3238         uint8_t chip;
3239         char *kld_name;
3240         char *fw_mod_name;
3241         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
3242 } fw_info[] = {
3243         {
3244                 .chip = CHELSIO_T4,
3245                 .kld_name = "t4fw_cfg",
3246                 .fw_mod_name = "t4fw",
3247                 .fw_hdr = {
3248                         .chip = FW_HDR_CHIP_T4,
3249                         .fw_ver = htobe32(FW_VERSION(T4)),
3250                         .intfver_nic = FW_INTFVER(T4, NIC),
3251                         .intfver_vnic = FW_INTFVER(T4, VNIC),
3252                         .intfver_ofld = FW_INTFVER(T4, OFLD),
3253                         .intfver_ri = FW_INTFVER(T4, RI),
3254                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
3255                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
3256                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
3257                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
3258                 },
3259         }, {
3260                 .chip = CHELSIO_T5,
3261                 .kld_name = "t5fw_cfg",
3262                 .fw_mod_name = "t5fw",
3263                 .fw_hdr = {
3264                         .chip = FW_HDR_CHIP_T5,
3265                         .fw_ver = htobe32(FW_VERSION(T5)),
3266                         .intfver_nic = FW_INTFVER(T5, NIC),
3267                         .intfver_vnic = FW_INTFVER(T5, VNIC),
3268                         .intfver_ofld = FW_INTFVER(T5, OFLD),
3269                         .intfver_ri = FW_INTFVER(T5, RI),
3270                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
3271                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
3272                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
3273                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
3274                 },
3275         }, {
3276                 .chip = CHELSIO_T6,
3277                 .kld_name = "t6fw_cfg",
3278                 .fw_mod_name = "t6fw",
3279                 .fw_hdr = {
3280                         .chip = FW_HDR_CHIP_T6,
3281                         .fw_ver = htobe32(FW_VERSION(T6)),
3282                         .intfver_nic = FW_INTFVER(T6, NIC),
3283                         .intfver_vnic = FW_INTFVER(T6, VNIC),
3284                         .intfver_ofld = FW_INTFVER(T6, OFLD),
3285                         .intfver_ri = FW_INTFVER(T6, RI),
3286                         .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
3287                         .intfver_iscsi = FW_INTFVER(T6, ISCSI),
3288                         .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
3289                         .intfver_fcoe = FW_INTFVER(T6, FCOE),
3290                 },
3291         }
3292 };
3293
3294 static struct fw_info *
3295 find_fw_info(int chip)
3296 {
3297         int i;
3298
3299         for (i = 0; i < nitems(fw_info); i++) {
3300                 if (fw_info[i].chip == chip)
3301                         return (&fw_info[i]);
3302         }
3303         return (NULL);
3304 }
3305
3306 /*
3307  * Is the given firmware API compatible with the one the driver was compiled
3308  * with?
3309  */
3310 static int
3311 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
3312 {
3313
3314         /* short circuit if it's the exact same firmware version */
3315         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
3316                 return (1);
3317
3318         /*
3319          * XXX: Is this too conservative?  Perhaps I should limit this to the
3320          * features that are supported in the driver.
3321          */
3322 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
3323         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
3324             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
3325             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
3326                 return (1);
3327 #undef SAME_INTF
3328
3329         return (0);
3330 }
3331
3332 /*
3333  * The firmware in the KLD is usable, but should it be installed?  This routine
3334  * explains itself in detail if it indicates the KLD firmware should be
3335  * installed.
3336  */
3337 static int
3338 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
3339 {
3340         const char *reason;
3341
3342         if (!card_fw_usable) {
3343                 reason = "incompatible or unusable";
3344                 goto install;
3345         }
3346
3347         if (k > c) {
3348                 reason = "older than the version bundled with this driver";
3349                 goto install;
3350         }
3351
3352         if (t4_fw_install == 2 && k != c) {
3353                 reason = "different than the version bundled with this driver";
3354                 goto install;
3355         }
3356
3357         return (0);
3358
3359 install:
3360         if (t4_fw_install == 0) {
3361                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
3362                     "but the driver is prohibited from installing a different "
3363                     "firmware on the card.\n",
3364                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
3365                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
3366
3367                 return (0);
3368         }
3369
3370         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
3371             "installing firmware %u.%u.%u.%u on card.\n",
3372             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
3373             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
3374             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
3375             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
3376
3377         return (1);
3378 }
3379
3380 /*
3381  * Establish contact with the firmware and determine if we are the master driver
3382  * or not, and whether we are responsible for chip initialization.
3383  */
3384 static int
3385 prep_firmware(struct adapter *sc)
3386 {
3387         const struct firmware *fw = NULL, *default_cfg;
3388         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
3389         enum dev_state state;
3390         struct fw_info *fw_info;
3391         struct fw_hdr *card_fw;         /* fw on the card */
3392         const struct fw_hdr *kld_fw;    /* fw in the KLD */
3393         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
3394                                            against */
3395
3396         /* This is the firmware whose headers the driver was compiled against */
3397         fw_info = find_fw_info(chip_id(sc));
3398         if (fw_info == NULL) {
3399                 device_printf(sc->dev,
3400                     "unable to look up firmware information for chip %d.\n",
3401                     chip_id(sc));
3402                 return (EINVAL);
3403         }
3404         drv_fw = &fw_info->fw_hdr;
3405
3406         /*
3407          * The firmware KLD contains many modules.  The KLD name is also the
3408          * name of the module that contains the default config file.
3409          */
3410         default_cfg = firmware_get(fw_info->kld_name);
3411
3412         /* This is the firmware in the KLD */
3413         fw = firmware_get(fw_info->fw_mod_name);
3414         if (fw != NULL) {
3415                 kld_fw = (const void *)fw->data;
3416                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
3417         } else {
3418                 kld_fw = NULL;
3419                 kld_fw_usable = 0;
3420         }
3421
3422         /* Read the header of the firmware on the card */
3423         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
3424         rc = -t4_read_flash(sc, FLASH_FW_START,
3425             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
3426         if (rc == 0) {
3427                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
3428                 if (card_fw->fw_ver == be32toh(0xffffffff)) {
3429                         uint32_t d = be32toh(kld_fw->fw_ver);
3430
3431                         if (!kld_fw_usable) {
3432                                 device_printf(sc->dev,
3433                                     "no firmware on the card and no usable "
3434                                     "firmware bundled with the driver.\n");
3435                                 rc = EIO;
3436                                 goto done;
3437                         } else if (t4_fw_install == 0) {
3438                                 device_printf(sc->dev,
3439                                     "no firmware on the card and the driver "
3440                                     "is prohibited from installing new "
3441                                     "firmware.\n");
3442                                 rc = EIO;
3443                                 goto done;
3444                         }
3445
3446                         device_printf(sc->dev, "no firmware on the card, "
3447                             "installing firmware %d.%d.%d.%d\n",
3448                             G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
3449                             G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d));
3450                         rc = t4_fw_forceinstall(sc, fw->data, fw->datasize);
3451                         if (rc < 0) {
3452                                 rc = -rc;
3453                                 device_printf(sc->dev,
3454                                     "firmware install failed: %d.\n", rc);
3455                                 goto done;
3456                         }
3457                         memcpy(card_fw, kld_fw, sizeof(*card_fw));
3458                         card_fw_usable = 1;
3459                         need_fw_reset = 0;
3460                 }
3461         } else {
3462                 device_printf(sc->dev,
3463                     "Unable to read card's firmware header: %d\n", rc);
3464                 card_fw_usable = 0;
3465         }
3466
3467         /* Contact firmware. */
3468         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
3469         if (rc < 0 || state == DEV_STATE_ERR) {
3470                 rc = -rc;
3471                 device_printf(sc->dev,
3472                     "failed to connect to the firmware: %d, %d.\n", rc, state);
3473                 goto done;
3474         }
3475         pf = rc;
3476         if (pf == sc->mbox)
3477                 sc->flags |= MASTER_PF;
3478         else if (state == DEV_STATE_UNINIT) {
3479                 /*
3480                  * We didn't get to be the master so we definitely won't be
3481                  * configuring the chip.  It's a bug if someone else hasn't
3482                  * configured it already.
3483                  */
3484                 device_printf(sc->dev, "couldn't be master(%d), "
3485                     "device not already initialized either(%d).\n", rc, state);
3486                 rc = EPROTO;
3487                 goto done;
3488         }
3489
3490         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
3491             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
3492                 /*
3493                  * Common case: the firmware on the card is an exact match and
3494                  * the KLD is an exact match too, or the KLD is
3495                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
3496                  * here -- use cxgbetool loadfw if you want to reinstall the
3497                  * same firmware as the one on the card.
3498                  */
3499         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
3500             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
3501             be32toh(card_fw->fw_ver))) {
3502
3503                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
3504                 if (rc != 0) {
3505                         device_printf(sc->dev,
3506                             "failed to install firmware: %d\n", rc);
3507                         goto done;
3508                 }
3509
3510                 /* Installed successfully, update the cached header too. */
3511                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
3512                 card_fw_usable = 1;
3513                 need_fw_reset = 0;      /* already reset as part of load_fw */
3514         }
3515
3516         if (!card_fw_usable) {
3517                 uint32_t d, c, k;
3518
3519                 d = ntohl(drv_fw->fw_ver);
3520                 c = ntohl(card_fw->fw_ver);
3521                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
3522
3523                 device_printf(sc->dev, "Cannot find a usable firmware: "
3524                     "fw_install %d, chip state %d, "
3525                     "driver compiled with %d.%d.%d.%d, "
3526                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
3527                     t4_fw_install, state,
3528                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
3529                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
3530                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
3531                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
3532                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
3533                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
3534                 rc = EINVAL;
3535                 goto done;
3536         }
3537
3538         /* Reset device */
3539         if (need_fw_reset &&
3540             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
3541                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
3542                 if (rc != ETIMEDOUT && rc != EIO)
3543                         t4_fw_bye(sc, sc->mbox);
3544                 goto done;
3545         }
3546         sc->flags |= FW_OK;
3547
3548         rc = get_params__pre_init(sc);
3549         if (rc != 0)
3550                 goto done; /* error message displayed already */
3551
3552         /* Partition adapter resources as specified in the config file. */
3553         if (state == DEV_STATE_UNINIT) {
3554
3555                 KASSERT(sc->flags & MASTER_PF,
3556                     ("%s: trying to change chip settings when not master.",
3557                     __func__));
3558
3559                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
3560                 if (rc != 0)
3561                         goto done;      /* error message displayed already */
3562
3563                 t4_tweak_chip_settings(sc);
3564
3565                 /* get basic stuff going */
3566                 rc = -t4_fw_initialize(sc, sc->mbox);
3567                 if (rc != 0) {
3568                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
3569                         goto done;
3570                 }
3571         } else {
3572                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
3573                 sc->cfcsum = 0;
3574         }
3575
3576 done:
3577         free(card_fw, M_CXGBE);
3578         if (fw != NULL)
3579                 firmware_put(fw, FIRMWARE_UNLOAD);
3580         if (default_cfg != NULL)
3581                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
3582
3583         return (rc);
3584 }
3585
3586 #define FW_PARAM_DEV(param) \
3587         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3588          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3589 #define FW_PARAM_PFVF(param) \
3590         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3591          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
3592
3593 /*
3594  * Partition chip resources for use between various PFs, VFs, etc.
3595  */
3596 static int
3597 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
3598     const char *name_prefix)
3599 {
3600         const struct firmware *cfg = NULL;
3601         int rc = 0;
3602         struct fw_caps_config_cmd caps;
3603         uint32_t mtype, moff, finicsum, cfcsum;
3604
3605         /*
3606          * Figure out what configuration file to use.  Pick the default config
3607          * file for the card if the user hasn't specified one explicitly.
3608          */
3609         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
3610         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
3611                 /* Card specific overrides go here. */
3612                 if (pci_get_device(sc->dev) == 0x440a)
3613                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
3614                 if (is_fpga(sc))
3615                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
3616         } else if (strncmp(t4_cfg_file, BUILTIN_CF, sizeof(t4_cfg_file)) == 0)
3617                 goto use_built_in_config;       /* go straight to config. */
3618
3619         /*
3620          * We need to load another module if the profile is anything except
3621          * "default" or "flash".
3622          */
3623         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
3624             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
3625                 char s[32];
3626
3627                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
3628                 cfg = firmware_get(s);
3629                 if (cfg == NULL) {
3630                         if (default_cfg != NULL) {
3631                                 device_printf(sc->dev,
3632                                     "unable to load module \"%s\" for "
3633                                     "configuration profile \"%s\", will use "
3634                                     "the default config file instead.\n",
3635                                     s, sc->cfg_file);
3636                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
3637                                     "%s", DEFAULT_CF);
3638                         } else {
3639                                 device_printf(sc->dev,
3640                                     "unable to load module \"%s\" for "
3641                                     "configuration profile \"%s\", will use "
3642                                     "the config file on the card's flash "
3643                                     "instead.\n", s, sc->cfg_file);
3644                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
3645                                     "%s", FLASH_CF);
3646                         }
3647                 }
3648         }
3649
3650         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
3651             default_cfg == NULL) {
3652                 device_printf(sc->dev,
3653                     "default config file not available, will use the config "
3654                     "file on the card's flash instead.\n");
3655                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
3656         }
3657
3658         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
3659                 u_int cflen;
3660                 const uint32_t *cfdata;
3661                 uint32_t param, val, addr;
3662
3663                 KASSERT(cfg != NULL || default_cfg != NULL,
3664                     ("%s: no config to upload", __func__));
3665
3666                 /*
3667                  * Ask the firmware where it wants us to upload the config file.
3668                  */
3669                 param = FW_PARAM_DEV(CF);
3670                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
3671                 if (rc != 0) {
3672                         /* No support for config file?  Shouldn't happen. */
3673                         device_printf(sc->dev,
3674                             "failed to query config file location: %d.\n", rc);
3675                         goto done;
3676                 }
3677                 mtype = G_FW_PARAMS_PARAM_Y(val);
3678                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
3679
3680                 /*
3681                  * XXX: sheer laziness.  We deliberately added 4 bytes of
3682                  * useless stuffing/comments at the end of the config file so
3683                  * it's ok to simply throw away the last remaining bytes when
3684                  * the config file is not an exact multiple of 4.  This also
3685                  * helps with the validate_mt_off_len check.
3686                  */
3687                 if (cfg != NULL) {
3688                         cflen = cfg->datasize & ~3;
3689                         cfdata = cfg->data;
3690                 } else {
3691                         cflen = default_cfg->datasize & ~3;
3692                         cfdata = default_cfg->data;
3693                 }
3694
3695                 if (cflen > FLASH_CFG_MAX_SIZE) {
3696                         device_printf(sc->dev,
3697                             "config file too long (%d, max allowed is %d).  "
3698                             "Will try to use the config on the card, if any.\n",
3699                             cflen, FLASH_CFG_MAX_SIZE);
3700                         goto use_config_on_flash;
3701                 }
3702
3703                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
3704                 if (rc != 0) {
3705                         device_printf(sc->dev,
3706                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
3707                             "Will try to use the config on the card, if any.\n",
3708                             __func__, mtype, moff, cflen, rc);
3709                         goto use_config_on_flash;
3710                 }
3711                 write_via_memwin(sc, 2, addr, cfdata, cflen);
3712         } else {
3713 use_config_on_flash:
3714                 mtype = FW_MEMTYPE_FLASH;
3715                 moff = t4_flash_cfg_addr(sc);
3716         }
3717
3718         bzero(&caps, sizeof(caps));
3719         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3720             F_FW_CMD_REQUEST | F_FW_CMD_READ);
3721         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
3722             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
3723             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
3724         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3725         if (rc != 0) {
3726                 device_printf(sc->dev,
3727                     "failed to pre-process config file: %d "
3728                     "(mtype %d, moff 0x%x).  Will reset the firmware and retry "
3729                     "with the built-in configuration.\n", rc, mtype, moff);
3730
3731                 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
3732                 if (rc != 0) {
3733                         device_printf(sc->dev,
3734                             "firmware reset failed: %d.\n", rc);
3735                         if (rc != ETIMEDOUT && rc != EIO) {
3736                                 t4_fw_bye(sc, sc->mbox);
3737                                 sc->flags &= ~FW_OK;
3738                         }
3739                         goto done;
3740                 }
3741                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", "built-in");
3742 use_built_in_config:
3743                 bzero(&caps, sizeof(caps));
3744                 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3745                     F_FW_CMD_REQUEST | F_FW_CMD_READ);
3746                 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
3747                 rc = t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3748                 if (rc != 0) {
3749                         device_printf(sc->dev,
3750                             "built-in configuration failed: %d.\n", rc);
3751                         goto done;
3752                 }
3753         }
3754
3755         finicsum = be32toh(caps.finicsum);
3756         cfcsum = be32toh(caps.cfcsum);
3757         if (finicsum != cfcsum) {
3758                 device_printf(sc->dev,
3759                     "WARNING: config file checksum mismatch: %08x %08x\n",
3760                     finicsum, cfcsum);
3761         }
3762         sc->cfcsum = cfcsum;
3763
3764 #define LIMIT_CAPS(x) do { \
3765         caps.x &= htobe16(t4_##x##_allowed); \
3766 } while (0)
3767
3768         /*
3769          * Let the firmware know what features will (not) be used so it can tune
3770          * things accordingly.
3771          */
3772         LIMIT_CAPS(nbmcaps);
3773         LIMIT_CAPS(linkcaps);
3774         LIMIT_CAPS(switchcaps);
3775         LIMIT_CAPS(niccaps);
3776         LIMIT_CAPS(toecaps);
3777         LIMIT_CAPS(rdmacaps);
3778         LIMIT_CAPS(cryptocaps);
3779         LIMIT_CAPS(iscsicaps);
3780         LIMIT_CAPS(fcoecaps);
3781 #undef LIMIT_CAPS
3782
3783         if (caps.niccaps & htobe16(FW_CAPS_CONFIG_NIC_HASHFILTER)) {
3784                 /*
3785                  * TOE and hashfilters are mutually exclusive.  It is a config
3786                  * file or firmware bug if both are reported as available.  Try
3787                  * to cope with the situation in non-debug builds by disabling
3788                  * TOE.
3789                  */
3790                 MPASS(caps.toecaps == 0);
3791
3792                 caps.toecaps = 0;
3793                 caps.rdmacaps = 0;
3794                 caps.iscsicaps = 0;
3795         }
3796
3797         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3798             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
3799         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
3800         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
3801         if (rc != 0) {
3802                 device_printf(sc->dev,
3803                     "failed to process config file: %d.\n", rc);
3804         }
3805 done:
3806         if (cfg != NULL)
3807                 firmware_put(cfg, FIRMWARE_UNLOAD);
3808         return (rc);
3809 }
3810
3811 /*
3812  * Retrieve parameters that are needed (or nice to have) very early.
3813  */
3814 static int
3815 get_params__pre_init(struct adapter *sc)
3816 {
3817         int rc;
3818         uint32_t param[2], val[2];
3819
3820         t4_get_version_info(sc);
3821
3822         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
3823             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
3824             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
3825             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
3826             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
3827
3828         snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
3829             G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
3830             G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
3831             G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
3832             G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));
3833
3834         snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
3835             G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
3836             G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
3837             G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
3838             G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
3839
3840         snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
3841             G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
3842             G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
3843             G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
3844             G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));
3845
3846         param[0] = FW_PARAM_DEV(PORTVEC);
3847         param[1] = FW_PARAM_DEV(CCLK);
3848         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3849         if (rc != 0) {
3850                 device_printf(sc->dev,
3851                     "failed to query parameters (pre_init): %d.\n", rc);
3852                 return (rc);
3853         }
3854
3855         sc->params.portvec = val[0];
3856         sc->params.nports = bitcount32(val[0]);
3857         sc->params.vpd.cclk = val[1];
3858
3859         /* Read device log parameters. */
3860         rc = -t4_init_devlog_params(sc, 1);
3861         if (rc == 0)
3862                 fixup_devlog_params(sc);
3863         else {
3864                 device_printf(sc->dev,
3865                     "failed to get devlog parameters: %d.\n", rc);
3866                 rc = 0; /* devlog isn't critical for device operation */
3867         }
3868
3869         return (rc);
3870 }
3871
3872 /*
3873  * Retrieve various parameters that are of interest to the driver.  The device
3874  * has been initialized by the firmware at this point.
3875  */
3876 static int
3877 get_params__post_init(struct adapter *sc)
3878 {
3879         int rc;
3880         uint32_t param[7], val[7];
3881         struct fw_caps_config_cmd caps;
3882
3883         param[0] = FW_PARAM_PFVF(IQFLINT_START);
3884         param[1] = FW_PARAM_PFVF(EQ_START);
3885         param[2] = FW_PARAM_PFVF(FILTER_START);
3886         param[3] = FW_PARAM_PFVF(FILTER_END);
3887         param[4] = FW_PARAM_PFVF(L2T_START);
3888         param[5] = FW_PARAM_PFVF(L2T_END);
3889         param[6] = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3890             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
3891             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD);
3892         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val);
3893         if (rc != 0) {
3894                 device_printf(sc->dev,
3895                     "failed to query parameters (post_init): %d.\n", rc);
3896                 return (rc);
3897         }
3898
3899         sc->sge.iq_start = val[0];
3900         sc->sge.eq_start = val[1];
3901         if ((int)val[3] > (int)val[2]) {
3902                 sc->tids.ftid_base = val[2];
3903                 sc->tids.ftid_end = val[3];
3904                 sc->tids.nftids = val[3] - val[2] + 1;
3905         }
3906         sc->vres.l2t.start = val[4];
3907         sc->vres.l2t.size = val[5] - val[4] + 1;
3908         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
3909             ("%s: L2 table size (%u) larger than expected (%u)",
3910             __func__, sc->vres.l2t.size, L2T_SIZE));
3911         sc->params.core_vdd = val[6];
3912
3913         if (chip_id(sc) >= CHELSIO_T6) {
3914
3915 #ifdef INVARIANTS
3916                 if (sc->params.fw_vers >=
3917                     (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) |
3918                     V_FW_HDR_FW_VER_MICRO(1) | V_FW_HDR_FW_VER_BUILD(0))) {
3919                         /*
3920                          * Note that the code to enable the region should run
3921                          * before t4_fw_initialize and not here.  This is just a
3922                          * reminder to add said code.
3923                          */
3924                         device_printf(sc->dev,
3925                             "hpfilter region not enabled.\n");
3926                 }
3927 #endif
3928
3929                 sc->tids.tid_base = t4_read_reg(sc,
3930                     A_LE_DB_ACTIVE_TABLE_START_INDEX);
3931
3932                 param[0] = FW_PARAM_PFVF(HPFILTER_START);
3933                 param[1] = FW_PARAM_PFVF(HPFILTER_END);
3934                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
3935                 if (rc != 0) {
3936                         device_printf(sc->dev,
3937                            "failed to query hpfilter parameters: %d.\n", rc);
3938                         return (rc);
3939                 }
3940                 if ((int)val[1] > (int)val[0]) {
3941                         sc->tids.hpftid_base = val[0];
3942                         sc->tids.hpftid_end = val[1];
3943                         sc->tids.nhpftids = val[1] - val[0] + 1;
3944
3945                         /*
3946                          * These should go off if the layout changes and the
3947                          * driver needs to catch up.
3948                          */
3949                         MPASS(sc->tids.hpftid_base == 0);
3950                         MPASS(sc->tids.tid_base == sc->tids.nhpftids);
3951                 }
3952         }
3953
3954         /*
3955          * MPSBGMAP is queried separately because only recent firmwares support
3956          * it as a parameter and we don't want the compound query above to fail
3957          * on older firmwares.
3958          */
3959         param[0] = FW_PARAM_DEV(MPSBGMAP);
3960         val[0] = 0;
3961         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
3962         if (rc == 0)
3963                 sc->params.mps_bg_map = val[0];
3964         else
3965                 sc->params.mps_bg_map = 0;
3966
3967         /*
3968          * Determine whether the firmware supports the filter2 work request.
3969          * This is queried separately for the same reason as MPSBGMAP above.
3970          */
3971         param[0] = FW_PARAM_DEV(FILTER2_WR);
3972         val[0] = 0;
3973         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
3974         if (rc == 0)
3975                 sc->params.filter2_wr_support = val[0] != 0;
3976         else
3977                 sc->params.filter2_wr_support = 0;
3978
3979         /* get capabilites */
3980         bzero(&caps, sizeof(caps));
3981         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3982             F_FW_CMD_REQUEST | F_FW_CMD_READ);
3983         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
3984         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
3985         if (rc != 0) {
3986                 device_printf(sc->dev,
3987                     "failed to get card capabilities: %d.\n", rc);
3988                 return (rc);
3989         }
3990
3991 #define READ_CAPS(x) do { \
3992         sc->x = htobe16(caps.x); \
3993 } while (0)
3994         READ_CAPS(nbmcaps);
3995         READ_CAPS(linkcaps);
3996         READ_CAPS(switchcaps);
3997         READ_CAPS(niccaps);
3998         READ_CAPS(toecaps);
3999         READ_CAPS(rdmacaps);
4000         READ_CAPS(cryptocaps);
4001         READ_CAPS(iscsicaps);
4002         READ_CAPS(fcoecaps);
4003
4004         if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) {
4005                 MPASS(chip_id(sc) > CHELSIO_T4);
4006                 MPASS(sc->toecaps == 0);
4007                 sc->toecaps = 0;
4008
4009                 param[0] = FW_PARAM_DEV(NTID);
4010                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
4011                 if (rc != 0) {
4012                         device_printf(sc->dev,
4013                             "failed to query HASHFILTER parameters: %d.\n", rc);
4014                         return (rc);
4015                 }
4016                 sc->tids.ntids = val[0];
4017                 if (sc->params.fw_vers <
4018                     (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) |
4019                     V_FW_HDR_FW_VER_MICRO(5) | V_FW_HDR_FW_VER_BUILD(0))) {
4020                         MPASS(sc->tids.ntids >= sc->tids.nhpftids);
4021                         sc->tids.ntids -= sc->tids.nhpftids;
4022                 }
4023                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
4024                 sc->params.hash_filter = 1;
4025         }
4026         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
4027                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
4028                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
4029                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
4030                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
4031                 if (rc != 0) {
4032                         device_printf(sc->dev,
4033                             "failed to query NIC parameters: %d.\n", rc);
4034                         return (rc);
4035                 }
4036                 if ((int)val[1] > (int)val[0]) {
4037                         sc->tids.etid_base = val[0];
4038                         sc->tids.etid_end = val[1];
4039                         sc->tids.netids = val[1] - val[0] + 1;
4040                         sc->params.eo_wr_cred = val[2];
4041                         sc->params.ethoffload = 1;
4042                 }
4043         }
4044         if (sc->toecaps) {
4045                 /* query offload-related parameters */
4046                 param[0] = FW_PARAM_DEV(NTID);
4047                 param[1] = FW_PARAM_PFVF(SERVER_START);
4048                 param[2] = FW_PARAM_PFVF(SERVER_END);
4049                 param[3] = FW_PARAM_PFVF(TDDP_START);
4050                 param[4] = FW_PARAM_PFVF(TDDP_END);
4051                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
4052                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
4053                 if (rc != 0) {
4054                         device_printf(sc->dev,
4055                             "failed to query TOE parameters: %d.\n", rc);
4056                         return (rc);
4057                 }
4058                 sc->tids.ntids = val[0];
4059                 if (sc->params.fw_vers <
4060                     (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) |
4061                     V_FW_HDR_FW_VER_MICRO(5) | V_FW_HDR_FW_VER_BUILD(0))) {
4062                         MPASS(sc->tids.ntids >= sc->tids.nhpftids);
4063                         sc->tids.ntids -= sc->tids.nhpftids;
4064                 }
4065                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
4066                 if ((int)val[2] > (int)val[1]) {
4067                         sc->tids.stid_base = val[1];
4068                         sc->tids.nstids = val[2] - val[1] + 1;
4069                 }
4070                 sc->vres.ddp.start = val[3];
4071                 sc->vres.ddp.size = val[4] - val[3] + 1;
4072                 sc->params.ofldq_wr_cred = val[5];
4073                 sc->params.offload = 1;
4074         } else {
4075                 /*
4076                  * The firmware attempts memfree TOE configuration for -SO cards
4077                  * and will report toecaps=0 if it runs out of resources (this
4078                  * depends on the config file).  It may not report 0 for other
4079                  * capabilities dependent on the TOE in this case.  Set them to
4080                  * 0 here so that the driver doesn't bother tracking resources
4081                  * that will never be used.
4082                  */
4083                 sc->iscsicaps = 0;
4084                 sc->rdmacaps = 0;
4085         }
4086         if (sc->rdmacaps) {
4087                 param[0] = FW_PARAM_PFVF(STAG_START);
4088                 param[1] = FW_PARAM_PFVF(STAG_END);
4089                 param[2] = FW_PARAM_PFVF(RQ_START);
4090                 param[3] = FW_PARAM_PFVF(RQ_END);
4091                 param[4] = FW_PARAM_PFVF(PBL_START);
4092                 param[5] = FW_PARAM_PFVF(PBL_END);
4093                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
4094                 if (rc != 0) {
4095                         device_printf(sc->dev,
4096                             "failed to query RDMA parameters(1): %d.\n", rc);
4097                         return (rc);
4098                 }
4099                 sc->vres.stag.start = val[0];
4100                 sc->vres.stag.size = val[1] - val[0] + 1;
4101                 sc->vres.rq.start = val[2];
4102                 sc->vres.rq.size = val[3] - val[2] + 1;
4103                 sc->vres.pbl.start = val[4];
4104                 sc->vres.pbl.size = val[5] - val[4] + 1;
4105
4106                 param[0] = FW_PARAM_PFVF(SQRQ_START);
4107                 param[1] = FW_PARAM_PFVF(SQRQ_END);
4108                 param[2] = FW_PARAM_PFVF(CQ_START);
4109                 param[3] = FW_PARAM_PFVF(CQ_END);
4110                 param[4] = FW_PARAM_PFVF(OCQ_START);
4111                 param[5] = FW_PARAM_PFVF(OCQ_END);
4112                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
4113                 if (rc != 0) {
4114                         device_printf(sc->dev,
4115                             "failed to query RDMA parameters(2): %d.\n", rc);
4116                         return (rc);
4117                 }
4118                 sc->vres.qp.start = val[0];
4119                 sc->vres.qp.size = val[1] - val[0] + 1;
4120                 sc->vres.cq.start = val[2];
4121                 sc->vres.cq.size = val[3] - val[2] + 1;
4122                 sc->vres.ocq.start = val[4];
4123                 sc->vres.ocq.size = val[5] - val[4] + 1;
4124
4125                 param[0] = FW_PARAM_PFVF(SRQ_START);
4126                 param[1] = FW_PARAM_PFVF(SRQ_END);
4127                 param[2] = FW_PARAM_DEV(MAXORDIRD_QP);
4128                 param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER);
4129                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
4130                 if (rc != 0) {
4131                         device_printf(sc->dev,
4132                             "failed to query RDMA parameters(3): %d.\n", rc);
4133                         return (rc);
4134                 }
4135                 sc->vres.srq.start = val[0];
4136                 sc->vres.srq.size = val[1] - val[0] + 1;
4137                 sc->params.max_ordird_qp = val[2];
4138                 sc->params.max_ird_adapter = val[3];
4139         }
4140         if (sc->iscsicaps) {
4141                 param[0] = FW_PARAM_PFVF(ISCSI_START);
4142                 param[1] = FW_PARAM_PFVF(ISCSI_END);
4143                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
4144                 if (rc != 0) {
4145                         device_printf(sc->dev,
4146                             "failed to query iSCSI parameters: %d.\n", rc);
4147                         return (rc);
4148                 }
4149                 sc->vres.iscsi.start = val[0];
4150                 sc->vres.iscsi.size = val[1] - val[0] + 1;
4151         }
4152         if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
4153                 param[0] = FW_PARAM_PFVF(TLS_START);
4154                 param[1] = FW_PARAM_PFVF(TLS_END);
4155                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
4156                 if (rc != 0) {
4157                         device_printf(sc->dev,
4158                             "failed to query TLS parameters: %d.\n", rc);
4159                         return (rc);
4160                 }
4161                 sc->vres.key.start = val[0];
4162                 sc->vres.key.size = val[1] - val[0] + 1;
4163         }
4164
4165         t4_init_sge_params(sc);
4166
4167         /*
4168          * We've got the params we wanted to query via the firmware.  Now grab
4169          * some others directly from the chip.
4170          */
4171         rc = t4_read_chip_settings(sc);
4172
4173         return (rc);
4174 }
4175
4176 static int
4177 set_params__post_init(struct adapter *sc)
4178 {
4179         uint32_t param, val;
4180 #ifdef TCP_OFFLOAD
4181         int i, v, shift;
4182 #endif
4183
4184         /* ask for encapsulated CPLs */
4185         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
4186         val = 1;
4187         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
4188
4189         /* Enable 32b port caps if the firmware supports it. */
4190         param = FW_PARAM_PFVF(PORT_CAPS32);
4191         val = 1;
4192         if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val) == 0)
4193                 sc->params.port_caps32 = 1;
4194
4195         /* Let filter + maskhash steer to a part of the VI's RSS region. */
4196         val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1);
4197         t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER),
4198             V_MASKFILTER(val - 1));
4199
4200 #ifdef TCP_OFFLOAD
4201         /*
4202          * Override the TOE timers with user provided tunables.  This is not the
4203          * recommended way to change the timers (the firmware config file is) so
4204          * these tunables are not documented.
4205          *
4206          * All the timer tunables are in microseconds.
4207          */
4208         if (t4_toe_keepalive_idle != 0) {
4209                 v = us_to_tcp_ticks(sc, t4_toe_keepalive_idle);
4210                 v &= M_KEEPALIVEIDLE;
4211                 t4_set_reg_field(sc, A_TP_KEEP_IDLE,
4212                     V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(v));
4213         }
4214         if (t4_toe_keepalive_interval != 0) {
4215                 v = us_to_tcp_ticks(sc, t4_toe_keepalive_interval);
4216                 v &= M_KEEPALIVEINTVL;
4217                 t4_set_reg_field(sc, A_TP_KEEP_INTVL,
4218                     V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(v));
4219         }
4220         if (t4_toe_keepalive_count != 0) {
4221                 v = t4_toe_keepalive_count & M_KEEPALIVEMAXR2;
4222                 t4_set_reg_field(sc, A_TP_SHIFT_CNT,
4223                     V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) |
4224                     V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2),
4225                     V_KEEPALIVEMAXR1(1) | V_KEEPALIVEMAXR2(v));
4226         }
4227         if (t4_toe_rexmt_min != 0) {
4228                 v = us_to_tcp_ticks(sc, t4_toe_rexmt_min);
4229                 v &= M_RXTMIN;
4230                 t4_set_reg_field(sc, A_TP_RXT_MIN,
4231                     V_RXTMIN(M_RXTMIN), V_RXTMIN(v));
4232         }
4233         if (t4_toe_rexmt_max != 0) {
4234                 v = us_to_tcp_ticks(sc, t4_toe_rexmt_max);
4235                 v &= M_RXTMAX;
4236                 t4_set_reg_field(sc, A_TP_RXT_MAX,
4237                     V_RXTMAX(M_RXTMAX), V_RXTMAX(v));
4238         }
4239         if (t4_toe_rexmt_count != 0) {
4240                 v = t4_toe_rexmt_count & M_RXTSHIFTMAXR2;
4241                 t4_set_reg_field(sc, A_TP_SHIFT_CNT,
4242                     V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) |
4243                     V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2),
4244                     V_RXTSHIFTMAXR1(1) | V_RXTSHIFTMAXR2(v));
4245         }
4246         for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) {
4247                 if (t4_toe_rexmt_backoff[i] != -1) {
4248                         v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0;
4249                         shift = (i & 3) << 3;
4250                         t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3),
4251                             M_TIMERBACKOFFINDEX0 << shift, v << shift);
4252                 }
4253         }
4254 #endif
4255         return (0);
4256 }
4257
4258 #undef FW_PARAM_PFVF
4259 #undef FW_PARAM_DEV
4260
4261 static void
4262 t4_set_desc(struct adapter *sc)
4263 {
4264         char buf[128];
4265         struct adapter_params *p = &sc->params;
4266
4267         snprintf(buf, sizeof(buf), "Chelsio %s", p->vpd.id);
4268
4269         device_set_desc_copy(sc->dev, buf);
4270 }
4271
4272 static inline void
4273 ifmedia_add4(struct ifmedia *ifm, int m)
4274 {
4275
4276         ifmedia_add(ifm, m, 0, NULL);
4277         ifmedia_add(ifm, m | IFM_ETH_TXPAUSE, 0, NULL);
4278         ifmedia_add(ifm, m | IFM_ETH_RXPAUSE, 0, NULL);
4279         ifmedia_add(ifm, m | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE, 0, NULL);
4280 }
4281
4282 /*
4283  * This is the selected media, which is not quite the same as the active media.
4284  * The media line in ifconfig is "media: Ethernet selected (active)" if selected
4285  * and active are not the same, and "media: Ethernet selected" otherwise.
4286  */
4287 static void
4288 set_current_media(struct port_info *pi)
4289 {
4290         struct link_config *lc;
4291         struct ifmedia *ifm;
4292         int mword;
4293         u_int speed;
4294
4295         PORT_LOCK_ASSERT_OWNED(pi);
4296
4297         /* Leave current media alone if it's already set to IFM_NONE. */
4298         ifm = &pi->media;
4299         if (ifm->ifm_cur != NULL &&
4300             IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE)
4301                 return;
4302
4303         lc = &pi->link_cfg;
4304         if (lc->requested_aneg != AUTONEG_DISABLE &&
4305             lc->supported & FW_PORT_CAP32_ANEG) {
4306                 ifmedia_set(ifm, IFM_ETHER | IFM_AUTO);
4307                 return;
4308         }
4309         mword = IFM_ETHER | IFM_FDX;
4310         if (lc->requested_fc & PAUSE_TX)
4311                 mword |= IFM_ETH_TXPAUSE;
4312         if (lc->requested_fc & PAUSE_RX)
4313                 mword |= IFM_ETH_RXPAUSE;
4314         if (lc->requested_speed == 0)
4315                 speed = port_top_speed(pi) * 1000;      /* Gbps -> Mbps */
4316         else
4317                 speed = lc->requested_speed;
4318         mword |= port_mword(pi, speed_to_fwcap(speed));
4319         ifmedia_set(ifm, mword);
4320 }
4321
4322 /*
4323  * Returns true if the ifmedia list for the port cannot change.
4324  */
4325 static bool
4326 fixed_ifmedia(struct port_info *pi)
4327 {
4328
4329         return (pi->port_type == FW_PORT_TYPE_BT_SGMII ||
4330             pi->port_type == FW_PORT_TYPE_BT_XFI ||
4331             pi->port_type == FW_PORT_TYPE_BT_XAUI ||
4332             pi->port_type == FW_PORT_TYPE_KX4 ||
4333             pi->port_type == FW_PORT_TYPE_KX ||
4334             pi->port_type == FW_PORT_TYPE_KR ||
4335             pi->port_type == FW_PORT_TYPE_BP_AP ||
4336             pi->port_type == FW_PORT_TYPE_BP4_AP ||
4337             pi->port_type == FW_PORT_TYPE_BP40_BA ||
4338             pi->port_type == FW_PORT_TYPE_KR4_100G ||
4339             pi->port_type == FW_PORT_TYPE_KR_SFP28 ||
4340             pi->port_type == FW_PORT_TYPE_KR_XLAUI);
4341 }
4342
4343 static void
4344 build_medialist(struct port_info *pi)
4345 {
4346         uint32_t ss, speed;
4347         int unknown, mword, bit;
4348         struct link_config *lc;
4349         struct ifmedia *ifm;
4350
4351         PORT_LOCK_ASSERT_OWNED(pi);
4352
4353         if (pi->flags & FIXED_IFMEDIA)
4354                 return;
4355
4356         /*
4357          * Rebuild the ifmedia list.
4358          */
4359         ifm = &pi->media;
4360         ifmedia_removeall(ifm);
4361         lc = &pi->link_cfg;
4362         ss = G_FW_PORT_CAP32_SPEED(lc->supported); /* Supported Speeds */
4363         if (__predict_false(ss == 0)) { /* not supposed to happen. */
4364                 MPASS(ss != 0);
4365 no_media:
4366                 MPASS(LIST_EMPTY(&ifm->ifm_list));
4367                 ifmedia_add(ifm, IFM_ETHER | IFM_NONE, 0, NULL);
4368                 ifmedia_set(ifm, IFM_ETHER | IFM_NONE);
4369                 return;
4370         }
4371
4372         unknown = 0;
4373         for (bit = S_FW_PORT_CAP32_SPEED; bit < fls(ss); bit++) {
4374                 speed = 1 << bit;
4375                 MPASS(speed & M_FW_PORT_CAP32_SPEED);
4376                 if (ss & speed) {
4377                         mword = port_mword(pi, speed);
4378                         if (mword == IFM_NONE) {
4379                                 goto no_media;
4380                         } else if (mword == IFM_UNKNOWN)
4381                                 unknown++;
4382                         else
4383                                 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | mword);
4384                 }
4385         }
4386         if (unknown > 0) /* Add one unknown for all unknown media types. */
4387                 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | IFM_UNKNOWN);
4388         if (lc->supported & FW_PORT_CAP32_ANEG)
4389                 ifmedia_add(ifm, IFM_ETHER | IFM_AUTO, 0, NULL);
4390
4391         set_current_media(pi);
4392 }
4393
4394 /*
4395  * Initialize the requested fields in the link config based on driver tunables.
4396  */
4397 static void
4398 init_link_config(struct port_info *pi)
4399 {
4400         struct link_config *lc = &pi->link_cfg;
4401
4402         PORT_LOCK_ASSERT_OWNED(pi);
4403
4404         lc->requested_speed = 0;
4405
4406         if (t4_autoneg == 0)
4407                 lc->requested_aneg = AUTONEG_DISABLE;
4408         else if (t4_autoneg == 1)
4409                 lc->requested_aneg = AUTONEG_ENABLE;
4410         else
4411                 lc->requested_aneg = AUTONEG_AUTO;
4412
4413         lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX |
4414             PAUSE_AUTONEG);
4415
4416         if (t4_fec == -1 || t4_fec & FEC_AUTO)
4417                 lc->requested_fec = FEC_AUTO;
4418         else {
4419                 lc->requested_fec = FEC_NONE;
4420                 if (t4_fec & FEC_RS)
4421                         lc->requested_fec |= FEC_RS;
4422                 if (t4_fec & FEC_BASER_RS)
4423                         lc->requested_fec |= FEC_BASER_RS;
4424         }
4425 }
4426
4427 /*
4428  * Makes sure that all requested settings comply with what's supported by the
4429  * port.  Returns the number of settings that were invalid and had to be fixed.
4430  */
4431 static int
4432 fixup_link_config(struct port_info *pi)
4433 {
4434         int n = 0;
4435         struct link_config *lc = &pi->link_cfg;
4436         uint32_t fwspeed;
4437
4438         PORT_LOCK_ASSERT_OWNED(pi);
4439
4440         /* Speed (when not autonegotiating) */
4441         if (lc->requested_speed != 0) {
4442                 fwspeed = speed_to_fwcap(lc->requested_speed);
4443                 if ((fwspeed & lc->supported) == 0) {
4444                         n++;
4445                         lc->requested_speed = 0;
4446                 }
4447         }
4448
4449         /* Link autonegotiation */
4450         MPASS(lc->requested_aneg == AUTONEG_ENABLE ||
4451             lc->requested_aneg == AUTONEG_DISABLE ||
4452             lc->requested_aneg == AUTONEG_AUTO);
4453         if (lc->requested_aneg == AUTONEG_ENABLE &&
4454             !(lc->supported & FW_PORT_CAP32_ANEG)) {
4455                 n++;
4456                 lc->requested_aneg = AUTONEG_AUTO;
4457         }
4458
4459         /* Flow control */
4460         MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0);
4461         if (lc->requested_fc & PAUSE_TX &&
4462             !(lc->supported & FW_PORT_CAP32_FC_TX)) {
4463                 n++;
4464                 lc->requested_fc &= ~PAUSE_TX;
4465         }
4466         if (lc->requested_fc & PAUSE_RX &&
4467             !(lc->supported & FW_PORT_CAP32_FC_RX)) {
4468                 n++;
4469                 lc->requested_fc &= ~PAUSE_RX;
4470         }
4471         if (!(lc->requested_fc & PAUSE_AUTONEG) &&
4472             !(lc->supported & FW_PORT_CAP32_FORCE_PAUSE)) {
4473                 n++;
4474                 lc->requested_fc |= PAUSE_AUTONEG;
4475         }
4476
4477         /* FEC */
4478         if ((lc->requested_fec & FEC_RS &&
4479             !(lc->supported & FW_PORT_CAP32_FEC_RS)) ||
4480             (lc->requested_fec & FEC_BASER_RS &&
4481             !(lc->supported & FW_PORT_CAP32_FEC_BASER_RS))) {
4482                 n++;
4483                 lc->requested_fec = FEC_AUTO;
4484         }
4485
4486         return (n);
4487 }
4488
4489 /*
4490  * Apply the requested L1 settings, which are expected to be valid, to the
4491  * hardware.
4492  */
4493 static int
4494 apply_link_config(struct port_info *pi)
4495 {
4496         struct adapter *sc = pi->adapter;
4497         struct link_config *lc = &pi->link_cfg;
4498         int rc;
4499
4500 #ifdef INVARIANTS
4501         ASSERT_SYNCHRONIZED_OP(sc);
4502         PORT_LOCK_ASSERT_OWNED(pi);
4503
4504         if (lc->requested_aneg == AUTONEG_ENABLE)
4505                 MPASS(lc->supported & FW_PORT_CAP32_ANEG);
4506         if (!(lc->requested_fc & PAUSE_AUTONEG))
4507                 MPASS(lc->supported & FW_PORT_CAP32_FORCE_PAUSE);
4508         if (lc->requested_fc & PAUSE_TX)
4509                 MPASS(lc->supported & FW_PORT_CAP32_FC_TX);
4510         if (lc->requested_fc & PAUSE_RX)
4511                 MPASS(lc->supported & FW_PORT_CAP32_FC_RX);
4512         if (lc->requested_fec & FEC_RS)
4513                 MPASS(lc->supported & FW_PORT_CAP32_FEC_RS);
4514         if (lc->requested_fec & FEC_BASER_RS)
4515                 MPASS(lc->supported & FW_PORT_CAP32_FEC_BASER_RS);
4516 #endif
4517         rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
4518         if (rc != 0) {
4519                 /* Don't complain if the VF driver gets back an EPERM. */
4520                 if (!(sc->flags & IS_VF) || rc != FW_EPERM)
4521                         device_printf(pi->dev, "l1cfg failed: %d\n", rc);
4522         } else {
4523                 /*
4524                  * An L1_CFG will almost always result in a link-change event if
4525                  * the link is up, and the driver will refresh the actual
4526                  * fec/fc/etc. when the notification is processed.  If the link
4527                  * is down then the actual settings are meaningless.
4528                  *
4529                  * This takes care of the case where a change in the L1 settings
4530                  * may not result in a notification.
4531                  */
4532                 if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG))
4533                         lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX);
4534         }
4535         return (rc);
4536 }
4537
4538 #define FW_MAC_EXACT_CHUNK      7
4539
4540 /*
4541  * Program the port's XGMAC based on parameters in ifnet.  The caller also
4542  * indicates which parameters should be programmed (the rest are left alone).
4543  */
4544 int
4545 update_mac_settings(struct ifnet *ifp, int flags)
4546 {
4547         int rc = 0;
4548         struct vi_info *vi = ifp->if_softc;
4549         struct port_info *pi = vi->pi;
4550         struct adapter *sc = pi->adapter;
4551         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
4552
4553         ASSERT_SYNCHRONIZED_OP(sc);
4554         KASSERT(flags, ("%s: not told what to update.", __func__));
4555
4556         if (flags & XGMAC_MTU)
4557                 mtu = ifp->if_mtu;
4558
4559         if (flags & XGMAC_PROMISC)
4560                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
4561
4562         if (flags & XGMAC_ALLMULTI)
4563                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
4564
4565         if (flags & XGMAC_VLANEX)
4566                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
4567
4568         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
4569                 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
4570                     allmulti, 1, vlanex, false);
4571                 if (rc) {
4572                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
4573                             rc);
4574                         return (rc);
4575                 }
4576         }
4577
4578         if (flags & XGMAC_UCADDR) {
4579                 uint8_t ucaddr[ETHER_ADDR_LEN];
4580
4581                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
4582                 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
4583                     ucaddr, true, true);
4584                 if (rc < 0) {
4585                         rc = -rc;
4586                         if_printf(ifp, "change_mac failed: %d\n", rc);
4587                         return (rc);
4588                 } else {
4589                         vi->xact_addr_filt = rc;
4590                         rc = 0;
4591                 }
4592         }
4593
4594         if (flags & XGMAC_MCADDRS) {
4595                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
4596                 int del = 1;
4597                 uint64_t hash = 0;
4598                 struct ifmultiaddr *ifma;
4599                 int i = 0, j;
4600
4601                 if_maddr_rlock(ifp);
4602                 CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
4603                         if (ifma->ifma_addr->sa_family != AF_LINK)
4604                                 continue;
4605                         mcaddr[i] =
4606                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
4607                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
4608                         i++;
4609
4610                         if (i == FW_MAC_EXACT_CHUNK) {
4611                                 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
4612                                     del, i, mcaddr, NULL, &hash, 0);
4613                                 if (rc < 0) {
4614                                         rc = -rc;
4615                                         for (j = 0; j < i; j++) {
4616                                                 if_printf(ifp,
4617                                                     "failed to add mc address"
4618                                                     " %02x:%02x:%02x:"
4619                                                     "%02x:%02x:%02x rc=%d\n",
4620                                                     mcaddr[j][0], mcaddr[j][1],
4621                                                     mcaddr[j][2], mcaddr[j][3],
4622                                                     mcaddr[j][4], mcaddr[j][5],
4623                                                     rc);
4624                                         }
4625                                         goto mcfail;
4626                                 }
4627                                 del = 0;
4628                                 i = 0;
4629                         }
4630                 }
4631                 if (i > 0) {
4632                         rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i,
4633                             mcaddr, NULL, &hash, 0);
4634                         if (rc < 0) {
4635                                 rc = -rc;
4636                                 for (j = 0; j < i; j++) {
4637                                         if_printf(ifp,
4638                                             "failed to add mc address"
4639                                             " %02x:%02x:%02x:"
4640                                             "%02x:%02x:%02x rc=%d\n",
4641                                             mcaddr[j][0], mcaddr[j][1],
4642                                             mcaddr[j][2], mcaddr[j][3],
4643                                             mcaddr[j][4], mcaddr[j][5],
4644                                             rc);
4645                                 }
4646                                 goto mcfail;
4647                         }
4648                 }
4649
4650                 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0);
4651                 if (rc != 0)
4652                         if_printf(ifp, "failed to set mc address hash: %d", rc);
4653 mcfail:
4654                 if_maddr_runlock(ifp);
4655         }
4656
4657         return (rc);
4658 }
4659
4660 /*
4661  * {begin|end}_synchronized_op must be called from the same thread.
4662  */
4663 int
4664 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
4665     char *wmesg)
4666 {
4667         int rc, pri;
4668
4669 #ifdef WITNESS
4670         /* the caller thinks it's ok to sleep, but is it really? */
4671         if (flags & SLEEP_OK)
4672                 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
4673                     "begin_synchronized_op");
4674 #endif
4675
4676         if (INTR_OK)
4677                 pri = PCATCH;
4678         else
4679                 pri = 0;
4680
4681         ADAPTER_LOCK(sc);
4682         for (;;) {
4683
4684                 if (vi && IS_DOOMED(vi)) {
4685                         rc = ENXIO;
4686                         goto done;
4687                 }
4688
4689                 if (!IS_BUSY(sc)) {
4690                         rc = 0;
4691                         break;
4692                 }
4693
4694                 if (!(flags & SLEEP_OK)) {
4695                         rc = EBUSY;
4696                         goto done;
4697                 }
4698
4699                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
4700                         rc = EINTR;
4701                         goto done;
4702                 }
4703         }
4704
4705         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
4706         SET_BUSY(sc);
4707 #ifdef INVARIANTS
4708         sc->last_op = wmesg;
4709         sc->last_op_thr = curthread;
4710         sc->last_op_flags = flags;
4711 #endif
4712
4713 done:
4714         if (!(flags & HOLD_LOCK) || rc)
4715                 ADAPTER_UNLOCK(sc);
4716
4717         return (rc);
4718 }
4719
4720 /*
4721  * Tell if_ioctl and if_init that the VI is going away.  This is
4722  * special variant of begin_synchronized_op and must be paired with a
4723  * call to end_synchronized_op.
4724  */
4725 void
4726 doom_vi(struct adapter *sc, struct vi_info *vi)
4727 {
4728
4729         ADAPTER_LOCK(sc);
4730         SET_DOOMED(vi);
4731         wakeup(&sc->flags);
4732         while (IS_BUSY(sc))
4733                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
4734         SET_BUSY(sc);
4735 #ifdef INVARIANTS
4736         sc->last_op = "t4detach";
4737         sc->last_op_thr = curthread;
4738         sc->last_op_flags = 0;
4739 #endif
4740         ADAPTER_UNLOCK(sc);
4741 }
4742
4743 /*
4744  * {begin|end}_synchronized_op must be called from the same thread.
4745  */
4746 void
4747 end_synchronized_op(struct adapter *sc, int flags)
4748 {
4749
4750         if (flags & LOCK_HELD)
4751                 ADAPTER_LOCK_ASSERT_OWNED(sc);
4752         else
4753                 ADAPTER_LOCK(sc);
4754
4755         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
4756         CLR_BUSY(sc);
4757         wakeup(&sc->flags);
4758         ADAPTER_UNLOCK(sc);
4759 }
4760
4761 static int
4762 cxgbe_init_synchronized(struct vi_info *vi)
4763 {
4764         struct port_info *pi = vi->pi;
4765         struct adapter *sc = pi->adapter;
4766         struct ifnet *ifp = vi->ifp;
4767         int rc = 0, i;
4768         struct sge_txq *txq;
4769
4770         ASSERT_SYNCHRONIZED_OP(sc);
4771
4772         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
4773                 return (0);     /* already running */
4774
4775         if (!(sc->flags & FULL_INIT_DONE) &&
4776             ((rc = adapter_full_init(sc)) != 0))
4777                 return (rc);    /* error message displayed already */
4778
4779         if (!(vi->flags & VI_INIT_DONE) &&
4780             ((rc = vi_full_init(vi)) != 0))
4781                 return (rc); /* error message displayed already */
4782
4783         rc = update_mac_settings(ifp, XGMAC_ALL);
4784         if (rc)
4785                 goto done;      /* error message displayed already */
4786
4787         PORT_LOCK(pi);
4788         if (pi->up_vis == 0) {
4789                 t4_update_port_info(pi);
4790                 fixup_link_config(pi);
4791                 build_medialist(pi);
4792                 apply_link_config(pi);
4793         }
4794
4795         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
4796         if (rc != 0) {
4797                 if_printf(ifp, "enable_vi failed: %d\n", rc);
4798                 PORT_UNLOCK(pi);
4799                 goto done;
4800         }
4801
4802         /*
4803          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
4804          * if this changes.
4805          */
4806
4807         for_each_txq(vi, i, txq) {
4808                 TXQ_LOCK(txq);
4809                 txq->eq.flags |= EQ_ENABLED;
4810                 TXQ_UNLOCK(txq);
4811         }
4812
4813         /*
4814          * The first iq of the first port to come up is used for tracing.
4815          */
4816         if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
4817                 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
4818                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
4819                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
4820                     V_QUEUENUMBER(sc->traceq));
4821                 pi->flags |= HAS_TRACEQ;
4822         }
4823
4824         /* all ok */
4825         pi->up_vis++;
4826         ifp->if_drv_flags |= IFF_DRV_RUNNING;
4827
4828         if (pi->nvi > 1 || sc->flags & IS_VF)
4829                 callout_reset(&vi->tick, hz, vi_tick, vi);
4830         else
4831                 callout_reset(&pi->tick, hz, cxgbe_tick, pi);
4832         PORT_UNLOCK(pi);
4833 done:
4834         if (rc != 0)
4835                 cxgbe_uninit_synchronized(vi);
4836
4837         return (rc);
4838 }
4839
4840 /*
4841  * Idempotent.
4842  */
4843 static int
4844 cxgbe_uninit_synchronized(struct vi_info *vi)
4845 {
4846         struct port_info *pi = vi->pi;
4847         struct adapter *sc = pi->adapter;
4848         struct ifnet *ifp = vi->ifp;
4849         int rc, i;
4850         struct sge_txq *txq;
4851
4852         ASSERT_SYNCHRONIZED_OP(sc);
4853
4854         if (!(vi->flags & VI_INIT_DONE)) {
4855                 if (__predict_false(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4856                         KASSERT(0, ("uninited VI is running"));
4857                         if_printf(ifp, "uninited VI with running ifnet.  "
4858                             "vi->flags 0x%016lx, if_flags 0x%08x, "
4859                             "if_drv_flags 0x%08x\n", vi->flags, ifp->if_flags,
4860                             ifp->if_drv_flags);
4861                 }
4862                 return (0);
4863         }
4864
4865         /*
4866          * Disable the VI so that all its data in either direction is discarded
4867          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
4868          * tick) intact as the TP can deliver negative advice or data that it's
4869          * holding in its RAM (for an offloaded connection) even after the VI is
4870          * disabled.
4871          */
4872         rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
4873         if (rc) {
4874                 if_printf(ifp, "disable_vi failed: %d\n", rc);
4875                 return (rc);
4876         }
4877
4878         for_each_txq(vi, i, txq) {
4879                 TXQ_LOCK(txq);
4880                 txq->eq.flags &= ~EQ_ENABLED;
4881                 TXQ_UNLOCK(txq);
4882         }
4883
4884         PORT_LOCK(pi);
4885         if (pi->nvi > 1 || sc->flags & IS_VF)
4886                 callout_stop(&vi->tick);
4887         else
4888                 callout_stop(&pi->tick);
4889         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4890                 PORT_UNLOCK(pi);
4891                 return (0);
4892         }
4893         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
4894         pi->up_vis--;
4895         if (pi->up_vis > 0) {
4896                 PORT_UNLOCK(pi);
4897                 return (0);
4898         }
4899
4900         pi->link_cfg.link_ok = false;
4901         pi->link_cfg.speed = 0;
4902         pi->link_cfg.link_down_rc = 255;
4903         t4_os_link_changed(pi);
4904         PORT_UNLOCK(pi);
4905
4906         return (0);
4907 }
4908
4909 /*
4910  * It is ok for this function to fail midway and return right away.  t4_detach
4911  * will walk the entire sc->irq list and clean up whatever is valid.
4912  */
4913 int
4914 t4_setup_intr_handlers(struct adapter *sc)
4915 {
4916         int rc, rid, p, q, v;
4917         char s[8];
4918         struct irq *irq;
4919         struct port_info *pi;
4920         struct vi_info *vi;
4921         struct sge *sge = &sc->sge;
4922         struct sge_rxq *rxq;
4923 #ifdef TCP_OFFLOAD
4924         struct sge_ofld_rxq *ofld_rxq;
4925 #endif
4926 #ifdef DEV_NETMAP
4927         struct sge_nm_rxq *nm_rxq;
4928 #endif
4929 #ifdef RSS
4930         int nbuckets = rss_getnumbuckets();
4931 #endif
4932
4933         /*
4934          * Setup interrupts.
4935          */
4936         irq = &sc->irq[0];
4937         rid = sc->intr_type == INTR_INTX ? 0 : 1;
4938         if (forwarding_intr_to_fwq(sc))
4939                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
4940
4941         /* Multiple interrupts. */
4942         if (sc->flags & IS_VF)
4943                 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
4944                     ("%s: too few intr.", __func__));
4945         else
4946                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
4947                     ("%s: too few intr.", __func__));
4948
4949         /* The first one is always error intr on PFs */
4950         if (!(sc->flags & IS_VF)) {
4951                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
4952                 if (rc != 0)
4953                         return (rc);
4954                 irq++;
4955                 rid++;
4956         }
4957
4958         /* The second one is always the firmware event queue (first on VFs) */
4959         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
4960         if (rc != 0)
4961                 return (rc);
4962         irq++;
4963         rid++;
4964
4965         for_each_port(sc, p) {
4966                 pi = sc->port[p];
4967                 for_each_vi(pi, v, vi) {
4968                         vi->first_intr = rid - 1;
4969
4970                         if (vi->nnmrxq > 0) {
4971                                 int n = max(vi->nrxq, vi->nnmrxq);
4972
4973                                 rxq = &sge->rxq[vi->first_rxq];
4974 #ifdef DEV_NETMAP
4975                                 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
4976 #endif
4977                                 for (q = 0; q < n; q++) {
4978                                         snprintf(s, sizeof(s), "%x%c%x", p,
4979                                             'a' + v, q);
4980                                         if (q < vi->nrxq)
4981                                                 irq->rxq = rxq++;
4982 #ifdef DEV_NETMAP
4983                                         if (q < vi->nnmrxq)
4984                                                 irq->nm_rxq = nm_rxq++;
4985
4986                                         if (irq->nm_rxq != NULL &&
4987                                             irq->rxq == NULL) {
4988                                                 /* Netmap rx only */
4989                                                 rc = t4_alloc_irq(sc, irq, rid,
4990                                                     t4_nm_intr, irq->nm_rxq, s);
4991                                         }
4992                                         if (irq->nm_rxq != NULL &&
4993                                             irq->rxq != NULL) {
4994                                                 /* NIC and Netmap rx */
4995                                                 rc = t4_alloc_irq(sc, irq, rid,
4996                                                     t4_vi_intr, irq, s);
4997                                         }
4998 #endif
4999                                         if (irq->rxq != NULL &&
5000                                             irq->nm_rxq == NULL) {
5001                                                 /* NIC rx only */
5002                                                 rc = t4_alloc_irq(sc, irq, rid,
5003                                                     t4_intr, irq->rxq, s);
5004                                         }
5005                                         if (rc != 0)
5006                                                 return (rc);
5007 #ifdef RSS
5008                                         if (q < vi->nrxq) {
5009                                                 bus_bind_intr(sc->dev, irq->res,
5010                                                     rss_getcpu(q % nbuckets));
5011                                         }
5012 #endif
5013                                         irq++;
5014                                         rid++;
5015                                         vi->nintr++;
5016                                 }
5017                         } else {
5018                                 for_each_rxq(vi, q, rxq) {
5019                                         snprintf(s, sizeof(s), "%x%c%x", p,
5020                                             'a' + v, q);
5021                                         rc = t4_alloc_irq(sc, irq, rid,
5022                                             t4_intr, rxq, s);
5023                                         if (rc != 0)
5024                                                 return (rc);
5025 #ifdef RSS
5026                                         bus_bind_intr(sc->dev, irq->res,
5027                                             rss_getcpu(q % nbuckets));
5028 #endif
5029                                         irq++;
5030                                         rid++;
5031                                         vi->nintr++;
5032                                 }
5033                         }
5034 #ifdef TCP_OFFLOAD
5035                         for_each_ofld_rxq(vi, q, ofld_rxq) {
5036                                 snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q);
5037                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
5038                                     ofld_rxq, s);
5039                                 if (rc != 0)
5040                                         return (rc);
5041                                 irq++;
5042                                 rid++;
5043                                 vi->nintr++;
5044                         }
5045 #endif
5046                 }
5047         }
5048         MPASS(irq == &sc->irq[sc->intr_count]);
5049
5050         return (0);
5051 }
5052
5053 int
5054 adapter_full_init(struct adapter *sc)
5055 {
5056         int rc, i;
5057 #ifdef RSS
5058         uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
5059         uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
5060 #endif
5061
5062         ASSERT_SYNCHRONIZED_OP(sc);
5063         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
5064         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
5065             ("%s: FULL_INIT_DONE already", __func__));
5066
5067         /*
5068          * queues that belong to the adapter (not any particular port).
5069          */
5070         rc = t4_setup_adapter_queues(sc);
5071         if (rc != 0)
5072                 goto done;
5073
5074         for (i = 0; i < nitems(sc->tq); i++) {
5075                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
5076                     taskqueue_thread_enqueue, &sc->tq[i]);
5077                 if (sc->tq[i] == NULL) {
5078                         device_printf(sc->dev,
5079                             "failed to allocate task queue %d\n", i);
5080                         rc = ENOMEM;
5081                         goto done;
5082                 }
5083                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
5084                     device_get_nameunit(sc->dev), i);
5085         }
5086 #ifdef RSS
5087         MPASS(RSS_KEYSIZE == 40);
5088         rss_getkey((void *)&raw_rss_key[0]);
5089         for (i = 0; i < nitems(rss_key); i++) {
5090                 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
5091         }
5092         t4_write_rss_key(sc, &rss_key[0], -1, 1);
5093 #endif
5094
5095         if (!(sc->flags & IS_VF))
5096                 t4_intr_enable(sc);
5097         sc->flags |= FULL_INIT_DONE;
5098 done:
5099         if (rc != 0)
5100                 adapter_full_uninit(sc);
5101
5102         return (rc);
5103 }
5104
5105 int
5106 adapter_full_uninit(struct adapter *sc)
5107 {
5108         int i;
5109
5110         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
5111
5112         t4_teardown_adapter_queues(sc);
5113
5114         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
5115                 taskqueue_free(sc->tq[i]);
5116                 sc->tq[i] = NULL;
5117         }
5118
5119         sc->flags &= ~FULL_INIT_DONE;
5120
5121         return (0);
5122 }
5123
5124 #ifdef RSS
5125 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
5126     RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
5127     RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
5128     RSS_HASHTYPE_RSS_UDP_IPV6)
5129
5130 /* Translates kernel hash types to hardware. */
5131 static int
5132 hashconfig_to_hashen(int hashconfig)
5133 {
5134         int hashen = 0;
5135
5136         if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
5137                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
5138         if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
5139                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
5140         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
5141                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
5142                     F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
5143         }
5144         if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
5145                 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
5146                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
5147         }
5148         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
5149                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
5150         if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
5151                 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
5152
5153         return (hashen);
5154 }
5155
5156 /* Translates hardware hash types to kernel. */
5157 static int
5158 hashen_to_hashconfig(int hashen)
5159 {
5160         int hashconfig = 0;
5161
5162         if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
5163                 /*
5164                  * If UDP hashing was enabled it must have been enabled for
5165                  * either IPv4 or IPv6 (inclusive or).  Enabling UDP without
5166                  * enabling any 4-tuple hash is nonsense configuration.
5167                  */
5168                 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
5169                     F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
5170
5171                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
5172                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
5173                 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
5174                         hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
5175         }
5176         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
5177                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
5178         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
5179                 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
5180         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
5181                 hashconfig |= RSS_HASHTYPE_RSS_IPV4;
5182         if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
5183                 hashconfig |= RSS_HASHTYPE_RSS_IPV6;
5184
5185         return (hashconfig);
5186 }
5187 #endif
5188
5189 int
5190 vi_full_init(struct vi_info *vi)
5191 {
5192         struct adapter *sc = vi->pi->adapter;
5193         struct ifnet *ifp = vi->ifp;
5194         uint16_t *rss;
5195         struct sge_rxq *rxq;
5196         int rc, i, j;
5197 #ifdef RSS
5198         int nbuckets = rss_getnumbuckets();
5199         int hashconfig = rss_gethashconfig();
5200         int extra;
5201 #endif
5202
5203         ASSERT_SYNCHRONIZED_OP(sc);
5204         KASSERT((vi->flags & VI_INIT_DONE) == 0,
5205             ("%s: VI_INIT_DONE already", __func__));
5206
5207         sysctl_ctx_init(&vi->ctx);
5208         vi->flags |= VI_SYSCTL_CTX;
5209
5210         /*
5211          * Allocate tx/rx/fl queues for this VI.
5212          */
5213         rc = t4_setup_vi_queues(vi);
5214         if (rc != 0)
5215                 goto done;      /* error message displayed already */
5216
5217         /*
5218          * Setup RSS for this VI.  Save a copy of the RSS table for later use.
5219          */
5220         if (vi->nrxq > vi->rss_size) {
5221                 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
5222                     "some queues will never receive traffic.\n", vi->nrxq,
5223                     vi->rss_size);
5224         } else if (vi->rss_size % vi->nrxq) {
5225                 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
5226                     "expect uneven traffic distribution.\n", vi->nrxq,
5227                     vi->rss_size);
5228         }
5229 #ifdef RSS
5230         if (vi->nrxq != nbuckets) {
5231                 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
5232                     "performance will be impacted.\n", vi->nrxq, nbuckets);
5233         }
5234 #endif
5235         rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
5236         for (i = 0; i < vi->rss_size;) {
5237 #ifdef RSS
5238                 j = rss_get_indirection_to_bucket(i);
5239                 j %= vi->nrxq;
5240                 rxq = &sc->sge.rxq[vi->first_rxq + j];
5241                 rss[i++] = rxq->iq.abs_id;
5242 #else
5243                 for_each_rxq(vi, j, rxq) {
5244                         rss[i++] = rxq->iq.abs_id;
5245                         if (i == vi->rss_size)
5246                                 break;
5247                 }
5248 #endif
5249         }
5250
5251         rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
5252             vi->rss_size);
5253         if (rc != 0) {
5254                 free(rss, M_CXGBE);
5255                 if_printf(ifp, "rss_config failed: %d\n", rc);
5256                 goto done;
5257         }
5258
5259 #ifdef RSS
5260         vi->hashen = hashconfig_to_hashen(hashconfig);
5261
5262         /*
5263          * We may have had to enable some hashes even though the global config
5264          * wants them disabled.  This is a potential problem that must be
5265          * reported to the user.
5266          */
5267         extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig;
5268
5269         /*
5270          * If we consider only the supported hash types, then the enabled hashes
5271          * are a superset of the requested hashes.  In other words, there cannot
5272          * be any supported hash that was requested but not enabled, but there
5273          * can be hashes that were not requested but had to be enabled.
5274          */
5275         extra &= SUPPORTED_RSS_HASHTYPES;
5276         MPASS((extra & hashconfig) == 0);
5277
5278         if (extra) {
5279                 if_printf(ifp,
5280                     "global RSS config (0x%x) cannot be accommodated.\n",
5281                     hashconfig);
5282         }
5283         if (extra & RSS_HASHTYPE_RSS_IPV4)
5284                 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
5285         if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
5286                 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
5287         if (extra & RSS_HASHTYPE_RSS_IPV6)
5288                 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
5289         if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
5290                 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
5291         if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
5292                 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
5293         if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
5294                 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
5295 #else
5296         vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
5297             F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
5298             F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
5299             F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
5300 #endif
5301         rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, rss[0], 0, 0);
5302         if (rc != 0) {
5303                 free(rss, M_CXGBE);
5304                 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
5305                 goto done;
5306         }
5307
5308         vi->rss = rss;
5309         vi->flags |= VI_INIT_DONE;
5310 done:
5311         if (rc != 0)
5312                 vi_full_uninit(vi);
5313
5314         return (rc);
5315 }
5316
5317 /*
5318  * Idempotent.
5319  */
5320 int
5321 vi_full_uninit(struct vi_info *vi)
5322 {
5323         struct port_info *pi = vi->pi;
5324         struct adapter *sc = pi->adapter;
5325         int i;
5326         struct sge_rxq *rxq;
5327         struct sge_txq *txq;
5328 #ifdef TCP_OFFLOAD
5329         struct sge_ofld_rxq *ofld_rxq;
5330         struct sge_wrq *ofld_txq;
5331 #endif
5332
5333         if (vi->flags & VI_INIT_DONE) {
5334
5335                 /* Need to quiesce queues.  */
5336
5337                 /* XXX: Only for the first VI? */
5338                 if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF))
5339                         quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
5340
5341                 for_each_txq(vi, i, txq) {
5342                         quiesce_txq(sc, txq);
5343                 }
5344
5345 #ifdef TCP_OFFLOAD
5346                 for_each_ofld_txq(vi, i, ofld_txq) {
5347                         quiesce_wrq(sc, ofld_txq);
5348                 }
5349 #endif
5350
5351                 for_each_rxq(vi, i, rxq) {
5352                         quiesce_iq(sc, &rxq->iq);
5353                         quiesce_fl(sc, &rxq->fl);
5354                 }
5355
5356 #ifdef TCP_OFFLOAD
5357                 for_each_ofld_rxq(vi, i, ofld_rxq) {
5358                         quiesce_iq(sc, &ofld_rxq->iq);
5359                         quiesce_fl(sc, &ofld_rxq->fl);
5360                 }
5361 #endif
5362                 free(vi->rss, M_CXGBE);
5363                 free(vi->nm_rss, M_CXGBE);
5364         }
5365
5366         t4_teardown_vi_queues(vi);
5367         vi->flags &= ~VI_INIT_DONE;
5368
5369         return (0);
5370 }
5371
5372 static void
5373 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
5374 {
5375         struct sge_eq *eq = &txq->eq;
5376         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
5377
5378         (void) sc;      /* unused */
5379
5380 #ifdef INVARIANTS
5381         TXQ_LOCK(txq);
5382         MPASS((eq->flags & EQ_ENABLED) == 0);
5383         TXQ_UNLOCK(txq);
5384 #endif
5385
5386         /* Wait for the mp_ring to empty. */
5387         while (!mp_ring_is_idle(txq->r)) {
5388                 mp_ring_check_drainage(txq->r, 0);
5389                 pause("rquiesce", 1);
5390         }
5391
5392         /* Then wait for the hardware to finish. */
5393         while (spg->cidx != htobe16(eq->pidx))
5394                 pause("equiesce", 1);
5395
5396         /* Finally, wait for the driver to reclaim all descriptors. */
5397         while (eq->cidx != eq->pidx)
5398                 pause("dquiesce", 1);
5399 }
5400
5401 static void
5402 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
5403 {
5404
5405         /* XXXTX */
5406 }
5407
5408 static void
5409 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
5410 {
5411         (void) sc;      /* unused */
5412
5413         /* Synchronize with the interrupt handler */
5414         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
5415                 pause("iqfree", 1);
5416 }
5417
5418 static void
5419 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
5420 {
5421         mtx_lock(&sc->sfl_lock);
5422         FL_LOCK(fl);
5423         fl->flags |= FL_DOOMED;
5424         FL_UNLOCK(fl);
5425         callout_stop(&sc->sfl_callout);
5426         mtx_unlock(&sc->sfl_lock);
5427
5428         KASSERT((fl->flags & FL_STARVING) == 0,
5429             ("%s: still starving", __func__));
5430 }
5431
5432 static int
5433 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
5434     driver_intr_t *handler, void *arg, char *name)
5435 {
5436         int rc;
5437
5438         irq->rid = rid;
5439         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
5440             RF_SHAREABLE | RF_ACTIVE);
5441         if (irq->res == NULL) {
5442                 device_printf(sc->dev,
5443                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
5444                 return (ENOMEM);
5445         }
5446
5447         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
5448             NULL, handler, arg, &irq->tag);
5449         if (rc != 0) {
5450                 device_printf(sc->dev,
5451                     "failed to setup interrupt for rid %d, name %s: %d\n",
5452                     rid, name, rc);
5453         } else if (name)
5454                 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);
5455
5456         return (rc);
5457 }
5458
5459 static int
5460 t4_free_irq(struct adapter *sc, struct irq *irq)
5461 {
5462         if (irq->tag)
5463                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
5464         if (irq->res)
5465                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
5466
5467         bzero(irq, sizeof(*irq));
5468
5469         return (0);
5470 }
5471
5472 static void
5473 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
5474 {
5475
5476         regs->version = chip_id(sc) | chip_rev(sc) << 10;
5477         t4_get_regs(sc, buf, regs->len);
5478 }
5479
5480 #define A_PL_INDIR_CMD  0x1f8
5481
5482 #define S_PL_AUTOINC    31
5483 #define M_PL_AUTOINC    0x1U
5484 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
5485 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
5486
5487 #define S_PL_VFID       20
5488 #define M_PL_VFID       0xffU
5489 #define V_PL_VFID(x)    ((x) << S_PL_VFID)
5490 #define G_PL_VFID(x)    (((x) >> S_PL_VFID) & M_PL_VFID)
5491
5492 #define S_PL_ADDR       0
5493 #define M_PL_ADDR       0xfffffU
5494 #define V_PL_ADDR(x)    ((x) << S_PL_ADDR)
5495 #define G_PL_ADDR(x)    (((x) >> S_PL_ADDR) & M_PL_ADDR)
5496
5497 #define A_PL_INDIR_DATA 0x1fc
5498
5499 static uint64_t
5500 read_vf_stat(struct adapter *sc, unsigned int viid, int reg)
5501 {
5502         u32 stats[2];
5503
5504         mtx_assert(&sc->reg_lock, MA_OWNED);
5505         if (sc->flags & IS_VF) {
5506                 stats[0] = t4_read_reg(sc, VF_MPS_REG(reg));
5507                 stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4));
5508         } else {
5509                 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
5510                     V_PL_VFID(G_FW_VIID_VIN(viid)) |
5511                     V_PL_ADDR(VF_MPS_REG(reg)));
5512                 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
5513                 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
5514         }
5515         return (((uint64_t)stats[1]) << 32 | stats[0]);
5516 }
5517
5518 static void
5519 t4_get_vi_stats(struct adapter *sc, unsigned int viid,
5520     struct fw_vi_stats_vf *stats)
5521 {
5522
5523 #define GET_STAT(name) \
5524         read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L)
5525
5526         stats->tx_bcast_bytes    = GET_STAT(TX_VF_BCAST_BYTES);
5527         stats->tx_bcast_frames   = GET_STAT(TX_VF_BCAST_FRAMES);
5528         stats->tx_mcast_bytes    = GET_STAT(TX_VF_MCAST_BYTES);
5529         stats->tx_mcast_frames   = GET_STAT(TX_VF_MCAST_FRAMES);
5530         stats->tx_ucast_bytes    = GET_STAT(TX_VF_UCAST_BYTES);
5531         stats->tx_ucast_frames   = GET_STAT(TX_VF_UCAST_FRAMES);
5532         stats->tx_drop_frames    = GET_STAT(TX_VF_DROP_FRAMES);
5533         stats->tx_offload_bytes  = GET_STAT(TX_VF_OFFLOAD_BYTES);
5534         stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
5535         stats->rx_bcast_bytes    = GET_STAT(RX_VF_BCAST_BYTES);
5536         stats->rx_bcast_frames   = GET_STAT(RX_VF_BCAST_FRAMES);
5537         stats->rx_mcast_bytes    = GET_STAT(RX_VF_MCAST_BYTES);
5538         stats->rx_mcast_frames   = GET_STAT(RX_VF_MCAST_FRAMES);
5539         stats->rx_ucast_bytes    = GET_STAT(RX_VF_UCAST_BYTES);
5540         stats->rx_ucast_frames   = GET_STAT(RX_VF_UCAST_FRAMES);
5541         stats->rx_err_frames     = GET_STAT(RX_VF_ERR_FRAMES);
5542
5543 #undef GET_STAT
5544 }
5545
5546 static void
5547 t4_clr_vi_stats(struct adapter *sc, unsigned int viid)
5548 {
5549         int reg;
5550
5551         t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
5552             V_PL_VFID(G_FW_VIID_VIN(viid)) |
5553             V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
5554         for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
5555              reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
5556                 t4_write_reg(sc, A_PL_INDIR_DATA, 0);
5557 }
5558
5559 static void
5560 vi_refresh_stats(struct adapter *sc, struct vi_info *vi)
5561 {
5562         struct timeval tv;
5563         const struct timeval interval = {0, 250000};    /* 250ms */
5564
5565         if (!(vi->flags & VI_INIT_DONE))
5566                 return;
5567
5568         getmicrotime(&tv);
5569         timevalsub(&tv, &interval);
5570         if (timevalcmp(&tv, &vi->last_refreshed, <))
5571                 return;
5572
5573         mtx_lock(&sc->reg_lock);
5574         t4_get_vi_stats(sc, vi->viid, &vi->stats);
5575         getmicrotime(&vi->last_refreshed);
5576         mtx_unlock(&sc->reg_lock);
5577 }
5578
5579 static void
5580 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
5581 {
5582         u_int i, v, tnl_cong_drops, bg_map;
5583         struct timeval tv;
5584         const struct timeval interval = {0, 250000};    /* 250ms */
5585
5586         getmicrotime(&tv);
5587         timevalsub(&tv, &interval);
5588         if (timevalcmp(&tv, &pi->last_refreshed, <))
5589                 return;
5590
5591         tnl_cong_drops = 0;
5592         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
5593         bg_map = pi->mps_bg_map;
5594         while (bg_map) {
5595                 i = ffs(bg_map) - 1;
5596                 mtx_lock(&sc->reg_lock);
5597                 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1,
5598                     A_TP_MIB_TNL_CNG_DROP_0 + i);
5599                 mtx_unlock(&sc->reg_lock);
5600                 tnl_cong_drops += v;
5601                 bg_map &= ~(1 << i);
5602         }
5603         pi->tnl_cong_drops = tnl_cong_drops;
5604         getmicrotime(&pi->last_refreshed);
5605 }
5606
5607 static void
5608 cxgbe_tick(void *arg)
5609 {
5610         struct port_info *pi = arg;
5611         struct adapter *sc = pi->adapter;
5612
5613         PORT_LOCK_ASSERT_OWNED(pi);
5614         cxgbe_refresh_stats(sc, pi);
5615
5616         callout_schedule(&pi->tick, hz);
5617 }
5618
5619 void
5620 vi_tick(void *arg)
5621 {
5622         struct vi_info *vi = arg;
5623         struct adapter *sc = vi->pi->adapter;
5624
5625         vi_refresh_stats(sc, vi);
5626
5627         callout_schedule(&vi->tick, hz);
5628 }
5629
5630 /*
5631  * Should match fw_caps_config_<foo> enums in t4fw_interface.h
5632  */
5633 static char *caps_decoder[] = {
5634         "\20\001IPMI\002NCSI",                          /* 0: NBM */
5635         "\20\001PPP\002QFC\003DCBX",                    /* 1: link */
5636         "\20\001INGRESS\002EGRESS",                     /* 2: switch */
5637         "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL"      /* 3: NIC */
5638             "\006HASHFILTER\007ETHOFLD",
5639         "\20\001TOE",                                   /* 4: TOE */
5640         "\20\001RDDP\002RDMAC",                         /* 5: RDMA */
5641         "\20\001INITIATOR_PDU\002TARGET_PDU"            /* 6: iSCSI */
5642             "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD"
5643             "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD"
5644             "\007T10DIF"
5645             "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD",
5646         "\20\001LOOKASIDE\002TLSKEYS",                  /* 7: Crypto */
5647         "\20\001INITIATOR\002TARGET\003CTRL_OFLD"       /* 8: FCoE */
5648                     "\004PO_INITIATOR\005PO_TARGET",
5649 };
5650
5651 void
5652 t4_sysctls(struct adapter *sc)
5653 {
5654         struct sysctl_ctx_list *ctx;
5655         struct sysctl_oid *oid;
5656         struct sysctl_oid_list *children, *c0;
5657         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
5658
5659         ctx = device_get_sysctl_ctx(sc->dev);
5660
5661         /*
5662          * dev.t4nex.X.
5663          */
5664         oid = device_get_sysctl_tree(sc->dev);
5665         c0 = children = SYSCTL_CHILDREN(oid);
5666
5667         sc->sc_do_rxcopy = 1;
5668         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
5669             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
5670
5671         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
5672             sc->params.nports, "# of ports");
5673
5674         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
5675             CTLTYPE_STRING | CTLFLAG_RD, doorbells, (uintptr_t)&sc->doorbells,
5676             sysctl_bitfield_8b, "A", "available doorbells");
5677
5678         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
5679             sc->params.vpd.cclk, "core clock frequency (in KHz)");
5680
5681         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
5682             CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.timer_val,
5683             sizeof(sc->params.sge.timer_val), sysctl_int_array, "A",
5684             "interrupt holdoff timer values (us)");
5685
5686         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
5687             CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.counter_val,
5688             sizeof(sc->params.sge.counter_val), sysctl_int_array, "A",
5689             "interrupt holdoff packet counter values");
5690
5691         t4_sge_sysctls(sc, ctx, children);
5692
5693         sc->lro_timeout = 100;
5694         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
5695             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
5696
5697         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW,
5698             &sc->debug_flags, 0, "flags to enable runtime debugging");
5699
5700         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version",
5701             CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
5702
5703         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
5704             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
5705
5706         if (sc->flags & IS_VF)
5707                 return;
5708
5709         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
5710             NULL, chip_rev(sc), "chip hardware revision");
5711
5712         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn",
5713             CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");
5714
5715         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn",
5716             CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");
5717
5718         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec",
5719             CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");
5720
5721         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "md_version",
5722             CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version");
5723
5724         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na",
5725             CTLFLAG_RD, sc->params.vpd.na, 0, "network address");
5726
5727         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD,
5728             sc->er_version, 0, "expansion ROM version");
5729
5730         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD,
5731             sc->bs_version, 0, "bootstrap firmware version");
5732
5733         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD,
5734             NULL, sc->params.scfg_vers, "serial config version");
5735
5736         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD,
5737             NULL, sc->params.vpd_vers, "VPD version");
5738
5739         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
5740             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
5741
5742         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
5743             sc->cfcsum, "config file checksum");
5744
5745 #define SYSCTL_CAP(name, n, text) \
5746         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \
5747             CTLTYPE_STRING | CTLFLAG_RD, caps_decoder[n], (uintptr_t)&sc->name, \
5748             sysctl_bitfield_16b, "A", "available " text " capabilities")
5749
5750         SYSCTL_CAP(nbmcaps, 0, "NBM");
5751         SYSCTL_CAP(linkcaps, 1, "link");
5752         SYSCTL_CAP(switchcaps, 2, "switch");
5753         SYSCTL_CAP(niccaps, 3, "NIC");
5754         SYSCTL_CAP(toecaps, 4, "TCP offload");
5755         SYSCTL_CAP(rdmacaps, 5, "RDMA");
5756         SYSCTL_CAP(iscsicaps, 6, "iSCSI");
5757         SYSCTL_CAP(cryptocaps, 7, "crypto");
5758         SYSCTL_CAP(fcoecaps, 8, "FCoE");
5759 #undef SYSCTL_CAP
5760
5761         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
5762             NULL, sc->tids.nftids, "number of filters");
5763
5764         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
5765             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
5766             "chip temperature (in Celsius)");
5767
5768         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "loadavg", CTLTYPE_STRING |
5769             CTLFLAG_RD, sc, 0, sysctl_loadavg, "A",
5770             "microprocessor load averages (debug firmwares only)");
5771
5772         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_vdd", CTLFLAG_RD,
5773             &sc->params.core_vdd, 0, "core Vdd (in mV)");
5774
5775         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "local_cpus",
5776             CTLTYPE_STRING | CTLFLAG_RD, sc, LOCAL_CPUS,
5777             sysctl_cpus, "A", "local CPUs");
5778
5779         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_cpus",
5780             CTLTYPE_STRING | CTLFLAG_RD, sc, INTR_CPUS,
5781             sysctl_cpus, "A", "preferred CPUs for interrupts");
5782
5783         /*
5784          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
5785          */
5786         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
5787             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
5788             "logs and miscellaneous information");
5789         children = SYSCTL_CHILDREN(oid);
5790
5791         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
5792             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5793             sysctl_cctrl, "A", "congestion control");
5794
5795         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
5796             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5797             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
5798
5799         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
5800             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
5801             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
5802
5803         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
5804             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
5805             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
5806
5807         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
5808             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
5809             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
5810
5811         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
5812             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
5813             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
5814
5815         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
5816             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
5817             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
5818
5819         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
5820             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5821             chip_id(sc) <= CHELSIO_T5 ? sysctl_cim_la : sysctl_cim_la_t6,
5822             "A", "CIM logic analyzer");
5823
5824         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
5825             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5826             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
5827
5828         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
5829             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
5830             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
5831
5832         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
5833             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
5834             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
5835
5836         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
5837             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
5838             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
5839
5840         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
5841             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
5842             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
5843
5844         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
5845             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
5846             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
5847
5848         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
5849             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
5850             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
5851
5852         if (chip_id(sc) > CHELSIO_T4) {
5853                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
5854                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
5855                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
5856
5857                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
5858                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
5859                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
5860         }
5861
5862         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
5863             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5864             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
5865
5866         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
5867             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5868             sysctl_cim_qcfg, "A", "CIM queue configuration");
5869
5870         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
5871             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5872             sysctl_cpl_stats, "A", "CPL statistics");
5873
5874         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
5875             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5876             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
5877
5878         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
5879             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5880             sysctl_devlog, "A", "firmware's device log");
5881
5882         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
5883             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5884             sysctl_fcoe_stats, "A", "FCoE statistics");
5885
5886         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
5887             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5888             sysctl_hw_sched, "A", "hardware scheduler ");
5889
5890         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
5891             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5892             sysctl_l2t, "A", "hardware L2 table");
5893
5894         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "smt",
5895             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5896             sysctl_smt, "A", "hardware source MAC table");
5897
5898         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
5899             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5900             sysctl_lb_stats, "A", "loopback statistics");
5901
5902         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
5903             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5904             sysctl_meminfo, "A", "memory regions");
5905
5906         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
5907             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5908             chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6,
5909             "A", "MPS TCAM entries");
5910
5911         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
5912             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5913             sysctl_path_mtus, "A", "path MTUs");
5914
5915         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
5916             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5917             sysctl_pm_stats, "A", "PM statistics");
5918
5919         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
5920             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5921             sysctl_rdma_stats, "A", "RDMA statistics");
5922
5923         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
5924             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5925             sysctl_tcp_stats, "A", "TCP statistics");
5926
5927         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
5928             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5929             sysctl_tids, "A", "TID information");
5930
5931         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
5932             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5933             sysctl_tp_err_stats, "A", "TP error statistics");
5934
5935         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask",
5936             CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tp_la_mask, "I",
5937             "TP logic analyzer event capture mask");
5938
5939         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
5940             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5941             sysctl_tp_la, "A", "TP logic analyzer");
5942
5943         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
5944             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5945             sysctl_tx_rate, "A", "Tx rate");
5946
5947         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
5948             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5949             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
5950
5951         if (chip_id(sc) >= CHELSIO_T5) {
5952                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
5953                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
5954                     sysctl_wcwr_stats, "A", "write combined work requests");
5955         }
5956
5957 #ifdef TCP_OFFLOAD
5958         if (is_offload(sc)) {
5959                 int i;
5960                 char s[4];
5961
5962                 /*
5963                  * dev.t4nex.X.toe.
5964                  */
5965                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
5966                     NULL, "TOE parameters");
5967                 children = SYSCTL_CHILDREN(oid);
5968
5969                 sc->tt.cong_algorithm = -1;
5970                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_algorithm",
5971                     CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control "
5972                     "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, "
5973                     "3 = highspeed)");
5974
5975                 sc->tt.sndbuf = 256 * 1024;
5976                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
5977                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
5978
5979                 sc->tt.ddp = 0;
5980                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
5981                     &sc->tt.ddp, 0, "DDP allowed");
5982
5983                 sc->tt.rx_coalesce = 1;
5984                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
5985                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
5986
5987                 sc->tt.tls = 0;
5988                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tls", CTLFLAG_RW,
5989                     &sc->tt.tls, 0, "Inline TLS allowed");
5990
5991                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls_rx_ports",
5992                     CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tls_rx_ports,
5993                     "I", "TCP ports that use inline TLS+TOE RX");
5994
5995                 sc->tt.tx_align = 1;
5996                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
5997                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
5998
5999                 sc->tt.tx_zcopy = 0;
6000                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy",
6001                     CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
6002                     "Enable zero-copy aio_write(2)");
6003
6004                 sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading;
6005                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
6006                     "cop_managed_offloading", CTLFLAG_RW,
6007                     &sc->tt.cop_managed_offloading, 0,
6008                     "COP (Connection Offload Policy) controls all TOE offload");
6009
6010                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick",
6011                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A",
6012                     "TP timer tick (us)");
6013
6014                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick",
6015                     CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_tp_tick, "A",
6016                     "TCP timestamp tick (us)");
6017
6018                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick",
6019                     CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_tp_tick, "A",
6020                     "DACK tick (us)");
6021
6022                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer",
6023                     CTLTYPE_UINT | CTLFLAG_RD, sc, 0, sysctl_tp_dack_timer,
6024                     "IU", "DACK timer (us)");
6025
6026                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min",
6027                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MIN,
6028                     sysctl_tp_timer, "LU", "Minimum retransmit interval (us)");
6029
6030                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max",
6031                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MAX,
6032                     sysctl_tp_timer, "LU", "Maximum retransmit interval (us)");
6033
6034                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min",
6035                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MIN,
6036                     sysctl_tp_timer, "LU", "Persist timer min (us)");
6037
6038                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max",
6039                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MAX,
6040                     sysctl_tp_timer, "LU", "Persist timer max (us)");
6041
6042                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle",
6043                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_IDLE,
6044                     sysctl_tp_timer, "LU", "Keepalive idle timer (us)");
6045
6046                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval",
6047                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_INTVL,
6048                     sysctl_tp_timer, "LU", "Keepalive interval timer (us)");
6049
6050                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt",
6051                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_INIT_SRTT,
6052                     sysctl_tp_timer, "LU", "Initial SRTT (us)");
6053
6054                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer",
6055                     CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_FINWAIT2_TIMER,
6056                     sysctl_tp_timer, "LU", "FINWAIT2 timer (us)");
6057
6058                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count",
6059                     CTLTYPE_UINT | CTLFLAG_RD, sc, S_SYNSHIFTMAX,
6060                     sysctl_tp_shift_cnt, "IU",
6061                     "Number of SYN retransmissions before abort");
6062
6063                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count",
6064                     CTLTYPE_UINT | CTLFLAG_RD, sc, S_RXTSHIFTMAXR2,
6065                     sysctl_tp_shift_cnt, "IU",
6066                     "Number of retransmissions before abort");
6067
6068                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count",
6069                     CTLTYPE_UINT | CTLFLAG_RD, sc, S_KEEPALIVEMAXR2,
6070                     sysctl_tp_shift_cnt, "IU",
6071                     "Number of keepalive probes before abort");
6072
6073                 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff",
6074                     CTLFLAG_RD, NULL, "TOE retransmit backoffs");
6075                 children = SYSCTL_CHILDREN(oid);
6076                 for (i = 0; i < 16; i++) {
6077                         snprintf(s, sizeof(s), "%u", i);
6078                         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s,
6079                             CTLTYPE_UINT | CTLFLAG_RD, sc, i, sysctl_tp_backoff,
6080                             "IU", "TOE retransmit backoff");
6081                 }
6082         }
6083 #endif
6084 }
6085
6086 void
6087 vi_sysctls(struct vi_info *vi)
6088 {
6089         struct sysctl_ctx_list *ctx;
6090         struct sysctl_oid *oid;
6091         struct sysctl_oid_list *children;
6092
6093         ctx = device_get_sysctl_ctx(vi->dev);
6094
6095         /*
6096          * dev.v?(cxgbe|cxl).X.
6097          */
6098         oid = device_get_sysctl_tree(vi->dev);
6099         children = SYSCTL_CHILDREN(oid);
6100
6101         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
6102             vi->viid, "VI identifer");
6103         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
6104             &vi->nrxq, 0, "# of rx queues");
6105         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
6106             &vi->ntxq, 0, "# of tx queues");
6107         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
6108             &vi->first_rxq, 0, "index of first rx queue");
6109         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
6110             &vi->first_txq, 0, "index of first tx queue");
6111         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_base", CTLFLAG_RD, NULL,
6112             vi->rss_base, "start of RSS indirection table");
6113         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL,
6114             vi->rss_size, "size of RSS indirection table");
6115
6116         if (IS_MAIN_VI(vi)) {
6117                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq",
6118                     CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU",
6119                     "Reserve queue 0 for non-flowid packets");
6120         }
6121
6122 #ifdef TCP_OFFLOAD
6123         if (vi->nofldrxq != 0) {
6124                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
6125                     &vi->nofldrxq, 0,
6126                     "# of rx queues for offloaded TCP connections");
6127                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
6128                     &vi->nofldtxq, 0,
6129                     "# of tx queues for offloaded TCP connections");
6130                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
6131                     CTLFLAG_RD, &vi->first_ofld_rxq, 0,
6132                     "index of first TOE rx queue");
6133                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
6134                     CTLFLAG_RD, &vi->first_ofld_txq, 0,
6135                     "index of first TOE tx queue");
6136                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx_ofld",
6137                     CTLTYPE_INT | CTLFLAG_RW, vi, 0,
6138                     sysctl_holdoff_tmr_idx_ofld, "I",
6139                     "holdoff timer index for TOE queues");
6140                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx_ofld",
6141                     CTLTYPE_INT | CTLFLAG_RW, vi, 0,
6142                     sysctl_holdoff_pktc_idx_ofld, "I",
6143                     "holdoff packet counter index for TOE queues");
6144         }
6145 #endif
6146 #ifdef DEV_NETMAP
6147         if (vi->nnmrxq != 0) {
6148                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
6149                     &vi->nnmrxq, 0, "# of netmap rx queues");
6150                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
6151                     &vi->nnmtxq, 0, "# of netmap tx queues");
6152                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
6153                     CTLFLAG_RD, &vi->first_nm_rxq, 0,
6154                     "index of first netmap rx queue");
6155                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
6156                     CTLFLAG_RD, &vi->first_nm_txq, 0,
6157                     "index of first netmap tx queue");
6158         }
6159 #endif
6160
6161         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
6162             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I",
6163             "holdoff timer index");
6164         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
6165             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I",
6166             "holdoff packet counter index");
6167
6168         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
6169             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I",
6170             "rx queue size");
6171         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
6172             CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I",
6173             "tx queue size");
6174 }
6175
6176 static void
6177 cxgbe_sysctls(struct port_info *pi)
6178 {
6179         struct sysctl_ctx_list *ctx;
6180         struct sysctl_oid *oid;
6181         struct sysctl_oid_list *children, *children2;
6182         struct adapter *sc = pi->adapter;
6183         int i;
6184         char name[16];
6185         static char *tc_flags = {"\20\1USER\2SYNC\3ASYNC\4ERR"};
6186
6187         ctx = device_get_sysctl_ctx(pi->dev);
6188
6189         /*
6190          * dev.cxgbe.X.
6191          */
6192         oid = device_get_sysctl_tree(pi->dev);
6193         children = SYSCTL_CHILDREN(oid);
6194
6195         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
6196            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
6197         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
6198                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
6199                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
6200                     "PHY temperature (in Celsius)");
6201                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
6202                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
6203                     "PHY firmware version");
6204         }
6205
6206         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
6207             CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_pause_settings, "A",
6208             "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
6209         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fec",
6210             CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_fec, "A",
6211             "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)");
6212         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg",
6213             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_autoneg, "I",
6214             "autonegotiation (-1 = not supported)");
6215
6216         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL,
6217             port_top_speed(pi), "max speed (in Gbps)");
6218         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mps_bg_map", CTLFLAG_RD, NULL,
6219             pi->mps_bg_map, "MPS buffer group map");
6220         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_e_chan_map", CTLFLAG_RD,
6221             NULL, pi->rx_e_chan_map, "TP rx e-channel map");
6222
6223         if (sc->flags & IS_VF)
6224                 return;
6225
6226         /*
6227          * dev.(cxgbe|cxl).X.tc.
6228          */
6229         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL,
6230             "Tx scheduler traffic classes (cl_rl)");
6231         children2 = SYSCTL_CHILDREN(oid);
6232         SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize",
6233             CTLFLAG_RW, &pi->sched_params->pktsize, 0,
6234             "pktsize for per-flow cl-rl (0 means up to the driver )");
6235         SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize",
6236             CTLFLAG_RW, &pi->sched_params->burstsize, 0,
6237             "burstsize for per-flow cl-rl (0 means up to the driver)");
6238         for (i = 0; i < sc->chip_params->nsched_cls; i++) {
6239                 struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i];
6240
6241                 snprintf(name, sizeof(name), "%d", i);
6242                 children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx,
6243                     SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL,
6244                     "traffic class"));
6245                 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "flags",
6246                     CTLTYPE_STRING | CTLFLAG_RD, tc_flags, (uintptr_t)&tc->flags,
6247                     sysctl_bitfield_8b, "A", "flags");
6248                 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount",
6249                     CTLFLAG_RD, &tc->refcount, 0, "references to this class");
6250                 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params",
6251                     CTLTYPE_STRING | CTLFLAG_RD, sc, (pi->port_id << 16) | i,
6252                     sysctl_tc_params, "A", "traffic class parameters");
6253         }
6254
6255         /*
6256          * dev.cxgbe.X.stats.
6257          */
6258         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
6259             NULL, "port statistics");
6260         children = SYSCTL_CHILDREN(oid);
6261         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
6262             &pi->tx_parse_error, 0,
6263             "# of tx packets with invalid length or # of segments");
6264
6265 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
6266         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
6267             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
6268             sysctl_handle_t4_reg64, "QU", desc)
6269
6270         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
6271             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
6272         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
6273             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
6274         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
6275             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
6276         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
6277             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
6278         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
6279             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
6280         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
6281             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
6282         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
6283             "# of tx frames in this range",
6284             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
6285         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
6286             "# of tx frames in this range",
6287             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
6288         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
6289             "# of tx frames in this range",
6290             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
6291         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
6292             "# of tx frames in this range",
6293             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
6294         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
6295             "# of tx frames in this range",
6296             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
6297         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
6298             "# of tx frames in this range",
6299             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
6300         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
6301             "# of tx frames in this range",
6302             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
6303         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
6304             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
6305         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
6306             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
6307         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
6308             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
6309         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
6310             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
6311         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
6312             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
6313         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
6314             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
6315         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
6316             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
6317         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
6318             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
6319         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
6320             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
6321         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
6322             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
6323
6324         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
6325             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
6326         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
6327             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
6328         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
6329             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
6330         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
6331             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
6332         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
6333             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
6334         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
6335             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
6336         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
6337             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
6338         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
6339             "# of frames received with bad FCS",
6340             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
6341         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
6342             "# of frames received with length error",
6343             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
6344         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
6345             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
6346         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
6347             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
6348         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
6349             "# of rx frames in this range",
6350             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
6351         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
6352             "# of rx frames in this range",
6353             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
6354         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
6355             "# of rx frames in this range",
6356             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
6357         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
6358             "# of rx frames in this range",
6359             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
6360         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
6361             "# of rx frames in this range",
6362             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
6363         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
6364             "# of rx frames in this range",
6365             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
6366         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
6367             "# of rx frames in this range",
6368             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
6369         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
6370             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
6371         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
6372             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
6373         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
6374             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
6375         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
6376             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
6377         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
6378             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
6379         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
6380             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
6381         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
6382             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
6383         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
6384             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
6385         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
6386             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
6387
6388 #undef SYSCTL_ADD_T4_REG64
6389
6390 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
6391         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
6392             &pi->stats.name, desc)
6393
6394         /* We get these from port_stats and they may be stale by up to 1s */
6395         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
6396             "# drops due to buffer-group 0 overflows");
6397         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
6398             "# drops due to buffer-group 1 overflows");
6399         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
6400             "# drops due to buffer-group 2 overflows");
6401         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
6402             "# drops due to buffer-group 3 overflows");
6403         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
6404             "# of buffer-group 0 truncated packets");
6405         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
6406             "# of buffer-group 1 truncated packets");
6407         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
6408             "# of buffer-group 2 truncated packets");
6409         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
6410             "# of buffer-group 3 truncated packets");
6411
6412 #undef SYSCTL_ADD_T4_PORTSTAT
6413
6414         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_records",
6415             CTLFLAG_RD, &pi->tx_tls_records,
6416             "# of TLS records transmitted");
6417         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_tls_octets",
6418             CTLFLAG_RD, &pi->tx_tls_octets,
6419             "# of payload octets in transmitted TLS records");
6420         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_records",
6421             CTLFLAG_RD, &pi->rx_tls_records,
6422             "# of TLS records received");
6423         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_tls_octets",
6424             CTLFLAG_RD, &pi->rx_tls_octets,
6425             "# of payload octets in received TLS records");
6426 }
6427
6428 static int
6429 sysctl_int_array(SYSCTL_HANDLER_ARGS)
6430 {
6431         int rc, *i, space = 0;
6432         struct sbuf sb;
6433
6434         sbuf_new_for_sysctl(&sb, NULL, 64, req);
6435         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
6436                 if (space)
6437                         sbuf_printf(&sb, " ");
6438                 sbuf_printf(&sb, "%d", *i);
6439                 space = 1;
6440         }
6441         rc = sbuf_finish(&sb);
6442         sbuf_delete(&sb);
6443         return (rc);
6444 }
6445
6446 static int
6447 sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS)
6448 {
6449         int rc;
6450         struct sbuf *sb;
6451
6452         rc = sysctl_wire_old_buffer(req, 0);
6453         if (rc != 0)
6454                 return(rc);
6455
6456         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
6457         if (sb == NULL)
6458                 return (ENOMEM);
6459
6460         sbuf_printf(sb, "%b", *(uint8_t *)(uintptr_t)arg2, (char *)arg1);
6461         rc = sbuf_finish(sb);
6462         sbuf_delete(sb);
6463
6464         return (rc);
6465 }
6466
6467 static int
6468 sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS)
6469 {
6470         int rc;
6471         struct sbuf *sb;
6472
6473         rc = sysctl_wire_old_buffer(req, 0);
6474         if (rc != 0)
6475                 return(rc);
6476
6477         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
6478         if (sb == NULL)
6479                 return (ENOMEM);
6480
6481         sbuf_printf(sb, "%b", *(uint16_t *)(uintptr_t)arg2, (char *)arg1);
6482         rc = sbuf_finish(sb);
6483         sbuf_delete(sb);
6484
6485         return (rc);
6486 }
6487
6488 static int
6489 sysctl_btphy(SYSCTL_HANDLER_ARGS)
6490 {
6491         struct port_info *pi = arg1;
6492         int op = arg2;
6493         struct adapter *sc = pi->adapter;
6494         u_int v;
6495         int rc;
6496
6497         rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
6498         if (rc)
6499                 return (rc);
6500         /* XXX: magic numbers */
6501         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
6502             &v);
6503         end_synchronized_op(sc, 0);
6504         if (rc)
6505                 return (rc);
6506         if (op == 0)
6507                 v /= 256;
6508
6509         rc = sysctl_handle_int(oidp, &v, 0, req);
6510         return (rc);
6511 }
6512
6513 static int
6514 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
6515 {
6516         struct vi_info *vi = arg1;
6517         int rc, val;
6518
6519         val = vi->rsrv_noflowq;
6520         rc = sysctl_handle_int(oidp, &val, 0, req);
6521         if (rc != 0 || req->newptr == NULL)
6522                 return (rc);
6523
6524         if ((val >= 1) && (vi->ntxq > 1))
6525                 vi->rsrv_noflowq = 1;
6526         else
6527                 vi->rsrv_noflowq = 0;
6528
6529         return (rc);
6530 }
6531
6532 static int
6533 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
6534 {
6535         struct vi_info *vi = arg1;
6536         struct adapter *sc = vi->pi->adapter;
6537         int idx, rc, i;
6538         struct sge_rxq *rxq;
6539         uint8_t v;
6540
6541         idx = vi->tmr_idx;
6542
6543         rc = sysctl_handle_int(oidp, &idx, 0, req);
6544         if (rc != 0 || req->newptr == NULL)
6545                 return (rc);
6546
6547         if (idx < 0 || idx >= SGE_NTIMERS)
6548                 return (EINVAL);
6549
6550         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
6551             "t4tmr");
6552         if (rc)
6553                 return (rc);
6554
6555         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
6556         for_each_rxq(vi, i, rxq) {
6557 #ifdef atomic_store_rel_8
6558                 atomic_store_rel_8(&rxq->iq.intr_params, v);
6559 #else
6560                 rxq->iq.intr_params = v;
6561 #endif
6562         }
6563         vi->tmr_idx = idx;
6564
6565         end_synchronized_op(sc, LOCK_HELD);
6566         return (0);
6567 }
6568
6569 static int
6570 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
6571 {
6572         struct vi_info *vi = arg1;
6573         struct adapter *sc = vi->pi->adapter;
6574         int idx, rc;
6575
6576         idx = vi->pktc_idx;
6577
6578         rc = sysctl_handle_int(oidp, &idx, 0, req);
6579         if (rc != 0 || req->newptr == NULL)
6580                 return (rc);
6581
6582         if (idx < -1 || idx >= SGE_NCOUNTERS)
6583                 return (EINVAL);
6584
6585         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
6586             "t4pktc");
6587         if (rc)
6588                 return (rc);
6589
6590         if (vi->flags & VI_INIT_DONE)
6591                 rc = EBUSY; /* cannot be changed once the queues are created */
6592         else
6593                 vi->pktc_idx = idx;
6594
6595         end_synchronized_op(sc, LOCK_HELD);
6596         return (rc);
6597 }
6598
6599 static int
6600 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
6601 {
6602         struct vi_info *vi = arg1;
6603         struct adapter *sc = vi->pi->adapter;
6604         int qsize, rc;
6605
6606         qsize = vi->qsize_rxq;
6607
6608         rc = sysctl_handle_int(oidp, &qsize, 0, req);
6609         if (rc != 0 || req->newptr == NULL)
6610                 return (rc);
6611
6612         if (qsize < 128 || (qsize & 7))
6613                 return (EINVAL);
6614
6615         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
6616             "t4rxqs");
6617         if (rc)
6618                 return (rc);
6619
6620         if (vi->flags & VI_INIT_DONE)
6621                 rc = EBUSY; /* cannot be changed once the queues are created */
6622         else
6623                 vi->qsize_rxq = qsize;
6624
6625         end_synchronized_op(sc, LOCK_HELD);
6626         return (rc);
6627 }
6628
6629 static int
6630 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
6631 {
6632         struct vi_info *vi = arg1;
6633         struct adapter *sc = vi->pi->adapter;
6634         int qsize, rc;
6635
6636         qsize = vi->qsize_txq;
6637
6638         rc = sysctl_handle_int(oidp, &qsize, 0, req);
6639         if (rc != 0 || req->newptr == NULL)
6640                 return (rc);
6641
6642         if (qsize < 128 || qsize > 65536)
6643                 return (EINVAL);
6644
6645         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
6646             "t4txqs");
6647         if (rc)
6648                 return (rc);
6649
6650         if (vi->flags & VI_INIT_DONE)
6651                 rc = EBUSY; /* cannot be changed once the queues are created */
6652         else
6653                 vi->qsize_txq = qsize;
6654
6655         end_synchronized_op(sc, LOCK_HELD);
6656         return (rc);
6657 }
6658
6659 static int
6660 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
6661 {
6662         struct port_info *pi = arg1;
6663         struct adapter *sc = pi->adapter;
6664         struct link_config *lc = &pi->link_cfg;
6665         int rc;
6666
6667         if (req->newptr == NULL) {
6668                 struct sbuf *sb;
6669                 static char *bits = "\20\1RX\2TX\3AUTO";
6670
6671                 rc = sysctl_wire_old_buffer(req, 0);
6672                 if (rc != 0)
6673                         return(rc);
6674
6675                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
6676                 if (sb == NULL)
6677                         return (ENOMEM);
6678
6679                 if (lc->link_ok) {
6680                         sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) |
6681                             (lc->requested_fc & PAUSE_AUTONEG), bits);
6682                 } else {
6683                         sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX |
6684                             PAUSE_RX | PAUSE_AUTONEG), bits);
6685                 }
6686                 rc = sbuf_finish(sb);
6687                 sbuf_delete(sb);
6688         } else {
6689                 char s[2];
6690                 int n;
6691
6692                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX |
6693                     PAUSE_AUTONEG));
6694                 s[1] = 0;
6695
6696                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
6697                 if (rc != 0)
6698                         return(rc);
6699
6700                 if (s[1] != 0)
6701                         return (EINVAL);
6702                 if (s[0] < '0' || s[0] > '9')
6703                         return (EINVAL);        /* not a number */
6704                 n = s[0] - '0';
6705                 if (n & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG))
6706                         return (EINVAL);        /* some other bit is set too */
6707
6708                 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
6709                     "t4PAUSE");
6710                 if (rc)
6711                         return (rc);
6712                 PORT_LOCK(pi);
6713                 lc->requested_fc = n;
6714                 fixup_link_config(pi);
6715                 if (pi->up_vis > 0)
6716                         rc = apply_link_config(pi);
6717                 set_current_media(pi);
6718                 PORT_UNLOCK(pi);
6719                 end_synchronized_op(sc, 0);
6720         }
6721
6722         return (rc);
6723 }
6724
6725 static int
6726 sysctl_fec(SYSCTL_HANDLER_ARGS)
6727 {
6728         struct port_info *pi = arg1;
6729         struct adapter *sc = pi->adapter;
6730         struct link_config *lc = &pi->link_cfg;
6731         int rc;
6732         int8_t old;
6733
6734         if (req->newptr == NULL) {
6735                 struct sbuf *sb;
6736                 static char *bits = "\20\1RS\2BASE-R\3RSVD1\4RSVD2\5RSVD3\6AUTO";
6737
6738                 rc = sysctl_wire_old_buffer(req, 0);
6739                 if (rc != 0)
6740                         return(rc);
6741
6742                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
6743                 if (sb == NULL)
6744                         return (ENOMEM);
6745
6746                 /*
6747                  * Display the requested_fec when the link is down -- the actual
6748                  * FEC makes sense only when the link is up.
6749                  */
6750                 if (lc->link_ok) {
6751                         sbuf_printf(sb, "%b", (lc->fec & M_FW_PORT_CAP32_FEC) |
6752                             (lc->requested_fec & FEC_AUTO), bits);
6753                 } else {
6754                         sbuf_printf(sb, "%b", lc->requested_fec, bits);
6755                 }
6756                 rc = sbuf_finish(sb);
6757                 sbuf_delete(sb);
6758         } else {
6759                 char s[3];
6760                 int n;
6761
6762                 snprintf(s, sizeof(s), "%d",
6763                     lc->requested_fec == FEC_AUTO ? -1 :
6764                     lc->requested_fec & M_FW_PORT_CAP32_FEC);
6765
6766                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
6767                 if (rc != 0)
6768                         return(rc);
6769
6770                 n = strtol(&s[0], NULL, 0);
6771                 if (n < 0 || n & FEC_AUTO)
6772                         n = FEC_AUTO;
6773                 else {
6774                         if (n & ~M_FW_PORT_CAP32_FEC)
6775                                 return (EINVAL);/* some other bit is set too */
6776                         if (!powerof2(n))
6777                                 return (EINVAL);/* one bit can be set at most */
6778                 }
6779
6780                 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
6781                     "t4fec");
6782                 if (rc)
6783                         return (rc);
6784                 PORT_LOCK(pi);
6785                 old = lc->requested_fec;
6786                 if (n == FEC_AUTO)
6787                         lc->requested_fec = FEC_AUTO;
6788                 else if (n == 0)
6789                         lc->requested_fec = FEC_NONE;
6790                 else {
6791                         if ((lc->supported | V_FW_PORT_CAP32_FEC(n)) !=
6792                             lc->supported) {
6793                                 rc = ENOTSUP;
6794                                 goto done;
6795                         }
6796                         lc->requested_fec = n;
6797                 }
6798                 fixup_link_config(pi);
6799                 if (pi->up_vis > 0) {
6800                         rc = apply_link_config(pi);
6801                         if (rc != 0) {
6802                                 lc->requested_fec = old;
6803                                 if (rc == FW_EPROTO)
6804                                         rc = ENOTSUP;
6805                         }
6806                 }
6807 done:
6808                 PORT_UNLOCK(pi);
6809                 end_synchronized_op(sc, 0);
6810         }
6811
6812         return (rc);
6813 }
6814
6815 static int
6816 sysctl_autoneg(SYSCTL_HANDLER_ARGS)
6817 {
6818         struct port_info *pi = arg1;
6819         struct adapter *sc = pi->adapter;
6820         struct link_config *lc = &pi->link_cfg;
6821         int rc, val;
6822
6823         if (lc->supported & FW_PORT_CAP32_ANEG)
6824                 val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1;
6825         else
6826                 val = -1;
6827         rc = sysctl_handle_int(oidp, &val, 0, req);
6828         if (rc != 0 || req->newptr == NULL)
6829                 return (rc);
6830         if (val == 0)
6831                 val = AUTONEG_DISABLE;
6832         else if (val == 1)
6833                 val = AUTONEG_ENABLE;
6834         else
6835                 val = AUTONEG_AUTO;
6836
6837         rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
6838             "t4aneg");
6839         if (rc)
6840                 return (rc);
6841         PORT_LOCK(pi);
6842         if (val == AUTONEG_ENABLE && !(lc->supported & FW_PORT_CAP32_ANEG)) {
6843                 rc = ENOTSUP;
6844                 goto done;
6845         }
6846         lc->requested_aneg = val;
6847         fixup_link_config(pi);
6848         if (pi->up_vis > 0)
6849                 rc = apply_link_config(pi);
6850         set_current_media(pi);
6851 done:
6852         PORT_UNLOCK(pi);
6853         end_synchronized_op(sc, 0);
6854         return (rc);
6855 }
6856
6857 static int
6858 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
6859 {
6860         struct adapter *sc = arg1;
6861         int reg = arg2;
6862         uint64_t val;
6863
6864         val = t4_read_reg64(sc, reg);
6865
6866         return (sysctl_handle_64(oidp, &val, 0, req));
6867 }
6868
6869 static int
6870 sysctl_temperature(SYSCTL_HANDLER_ARGS)
6871 {
6872         struct adapter *sc = arg1;
6873         int rc, t;
6874         uint32_t param, val;
6875
6876         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
6877         if (rc)
6878                 return (rc);
6879         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
6880             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
6881             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
6882         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
6883         end_synchronized_op(sc, 0);
6884         if (rc)
6885                 return (rc);
6886
6887         /* unknown is returned as 0 but we display -1 in that case */
6888         t = val == 0 ? -1 : val;
6889
6890         rc = sysctl_handle_int(oidp, &t, 0, req);
6891         return (rc);
6892 }
6893
6894 static int
6895 sysctl_loadavg(SYSCTL_HANDLER_ARGS)
6896 {
6897         struct adapter *sc = arg1;
6898         struct sbuf *sb;
6899         int rc;
6900         uint32_t param, val;
6901
6902         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4lavg");
6903         if (rc)
6904                 return (rc);
6905         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
6906             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_LOAD);
6907         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
6908         end_synchronized_op(sc, 0);
6909         if (rc)
6910                 return (rc);
6911
6912         rc = sysctl_wire_old_buffer(req, 0);
6913         if (rc != 0)
6914                 return (rc);
6915
6916         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6917         if (sb == NULL)
6918                 return (ENOMEM);
6919
6920         if (val == 0xffffffff) {
6921                 /* Only debug and custom firmwares report load averages. */
6922                 sbuf_printf(sb, "not available");
6923         } else {
6924                 sbuf_printf(sb, "%d %d %d", val & 0xff, (val >> 8) & 0xff,
6925                     (val >> 16) & 0xff);
6926         }
6927         rc = sbuf_finish(sb);
6928         sbuf_delete(sb);
6929
6930         return (rc);
6931 }
6932
6933 static int
6934 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
6935 {
6936         struct adapter *sc = arg1;
6937         struct sbuf *sb;
6938         int rc, i;
6939         uint16_t incr[NMTUS][NCCTRL_WIN];
6940         static const char *dec_fac[] = {
6941                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
6942                 "0.9375"
6943         };
6944
6945         rc = sysctl_wire_old_buffer(req, 0);
6946         if (rc != 0)
6947                 return (rc);
6948
6949         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6950         if (sb == NULL)
6951                 return (ENOMEM);
6952
6953         t4_read_cong_tbl(sc, incr);
6954
6955         for (i = 0; i < NCCTRL_WIN; ++i) {
6956                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
6957                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
6958                     incr[5][i], incr[6][i], incr[7][i]);
6959                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
6960                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
6961                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
6962                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
6963         }
6964
6965         rc = sbuf_finish(sb);
6966         sbuf_delete(sb);
6967
6968         return (rc);
6969 }
6970
6971 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
6972         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
6973         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
6974         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
6975 };
6976
6977 static int
6978 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
6979 {
6980         struct adapter *sc = arg1;
6981         struct sbuf *sb;
6982         int rc, i, n, qid = arg2;
6983         uint32_t *buf, *p;
6984         char *qtype;
6985         u_int cim_num_obq = sc->chip_params->cim_num_obq;
6986
6987         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
6988             ("%s: bad qid %d\n", __func__, qid));
6989
6990         if (qid < CIM_NUM_IBQ) {
6991                 /* inbound queue */
6992                 qtype = "IBQ";
6993                 n = 4 * CIM_IBQ_SIZE;
6994                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
6995                 rc = t4_read_cim_ibq(sc, qid, buf, n);
6996         } else {
6997                 /* outbound queue */
6998                 qtype = "OBQ";
6999                 qid -= CIM_NUM_IBQ;
7000                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
7001                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
7002                 rc = t4_read_cim_obq(sc, qid, buf, n);
7003         }
7004
7005         if (rc < 0) {
7006                 rc = -rc;
7007                 goto done;
7008         }
7009         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
7010
7011         rc = sysctl_wire_old_buffer(req, 0);
7012         if (rc != 0)
7013                 goto done;
7014
7015         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
7016         if (sb == NULL) {
7017                 rc = ENOMEM;
7018                 goto done;
7019         }
7020
7021         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
7022         for (i = 0, p = buf; i < n; i += 16, p += 4)
7023                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
7024                     p[2], p[3]);
7025
7026         rc = sbuf_finish(sb);
7027         sbuf_delete(sb);
7028 done:
7029         free(buf, M_CXGBE);
7030         return (rc);
7031 }
7032
7033 static int
7034 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
7035 {
7036         struct adapter *sc = arg1;
7037         u_int cfg;
7038         struct sbuf *sb;
7039         uint32_t *buf, *p;
7040         int rc;
7041
7042         MPASS(chip_id(sc) <= CHELSIO_T5);
7043
7044         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
7045         if (rc != 0)
7046                 return (rc);
7047
7048         rc = sysctl_wire_old_buffer(req, 0);
7049         if (rc != 0)
7050                 return (rc);
7051
7052         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7053         if (sb == NULL)
7054                 return (ENOMEM);
7055
7056         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
7057             M_ZERO | M_WAITOK);
7058
7059         rc = -t4_cim_read_la(sc, buf, NULL);
7060         if (rc != 0)
7061                 goto done;
7062
7063         sbuf_printf(sb, "Status   Data      PC%s",
7064             cfg & F_UPDBGLACAPTPCONLY ? "" :
7065             "     LS0Stat  LS0Addr             LS0Data");
7066
7067         for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
7068                 if (cfg & F_UPDBGLACAPTPCONLY) {
7069                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
7070                             p[6], p[7]);
7071                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
7072                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
7073                             p[4] & 0xff, p[5] >> 8);
7074                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
7075                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
7076                             p[1] & 0xf, p[2] >> 4);
7077                 } else {
7078                         sbuf_printf(sb,
7079                             "\n  %02x   %x%07x %x%07x %08x %08x "
7080                             "%08x%08x%08x%08x",
7081                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
7082                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
7083                             p[6], p[7]);
7084                 }
7085         }
7086
7087         rc = sbuf_finish(sb);
7088         sbuf_delete(sb);
7089 done:
7090         free(buf, M_CXGBE);
7091         return (rc);
7092 }
7093
7094 static int
7095 sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS)
7096 {
7097         struct adapter *sc = arg1;
7098         u_int cfg;
7099         struct sbuf *sb;
7100         uint32_t *buf, *p;
7101         int rc;
7102
7103         MPASS(chip_id(sc) > CHELSIO_T5);
7104
7105         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
7106         if (rc != 0)
7107                 return (rc);
7108
7109         rc = sysctl_wire_old_buffer(req, 0);
7110         if (rc != 0)
7111                 return (rc);
7112
7113         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7114         if (sb == NULL)
7115                 return (ENOMEM);
7116
7117         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
7118             M_ZERO | M_WAITOK);
7119
7120         rc = -t4_cim_read_la(sc, buf, NULL);
7121         if (rc != 0)
7122                 goto done;
7123
7124         sbuf_printf(sb, "Status   Inst    Data      PC%s",
7125             cfg & F_UPDBGLACAPTPCONLY ? "" :
7126             "     LS0Stat  LS0Addr  LS0Data  LS1Stat  LS1Addr  LS1Data");
7127
7128         for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
7129                 if (cfg & F_UPDBGLACAPTPCONLY) {
7130                         sbuf_printf(sb, "\n  %02x   %08x %08x %08x",
7131                             p[3] & 0xff, p[2], p[1], p[0]);
7132                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x %02x%06x",
7133                             (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8,
7134                             p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8);
7135                         sbuf_printf(sb, "\n  %02x   %04x%04x %04x%04x %04x%04x",
7136                             (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16,
7137                             p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff,
7138                             p[6] >> 16);
7139                 } else {
7140                         sbuf_printf(sb, "\n  %02x   %04x%04x %04x%04x %04x%04x "
7141                             "%08x %08x %08x %08x %08x %08x",
7142                             (p[9] >> 16) & 0xff,
7143                             p[9] & 0xffff, p[8] >> 16,
7144                             p[8] & 0xffff, p[7] >> 16,
7145                             p[7] & 0xffff, p[6] >> 16,
7146                             p[2], p[1], p[0], p[5], p[4], p[3]);
7147                 }
7148         }
7149
7150         rc = sbuf_finish(sb);
7151         sbuf_delete(sb);
7152 done:
7153         free(buf, M_CXGBE);
7154         return (rc);
7155 }
7156
7157 static int
7158 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
7159 {
7160         struct adapter *sc = arg1;
7161         u_int i;
7162         struct sbuf *sb;
7163         uint32_t *buf, *p;
7164         int rc;
7165
7166         rc = sysctl_wire_old_buffer(req, 0);
7167         if (rc != 0)
7168                 return (rc);
7169
7170         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7171         if (sb == NULL)
7172                 return (ENOMEM);
7173
7174         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
7175             M_ZERO | M_WAITOK);
7176
7177         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
7178         p = buf;
7179
7180         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
7181                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
7182                     p[1], p[0]);
7183         }
7184
7185         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
7186         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
7187                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
7188                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
7189                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
7190                     (p[1] >> 2) | ((p[2] & 3) << 30),
7191                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
7192                     p[0] & 1);
7193         }
7194
7195         rc = sbuf_finish(sb);
7196         sbuf_delete(sb);
7197         free(buf, M_CXGBE);
7198         return (rc);
7199 }
7200
7201 static int
7202 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
7203 {
7204         struct adapter *sc = arg1;
7205         u_int i;
7206         struct sbuf *sb;
7207         uint32_t *buf, *p;
7208         int rc;
7209
7210         rc = sysctl_wire_old_buffer(req, 0);
7211         if (rc != 0)
7212                 return (rc);
7213
7214         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7215         if (sb == NULL)
7216                 return (ENOMEM);
7217
7218         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
7219             M_ZERO | M_WAITOK);
7220
7221         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
7222         p = buf;
7223
7224         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
7225         for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
7226                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
7227                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
7228                     p[4], p[3], p[2], p[1], p[0]);
7229         }
7230
7231         sbuf_printf(sb, "\n\nCntl ID               Data");
7232         for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
7233                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
7234                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
7235         }
7236
7237         rc = sbuf_finish(sb);
7238         sbuf_delete(sb);
7239         free(buf, M_CXGBE);
7240         return (rc);
7241 }
7242
7243 static int
7244 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
7245 {
7246         struct adapter *sc = arg1;
7247         struct sbuf *sb;
7248         int rc, i;
7249         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
7250         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
7251         uint16_t thres[CIM_NUM_IBQ];
7252         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
7253         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
7254         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
7255
7256         cim_num_obq = sc->chip_params->cim_num_obq;
7257         if (is_t4(sc)) {
7258                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
7259                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
7260         } else {
7261                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
7262                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
7263         }
7264         nq = CIM_NUM_IBQ + cim_num_obq;
7265
7266         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
7267         if (rc == 0)
7268                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
7269         if (rc != 0)
7270                 return (rc);
7271
7272         t4_read_cimq_cfg(sc, base, size, thres);
7273
7274         rc = sysctl_wire_old_buffer(req, 0);
7275         if (rc != 0)
7276                 return (rc);
7277
7278         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
7279         if (sb == NULL)
7280                 return (ENOMEM);
7281
7282         sbuf_printf(sb,
7283             "  Queue  Base  Size Thres  RdPtr WrPtr  SOP  EOP Avail");
7284
7285         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
7286                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
7287                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
7288                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
7289                     G_QUEREMFLITS(p[2]) * 16);
7290         for ( ; i < nq; i++, p += 4, wr += 2)
7291                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
7292                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
7293                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
7294                     G_QUEREMFLITS(p[2]) * 16);
7295
7296         rc = sbuf_finish(sb);
7297         sbuf_delete(sb);
7298
7299         return (rc);
7300 }
7301
7302 static int
7303 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
7304 {
7305         struct adapter *sc = arg1;
7306         struct sbuf *sb;
7307         int rc;
7308         struct tp_cpl_stats stats;
7309
7310         rc = sysctl_wire_old_buffer(req, 0);
7311         if (rc != 0)
7312                 return (rc);
7313
7314         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7315         if (sb == NULL)
7316                 return (ENOMEM);
7317
7318         mtx_lock(&sc->reg_lock);
7319         t4_tp_get_cpl_stats(sc, &stats, 0);
7320         mtx_unlock(&sc->reg_lock);
7321
7322         if (sc->chip_params->nchan > 2) {
7323                 sbuf_printf(sb, "                 channel 0  channel 1"
7324                     "  channel 2  channel 3");
7325                 sbuf_printf(sb, "\nCPL requests:   %10u %10u %10u %10u",
7326                     stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
7327                 sbuf_printf(sb, "\nCPL responses:   %10u %10u %10u %10u",
7328                     stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
7329         } else {
7330                 sbuf_printf(sb, "                 channel 0  channel 1");
7331                 sbuf_printf(sb, "\nCPL requests:   %10u %10u",
7332                     stats.req[0], stats.req[1]);
7333                 sbuf_printf(sb, "\nCPL responses:   %10u %10u",
7334                     stats.rsp[0], stats.rsp[1]);
7335         }
7336
7337         rc = sbuf_finish(sb);
7338         sbuf_delete(sb);
7339
7340         return (rc);
7341 }
7342
7343 static int
7344 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
7345 {
7346         struct adapter *sc = arg1;
7347         struct sbuf *sb;
7348         int rc;
7349         struct tp_usm_stats stats;
7350
7351         rc = sysctl_wire_old_buffer(req, 0);
7352         if (rc != 0)
7353                 return(rc);
7354
7355         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7356         if (sb == NULL)
7357                 return (ENOMEM);
7358
7359         t4_get_usm_stats(sc, &stats, 1);
7360
7361         sbuf_printf(sb, "Frames: %u\n", stats.frames);
7362         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
7363         sbuf_printf(sb, "Drops:  %u", stats.drops);
7364
7365         rc = sbuf_finish(sb);
7366         sbuf_delete(sb);
7367
7368         return (rc);
7369 }
7370
7371 static const char * const devlog_level_strings[] = {
7372         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
7373         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
7374         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
7375         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
7376         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
7377         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
7378 };
7379
7380 static const char * const devlog_facility_strings[] = {
7381         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
7382         [FW_DEVLOG_FACILITY_CF]         = "CF",
7383         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
7384         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
7385         [FW_DEVLOG_FACILITY_RES]        = "RES",
7386         [FW_DEVLOG_FACILITY_HW]         = "HW",
7387         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
7388         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
7389         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
7390         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
7391         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
7392         [FW_DEVLOG_FACILITY_VI]         = "VI",
7393         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
7394         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
7395         [FW_DEVLOG_FACILITY_TM]         = "TM",
7396         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
7397         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
7398         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
7399         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
7400         [FW_DEVLOG_FACILITY_RI]         = "RI",
7401         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
7402         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
7403         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
7404         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE",
7405         [FW_DEVLOG_FACILITY_CHNET]      = "CHNET",
7406 };
7407
7408 static int
7409 sysctl_devlog(SYSCTL_HANDLER_ARGS)
7410 {
7411         struct adapter *sc = arg1;
7412         struct devlog_params *dparams = &sc->params.devlog;
7413         struct fw_devlog_e *buf, *e;
7414         int i, j, rc, nentries, first = 0;
7415         struct sbuf *sb;
7416         uint64_t ftstamp = UINT64_MAX;
7417
7418         if (dparams->addr == 0)
7419                 return (ENXIO);
7420
7421         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
7422         if (buf == NULL)
7423                 return (ENOMEM);
7424
7425         rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size);
7426         if (rc != 0)
7427                 goto done;
7428
7429         nentries = dparams->size / sizeof(struct fw_devlog_e);
7430         for (i = 0; i < nentries; i++) {
7431                 e = &buf[i];
7432
7433                 if (e->timestamp == 0)
7434                         break;  /* end */
7435
7436                 e->timestamp = be64toh(e->timestamp);
7437                 e->seqno = be32toh(e->seqno);
7438                 for (j = 0; j < 8; j++)
7439                         e->params[j] = be32toh(e->params[j]);
7440
7441                 if (e->timestamp < ftstamp) {
7442                         ftstamp = e->timestamp;
7443                         first = i;
7444                 }
7445         }
7446
7447         if (buf[first].timestamp == 0)
7448                 goto done;      /* nothing in the log */
7449
7450         rc = sysctl_wire_old_buffer(req, 0);
7451         if (rc != 0)
7452                 goto done;
7453
7454         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7455         if (sb == NULL) {
7456                 rc = ENOMEM;
7457                 goto done;
7458         }
7459         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
7460             "Seq#", "Tstamp", "Level", "Facility", "Message");
7461
7462         i = first;
7463         do {
7464                 e = &buf[i];
7465                 if (e->timestamp == 0)
7466                         break;  /* end */
7467
7468                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
7469                     e->seqno, e->timestamp,
7470                     (e->level < nitems(devlog_level_strings) ?
7471                         devlog_level_strings[e->level] : "UNKNOWN"),
7472                     (e->facility < nitems(devlog_facility_strings) ?
7473                         devlog_facility_strings[e->facility] : "UNKNOWN"));
7474                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
7475                     e->params[2], e->params[3], e->params[4],
7476                     e->params[5], e->params[6], e->params[7]);
7477
7478                 if (++i == nentries)
7479                         i = 0;
7480         } while (i != first);
7481
7482         rc = sbuf_finish(sb);
7483         sbuf_delete(sb);
7484 done:
7485         free(buf, M_CXGBE);
7486         return (rc);
7487 }
7488
7489 static int
7490 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
7491 {
7492         struct adapter *sc = arg1;
7493         struct sbuf *sb;
7494         int rc;
7495         struct tp_fcoe_stats stats[MAX_NCHAN];
7496         int i, nchan = sc->chip_params->nchan;
7497
7498         rc = sysctl_wire_old_buffer(req, 0);
7499         if (rc != 0)
7500                 return (rc);
7501
7502         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7503         if (sb == NULL)
7504                 return (ENOMEM);
7505
7506         for (i = 0; i < nchan; i++)
7507                 t4_get_fcoe_stats(sc, i, &stats[i], 1);
7508
7509         if (nchan > 2) {
7510                 sbuf_printf(sb, "                   channel 0        channel 1"
7511                     "        channel 2        channel 3");
7512                 sbuf_printf(sb, "\noctetsDDP:  %16ju %16ju %16ju %16ju",
7513                     stats[0].octets_ddp, stats[1].octets_ddp,
7514                     stats[2].octets_ddp, stats[3].octets_ddp);
7515                 sbuf_printf(sb, "\nframesDDP:  %16u %16u %16u %16u",
7516                     stats[0].frames_ddp, stats[1].frames_ddp,
7517                     stats[2].frames_ddp, stats[3].frames_ddp);
7518                 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u",
7519                     stats[0].frames_drop, stats[1].frames_drop,
7520                     stats[2].frames_drop, stats[3].frames_drop);
7521         } else {
7522                 sbuf_printf(sb, "                   channel 0        channel 1");
7523                 sbuf_printf(sb, "\noctetsDDP:  %16ju %16ju",
7524                     stats[0].octets_ddp, stats[1].octets_ddp);
7525                 sbuf_printf(sb, "\nframesDDP:  %16u %16u",
7526                     stats[0].frames_ddp, stats[1].frames_ddp);
7527                 sbuf_printf(sb, "\nframesDrop: %16u %16u",
7528                     stats[0].frames_drop, stats[1].frames_drop);
7529         }
7530
7531         rc = sbuf_finish(sb);
7532         sbuf_delete(sb);
7533
7534         return (rc);
7535 }
7536
7537 static int
7538 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
7539 {
7540         struct adapter *sc = arg1;
7541         struct sbuf *sb;
7542         int rc, i;
7543         unsigned int map, kbps, ipg, mode;
7544         unsigned int pace_tab[NTX_SCHED];
7545
7546         rc = sysctl_wire_old_buffer(req, 0);
7547         if (rc != 0)
7548                 return (rc);
7549
7550         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
7551         if (sb == NULL)
7552                 return (ENOMEM);
7553
7554         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
7555         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
7556         t4_read_pace_tbl(sc, pace_tab);
7557
7558         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
7559             "Class IPG (0.1 ns)   Flow IPG (us)");
7560
7561         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
7562                 t4_get_tx_sched(sc, i, &kbps, &ipg, 1);
7563                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
7564                     (mode & (1 << i)) ? "flow" : "class", map & 3);
7565                 if (kbps)
7566                         sbuf_printf(sb, "%9u     ", kbps);
7567                 else
7568                         sbuf_printf(sb, " disabled     ");
7569
7570                 if (ipg)
7571                         sbuf_printf(sb, "%13u        ", ipg);
7572                 else
7573                         sbuf_printf(sb, "     disabled        ");
7574
7575                 if (pace_tab[i])
7576                         sbuf_printf(sb, "%10u", pace_tab[i]);
7577                 else
7578                         sbuf_printf(sb, "  disabled");
7579         }
7580
7581         rc = sbuf_finish(sb);
7582         sbuf_delete(sb);
7583
7584         return (rc);
7585 }
7586
7587 static int
7588 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
7589 {
7590         struct adapter *sc = arg1;
7591         struct sbuf *sb;
7592         int rc, i, j;
7593         uint64_t *p0, *p1;
7594         struct lb_port_stats s[2];
7595         static const char *stat_name[] = {
7596                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
7597                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
7598                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
7599                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
7600                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
7601                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
7602                 "BG2FramesTrunc:", "BG3FramesTrunc:"
7603         };
7604
7605         rc = sysctl_wire_old_buffer(req, 0);
7606         if (rc != 0)
7607                 return (rc);
7608
7609         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7610         if (sb == NULL)
7611                 return (ENOMEM);
7612
7613         memset(s, 0, sizeof(s));
7614
7615         for (i = 0; i < sc->chip_params->nchan; i += 2) {
7616                 t4_get_lb_stats(sc, i, &s[0]);
7617                 t4_get_lb_stats(sc, i + 1, &s[1]);
7618
7619                 p0 = &s[0].octets;
7620                 p1 = &s[1].octets;
7621                 sbuf_printf(sb, "%s                       Loopback %u"
7622                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
7623
7624                 for (j = 0; j < nitems(stat_name); j++)
7625                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
7626                                    *p0++, *p1++);
7627         }
7628
7629         rc = sbuf_finish(sb);
7630         sbuf_delete(sb);
7631
7632         return (rc);
7633 }
7634
7635 static int
7636 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
7637 {
7638         int rc = 0;
7639         struct port_info *pi = arg1;
7640         struct link_config *lc = &pi->link_cfg;
7641         struct sbuf *sb;
7642
7643         rc = sysctl_wire_old_buffer(req, 0);
7644         if (rc != 0)
7645                 return(rc);
7646         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
7647         if (sb == NULL)
7648                 return (ENOMEM);
7649
7650         if (lc->link_ok || lc->link_down_rc == 255)
7651                 sbuf_printf(sb, "n/a");
7652         else
7653                 sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc));
7654
7655         rc = sbuf_finish(sb);
7656         sbuf_delete(sb);
7657
7658         return (rc);
7659 }
7660
7661 struct mem_desc {
7662         unsigned int base;
7663         unsigned int limit;
7664         unsigned int idx;
7665 };
7666
7667 static int
7668 mem_desc_cmp(const void *a, const void *b)
7669 {
7670         return ((const struct mem_desc *)a)->base -
7671                ((const struct mem_desc *)b)->base;
7672 }
7673
7674 static void
7675 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
7676     unsigned int to)
7677 {
7678         unsigned int size;
7679
7680         if (from == to)
7681                 return;
7682
7683         size = to - from + 1;
7684         if (size == 0)
7685                 return;
7686
7687         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
7688         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
7689 }
7690
7691 static int
7692 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
7693 {
7694         struct adapter *sc = arg1;
7695         struct sbuf *sb;
7696         int rc, i, n;
7697         uint32_t lo, hi, used, alloc;
7698         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
7699         static const char *region[] = {
7700                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
7701                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
7702                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
7703                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
7704                 "RQUDP region:", "PBL region:", "TXPBL region:",
7705                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
7706                 "On-chip queues:", "TLS keys:",
7707         };
7708         struct mem_desc avail[4];
7709         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
7710         struct mem_desc *md = mem;
7711
7712         rc = sysctl_wire_old_buffer(req, 0);
7713         if (rc != 0)
7714                 return (rc);
7715
7716         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7717         if (sb == NULL)
7718                 return (ENOMEM);
7719
7720         for (i = 0; i < nitems(mem); i++) {
7721                 mem[i].limit = 0;
7722                 mem[i].idx = i;
7723         }
7724
7725         /* Find and sort the populated memory ranges */
7726         i = 0;
7727         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
7728         if (lo & F_EDRAM0_ENABLE) {
7729                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
7730                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
7731                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
7732                 avail[i].idx = 0;
7733                 i++;
7734         }
7735         if (lo & F_EDRAM1_ENABLE) {
7736                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
7737                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
7738                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
7739                 avail[i].idx = 1;
7740                 i++;
7741         }
7742         if (lo & F_EXT_MEM_ENABLE) {
7743                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
7744                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
7745                 avail[i].limit = avail[i].base +
7746                     (G_EXT_MEM_SIZE(hi) << 20);
7747                 avail[i].idx = is_t5(sc) ? 3 : 2;       /* Call it MC0 for T5 */
7748                 i++;
7749         }
7750         if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) {
7751                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
7752                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
7753                 avail[i].limit = avail[i].base +
7754                     (G_EXT_MEM1_SIZE(hi) << 20);
7755                 avail[i].idx = 4;
7756                 i++;
7757         }
7758         if (!i)                                    /* no memory available */
7759                 return 0;
7760         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
7761
7762         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
7763         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
7764         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
7765         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7766         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
7767         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
7768         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
7769         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
7770         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
7771
7772         /* the next few have explicit upper bounds */
7773         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
7774         md->limit = md->base - 1 +
7775                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
7776                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
7777         md++;
7778
7779         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
7780         md->limit = md->base - 1 +
7781                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
7782                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
7783         md++;
7784
7785         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
7786                 if (chip_id(sc) <= CHELSIO_T5)
7787                         md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
7788                 else
7789                         md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR);
7790                 md->limit = 0;
7791         } else {
7792                 md->base = 0;
7793                 md->idx = nitems(region);  /* hide it */
7794         }
7795         md++;
7796
7797 #define ulp_region(reg) \
7798         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
7799         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
7800
7801         ulp_region(RX_ISCSI);
7802         ulp_region(RX_TDDP);
7803         ulp_region(TX_TPT);
7804         ulp_region(RX_STAG);
7805         ulp_region(RX_RQ);
7806         ulp_region(RX_RQUDP);
7807         ulp_region(RX_PBL);
7808         ulp_region(TX_PBL);
7809 #undef ulp_region
7810
7811         md->base = 0;
7812         md->idx = nitems(region);
7813         if (!is_t4(sc)) {
7814                 uint32_t size = 0;
7815                 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2);
7816                 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE);
7817
7818                 if (is_t5(sc)) {
7819                         if (sge_ctrl & F_VFIFO_ENABLE)
7820                                 size = G_DBVFIFO_SIZE(fifo_size);
7821                 } else
7822                         size = G_T6_DBVFIFO_SIZE(fifo_size);
7823
7824                 if (size) {
7825                         md->base = G_BASEADDR(t4_read_reg(sc,
7826                             A_SGE_DBVFIFO_BADDR));
7827                         md->limit = md->base + (size << 2) - 1;
7828                 }
7829         }
7830         md++;
7831
7832         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
7833         md->limit = 0;
7834         md++;
7835         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
7836         md->limit = 0;
7837         md++;
7838
7839         md->base = sc->vres.ocq.start;
7840         if (sc->vres.ocq.size)
7841                 md->limit = md->base + sc->vres.ocq.size - 1;
7842         else
7843                 md->idx = nitems(region);  /* hide it */
7844         md++;
7845
7846         md->base = sc->vres.key.start;
7847         if (sc->vres.key.size)
7848                 md->limit = md->base + sc->vres.key.size - 1;
7849         else
7850                 md->idx = nitems(region);  /* hide it */
7851         md++;
7852
7853         /* add any address-space holes, there can be up to 3 */
7854         for (n = 0; n < i - 1; n++)
7855                 if (avail[n].limit < avail[n + 1].base)
7856                         (md++)->base = avail[n].limit;
7857         if (avail[n].limit)
7858                 (md++)->base = avail[n].limit;
7859
7860         n = md - mem;
7861         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
7862
7863         for (lo = 0; lo < i; lo++)
7864                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
7865                                 avail[lo].limit - 1);
7866
7867         sbuf_printf(sb, "\n");
7868         for (i = 0; i < n; i++) {
7869                 if (mem[i].idx >= nitems(region))
7870                         continue;                        /* skip holes */
7871                 if (!mem[i].limit)
7872                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
7873                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
7874                                 mem[i].limit);
7875         }
7876
7877         sbuf_printf(sb, "\n");
7878         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
7879         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
7880         mem_region_show(sb, "uP RAM:", lo, hi);
7881
7882         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
7883         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
7884         mem_region_show(sb, "uP Extmem2:", lo, hi);
7885
7886         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
7887         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
7888                    G_PMRXMAXPAGE(lo),
7889                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
7890                    (lo & F_PMRXNUMCHN) ? 2 : 1);
7891
7892         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
7893         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
7894         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
7895                    G_PMTXMAXPAGE(lo),
7896                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
7897                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
7898         sbuf_printf(sb, "%u p-structs\n",
7899                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
7900
7901         for (i = 0; i < 4; i++) {
7902                 if (chip_id(sc) > CHELSIO_T5)
7903                         lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4);
7904                 else
7905                         lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
7906                 if (is_t5(sc)) {
7907                         used = G_T5_USED(lo);
7908                         alloc = G_T5_ALLOC(lo);
7909                 } else {
7910                         used = G_USED(lo);
7911                         alloc = G_ALLOC(lo);
7912                 }
7913                 /* For T6 these are MAC buffer groups */
7914                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
7915                     i, used, alloc);
7916         }
7917         for (i = 0; i < sc->chip_params->nchan; i++) {
7918                 if (chip_id(sc) > CHELSIO_T5)
7919                         lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4);
7920                 else
7921                         lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
7922                 if (is_t5(sc)) {
7923                         used = G_T5_USED(lo);
7924                         alloc = G_T5_ALLOC(lo);
7925                 } else {
7926                         used = G_USED(lo);
7927                         alloc = G_ALLOC(lo);
7928                 }
7929                 /* For T6 these are MAC buffer groups */
7930                 sbuf_printf(sb,
7931                     "\nLoopback %d using %u pages out of %u allocated",
7932                     i, used, alloc);
7933         }
7934
7935         rc = sbuf_finish(sb);
7936         sbuf_delete(sb);
7937
7938         return (rc);
7939 }
7940
7941 static inline void
7942 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
7943 {
7944         *mask = x | y;
7945         y = htobe64(y);
7946         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
7947 }
7948
7949 static int
7950 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
7951 {
7952         struct adapter *sc = arg1;
7953         struct sbuf *sb;
7954         int rc, i;
7955
7956         MPASS(chip_id(sc) <= CHELSIO_T5);
7957
7958         rc = sysctl_wire_old_buffer(req, 0);
7959         if (rc != 0)
7960                 return (rc);
7961
7962         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
7963         if (sb == NULL)
7964                 return (ENOMEM);
7965
7966         sbuf_printf(sb,
7967             "Idx  Ethernet address     Mask     Vld Ports PF"
7968             "  VF              Replication             P0 P1 P2 P3  ML");
7969         for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
7970                 uint64_t tcamx, tcamy, mask;
7971                 uint32_t cls_lo, cls_hi;
7972                 uint8_t addr[ETHER_ADDR_LEN];
7973
7974                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
7975                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
7976                 if (tcamx & tcamy)
7977                         continue;
7978                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
7979                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
7980                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
7981                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
7982                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
7983                            addr[3], addr[4], addr[5], (uintmax_t)mask,
7984                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
7985                            G_PORTMAP(cls_hi), G_PF(cls_lo),
7986                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
7987
7988                 if (cls_lo & F_REPLICATE) {
7989                         struct fw_ldst_cmd ldst_cmd;
7990
7991                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
7992                         ldst_cmd.op_to_addrspace =
7993                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
7994                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
7995                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
7996                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
7997                         ldst_cmd.u.mps.rplc.fid_idx =
7998                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
7999                                 V_FW_LDST_CMD_IDX(i));
8000
8001                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
8002                             "t4mps");
8003                         if (rc)
8004                                 break;
8005                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
8006                             sizeof(ldst_cmd), &ldst_cmd);
8007                         end_synchronized_op(sc, 0);
8008
8009                         if (rc != 0) {
8010                                 sbuf_printf(sb, "%36d", rc);
8011                                 rc = 0;
8012                         } else {
8013                                 sbuf_printf(sb, " %08x %08x %08x %08x",
8014                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
8015                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
8016                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
8017                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
8018                         }
8019                 } else
8020                         sbuf_printf(sb, "%36s", "");
8021
8022                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
8023                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
8024                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
8025         }
8026
8027         if (rc)
8028                 (void) sbuf_finish(sb);
8029         else
8030                 rc = sbuf_finish(sb);
8031         sbuf_delete(sb);
8032
8033         return (rc);
8034 }
8035
8036 static int
8037 sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS)
8038 {
8039         struct adapter *sc = arg1;
8040         struct sbuf *sb;
8041         int rc, i;
8042
8043         MPASS(chip_id(sc) > CHELSIO_T5);
8044
8045         rc = sysctl_wire_old_buffer(req, 0);
8046         if (rc != 0)
8047                 return (rc);
8048
8049         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
8050         if (sb == NULL)
8051                 return (ENOMEM);
8052
8053         sbuf_printf(sb, "Idx  Ethernet address     Mask       VNI   Mask"
8054             "   IVLAN Vld DIP_Hit   Lookup  Port Vld Ports PF  VF"
8055             "                           Replication"
8056             "                                    P0 P1 P2 P3  ML\n");
8057
8058         for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
8059                 uint8_t dip_hit, vlan_vld, lookup_type, port_num;
8060                 uint16_t ivlan;
8061                 uint64_t tcamx, tcamy, val, mask;
8062                 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy;
8063                 uint8_t addr[ETHER_ADDR_LEN];
8064
8065                 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0);
8066                 if (i < 256)
8067                         ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0);
8068                 else
8069                         ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1);
8070                 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
8071                 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
8072                 tcamy = G_DMACH(val) << 32;
8073                 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
8074                 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
8075                 lookup_type = G_DATALKPTYPE(data2);
8076                 port_num = G_DATAPORTNUM(data2);
8077                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
8078                         /* Inner header VNI */
8079                         vniy = ((data2 & F_DATAVIDH2) << 23) |
8080                                        (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
8081                         dip_hit = data2 & F_DATADIPHIT;
8082                         vlan_vld = 0;
8083                 } else {
8084                         vniy = 0;
8085                         dip_hit = 0;
8086                         vlan_vld = data2 & F_DATAVIDH2;
8087                         ivlan = G_VIDL(val);
8088                 }
8089
8090                 ctl |= V_CTLXYBITSEL(1);
8091                 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
8092                 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
8093                 tcamx = G_DMACH(val) << 32;
8094                 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
8095                 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
8096                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
8097                         /* Inner header VNI mask */
8098                         vnix = ((data2 & F_DATAVIDH2) << 23) |
8099                                (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
8100                 } else
8101                         vnix = 0;
8102
8103                 if (tcamx & tcamy)
8104                         continue;
8105                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
8106
8107                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
8108                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
8109
8110                 if (lookup_type && lookup_type != M_DATALKPTYPE) {
8111                         sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
8112                             "%012jx %06x %06x    -    -   %3c"
8113                             "      'I'  %4x   %3c   %#x%4u%4d", i, addr[0],
8114                             addr[1], addr[2], addr[3], addr[4], addr[5],
8115                             (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N',
8116                             port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
8117                             G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
8118                             cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
8119                 } else {
8120                         sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
8121                             "%012jx    -       -   ", i, addr[0], addr[1],
8122                             addr[2], addr[3], addr[4], addr[5],
8123                             (uintmax_t)mask);
8124
8125                         if (vlan_vld)
8126                                 sbuf_printf(sb, "%4u   Y     ", ivlan);
8127                         else
8128                                 sbuf_printf(sb, "  -    N     ");
8129
8130                         sbuf_printf(sb, "-      %3c  %4x   %3c   %#x%4u%4d",
8131                             lookup_type ? 'I' : 'O', port_num,
8132                             cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
8133                             G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
8134                             cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
8135                 }
8136
8137
8138                 if (cls_lo & F_T6_REPLICATE) {
8139                         struct fw_ldst_cmd ldst_cmd;
8140
8141                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
8142                         ldst_cmd.op_to_addrspace =
8143                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
8144                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
8145                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
8146                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
8147                         ldst_cmd.u.mps.rplc.fid_idx =
8148                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
8149                                 V_FW_LDST_CMD_IDX(i));
8150
8151                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
8152                             "t6mps");
8153                         if (rc)
8154                                 break;
8155                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
8156                             sizeof(ldst_cmd), &ldst_cmd);
8157                         end_synchronized_op(sc, 0);
8158
8159                         if (rc != 0) {
8160                                 sbuf_printf(sb, "%72d", rc);
8161                                 rc = 0;
8162                         } else {
8163                                 sbuf_printf(sb, " %08x %08x %08x %08x"
8164                                     " %08x %08x %08x %08x",
8165                                     be32toh(ldst_cmd.u.mps.rplc.rplc255_224),
8166                                     be32toh(ldst_cmd.u.mps.rplc.rplc223_192),
8167                                     be32toh(ldst_cmd.u.mps.rplc.rplc191_160),
8168                                     be32toh(ldst_cmd.u.mps.rplc.rplc159_128),
8169                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
8170                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
8171                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
8172                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
8173                         }
8174                 } else
8175                         sbuf_printf(sb, "%72s", "");
8176
8177                 sbuf_printf(sb, "%4u%3u%3u%3u %#x",
8178                     G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo),
8179                     G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo),
8180                     (cls_lo >> S_T6_MULTILISTEN0) & 0xf);
8181         }
8182
8183         if (rc)
8184                 (void) sbuf_finish(sb);
8185         else
8186                 rc = sbuf_finish(sb);
8187         sbuf_delete(sb);
8188
8189         return (rc);
8190 }
8191
8192 static int
8193 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
8194 {
8195         struct adapter *sc = arg1;
8196         struct sbuf *sb;
8197         int rc;
8198         uint16_t mtus[NMTUS];
8199
8200         rc = sysctl_wire_old_buffer(req, 0);
8201         if (rc != 0)
8202                 return (rc);
8203
8204         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
8205         if (sb == NULL)
8206                 return (ENOMEM);
8207
8208         t4_read_mtu_tbl(sc, mtus, NULL);
8209
8210         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
8211             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
8212             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
8213             mtus[14], mtus[15]);
8214
8215         rc = sbuf_finish(sb);
8216         sbuf_delete(sb);
8217
8218         return (rc);
8219 }
8220
8221 static int
8222 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
8223 {
8224         struct adapter *sc = arg1;
8225         struct sbuf *sb;
8226         int rc, i;
8227         uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS];
8228         uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS];
8229         static const char *tx_stats[MAX_PM_NSTATS] = {
8230                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:",
8231                 "Tx FIFO wait", NULL, "Tx latency"
8232         };
8233         static const char *rx_stats[MAX_PM_NSTATS] = {
8234                 "Read:", "Write bypass:", "Write mem:", "Flush:",
8235                 "Rx FIFO wait", NULL, "Rx latency"
8236         };
8237
8238         rc = sysctl_wire_old_buffer(req, 0);
8239         if (rc != 0)
8240                 return (rc);
8241
8242         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
8243         if (sb == NULL)
8244                 return (ENOMEM);
8245
8246         t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
8247         t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
8248
8249         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
8250         for (i = 0; i < 4; i++) {
8251                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
8252                     tx_cyc[i]);
8253         }
8254
8255         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
8256         for (i = 0; i < 4; i++) {
8257                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
8258                     rx_cyc[i]);
8259         }
8260
8261         if (chip_id(sc) > CHELSIO_T5) {
8262                 sbuf_printf(sb,
8263                     "\n              Total wait      Total occupancy");
8264                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
8265                     tx_cyc[i]);
8266                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
8267                     rx_cyc[i]);
8268
8269                 i += 2;
8270                 MPASS(i < nitems(tx_stats));
8271
8272                 sbuf_printf(sb,
8273                     "\n                   Reads           Total wait");
8274                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
8275                     tx_cyc[i]);
8276                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
8277                     rx_cyc[i]);
8278         }
8279
8280         rc = sbuf_finish(sb);
8281         sbuf_delete(sb);
8282
8283         return (rc);
8284 }
8285
8286 static int
8287 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
8288 {
8289         struct adapter *sc = arg1;
8290         struct sbuf *sb;
8291         int rc;
8292         struct tp_rdma_stats stats;
8293
8294         rc = sysctl_wire_old_buffer(req, 0);
8295         if (rc != 0)
8296                 return (rc);
8297
8298         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
8299         if (sb == NULL)
8300                 return (ENOMEM);
8301
8302         mtx_lock(&sc->reg_lock);
8303         t4_tp_get_rdma_stats(sc, &stats, 0);
8304         mtx_unlock(&sc->reg_lock);
8305
8306         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
8307         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
8308
8309         rc = sbuf_finish(sb);
8310         sbuf_delete(sb);
8311
8312         return (rc);
8313 }
8314
8315 static int
8316 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
8317 {
8318         struct adapter *sc = arg1;
8319         struct sbuf *sb;
8320         int rc;
8321         struct tp_tcp_stats v4, v6;
8322
8323         rc = sysctl_wire_old_buffer(req, 0);
8324         if (rc != 0)
8325                 return (rc);
8326
8327         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
8328         if (sb == NULL)
8329                 return (ENOMEM);
8330
8331         mtx_lock(&sc->reg_lock);
8332         t4_tp_get_tcp_stats(sc, &v4, &v6, 0);
8333         mtx_unlock(&sc->reg_lock);
8334
8335         sbuf_printf(sb,
8336             "                                IP                 IPv6\n");
8337         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
8338             v4.tcp_out_rsts, v6.tcp_out_rsts);
8339         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
8340             v4.tcp_in_segs, v6.tcp_in_segs);
8341         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
8342             v4.tcp_out_segs, v6.tcp_out_segs);
8343         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
8344             v4.tcp_retrans_segs, v6.tcp_retrans_segs);
8345
8346         rc = sbuf_finish(sb);
8347         sbuf_delete(sb);
8348
8349         return (rc);
8350 }
8351
8352 static int
8353 sysctl_tids(SYSCTL_HANDLER_ARGS)
8354 {
8355         struct adapter *sc = arg1;
8356         struct sbuf *sb;
8357         int rc;
8358         struct tid_info *t = &sc->tids;
8359
8360         rc = sysctl_wire_old_buffer(req, 0);
8361         if (rc != 0)
8362                 return (rc);
8363
8364         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
8365         if (sb == NULL)
8366                 return (ENOMEM);
8367
8368         if (t->natids) {
8369                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
8370                     t->atids_in_use);
8371         }
8372
8373         if (t->nhpftids) {
8374                 sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n",
8375                     t->hpftid_base, t->hpftid_end, t->hpftids_in_use);
8376         }
8377
8378         if (t->ntids) {
8379                 sbuf_printf(sb, "TID range: ");
8380                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
8381                         uint32_t b, hb;
8382
8383                         if (chip_id(sc) <= CHELSIO_T5) {
8384                                 b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
8385                                 hb = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
8386                         } else {
8387                                 b = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX);
8388                                 hb = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE);
8389                         }
8390
8391                         if (b)
8392                                 sbuf_printf(sb, "%u-%u, ", t->tid_base, b - 1);
8393                         sbuf_printf(sb, "%u-%u", hb, t->ntids - 1);
8394                 } else
8395                         sbuf_printf(sb, "%u-%u", t->tid_base, t->ntids - 1);
8396                 sbuf_printf(sb, ", in use: %u\n",
8397                     atomic_load_acq_int(&t->tids_in_use));
8398         }
8399
8400         if (t->nstids) {
8401                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
8402                     t->stid_base + t->nstids - 1, t->stids_in_use);
8403         }
8404
8405         if (t->nftids) {
8406                 sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base,
8407                     t->ftid_end, t->ftids_in_use);
8408         }
8409
8410         if (t->netids) {
8411                 sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base,
8412                     t->etid_base + t->netids - 1, t->etids_in_use);
8413         }
8414
8415         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
8416             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
8417             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
8418
8419         rc = sbuf_finish(sb);
8420         sbuf_delete(sb);
8421
8422         return (rc);
8423 }
8424
8425 static int
8426 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
8427 {
8428         struct adapter *sc = arg1;
8429         struct sbuf *sb;
8430         int rc;
8431         struct tp_err_stats stats;
8432
8433         rc = sysctl_wire_old_buffer(req, 0);
8434         if (rc != 0)
8435                 return (rc);
8436
8437         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
8438         if (sb == NULL)
8439                 return (ENOMEM);
8440
8441         mtx_lock(&sc->reg_lock);
8442         t4_tp_get_err_stats(sc, &stats, 0);
8443         mtx_unlock(&sc->reg_lock);
8444
8445         if (sc->chip_params->nchan > 2) {
8446                 sbuf_printf(sb, "                 channel 0  channel 1"
8447                     "  channel 2  channel 3\n");
8448                 sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
8449                     stats.mac_in_errs[0], stats.mac_in_errs[1],
8450                     stats.mac_in_errs[2], stats.mac_in_errs[3]);
8451                 sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
8452                     stats.hdr_in_errs[0], stats.hdr_in_errs[1],
8453                     stats.hdr_in_errs[2], stats.hdr_in_errs[3]);
8454                 sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
8455                     stats.tcp_in_errs[0], stats.tcp_in_errs[1],
8456                     stats.tcp_in_errs[2], stats.tcp_in_errs[3]);
8457                 sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
8458                     stats.tcp6_in_errs[0], stats.tcp6_in_errs[1],
8459                     stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]);
8460                 sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
8461                     stats.tnl_cong_drops[0], stats.tnl_cong_drops[1],
8462                     stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]);
8463                 sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
8464                     stats.tnl_tx_drops[0], stats.tnl_tx_drops[1],
8465                     stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]);
8466                 sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
8467                     stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1],
8468                     stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]);
8469                 sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
8470                     stats.ofld_chan_drops[0], stats.ofld_chan_drops[1],
8471                     stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]);
8472         } else {
8473                 sbuf_printf(sb, "                 channel 0  channel 1\n");
8474                 sbuf_printf(sb, "macInErrs:      %10u %10u\n",
8475                     stats.mac_in_errs[0], stats.mac_in_errs[1]);
8476                 sbuf_printf(sb, "hdrInErrs:      %10u %10u\n",
8477                     stats.hdr_in_errs[0], stats.hdr_in_errs[1]);
8478                 sbuf_printf(sb, "tcpInErrs:      %10u %10u\n",
8479                     stats.tcp_in_errs[0], stats.tcp_in_errs[1]);
8480                 sbuf_printf(sb, "tcp6InErrs:     %10u %10u\n",
8481                     stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]);
8482                 sbuf_printf(sb, "tnlCongDrops:   %10u %10u\n",
8483                     stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]);
8484                 sbuf_printf(sb, "tnlTxDrops:     %10u %10u\n",
8485                     stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]);
8486                 sbuf_printf(sb, "ofldVlanDrops:  %10u %10u\n",
8487                     stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]);
8488                 sbuf_printf(sb, "ofldChanDrops:  %10u %10u\n\n",
8489                     stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]);
8490         }
8491
8492         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
8493             stats.ofld_no_neigh, stats.ofld_cong_defer);
8494
8495         rc = sbuf_finish(sb);
8496         sbuf_delete(sb);
8497
8498         return (rc);
8499 }
8500
8501 static int
8502 sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS)
8503 {
8504         struct adapter *sc = arg1;
8505         struct tp_params *tpp = &sc->params.tp;
8506         u_int mask;
8507         int rc;
8508
8509         mask = tpp->la_mask >> 16;
8510         rc = sysctl_handle_int(oidp, &mask, 0, req);
8511         if (rc != 0 || req->newptr == NULL)
8512                 return (rc);
8513         if (mask > 0xffff)
8514                 return (EINVAL);
8515         tpp->la_mask = mask << 16;
8516         t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask);
8517
8518         return (0);
8519 }
8520
8521 struct field_desc {
8522         const char *name;
8523         u_int start;
8524         u_int width;
8525 };
8526
8527 static void
8528 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
8529 {
8530         char buf[32];
8531         int line_size = 0;
8532
8533         while (f->name) {
8534                 uint64_t mask = (1ULL << f->width) - 1;
8535                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
8536                     ((uintmax_t)v >> f->start) & mask);
8537
8538                 if (line_size + len >= 79) {
8539                         line_size = 8;
8540                         sbuf_printf(sb, "\n        ");
8541                 }
8542                 sbuf_printf(sb, "%s ", buf);
8543                 line_size += len + 1;
8544                 f++;
8545         }
8546         sbuf_printf(sb, "\n");
8547 }
8548
8549 static const struct field_desc tp_la0[] = {
8550         { "RcfOpCodeOut", 60, 4 },
8551         { "State", 56, 4 },
8552         { "WcfState", 52, 4 },
8553         { "RcfOpcSrcOut", 50, 2 },
8554         { "CRxError", 49, 1 },
8555         { "ERxError", 48, 1 },
8556         { "SanityFailed", 47, 1 },
8557         { "SpuriousMsg", 46, 1 },
8558         { "FlushInputMsg", 45, 1 },
8559         { "FlushInputCpl", 44, 1 },
8560         { "RssUpBit", 43, 1 },
8561         { "RssFilterHit", 42, 1 },
8562         { "Tid", 32, 10 },
8563         { "InitTcb", 31, 1 },
8564         { "LineNumber", 24, 7 },
8565         { "Emsg", 23, 1 },
8566         { "EdataOut", 22, 1 },
8567         { "Cmsg", 21, 1 },
8568         { "CdataOut", 20, 1 },
8569         { "EreadPdu", 19, 1 },
8570         { "CreadPdu", 18, 1 },
8571         { "TunnelPkt", 17, 1 },
8572         { "RcfPeerFin", 16, 1 },
8573         { "RcfReasonOut", 12, 4 },
8574         { "TxCchannel", 10, 2 },
8575         { "RcfTxChannel", 8, 2 },
8576         { "RxEchannel", 6, 2 },
8577         { "RcfRxChannel", 5, 1 },
8578         { "RcfDataOutSrdy", 4, 1 },
8579         { "RxDvld", 3, 1 },
8580         { "RxOoDvld", 2, 1 },
8581         { "RxCongestion", 1, 1 },
8582         { "TxCongestion", 0, 1 },
8583         { NULL }
8584 };
8585
8586 static const struct field_desc tp_la1[] = {
8587         { "CplCmdIn", 56, 8 },
8588         { "CplCmdOut", 48, 8 },
8589         { "ESynOut", 47, 1 },
8590         { "EAckOut", 46, 1 },
8591         { "EFinOut", 45, 1 },
8592         { "ERstOut", 44, 1 },
8593         { "SynIn", 43, 1 },
8594         { "AckIn", 42, 1 },
8595         { "FinIn", 41, 1 },
8596         { "RstIn", 40, 1 },
8597         { "DataIn", 39, 1 },
8598         { "DataInVld", 38, 1 },
8599         { "PadIn", 37, 1 },
8600         { "RxBufEmpty", 36, 1 },
8601         { "RxDdp", 35, 1 },
8602         { "RxFbCongestion", 34, 1 },
8603         { "TxFbCongestion", 33, 1 },
8604         { "TxPktSumSrdy", 32, 1 },
8605         { "RcfUlpType", 28, 4 },
8606         { "Eread", 27, 1 },
8607         { "Ebypass", 26, 1 },
8608         { "Esave", 25, 1 },
8609         { "Static0", 24, 1 },
8610         { "Cread", 23, 1 },
8611         { "Cbypass", 22, 1 },
8612         { "Csave", 21, 1 },
8613         { "CPktOut", 20, 1 },
8614         { "RxPagePoolFull", 18, 2 },
8615         { "RxLpbkPkt", 17, 1 },
8616         { "TxLpbkPkt", 16, 1 },
8617         { "RxVfValid", 15, 1 },
8618         { "SynLearned", 14, 1 },
8619         { "SetDelEntry", 13, 1 },
8620         { "SetInvEntry", 12, 1 },
8621         { "CpcmdDvld", 11, 1 },
8622         { "CpcmdSave", 10, 1 },
8623         { "RxPstructsFull", 8, 2 },
8624         { "EpcmdDvld", 7, 1 },
8625         { "EpcmdFlush", 6, 1 },
8626         { "EpcmdTrimPrefix", 5, 1 },
8627         { "EpcmdTrimPostfix", 4, 1 },
8628         { "ERssIp4Pkt", 3, 1 },
8629         { "ERssIp6Pkt", 2, 1 },
8630         { "ERssTcpUdpPkt", 1, 1 },
8631         { "ERssFceFipPkt", 0, 1 },
8632         { NULL }
8633 };
8634
8635 static const struct field_desc tp_la2[] = {
8636         { "CplCmdIn", 56, 8 },
8637         { "MpsVfVld", 55, 1 },
8638         { "MpsPf", 52, 3 },
8639         { "MpsVf", 44, 8 },
8640         { "SynIn", 43, 1 },
8641         { "AckIn", 42, 1 },
8642         { "FinIn", 41, 1 },
8643         { "RstIn", 40, 1 },
8644         { "DataIn", 39, 1 },
8645         { "DataInVld", 38, 1 },
8646         { "PadIn", 37, 1 },
8647         { "RxBufEmpty", 36, 1 },
8648         { "RxDdp", 35, 1 },
8649         { "RxFbCongestion", 34, 1 },
8650         { "TxFbCongestion", 33, 1 },
8651         { "TxPktSumSrdy", 32, 1 },
8652         { "RcfUlpType", 28, 4 },
8653         { "Eread", 27, 1 },
8654         { "Ebypass", 26, 1 },
8655         { "Esave", 25, 1 },
8656         { "Static0", 24, 1 },
8657         { "Cread", 23, 1 },
8658         { "Cbypass", 22, 1 },
8659         { "Csave", 21, 1 },
8660         { "CPktOut", 20, 1 },
8661         { "RxPagePoolFull", 18, 2 },
8662         { "RxLpbkPkt", 17, 1 },
8663         { "TxLpbkPkt", 16, 1 },
8664         { "RxVfValid", 15, 1 },
8665         { "SynLearned", 14, 1 },
8666         { "SetDelEntry", 13, 1 },
8667         { "SetInvEntry", 12, 1 },
8668         { "CpcmdDvld", 11, 1 },
8669         { "CpcmdSave", 10, 1 },
8670         { "RxPstructsFull", 8, 2 },
8671         { "EpcmdDvld", 7, 1 },
8672         { "EpcmdFlush", 6, 1 },
8673         { "EpcmdTrimPrefix", 5, 1 },
8674         { "EpcmdTrimPostfix", 4, 1 },
8675         { "ERssIp4Pkt", 3, 1 },
8676         { "ERssIp6Pkt", 2, 1 },
8677         { "ERssTcpUdpPkt", 1, 1 },
8678         { "ERssFceFipPkt", 0, 1 },
8679         { NULL }
8680 };
8681
8682 static void
8683 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
8684 {
8685
8686         field_desc_show(sb, *p, tp_la0);
8687 }
8688
8689 static void
8690 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
8691 {
8692
8693         if (idx)
8694                 sbuf_printf(sb, "\n");
8695         field_desc_show(sb, p[0], tp_la0);
8696         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
8697                 field_desc_show(sb, p[1], tp_la0);
8698 }
8699
8700 static void
8701 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
8702 {
8703
8704         if (idx)
8705                 sbuf_printf(sb, "\n");
8706         field_desc_show(sb, p[0], tp_la0);
8707         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
8708                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
8709 }
8710
8711 static int
8712 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
8713 {
8714         struct adapter *sc = arg1;
8715         struct sbuf *sb;
8716         uint64_t *buf, *p;
8717         int rc;
8718         u_int i, inc;
8719         void (*show_func)(struct sbuf *, uint64_t *, int);
8720
8721         rc = sysctl_wire_old_buffer(req, 0);
8722         if (rc != 0)
8723                 return (rc);
8724
8725         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
8726         if (sb == NULL)
8727                 return (ENOMEM);
8728
8729         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
8730
8731         t4_tp_read_la(sc, buf, NULL);
8732         p = buf;
8733
8734         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
8735         case 2:
8736                 inc = 2;
8737                 show_func = tp_la_show2;
8738                 break;
8739         case 3:
8740                 inc = 2;
8741                 show_func = tp_la_show3;
8742                 break;
8743         default:
8744                 inc = 1;
8745                 show_func = tp_la_show;
8746         }
8747
8748         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
8749                 (*show_func)(sb, p, i);
8750
8751         rc = sbuf_finish(sb);
8752         sbuf_delete(sb);
8753         free(buf, M_CXGBE);
8754         return (rc);
8755 }
8756
8757 static int
8758 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
8759 {
8760         struct adapter *sc = arg1;
8761         struct sbuf *sb;
8762         int rc;
8763         u64 nrate[MAX_NCHAN], orate[MAX_NCHAN];
8764
8765         rc = sysctl_wire_old_buffer(req, 0);
8766         if (rc != 0)
8767                 return (rc);
8768
8769         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
8770         if (sb == NULL)
8771                 return (ENOMEM);
8772
8773         t4_get_chan_txrate(sc, nrate, orate);
8774
8775         if (sc->chip_params->nchan > 2) {
8776                 sbuf_printf(sb, "              channel 0   channel 1"
8777                     "   channel 2   channel 3\n");
8778                 sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
8779                     nrate[0], nrate[1], nrate[2], nrate[3]);
8780                 sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
8781                     orate[0], orate[1], orate[2], orate[3]);
8782         } else {
8783                 sbuf_printf(sb, "              channel 0   channel 1\n");
8784                 sbuf_printf(sb, "NIC B/s:     %10ju  %10ju\n",
8785                     nrate[0], nrate[1]);
8786                 sbuf_printf(sb, "Offload B/s: %10ju  %10ju",
8787                     orate[0], orate[1]);
8788         }
8789
8790         rc = sbuf_finish(sb);
8791         sbuf_delete(sb);
8792
8793         return (rc);
8794 }
8795
8796 static int
8797 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
8798 {
8799         struct adapter *sc = arg1;
8800         struct sbuf *sb;
8801         uint32_t *buf, *p;
8802         int rc, i;
8803
8804         rc = sysctl_wire_old_buffer(req, 0);
8805         if (rc != 0)
8806                 return (rc);
8807
8808         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
8809         if (sb == NULL)
8810                 return (ENOMEM);
8811
8812         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
8813             M_ZERO | M_WAITOK);
8814
8815         t4_ulprx_read_la(sc, buf);
8816         p = buf;
8817
8818         sbuf_printf(sb, "      Pcmd        Type   Message"
8819             "                Data");
8820         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
8821                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
8822                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
8823         }
8824
8825         rc = sbuf_finish(sb);
8826         sbuf_delete(sb);
8827         free(buf, M_CXGBE);
8828         return (rc);
8829 }
8830
8831 static int
8832 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
8833 {
8834         struct adapter *sc = arg1;
8835         struct sbuf *sb;
8836         int rc, v;
8837
8838         MPASS(chip_id(sc) >= CHELSIO_T5);
8839
8840         rc = sysctl_wire_old_buffer(req, 0);
8841         if (rc != 0)
8842                 return (rc);
8843
8844         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
8845         if (sb == NULL)
8846                 return (ENOMEM);
8847
8848         v = t4_read_reg(sc, A_SGE_STAT_CFG);
8849         if (G_STATSOURCE_T5(v) == 7) {
8850                 int mode;
8851
8852                 mode = is_t5(sc) ? G_STATMODE(v) : G_T6_STATMODE(v);
8853                 if (mode == 0) {
8854                         sbuf_printf(sb, "total %d, incomplete %d",
8855                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
8856                             t4_read_reg(sc, A_SGE_STAT_MATCH));
8857                 } else if (mode == 1) {
8858                         sbuf_printf(sb, "total %d, data overflow %d",
8859                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
8860                             t4_read_reg(sc, A_SGE_STAT_MATCH));
8861                 } else {
8862                         sbuf_printf(sb, "unknown mode %d", mode);
8863                 }
8864         }
8865         rc = sbuf_finish(sb);
8866         sbuf_delete(sb);
8867
8868         return (rc);
8869 }
8870
8871 static int
8872 sysctl_cpus(SYSCTL_HANDLER_ARGS)
8873 {
8874         struct adapter *sc = arg1;
8875         enum cpu_sets op = arg2;
8876         cpuset_t cpuset;
8877         struct sbuf *sb;
8878         int i, rc;
8879
8880         MPASS(op == LOCAL_CPUS || op == INTR_CPUS);
8881
8882         CPU_ZERO(&cpuset);
8883         rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset);
8884         if (rc != 0)
8885                 return (rc);
8886
8887         rc = sysctl_wire_old_buffer(req, 0);
8888         if (rc != 0)
8889                 return (rc);
8890
8891         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
8892         if (sb == NULL)
8893                 return (ENOMEM);
8894
8895         CPU_FOREACH(i)
8896                 sbuf_printf(sb, "%d ", i);
8897         rc = sbuf_finish(sb);
8898         sbuf_delete(sb);
8899
8900         return (rc);
8901 }
8902
8903 #ifdef TCP_OFFLOAD
8904 static int
8905 sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS)
8906 {
8907         struct adapter *sc = arg1;
8908         int *old_ports, *new_ports;
8909         int i, new_count, rc;
8910
8911         if (req->newptr == NULL && req->oldptr == NULL)
8912                 return (SYSCTL_OUT(req, NULL, imax(sc->tt.num_tls_rx_ports, 1) *
8913                     sizeof(sc->tt.tls_rx_ports[0])));
8914
8915         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4tlsrx");
8916         if (rc)
8917                 return (rc);
8918
8919         if (sc->tt.num_tls_rx_ports == 0) {
8920                 i = -1;
8921                 rc = SYSCTL_OUT(req, &i, sizeof(i));
8922         } else
8923                 rc = SYSCTL_OUT(req, sc->tt.tls_rx_ports,
8924                     sc->tt.num_tls_rx_ports * sizeof(sc->tt.tls_rx_ports[0]));
8925         if (rc == 0 && req->newptr != NULL) {
8926                 new_count = req->newlen / sizeof(new_ports[0]);
8927                 new_ports = malloc(new_count * sizeof(new_ports[0]), M_CXGBE,
8928                     M_WAITOK);
8929                 rc = SYSCTL_IN(req, new_ports, new_count *
8930                     sizeof(new_ports[0]));
8931                 if (rc)
8932                         goto err;
8933
8934                 /* Allow setting to a single '-1' to clear the list. */
8935                 if (new_count == 1 && new_ports[0] == -1) {
8936                         ADAPTER_LOCK(sc);
8937                         old_ports = sc->tt.tls_rx_ports;
8938                         sc->tt.tls_rx_ports = NULL;
8939                         sc->tt.num_tls_rx_ports = 0;
8940                         ADAPTER_UNLOCK(sc);
8941                         free(old_ports, M_CXGBE);
8942                 } else {
8943                         for (i = 0; i < new_count; i++) {
8944                                 if (new_ports[i] < 1 ||
8945                                     new_ports[i] > IPPORT_MAX) {
8946                                         rc = EINVAL;
8947                                         goto err;
8948                                 }
8949                         }
8950
8951                         ADAPTER_LOCK(sc);
8952                         old_ports = sc->tt.tls_rx_ports;
8953                         sc->tt.tls_rx_ports = new_ports;
8954                         sc->tt.num_tls_rx_ports = new_count;
8955                         ADAPTER_UNLOCK(sc);
8956                         free(old_ports, M_CXGBE);
8957                         new_ports = NULL;
8958                 }
8959         err:
8960                 free(new_ports, M_CXGBE);
8961         }
8962         end_synchronized_op(sc, 0);
8963         return (rc);
8964 }
8965
8966 static void
8967 unit_conv(char *buf, size_t len, u_int val, u_int factor)
8968 {
8969         u_int rem = val % factor;
8970
8971         if (rem == 0)
8972                 snprintf(buf, len, "%u", val / factor);
8973         else {
8974                 while (rem % 10 == 0)
8975                         rem /= 10;
8976                 snprintf(buf, len, "%u.%u", val / factor, rem);
8977         }
8978 }
8979
8980 static int
8981 sysctl_tp_tick(SYSCTL_HANDLER_ARGS)
8982 {
8983         struct adapter *sc = arg1;
8984         char buf[16];
8985         u_int res, re;
8986         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
8987
8988         res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
8989         switch (arg2) {
8990         case 0:
8991                 /* timer_tick */
8992                 re = G_TIMERRESOLUTION(res);
8993                 break;
8994         case 1:
8995                 /* TCP timestamp tick */
8996                 re = G_TIMESTAMPRESOLUTION(res);
8997                 break;
8998         case 2:
8999                 /* DACK tick */
9000                 re = G_DELAYEDACKRESOLUTION(res);
9001                 break;
9002         default:
9003                 return (EDOOFUS);
9004         }
9005
9006         unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000);
9007
9008         return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
9009 }
9010
9011 static int
9012 sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS)
9013 {
9014         struct adapter *sc = arg1;
9015         u_int res, dack_re, v;
9016         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
9017
9018         res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
9019         dack_re = G_DELAYEDACKRESOLUTION(res);
9020         v = ((cclk_ps << dack_re) / 1000000) * t4_read_reg(sc, A_TP_DACK_TIMER);
9021
9022         return (sysctl_handle_int(oidp, &v, 0, req));
9023 }
9024
9025 static int
9026 sysctl_tp_timer(SYSCTL_HANDLER_ARGS)
9027 {
9028         struct adapter *sc = arg1;
9029         int reg = arg2;
9030         u_int tre;
9031         u_long tp_tick_us, v;
9032         u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
9033
9034         MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX ||
9035             reg == A_TP_PERS_MIN  || reg == A_TP_PERS_MAX ||
9036             reg == A_TP_KEEP_IDLE || reg == A_TP_KEEP_INTVL ||
9037             reg == A_TP_INIT_SRTT || reg == A_TP_FINWAIT2_TIMER);
9038
9039         tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION));
9040         tp_tick_us = (cclk_ps << tre) / 1000000;
9041
9042         if (reg == A_TP_INIT_SRTT)
9043                 v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg));
9044         else
9045                 v = tp_tick_us * t4_read_reg(sc, reg);
9046
9047         return (sysctl_handle_long(oidp, &v, 0, req));
9048 }
9049
9050 /*
9051  * All fields in TP_SHIFT_CNT are 4b and the starting location of the field is
9052  * passed to this function.
9053  */
9054 static int
9055 sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS)
9056 {
9057         struct adapter *sc = arg1;
9058         int idx = arg2;
9059         u_int v;
9060
9061         MPASS(idx >= 0 && idx <= 24);
9062
9063         v = (t4_read_reg(sc, A_TP_SHIFT_CNT) >> idx) & 0xf;
9064
9065         return (sysctl_handle_int(oidp, &v, 0, req));
9066 }
9067
9068 static int
9069 sysctl_tp_backoff(SYSCTL_HANDLER_ARGS)
9070 {
9071         struct adapter *sc = arg1;
9072         int idx = arg2;
9073         u_int shift, v, r;
9074
9075         MPASS(idx >= 0 && idx < 16);
9076
9077         r = A_TP_TCP_BACKOFF_REG0 + (idx & ~3);
9078         shift = (idx & 3) << 3;
9079         v = (t4_read_reg(sc, r) >> shift) & M_TIMERBACKOFFINDEX0;
9080
9081         return (sysctl_handle_int(oidp, &v, 0, req));
9082 }
9083
9084 static int
9085 sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS)
9086 {
9087         struct vi_info *vi = arg1;
9088         struct adapter *sc = vi->pi->adapter;
9089         int idx, rc, i;
9090         struct sge_ofld_rxq *ofld_rxq;
9091         uint8_t v;
9092
9093         idx = vi->ofld_tmr_idx;
9094
9095         rc = sysctl_handle_int(oidp, &idx, 0, req);
9096         if (rc != 0 || req->newptr == NULL)
9097                 return (rc);
9098
9099         if (idx < 0 || idx >= SGE_NTIMERS)
9100                 return (EINVAL);
9101
9102         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
9103             "t4otmr");
9104         if (rc)
9105                 return (rc);
9106
9107         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1);
9108         for_each_ofld_rxq(vi, i, ofld_rxq) {
9109 #ifdef atomic_store_rel_8
9110                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
9111 #else
9112                 ofld_rxq->iq.intr_params = v;
9113 #endif
9114         }
9115         vi->ofld_tmr_idx = idx;
9116
9117         end_synchronized_op(sc, LOCK_HELD);
9118         return (0);
9119 }
9120
9121 static int
9122 sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS)
9123 {
9124         struct vi_info *vi = arg1;
9125         struct adapter *sc = vi->pi->adapter;
9126         int idx, rc;
9127
9128         idx = vi->ofld_pktc_idx;
9129
9130         rc = sysctl_handle_int(oidp, &idx, 0, req);
9131         if (rc != 0 || req->newptr == NULL)
9132                 return (rc);
9133
9134         if (idx < -1 || idx >= SGE_NCOUNTERS)
9135                 return (EINVAL);
9136
9137         rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
9138             "t4opktc");
9139         if (rc)
9140                 return (rc);
9141
9142         if (vi->flags & VI_INIT_DONE)
9143                 rc = EBUSY; /* cannot be changed once the queues are created */
9144         else
9145                 vi->ofld_pktc_idx = idx;
9146
9147         end_synchronized_op(sc, LOCK_HELD);
9148         return (rc);
9149 }
9150 #endif
9151
9152 static int
9153 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
9154 {
9155         int rc;
9156
9157         if (cntxt->cid > M_CTXTQID)
9158                 return (EINVAL);
9159
9160         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
9161             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
9162                 return (EINVAL);
9163
9164         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
9165         if (rc)
9166                 return (rc);
9167
9168         if (sc->flags & FW_OK) {
9169                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
9170                     &cntxt->data[0]);
9171                 if (rc == 0)
9172                         goto done;
9173         }
9174
9175         /*
9176          * Read via firmware failed or wasn't even attempted.  Read directly via
9177          * the backdoor.
9178          */
9179         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
9180 done:
9181         end_synchronized_op(sc, 0);
9182         return (rc);
9183 }
9184
9185 static int
9186 load_fw(struct adapter *sc, struct t4_data *fw)
9187 {
9188         int rc;
9189         uint8_t *fw_data;
9190
9191         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
9192         if (rc)
9193                 return (rc);
9194
9195         /*
9196          * The firmware, with the sole exception of the memory parity error
9197          * handler, runs from memory and not flash.  It is almost always safe to
9198          * install a new firmware on a running system.  Just set bit 1 in
9199          * hw.cxgbe.dflags or dev.<nexus>.<n>.dflags first.
9200          */
9201         if (sc->flags & FULL_INIT_DONE &&
9202             (sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) {
9203                 rc = EBUSY;
9204                 goto done;
9205         }
9206
9207         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
9208         if (fw_data == NULL) {
9209                 rc = ENOMEM;
9210                 goto done;
9211         }
9212
9213         rc = copyin(fw->data, fw_data, fw->len);
9214         if (rc == 0)
9215                 rc = -t4_load_fw(sc, fw_data, fw->len);
9216
9217         free(fw_data, M_CXGBE);
9218 done:
9219         end_synchronized_op(sc, 0);
9220         return (rc);
9221 }
9222
9223 static int
9224 load_cfg(struct adapter *sc, struct t4_data *cfg)
9225 {
9226         int rc;
9227         uint8_t *cfg_data = NULL;
9228
9229         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf");
9230         if (rc)
9231                 return (rc);
9232
9233         if (cfg->len == 0) {
9234                 /* clear */
9235                 rc = -t4_load_cfg(sc, NULL, 0);
9236                 goto done;
9237         }
9238
9239         cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK);
9240         if (cfg_data == NULL) {
9241                 rc = ENOMEM;
9242                 goto done;
9243         }
9244
9245         rc = copyin(cfg->data, cfg_data, cfg->len);
9246         if (rc == 0)
9247                 rc = -t4_load_cfg(sc, cfg_data, cfg->len);
9248
9249         free(cfg_data, M_CXGBE);
9250 done:
9251         end_synchronized_op(sc, 0);
9252         return (rc);
9253 }
9254
9255 static int
9256 load_boot(struct adapter *sc, struct t4_bootrom *br)
9257 {
9258         int rc;
9259         uint8_t *br_data = NULL;
9260         u_int offset;
9261
9262         if (br->len > 1024 * 1024)
9263                 return (EFBIG);
9264
9265         if (br->pf_offset == 0) {
9266                 /* pfidx */
9267                 if (br->pfidx_addr > 7)
9268                         return (EINVAL);
9269                 offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr,
9270                     A_PCIE_PF_EXPROM_OFST)));
9271         } else if (br->pf_offset == 1) {
9272                 /* offset */
9273                 offset = G_OFFSET(br->pfidx_addr);
9274         } else {
9275                 return (EINVAL);
9276         }
9277
9278         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldbr");
9279         if (rc)
9280                 return (rc);
9281
9282         if (br->len == 0) {
9283                 /* clear */
9284                 rc = -t4_load_boot(sc, NULL, offset, 0);
9285                 goto done;
9286         }
9287
9288         br_data = malloc(br->len, M_CXGBE, M_WAITOK);
9289         if (br_data == NULL) {
9290                 rc = ENOMEM;
9291                 goto done;
9292         }
9293
9294         rc = copyin(br->data, br_data, br->len);
9295         if (rc == 0)
9296                 rc = -t4_load_boot(sc, br_data, offset, br->len);
9297
9298         free(br_data, M_CXGBE);
9299 done:
9300         end_synchronized_op(sc, 0);
9301         return (rc);
9302 }
9303
9304 static int
9305 load_bootcfg(struct adapter *sc, struct t4_data *bc)
9306 {
9307         int rc;
9308         uint8_t *bc_data = NULL;
9309
9310         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf");
9311         if (rc)
9312                 return (rc);
9313
9314         if (bc->len == 0) {
9315                 /* clear */
9316                 rc = -t4_load_bootcfg(sc, NULL, 0);
9317                 goto done;
9318         }
9319
9320         bc_data = malloc(bc->len, M_CXGBE, M_WAITOK);
9321         if (bc_data == NULL) {
9322                 rc = ENOMEM;
9323                 goto done;
9324         }
9325
9326         rc = copyin(bc->data, bc_data, bc->len);
9327         if (rc == 0)
9328                 rc = -t4_load_bootcfg(sc, bc_data, bc->len);
9329
9330         free(bc_data, M_CXGBE);
9331 done:
9332         end_synchronized_op(sc, 0);
9333         return (rc);
9334 }
9335
9336 static int
9337 cudbg_dump(struct adapter *sc, struct t4_cudbg_dump *dump)
9338 {
9339         int rc;
9340         struct cudbg_init *cudbg;
9341         void *handle, *buf;
9342
9343         /* buf is large, don't block if no memory is available */
9344         buf = malloc(dump->len, M_CXGBE, M_NOWAIT | M_ZERO);
9345         if (buf == NULL)
9346                 return (ENOMEM);
9347
9348         handle = cudbg_alloc_handle();
9349         if (handle == NULL) {
9350                 rc = ENOMEM;
9351                 goto done;
9352         }
9353
9354         cudbg = cudbg_get_init(handle);
9355         cudbg->adap = sc;
9356         cudbg->print = (cudbg_print_cb)printf;
9357
9358 #ifndef notyet
9359         device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n",
9360             __func__, dump->wr_flash, dump->len, dump->data);
9361 #endif
9362
9363         if (dump->wr_flash)
9364                 cudbg->use_flash = 1;
9365         MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap));
9366         memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap));
9367
9368         rc = cudbg_collect(handle, buf, &dump->len);
9369         if (rc != 0)
9370                 goto done;
9371
9372         rc = copyout(buf, dump->data, dump->len);
9373 done:
9374         cudbg_free_handle(handle);
9375         free(buf, M_CXGBE);
9376         return (rc);
9377 }
9378
9379 static void
9380 free_offload_policy(struct t4_offload_policy *op)
9381 {
9382         struct offload_rule *r;
9383         int i;
9384
9385         if (op == NULL)
9386                 return;
9387
9388         r = &op->rule[0];
9389         for (i = 0; i < op->nrules; i++, r++) {
9390                 free(r->bpf_prog.bf_insns, M_CXGBE);
9391         }
9392         free(op->rule, M_CXGBE);
9393         free(op, M_CXGBE);
9394 }
9395
9396 static int
9397 set_offload_policy(struct adapter *sc, struct t4_offload_policy *uop)
9398 {
9399         int i, rc, len;
9400         struct t4_offload_policy *op, *old;
9401         struct bpf_program *bf;
9402         const struct offload_settings *s;
9403         struct offload_rule *r;
9404         void *u;
9405
9406         if (!is_offload(sc))
9407                 return (ENODEV);
9408
9409         if (uop->nrules == 0) {
9410                 /* Delete installed policies. */
9411                 op = NULL;
9412                 goto set_policy;
9413         } if (uop->nrules > 256) { /* arbitrary */
9414                 return (E2BIG);
9415         }
9416
9417         /* Copy userspace offload policy to kernel */
9418         op = malloc(sizeof(*op), M_CXGBE, M_ZERO | M_WAITOK);
9419         op->nrules = uop->nrules;
9420         len = op->nrules * sizeof(struct offload_rule);
9421         op->rule = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
9422         rc = copyin(uop->rule, op->rule, len);
9423         if (rc) {
9424                 free(op->rule, M_CXGBE);
9425                 free(op, M_CXGBE);
9426                 return (rc);
9427         }
9428
9429         r = &op->rule[0];
9430         for (i = 0; i < op->nrules; i++, r++) {
9431
9432                 /* Validate open_type */
9433                 if (r->open_type != OPEN_TYPE_LISTEN &&
9434                     r->open_type != OPEN_TYPE_ACTIVE &&
9435                     r->open_type != OPEN_TYPE_PASSIVE &&
9436                     r->open_type != OPEN_TYPE_DONTCARE) {
9437 error:
9438                         /*
9439                          * Rules 0 to i have malloc'd filters that need to be
9440                          * freed.  Rules i+1 to nrules have userspace pointers
9441                          * and should be left alone.
9442                          */
9443                         op->nrules = i;
9444                         free_offload_policy(op);
9445                         return (rc);
9446                 }
9447
9448                 /* Validate settings */
9449                 s = &r->settings;
9450                 if ((s->offload != 0 && s->offload != 1) ||
9451                     s->cong_algo < -1 || s->cong_algo > CONG_ALG_HIGHSPEED ||
9452                     s->sched_class < -1 ||
9453                     s->sched_class >= sc->chip_params->nsched_cls) {
9454                         rc = EINVAL;
9455                         goto error;
9456                 }
9457
9458                 bf = &r->bpf_prog;
9459                 u = bf->bf_insns;       /* userspace ptr */
9460                 bf->bf_insns = NULL;
9461                 if (bf->bf_len == 0) {
9462                         /* legal, matches everything */
9463                         continue;
9464                 }
9465                 len = bf->bf_len * sizeof(*bf->bf_insns);
9466                 bf->bf_insns = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
9467                 rc = copyin(u, bf->bf_insns, len);
9468                 if (rc != 0)
9469                         goto error;
9470
9471                 if (!bpf_validate(bf->bf_insns, bf->bf_len)) {
9472                         rc = EINVAL;
9473                         goto error;
9474                 }
9475         }
9476 set_policy:
9477         rw_wlock(&sc->policy_lock);
9478         old = sc->policy;
9479         sc->policy = op;
9480         rw_wunlock(&sc->policy_lock);
9481         free_offload_policy(old);
9482
9483         return (0);
9484 }
9485
9486 #define MAX_READ_BUF_SIZE (128 * 1024)
9487 static int
9488 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
9489 {
9490         uint32_t addr, remaining, n;
9491         uint32_t *buf;
9492         int rc;
9493         uint8_t *dst;
9494
9495         rc = validate_mem_range(sc, mr->addr, mr->len);
9496         if (rc != 0)
9497                 return (rc);
9498
9499         buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK);
9500         addr = mr->addr;
9501         remaining = mr->len;
9502         dst = (void *)mr->data;
9503
9504         while (remaining) {
9505                 n = min(remaining, MAX_READ_BUF_SIZE);
9506                 read_via_memwin(sc, 2, addr, buf, n);
9507
9508                 rc = copyout(buf, dst, n);
9509                 if (rc != 0)
9510                         break;
9511
9512                 dst += n;
9513                 remaining -= n;
9514                 addr += n;
9515         }
9516
9517         free(buf, M_CXGBE);
9518         return (rc);
9519 }
9520 #undef MAX_READ_BUF_SIZE
9521
9522 static int
9523 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
9524 {
9525         int rc;
9526
9527         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
9528                 return (EINVAL);
9529
9530         if (i2cd->len > sizeof(i2cd->data))
9531                 return (EFBIG);
9532
9533         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
9534         if (rc)
9535                 return (rc);
9536         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
9537             i2cd->offset, i2cd->len, &i2cd->data[0]);
9538         end_synchronized_op(sc, 0);
9539
9540         return (rc);
9541 }
9542
9543 int
9544 t4_os_find_pci_capability(struct adapter *sc, int cap)
9545 {
9546         int i;
9547
9548         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
9549 }
9550
9551 int
9552 t4_os_pci_save_state(struct adapter *sc)
9553 {
9554         device_t dev;
9555         struct pci_devinfo *dinfo;
9556
9557         dev = sc->dev;
9558         dinfo = device_get_ivars(dev);
9559
9560         pci_cfg_save(dev, dinfo, 0);
9561         return (0);
9562 }
9563
9564 int
9565 t4_os_pci_restore_state(struct adapter *sc)
9566 {
9567         device_t dev;
9568         struct pci_devinfo *dinfo;
9569
9570         dev = sc->dev;
9571         dinfo = device_get_ivars(dev);
9572
9573         pci_cfg_restore(dev, dinfo);
9574         return (0);
9575 }
9576
9577 void
9578 t4_os_portmod_changed(struct port_info *pi)
9579 {
9580         struct adapter *sc = pi->adapter;
9581         struct vi_info *vi;
9582         struct ifnet *ifp;
9583         static const char *mod_str[] = {
9584                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
9585         };
9586
9587         KASSERT((pi->flags & FIXED_IFMEDIA) == 0,
9588             ("%s: port_type %u", __func__, pi->port_type));
9589
9590         vi = &pi->vi[0];
9591         if (begin_synchronized_op(sc, vi, HOLD_LOCK, "t4mod") == 0) {
9592                 PORT_LOCK(pi);
9593                 build_medialist(pi);
9594                 if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) {
9595                         fixup_link_config(pi);
9596                         apply_link_config(pi);
9597                 }
9598                 PORT_UNLOCK(pi);
9599                 end_synchronized_op(sc, LOCK_HELD);
9600         }
9601
9602         ifp = vi->ifp;
9603         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
9604                 if_printf(ifp, "transceiver unplugged.\n");
9605         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
9606                 if_printf(ifp, "unknown transceiver inserted.\n");
9607         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
9608                 if_printf(ifp, "unsupported transceiver inserted.\n");
9609         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
9610                 if_printf(ifp, "%dGbps %s transceiver inserted.\n",
9611                     port_top_speed(pi), mod_str[pi->mod_type]);
9612         } else {
9613                 if_printf(ifp, "transceiver (type %d) inserted.\n",
9614                     pi->mod_type);
9615         }
9616 }
9617
9618 void
9619 t4_os_link_changed(struct port_info *pi)
9620 {
9621         struct vi_info *vi;
9622         struct ifnet *ifp;
9623         struct link_config *lc;
9624         int v;
9625
9626         PORT_LOCK_ASSERT_OWNED(pi);
9627
9628         for_each_vi(pi, v, vi) {
9629                 ifp = vi->ifp;
9630                 if (ifp == NULL)
9631                         continue;
9632
9633                 lc = &pi->link_cfg;
9634                 if (lc->link_ok) {
9635                         ifp->if_baudrate = IF_Mbps(lc->speed);
9636                         if_link_state_change(ifp, LINK_STATE_UP);
9637                 } else {
9638                         if_link_state_change(ifp, LINK_STATE_DOWN);
9639                 }
9640         }
9641 }
9642
9643 void
9644 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
9645 {
9646         struct adapter *sc;
9647
9648         sx_slock(&t4_list_lock);
9649         SLIST_FOREACH(sc, &t4_list, link) {
9650                 /*
9651                  * func should not make any assumptions about what state sc is
9652                  * in - the only guarantee is that sc->sc_lock is a valid lock.
9653                  */
9654                 func(sc, arg);
9655         }
9656         sx_sunlock(&t4_list_lock);
9657 }
9658
9659 static int
9660 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
9661     struct thread *td)
9662 {
9663         int rc;
9664         struct adapter *sc = dev->si_drv1;
9665
9666         rc = priv_check(td, PRIV_DRIVER);
9667         if (rc != 0)
9668                 return (rc);
9669
9670         switch (cmd) {
9671         case CHELSIO_T4_GETREG: {
9672                 struct t4_reg *edata = (struct t4_reg *)data;
9673
9674                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
9675                         return (EFAULT);
9676
9677                 if (edata->size == 4)
9678                         edata->val = t4_read_reg(sc, edata->addr);
9679                 else if (edata->size == 8)
9680                         edata->val = t4_read_reg64(sc, edata->addr);
9681                 else
9682                         return (EINVAL);
9683
9684                 break;
9685         }
9686         case CHELSIO_T4_SETREG: {
9687                 struct t4_reg *edata = (struct t4_reg *)data;
9688
9689                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
9690                         return (EFAULT);
9691
9692                 if (edata->size == 4) {
9693                         if (edata->val & 0xffffffff00000000)
9694                                 return (EINVAL);
9695                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
9696                 } else if (edata->size == 8)
9697                         t4_write_reg64(sc, edata->addr, edata->val);
9698                 else
9699                         return (EINVAL);
9700                 break;
9701         }
9702         case CHELSIO_T4_REGDUMP: {
9703                 struct t4_regdump *regs = (struct t4_regdump *)data;
9704                 int reglen = t4_get_regs_len(sc);
9705                 uint8_t *buf;
9706
9707                 if (regs->len < reglen) {
9708                         regs->len = reglen; /* hint to the caller */
9709                         return (ENOBUFS);
9710                 }
9711
9712                 regs->len = reglen;
9713                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
9714                 get_regs(sc, regs, buf);
9715                 rc = copyout(buf, regs->data, reglen);
9716                 free(buf, M_CXGBE);
9717                 break;
9718         }
9719         case CHELSIO_T4_GET_FILTER_MODE:
9720                 rc = get_filter_mode(sc, (uint32_t *)data);
9721                 break;
9722         case CHELSIO_T4_SET_FILTER_MODE:
9723                 rc = set_filter_mode(sc, *(uint32_t *)data);
9724                 break;
9725         case CHELSIO_T4_GET_FILTER:
9726                 rc = get_filter(sc, (struct t4_filter *)data);
9727                 break;
9728         case CHELSIO_T4_SET_FILTER:
9729                 rc = set_filter(sc, (struct t4_filter *)data);
9730                 break;
9731         case CHELSIO_T4_DEL_FILTER:
9732                 rc = del_filter(sc, (struct t4_filter *)data);
9733                 break;
9734         case CHELSIO_T4_GET_SGE_CONTEXT:
9735                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
9736                 break;
9737         case CHELSIO_T4_LOAD_FW:
9738                 rc = load_fw(sc, (struct t4_data *)data);
9739                 break;
9740         case CHELSIO_T4_GET_MEM:
9741                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
9742                 break;
9743         case CHELSIO_T4_GET_I2C:
9744                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
9745                 break;
9746         case CHELSIO_T4_CLEAR_STATS: {
9747                 int i, v, bg_map;
9748                 u_int port_id = *(uint32_t *)data;
9749                 struct port_info *pi;
9750                 struct vi_info *vi;
9751
9752                 if (port_id >= sc->params.nports)
9753                         return (EINVAL);
9754                 pi = sc->port[port_id];
9755                 if (pi == NULL)
9756                         return (EIO);
9757
9758                 /* MAC stats */
9759                 t4_clr_port_stats(sc, pi->tx_chan);
9760                 pi->tx_parse_error = 0;
9761                 pi->tnl_cong_drops = 0;
9762                 mtx_lock(&sc->reg_lock);
9763                 for_each_vi(pi, v, vi) {
9764                         if (vi->flags & VI_INIT_DONE)
9765                                 t4_clr_vi_stats(sc, vi->viid);
9766                 }
9767                 bg_map = pi->mps_bg_map;
9768                 v = 0;  /* reuse */
9769                 while (bg_map) {
9770                         i = ffs(bg_map) - 1;
9771                         t4_write_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
9772                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
9773                         bg_map &= ~(1 << i);
9774                 }
9775                 mtx_unlock(&sc->reg_lock);
9776
9777                 /*
9778                  * Since this command accepts a port, clear stats for
9779                  * all VIs on this port.
9780                  */
9781                 for_each_vi(pi, v, vi) {
9782                         if (vi->flags & VI_INIT_DONE) {
9783                                 struct sge_rxq *rxq;
9784                                 struct sge_txq *txq;
9785                                 struct sge_wrq *wrq;
9786
9787                                 for_each_rxq(vi, i, rxq) {
9788 #if defined(INET) || defined(INET6)
9789                                         rxq->lro.lro_queued = 0;
9790                                         rxq->lro.lro_flushed = 0;
9791 #endif
9792                                         rxq->rxcsum = 0;
9793                                         rxq->vlan_extraction = 0;
9794                                 }
9795
9796                                 for_each_txq(vi, i, txq) {
9797                                         txq->txcsum = 0;
9798                                         txq->tso_wrs = 0;
9799                                         txq->vlan_insertion = 0;
9800                                         txq->imm_wrs = 0;
9801                                         txq->sgl_wrs = 0;
9802                                         txq->txpkt_wrs = 0;
9803                                         txq->txpkts0_wrs = 0;
9804                                         txq->txpkts1_wrs = 0;
9805                                         txq->txpkts0_pkts = 0;
9806                                         txq->txpkts1_pkts = 0;
9807                                         mp_ring_reset_stats(txq->r);
9808                                 }
9809
9810 #ifdef TCP_OFFLOAD
9811                                 /* nothing to clear for each ofld_rxq */
9812
9813                                 for_each_ofld_txq(vi, i, wrq) {
9814                                         wrq->tx_wrs_direct = 0;
9815                                         wrq->tx_wrs_copied = 0;
9816                                 }
9817 #endif
9818
9819                                 if (IS_MAIN_VI(vi)) {
9820                                         wrq = &sc->sge.ctrlq[pi->port_id];
9821                                         wrq->tx_wrs_direct = 0;
9822                                         wrq->tx_wrs_copied = 0;
9823                                 }
9824                         }
9825                 }
9826                 break;
9827         }
9828         case CHELSIO_T4_SCHED_CLASS:
9829                 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data);
9830                 break;
9831         case CHELSIO_T4_SCHED_QUEUE:
9832                 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data);
9833                 break;
9834         case CHELSIO_T4_GET_TRACER:
9835                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
9836                 break;
9837         case CHELSIO_T4_SET_TRACER:
9838                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
9839                 break;
9840         case CHELSIO_T4_LOAD_CFG:
9841                 rc = load_cfg(sc, (struct t4_data *)data);
9842                 break;
9843         case CHELSIO_T4_LOAD_BOOT:
9844                 rc = load_boot(sc, (struct t4_bootrom *)data);
9845                 break;
9846         case CHELSIO_T4_LOAD_BOOTCFG:
9847                 rc = load_bootcfg(sc, (struct t4_data *)data);
9848                 break;
9849         case CHELSIO_T4_CUDBG_DUMP:
9850                 rc = cudbg_dump(sc, (struct t4_cudbg_dump *)data);
9851                 break;
9852         case CHELSIO_T4_SET_OFLD_POLICY:
9853                 rc = set_offload_policy(sc, (struct t4_offload_policy *)data);
9854                 break;
9855         default:
9856                 rc = ENOTTY;
9857         }
9858
9859         return (rc);
9860 }
9861
9862 void
9863 t4_db_full(struct adapter *sc)
9864 {
9865
9866         CXGBE_UNIMPLEMENTED(__func__);
9867 }
9868
9869 void
9870 t4_db_dropped(struct adapter *sc)
9871 {
9872
9873         CXGBE_UNIMPLEMENTED(__func__);
9874 }
9875
9876 #ifdef TCP_OFFLOAD
9877 static int
9878 toe_capability(struct vi_info *vi, int enable)
9879 {
9880         int rc;
9881         struct port_info *pi = vi->pi;
9882         struct adapter *sc = pi->adapter;
9883
9884         ASSERT_SYNCHRONIZED_OP(sc);
9885
9886         if (!is_offload(sc))
9887                 return (ENODEV);
9888
9889         if (enable) {
9890                 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) {
9891                         /* TOE is already enabled. */
9892                         return (0);
9893                 }
9894
9895                 /*
9896                  * We need the port's queues around so that we're able to send
9897                  * and receive CPLs to/from the TOE even if the ifnet for this
9898                  * port has never been UP'd administratively.
9899                  */
9900                 if (!(vi->flags & VI_INIT_DONE)) {
9901                         rc = vi_full_init(vi);
9902                         if (rc)
9903                                 return (rc);
9904                 }
9905                 if (!(pi->vi[0].flags & VI_INIT_DONE)) {
9906                         rc = vi_full_init(&pi->vi[0]);
9907                         if (rc)
9908                                 return (rc);
9909                 }
9910
9911                 if (isset(&sc->offload_map, pi->port_id)) {
9912                         /* TOE is enabled on another VI of this port. */
9913                         pi->uld_vis++;
9914                         return (0);
9915                 }
9916
9917                 if (!uld_active(sc, ULD_TOM)) {
9918                         rc = t4_activate_uld(sc, ULD_TOM);
9919                         if (rc == EAGAIN) {
9920                                 log(LOG_WARNING,
9921                                     "You must kldload t4_tom.ko before trying "
9922                                     "to enable TOE on a cxgbe interface.\n");
9923                         }
9924                         if (rc != 0)
9925                                 return (rc);
9926                         KASSERT(sc->tom_softc != NULL,
9927                             ("%s: TOM activated but softc NULL", __func__));
9928                         KASSERT(uld_active(sc, ULD_TOM),
9929                             ("%s: TOM activated but flag not set", __func__));
9930                 }
9931
9932                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
9933                 if (!uld_active(sc, ULD_IWARP))
9934                         (void) t4_activate_uld(sc, ULD_IWARP);
9935                 if (!uld_active(sc, ULD_ISCSI))
9936                         (void) t4_activate_uld(sc, ULD_ISCSI);
9937
9938                 pi->uld_vis++;
9939                 setbit(&sc->offload_map, pi->port_id);
9940         } else {
9941                 pi->uld_vis--;
9942
9943                 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
9944                         return (0);
9945
9946                 KASSERT(uld_active(sc, ULD_TOM),
9947                     ("%s: TOM never initialized?", __func__));
9948                 clrbit(&sc->offload_map, pi->port_id);
9949         }
9950
9951         return (0);
9952 }
9953
9954 /*
9955  * Add an upper layer driver to the global list.
9956  */
9957 int
9958 t4_register_uld(struct uld_info *ui)
9959 {
9960         int rc = 0;
9961         struct uld_info *u;
9962
9963         sx_xlock(&t4_uld_list_lock);
9964         SLIST_FOREACH(u, &t4_uld_list, link) {
9965             if (u->uld_id == ui->uld_id) {
9966                     rc = EEXIST;
9967                     goto done;
9968             }
9969         }
9970
9971         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
9972         ui->refcount = 0;
9973 done:
9974         sx_xunlock(&t4_uld_list_lock);
9975         return (rc);
9976 }
9977
9978 int
9979 t4_unregister_uld(struct uld_info *ui)
9980 {
9981         int rc = EINVAL;
9982         struct uld_info *u;
9983
9984         sx_xlock(&t4_uld_list_lock);
9985
9986         SLIST_FOREACH(u, &t4_uld_list, link) {
9987             if (u == ui) {
9988                     if (ui->refcount > 0) {
9989                             rc = EBUSY;
9990                             goto done;
9991                     }
9992
9993                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
9994                     rc = 0;
9995                     goto done;
9996             }
9997         }
9998 done:
9999         sx_xunlock(&t4_uld_list_lock);
10000         return (rc);
10001 }
10002
10003 int
10004 t4_activate_uld(struct adapter *sc, int id)
10005 {
10006         int rc;
10007         struct uld_info *ui;
10008
10009         ASSERT_SYNCHRONIZED_OP(sc);
10010
10011         if (id < 0 || id > ULD_MAX)
10012                 return (EINVAL);
10013         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
10014
10015         sx_slock(&t4_uld_list_lock);
10016
10017         SLIST_FOREACH(ui, &t4_uld_list, link) {
10018                 if (ui->uld_id == id) {
10019                         if (!(sc->flags & FULL_INIT_DONE)) {
10020                                 rc = adapter_full_init(sc);
10021                                 if (rc != 0)
10022                                         break;
10023                         }
10024
10025                         rc = ui->activate(sc);
10026                         if (rc == 0) {
10027                                 setbit(&sc->active_ulds, id);
10028                                 ui->refcount++;
10029                         }
10030                         break;
10031                 }
10032         }
10033
10034         sx_sunlock(&t4_uld_list_lock);
10035
10036         return (rc);
10037 }
10038
10039 int
10040 t4_deactivate_uld(struct adapter *sc, int id)
10041 {
10042         int rc;
10043         struct uld_info *ui;
10044
10045         ASSERT_SYNCHRONIZED_OP(sc);
10046
10047         if (id < 0 || id > ULD_MAX)
10048                 return (EINVAL);
10049         rc = ENXIO;
10050
10051         sx_slock(&t4_uld_list_lock);
10052
10053         SLIST_FOREACH(ui, &t4_uld_list, link) {
10054                 if (ui->uld_id == id) {
10055                         rc = ui->deactivate(sc);
10056                         if (rc == 0) {
10057                                 clrbit(&sc->active_ulds, id);
10058                                 ui->refcount--;
10059                         }
10060                         break;
10061                 }
10062         }
10063
10064         sx_sunlock(&t4_uld_list_lock);
10065
10066         return (rc);
10067 }
10068
10069 int
10070 uld_active(struct adapter *sc, int uld_id)
10071 {
10072
10073         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
10074
10075         return (isset(&sc->active_ulds, uld_id));
10076 }
10077 #endif
10078
10079 /*
10080  * t  = ptr to tunable.
10081  * nc = number of CPUs.
10082  * c  = compiled in default for that tunable.
10083  */
10084 static void
10085 calculate_nqueues(int *t, int nc, const int c)
10086 {
10087         int nq;
10088
10089         if (*t > 0)
10090                 return;
10091         nq = *t < 0 ? -*t : c;
10092         *t = min(nc, nq);
10093 }
10094
10095 /*
10096  * Come up with reasonable defaults for some of the tunables, provided they're
10097  * not set by the user (in which case we'll use the values as is).
10098  */
10099 static void
10100 tweak_tunables(void)
10101 {
10102         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
10103
10104         if (t4_ntxq < 1) {
10105 #ifdef RSS
10106                 t4_ntxq = rss_getnumbuckets();
10107 #else
10108                 calculate_nqueues(&t4_ntxq, nc, NTXQ);
10109 #endif
10110         }
10111
10112         calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI);
10113
10114         if (t4_nrxq < 1) {
10115 #ifdef RSS
10116                 t4_nrxq = rss_getnumbuckets();
10117 #else
10118                 calculate_nqueues(&t4_nrxq, nc, NRXQ);
10119 #endif
10120         }
10121
10122         calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI);
10123
10124 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
10125         calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ);
10126         calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI);
10127 #endif
10128 #ifdef TCP_OFFLOAD
10129         calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ);
10130         calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI);
10131
10132         if (t4_toecaps_allowed == -1)
10133                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
10134
10135         if (t4_rdmacaps_allowed == -1) {
10136                 t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP |
10137                     FW_CAPS_CONFIG_RDMA_RDMAC;
10138         }
10139
10140         if (t4_iscsicaps_allowed == -1) {
10141                 t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU |
10142                     FW_CAPS_CONFIG_ISCSI_TARGET_PDU |
10143                     FW_CAPS_CONFIG_ISCSI_T10DIF;
10144         }
10145
10146         if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS)
10147                 t4_tmr_idx_ofld = TMR_IDX_OFLD;
10148
10149         if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS)
10150                 t4_pktc_idx_ofld = PKTC_IDX_OFLD;
10151 #else
10152         if (t4_toecaps_allowed == -1)
10153                 t4_toecaps_allowed = 0;
10154
10155         if (t4_rdmacaps_allowed == -1)
10156                 t4_rdmacaps_allowed = 0;
10157
10158         if (t4_iscsicaps_allowed == -1)
10159                 t4_iscsicaps_allowed = 0;
10160 #endif
10161
10162 #ifdef DEV_NETMAP
10163         calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI);
10164         calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI);
10165 #endif
10166
10167         if (t4_tmr_idx < 0 || t4_tmr_idx >= SGE_NTIMERS)
10168                 t4_tmr_idx = TMR_IDX;
10169
10170         if (t4_pktc_idx < -1 || t4_pktc_idx >= SGE_NCOUNTERS)
10171                 t4_pktc_idx = PKTC_IDX;
10172
10173         if (t4_qsize_txq < 128)
10174                 t4_qsize_txq = 128;
10175
10176         if (t4_qsize_rxq < 128)
10177                 t4_qsize_rxq = 128;
10178         while (t4_qsize_rxq & 7)
10179                 t4_qsize_rxq++;
10180
10181         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
10182
10183         /*
10184          * Number of VIs to create per-port.  The first VI is the "main" regular
10185          * VI for the port.  The rest are additional virtual interfaces on the
10186          * same physical port.  Note that the main VI does not have native
10187          * netmap support but the extra VIs do.
10188          *
10189          * Limit the number of VIs per port to the number of available
10190          * MAC addresses per port.
10191          */
10192         if (t4_num_vis < 1)
10193                 t4_num_vis = 1;
10194         if (t4_num_vis > nitems(vi_mac_funcs)) {
10195                 t4_num_vis = nitems(vi_mac_funcs);
10196                 printf("cxgbe: number of VIs limited to %d\n", t4_num_vis);
10197         }
10198
10199         if (pcie_relaxed_ordering < 0 || pcie_relaxed_ordering > 2) {
10200                 pcie_relaxed_ordering = 1;
10201 #if defined(__i386__) || defined(__amd64__)
10202                 if (cpu_vendor_id == CPU_VENDOR_INTEL)
10203                         pcie_relaxed_ordering = 0;
10204 #endif
10205         }
10206 }
10207
10208 #ifdef DDB
10209 static void
10210 t4_dump_tcb(struct adapter *sc, int tid)
10211 {
10212         uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos;
10213
10214         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2);
10215         save = t4_read_reg(sc, reg);
10216         base = sc->memwin[2].mw_base;
10217
10218         /* Dump TCB for the tid */
10219         tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
10220         tcb_addr += tid * TCB_SIZE;
10221
10222         if (is_t4(sc)) {
10223                 pf = 0;
10224                 win_pos = tcb_addr & ~0xf;      /* start must be 16B aligned */
10225         } else {
10226                 pf = V_PFNUM(sc->pf);
10227                 win_pos = tcb_addr & ~0x7f;     /* start must be 128B aligned */
10228         }
10229         t4_write_reg(sc, reg, win_pos | pf);
10230         t4_read_reg(sc, reg);
10231
10232         off = tcb_addr - win_pos;
10233         for (i = 0; i < 4; i++) {
10234                 uint32_t buf[8];
10235                 for (j = 0; j < 8; j++, off += 4)
10236                         buf[j] = htonl(t4_read_reg(sc, base + off));
10237
10238                 db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n",
10239                     buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
10240                     buf[7]);
10241         }
10242
10243         t4_write_reg(sc, reg, save);
10244         t4_read_reg(sc, reg);
10245 }
10246
10247 static void
10248 t4_dump_devlog(struct adapter *sc)
10249 {
10250         struct devlog_params *dparams = &sc->params.devlog;
10251         struct fw_devlog_e e;
10252         int i, first, j, m, nentries, rc;
10253         uint64_t ftstamp = UINT64_MAX;
10254
10255         if (dparams->start == 0) {
10256                 db_printf("devlog params not valid\n");
10257                 return;
10258         }
10259
10260         nentries = dparams->size / sizeof(struct fw_devlog_e);
10261         m = fwmtype_to_hwmtype(dparams->memtype);
10262
10263         /* Find the first entry. */
10264         first = -1;
10265         for (i = 0; i < nentries && !db_pager_quit; i++) {
10266                 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
10267                     sizeof(e), (void *)&e);
10268                 if (rc != 0)
10269                         break;
10270
10271                 if (e.timestamp == 0)
10272                         break;
10273
10274                 e.timestamp = be64toh(e.timestamp);
10275                 if (e.timestamp < ftstamp) {
10276                         ftstamp = e.timestamp;
10277                         first = i;
10278                 }
10279         }
10280
10281         if (first == -1)
10282                 return;
10283
10284         i = first;
10285         do {
10286                 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
10287                     sizeof(e), (void *)&e);
10288                 if (rc != 0)
10289                         return;
10290
10291                 if (e.timestamp == 0)
10292                         return;
10293
10294                 e.timestamp = be64toh(e.timestamp);
10295                 e.seqno = be32toh(e.seqno);
10296                 for (j = 0; j < 8; j++)
10297                         e.params[j] = be32toh(e.params[j]);
10298
10299                 db_printf("%10d  %15ju  %8s  %8s  ",
10300                     e.seqno, e.timestamp,
10301                     (e.level < nitems(devlog_level_strings) ?
10302                         devlog_level_strings[e.level] : "UNKNOWN"),
10303                     (e.facility < nitems(devlog_facility_strings) ?
10304                         devlog_facility_strings[e.facility] : "UNKNOWN"));
10305                 db_printf(e.fmt, e.params[0], e.params[1], e.params[2],
10306                     e.params[3], e.params[4], e.params[5], e.params[6],
10307                     e.params[7]);
10308
10309                 if (++i == nentries)
10310                         i = 0;
10311         } while (i != first && !db_pager_quit);
10312 }
10313
10314 static struct command_table db_t4_table = LIST_HEAD_INITIALIZER(db_t4_table);
10315 _DB_SET(_show, t4, NULL, db_show_table, 0, &db_t4_table);
10316
10317 DB_FUNC(devlog, db_show_devlog, db_t4_table, CS_OWN, NULL)
10318 {
10319         device_t dev;
10320         int t;
10321         bool valid;
10322
10323         valid = false;
10324         t = db_read_token();
10325         if (t == tIDENT) {
10326                 dev = device_lookup_by_name(db_tok_string);
10327                 valid = true;
10328         }
10329         db_skip_to_eol();
10330         if (!valid) {
10331                 db_printf("usage: show t4 devlog <nexus>\n");
10332                 return;
10333         }
10334
10335         if (dev == NULL) {
10336                 db_printf("device not found\n");
10337                 return;
10338         }
10339
10340         t4_dump_devlog(device_get_softc(dev));
10341 }
10342
10343 DB_FUNC(tcb, db_show_t4tcb, db_t4_table, CS_OWN, NULL)
10344 {
10345         device_t dev;
10346         int radix, tid, t;
10347         bool valid;
10348
10349         valid = false;
10350         radix = db_radix;
10351         db_radix = 10;
10352         t = db_read_token();
10353         if (t == tIDENT) {
10354                 dev = device_lookup_by_name(db_tok_string);
10355                 t = db_read_token();
10356                 if (t == tNUMBER) {
10357                         tid = db_tok_number;
10358                         valid = true;
10359                 }
10360         }
10361         db_radix = radix;
10362         db_skip_to_eol();
10363         if (!valid) {
10364                 db_printf("usage: show t4 tcb <nexus> <tid>\n");
10365                 return;
10366         }
10367
10368         if (dev == NULL) {
10369                 db_printf("device not found\n");
10370                 return;
10371         }
10372         if (tid < 0) {
10373                 db_printf("invalid tid\n");
10374                 return;
10375         }
10376
10377         t4_dump_tcb(device_get_softc(dev), tid);
10378 }
10379 #endif
10380
10381 /*
10382  * Borrowed from cesa_prep_aes_key().
10383  *
10384  * NB: The crypto engine wants the words in the decryption key in reverse
10385  * order.
10386  */
10387 void
10388 t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits)
10389 {
10390         uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
10391         uint32_t *dkey;
10392         int i;
10393
10394         rijndaelKeySetupEnc(ek, enc_key, kbits);
10395         dkey = dec_key;
10396         dkey += (kbits / 8) / 4;
10397
10398         switch (kbits) {
10399         case 128:
10400                 for (i = 0; i < 4; i++)
10401                         *--dkey = htobe32(ek[4 * 10 + i]);
10402                 break;
10403         case 192:
10404                 for (i = 0; i < 2; i++)
10405                         *--dkey = htobe32(ek[4 * 11 + 2 + i]);
10406                 for (i = 0; i < 4; i++)
10407                         *--dkey = htobe32(ek[4 * 12 + i]);
10408                 break;
10409         case 256:
10410                 for (i = 0; i < 4; i++)
10411                         *--dkey = htobe32(ek[4 * 13 + i]);
10412                 for (i = 0; i < 4; i++)
10413                         *--dkey = htobe32(ek[4 * 14 + i]);
10414                 break;
10415         }
10416         MPASS(dkey == dec_key);
10417 }
10418
10419 static struct sx mlu;   /* mod load unload */
10420 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
10421
10422 static int
10423 mod_event(module_t mod, int cmd, void *arg)
10424 {
10425         int rc = 0;
10426         static int loaded = 0;
10427
10428         switch (cmd) {
10429         case MOD_LOAD:
10430                 sx_xlock(&mlu);
10431                 if (loaded++ == 0) {
10432                         t4_sge_modload();
10433                         t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
10434                             t4_filter_rpl, CPL_COOKIE_FILTER);
10435                         t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL,
10436                             do_l2t_write_rpl, CPL_COOKIE_FILTER);
10437                         t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL,
10438                             t4_hashfilter_ao_rpl, CPL_COOKIE_HASHFILTER);
10439                         t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
10440                             t4_hashfilter_tcb_rpl, CPL_COOKIE_HASHFILTER);
10441                         t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS,
10442                             t4_del_hashfilter_rpl, CPL_COOKIE_HASHFILTER);
10443                         t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt);
10444                         t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt);
10445                         t4_register_cpl_handler(CPL_SMT_WRITE_RPL,
10446                             do_smt_write_rpl);
10447                         sx_init(&t4_list_lock, "T4/T5 adapters");
10448                         SLIST_INIT(&t4_list);
10449 #ifdef TCP_OFFLOAD
10450                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
10451                         SLIST_INIT(&t4_uld_list);
10452 #endif
10453                         t4_tracer_modload();
10454                         tweak_tunables();
10455                 }
10456                 sx_xunlock(&mlu);
10457                 break;
10458
10459         case MOD_UNLOAD:
10460                 sx_xlock(&mlu);
10461                 if (--loaded == 0) {
10462                         int tries;
10463
10464                         sx_slock(&t4_list_lock);
10465                         if (!SLIST_EMPTY(&t4_list)) {
10466                                 rc = EBUSY;
10467                                 sx_sunlock(&t4_list_lock);
10468                                 goto done_unload;
10469                         }
10470 #ifdef TCP_OFFLOAD
10471                         sx_slock(&t4_uld_list_lock);
10472                         if (!SLIST_EMPTY(&t4_uld_list)) {
10473                                 rc = EBUSY;
10474                                 sx_sunlock(&t4_uld_list_lock);
10475                                 sx_sunlock(&t4_list_lock);
10476                                 goto done_unload;
10477                         }
10478 #endif
10479                         tries = 0;
10480                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
10481                                 uprintf("%ju clusters with custom free routine "
10482                                     "still is use.\n", t4_sge_extfree_refs());
10483                                 pause("t4unload", 2 * hz);
10484                         }
10485 #ifdef TCP_OFFLOAD
10486                         sx_sunlock(&t4_uld_list_lock);
10487 #endif
10488                         sx_sunlock(&t4_list_lock);
10489
10490                         if (t4_sge_extfree_refs() == 0) {
10491                                 t4_tracer_modunload();
10492 #ifdef TCP_OFFLOAD
10493                                 sx_destroy(&t4_uld_list_lock);
10494 #endif
10495                                 sx_destroy(&t4_list_lock);
10496                                 t4_sge_modunload();
10497                                 loaded = 0;
10498                         } else {
10499                                 rc = EBUSY;
10500                                 loaded++;       /* undo earlier decrement */
10501                         }
10502                 }
10503 done_unload:
10504                 sx_xunlock(&mlu);
10505                 break;
10506         }
10507
10508         return (rc);
10509 }
10510
10511 static devclass_t t4_devclass, t5_devclass, t6_devclass;
10512 static devclass_t cxgbe_devclass, cxl_devclass, cc_devclass;
10513 static devclass_t vcxgbe_devclass, vcxl_devclass, vcc_devclass;
10514
10515 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
10516 MODULE_VERSION(t4nex, 1);
10517 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
10518 #ifdef DEV_NETMAP
10519 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
10520 #endif /* DEV_NETMAP */
10521
10522 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
10523 MODULE_VERSION(t5nex, 1);
10524 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
10525 #ifdef DEV_NETMAP
10526 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
10527 #endif /* DEV_NETMAP */
10528
10529 DRIVER_MODULE(t6nex, pci, t6_driver, t6_devclass, mod_event, 0);
10530 MODULE_VERSION(t6nex, 1);
10531 MODULE_DEPEND(t6nex, firmware, 1, 1, 1);
10532 #ifdef DEV_NETMAP
10533 MODULE_DEPEND(t6nex, netmap, 1, 1, 1);
10534 #endif /* DEV_NETMAP */
10535
10536 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
10537 MODULE_VERSION(cxgbe, 1);
10538
10539 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
10540 MODULE_VERSION(cxl, 1);
10541
10542 DRIVER_MODULE(cc, t6nex, cc_driver, cc_devclass, 0, 0);
10543 MODULE_VERSION(cc, 1);
10544
10545 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0);
10546 MODULE_VERSION(vcxgbe, 1);
10547
10548 DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0);
10549 MODULE_VERSION(vcxl, 1);
10550
10551 DRIVER_MODULE(vcc, cc, vcc_driver, vcc_devclass, 0, 0);
10552 MODULE_VERSION(vcc, 1);