sys/dev/cxgb/cxgb_main.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2007-2008, Chelsio Inc.
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12  2. Neither the name of the Chelsio Corporation nor the names of its
  13     contributors may be used to endorse or promote products derived from
  14     this software without specific prior written permission.
  15
  16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  26 POSSIBILITY OF SUCH DAMAGE.
  27
  28 ***************************************************************************/
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/module.h>
  38 #include <sys/pciio.h>
  39 #include <sys/conf.h>
  40 #include <machine/bus.h>
  41 #include <machine/resource.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/ktr.h>
  44 #include <sys/rman.h>
  45 #include <sys/ioccom.h>
  46 #include <sys/mbuf.h>
  47 #include <sys/linker.h>
  48 #include <sys/firmware.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54 #include <sys/queue.h>
  55 #include <sys/taskqueue.h>
  56 #include <sys/proc.h>
  57
  58 #include <net/bpf.h>
  59 #include <net/ethernet.h>
  60 #include <net/if.h>
  61 #include <net/if_arp.h>
  62 #include <net/if_dl.h>
  63 #include <net/if_media.h>
  64 #include <net/if_types.h>
  65
  66 #include <netinet/in_systm.h>
  67 #include <netinet/in.h>
  68 #include <netinet/if_ether.h>
  69 #include <netinet/ip.h>
  70 #include <netinet/ip.h>
  71 #include <netinet/tcp.h>
  72 #include <netinet/udp.h>
  73
  74 #include <dev/pci/pcireg.h>
  75 #include <dev/pci/pcivar.h>
  76 #include <dev/pci/pci_private.h>
  77
  78 #ifdef CONFIG_DEFINED
  79 #include <cxgb_include.h>
  80 #else
  81 #include <dev/cxgb/cxgb_include.h>
  82 #endif
  83
  84 #ifdef PRIV_SUPPORTED
  85 #include <sys/priv.h>
  86 #endif
  87
  88 #ifdef IFNET_MULTIQUEUE
  89 #include <machine/intr_machdep.h>
  90 #endif
  91
  92 static int cxgb_setup_msix(adapter_t *, int);
  93 static void cxgb_teardown_msix(adapter_t *);
  94 static void cxgb_init(void *);
  95 static void cxgb_init_locked(struct port_info *);
  96 static void cxgb_stop_locked(struct port_info *);
  97 static void cxgb_set_rxmode(struct port_info *);
  98 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
  99 static int cxgb_media_change(struct ifnet *);
 100 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
 101 static int setup_sge_qsets(adapter_t *);
 102 static void cxgb_async_intr(void *);
 103 static void cxgb_ext_intr_handler(void *, int);
 104 static void cxgb_tick_handler(void *, int);
 105 static void cxgb_down_locked(struct adapter *sc);
 106 static void cxgb_tick(void *);
 107 static void setup_rss(adapter_t *sc);
 108
 109 /* Attachment glue for the PCI controller end of the device.  Each port of
 110  * the device is attached separately, as defined later.
 111  */
 112 static int cxgb_controller_probe(device_t);
 113 static int cxgb_controller_attach(device_t);
 114 static int cxgb_controller_detach(device_t);
 115 static void cxgb_free(struct adapter *);
 116 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
 117     unsigned int end);
 118 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf);
 119 static int cxgb_get_regs_len(void);
 120 static int offload_open(struct port_info *pi);
 121 static void touch_bars(device_t dev);
 122 static int offload_close(struct t3cdev *tdev);
 123 static void cxgb_link_start(struct port_info *p);
 124
 125 static device_method_t cxgb_controller_methods[] = {
 126         DEVMETHOD(device_probe,         cxgb_controller_probe),
 127         DEVMETHOD(device_attach,        cxgb_controller_attach),
 128         DEVMETHOD(device_detach,        cxgb_controller_detach),
 129
 130         /* bus interface */
 131         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 132         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 133
 134         { 0, 0 }
 135 };
 136
 137 static driver_t cxgb_controller_driver = {
 138         "cxgbc",
 139         cxgb_controller_methods,
 140         sizeof(struct adapter)
 141 };
 142
 143 static devclass_t       cxgb_controller_devclass;
 144 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass, 0, 0);
 145
 146 /*
 147  * Attachment glue for the ports.  Attachment is done directly to the
 148  * controller device.
 149  */
 150 static int cxgb_port_probe(device_t);
 151 static int cxgb_port_attach(device_t);
 152 static int cxgb_port_detach(device_t);
 153
 154 static device_method_t cxgb_port_methods[] = {
 155         DEVMETHOD(device_probe,         cxgb_port_probe),
 156         DEVMETHOD(device_attach,        cxgb_port_attach),
 157         DEVMETHOD(device_detach,        cxgb_port_detach),
 158         { 0, 0 }
 159 };
 160
 161 static driver_t cxgb_port_driver = {
 162         "cxgb",
 163         cxgb_port_methods,
 164         0
 165 };
 166
 167 static d_ioctl_t cxgb_extension_ioctl;
 168 static d_open_t cxgb_extension_open;
 169 static d_close_t cxgb_extension_close;
 170
 171 static struct cdevsw cxgb_cdevsw = {
 172        .d_version =    D_VERSION,
 173        .d_flags =      0,
 174        .d_open =       cxgb_extension_open,
 175        .d_close =      cxgb_extension_close,
 176        .d_ioctl =      cxgb_extension_ioctl,
 177        .d_name =       "cxgb",
 178 };
 179
 180 static devclass_t       cxgb_port_devclass;
 181 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
 182
 183 #define SGE_MSIX_COUNT (SGE_QSETS + 1)
 184
 185 /*
 186  * The driver uses the best interrupt scheme available on a platform in the
 187  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 188  * of these schemes the driver may consider as follows:
 189  *
 190  * msi = 2: choose from among all three options
 191  * msi = 1 : only consider MSI and pin interrupts
 192  * msi = 0: force pin interrupts
 193  */
 194 static int msi_allowed = 2;
 195
 196 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
 197 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
 198 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
 199     "MSI-X, MSI, INTx selector");
 200
 201 /*
 202  * The driver enables offload as a default.
 203  * To disable it, use ofld_disable = 1.
 204  */
 205 static int ofld_disable = 0;
 206 TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
 207 SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
 208     "disable ULP offload");
 209
 210 /*
 211  * The driver uses an auto-queue algorithm by default.
 212  * To disable it and force a single queue-set per port, use singleq = 1.
 213  */
 214 static int singleq = 0;
 215 TUNABLE_INT("hw.cxgb.singleq", &singleq);
 216 SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
 217     "use a single queue-set per port");
 218
 219
 220 /*
 221  * The driver uses an auto-queue algorithm by default.
 222  * To disable it and force a single queue-set per port, use singleq = 1.
 223  */
 224 static int force_fw_update = 0;
 225 TUNABLE_INT("hw.cxgb.force_fw_update", &force_fw_update);
 226 SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
 227     "update firmware even if up to date");
 228
 229 int cxgb_use_16k_clusters = 0;
 230 TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
 231 SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
 232     &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
 233
 234 enum {
 235         MAX_TXQ_ENTRIES      = 16384,
 236         MAX_CTRL_TXQ_ENTRIES = 1024,
 237         MAX_RSPQ_ENTRIES     = 16384,
 238         MAX_RX_BUFFERS       = 16384,
 239         MAX_RX_JUMBO_BUFFERS = 16384,
 240         MIN_TXQ_ENTRIES      = 4,
 241         MIN_CTRL_TXQ_ENTRIES = 4,
 242         MIN_RSPQ_ENTRIES     = 32,
 243         MIN_FL_ENTRIES       = 32,
 244         MIN_FL_JUMBO_ENTRIES = 32
 245 };
 246
 247 struct filter_info {
 248         u32 sip;
 249         u32 sip_mask;
 250         u32 dip;
 251         u16 sport;
 252         u16 dport;
 253         u32 vlan:12;
 254         u32 vlan_prio:3;
 255         u32 mac_hit:1;
 256         u32 mac_idx:4;
 257         u32 mac_vld:1;
 258         u32 pkt_type:2;
 259         u32 report_filter_id:1;
 260         u32 pass:1;
 261         u32 rss:1;
 262         u32 qset:3;
 263         u32 locked:1;
 264         u32 valid:1;
 265 };
 266
 267 enum { FILTER_NO_VLAN_PRI = 7 };
 268
 269 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
 270
 271 /* Table for probing the cards.  The desc field isn't actually used */
 272 struct cxgb_ident {
 273         uint16_t        vendor;
 274         uint16_t        device;
 275         int             index;
 276         char            *desc;
 277 } cxgb_identifiers[] = {
 278         {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
 279         {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
 280         {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
 281         {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
 282         {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
 283         {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
 284         {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
 285         {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
 286         {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
 287         {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
 288         {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
 289         {0, 0, 0, NULL}
 290 };
 291
 292 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
 293
 294
 295 void
 296 cxgb_log_tcb(struct adapter *sc, unsigned int tid)
 297 {
 298         char buf[TCB_SIZE];
 299         uint64_t *tcb = (uint64_t *)buf;
 300         int i, error;
 301         struct mc7 *mem = &sc->cm;
 302
 303         error = t3_mc7_bd_read(mem, tid*TCB_SIZE/8, TCB_SIZE/8, tcb);
 304         if (error)
 305                 printf("cxgb_tcb_log failed\n");
 306
 307         CTR1(KTR_CXGB, "TCB tid=%u", tid);
 308         for (i = 0; i < TCB_SIZE / 32; i++) {
 309                 CTR5(KTR_CXGB, "%1d: %08x %08x %08x %08x",
 310                     i, (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
 311                     (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
 312                 tcb += 2;
 313                 CTR4(KTR_CXGB, "   %08x %08x %08x %08x",
 314                     (uint32_t)tcb[1], (uint32_t)(tcb[1] >> 32),
 315                     (uint32_t)tcb[0], (uint32_t)(tcb[0] >> 32));
 316                 tcb += 2;
 317         }
 318 }
 319
 320 static __inline char
 321 t3rev2char(struct adapter *adapter)
 322 {
 323         char rev = 'z';
 324
 325         switch(adapter->params.rev) {
 326         case T3_REV_A:
 327                 rev = 'a';
 328                 break;
 329         case T3_REV_B:
 330         case T3_REV_B2:
 331                 rev = 'b';
 332                 break;
 333         case T3_REV_C:
 334                 rev = 'c';
 335                 break;
 336         }
 337         return rev;
 338 }
 339
 340 static struct cxgb_ident *
 341 cxgb_get_ident(device_t dev)
 342 {
 343         struct cxgb_ident *id;
 344
 345         for (id = cxgb_identifiers; id->desc != NULL; id++) {
 346                 if ((id->vendor == pci_get_vendor(dev)) &&
 347                     (id->device == pci_get_device(dev))) {
 348                         return (id);
 349                 }
 350         }
 351         return (NULL);
 352 }
 353
 354 static const struct adapter_info *
 355 cxgb_get_adapter_info(device_t dev)
 356 {
 357         struct cxgb_ident *id;
 358         const struct adapter_info *ai;
 359
 360         id = cxgb_get_ident(dev);
 361         if (id == NULL)
 362                 return (NULL);
 363
 364         ai = t3_get_adapter_info(id->index);
 365
 366         return (ai);
 367 }
 368
 369 static int
 370 cxgb_controller_probe(device_t dev)
 371 {
 372         const struct adapter_info *ai;
 373         char *ports, buf[80];
 374         int nports;
 375
 376         ai = cxgb_get_adapter_info(dev);
 377         if (ai == NULL)
 378                 return (ENXIO);
 379
 380         nports = ai->nports0 + ai->nports1;
 381         if (nports == 1)
 382                 ports = "port";
 383         else
 384                 ports = "ports";
 385
 386         snprintf(buf, sizeof(buf), "%s RNIC, %d %s", ai->desc, nports, ports);
 387         device_set_desc_copy(dev, buf);
 388         return (BUS_PROBE_DEFAULT);
 389 }
 390
 391 #define FW_FNAME "cxgb_t3fw"
 392 #define TPEEPROM_NAME "t3b_tp_eeprom"
 393 #define TPSRAM_NAME "t3b_protocol_sram"
 394
 395 static int
 396 upgrade_fw(adapter_t *sc)
 397 {
 398 #ifdef FIRMWARE_LATEST
 399         const struct firmware *fw;
 400 #else
 401         struct firmware *fw;
 402 #endif
 403         int status;
 404
 405         if ((fw = firmware_get(FW_FNAME)) == NULL)  {
 406                 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
 407                 return (ENOENT);
 408         } else
 409                 device_printf(sc->dev, "updating firmware on card\n");
 410         status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
 411
 412         device_printf(sc->dev, "firmware update returned %s %d\n", (status == 0) ? "success" : "fail", status);
 413
 414         firmware_put(fw, FIRMWARE_UNLOAD);
 415
 416         return (status);
 417 }
 418
 419 static int
 420 cxgb_controller_attach(device_t dev)
 421 {
 422         device_t child;
 423         const struct adapter_info *ai;
 424         struct adapter *sc;
 425         int i, error = 0;
 426         uint32_t vers;
 427         int port_qsets = 1;
 428 #ifdef MSI_SUPPORTED
 429         int msi_needed, reg;
 430 #endif
 431         int must_load = 0;
 432         sc = device_get_softc(dev);
 433         sc->dev = dev;
 434         sc->msi_count = 0;
 435         ai = cxgb_get_adapter_info(dev);
 436
 437         /*
 438          * XXX not really related but a recent addition
 439          */
 440 #ifdef MSI_SUPPORTED
 441         /* find the PCIe link width and set max read request to 4KB*/
 442         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
 443                 uint16_t lnk, pectl;
 444                 lnk = pci_read_config(dev, reg + 0x12, 2);
 445                 sc->link_width = (lnk >> 4) & 0x3f;
 446
 447                 pectl = pci_read_config(dev, reg + 0x8, 2);
 448                 pectl = (pectl & ~0x7000) | (5 << 12);
 449                 pci_write_config(dev, reg + 0x8, pectl, 2);
 450         }
 451
 452         if (sc->link_width != 0 && sc->link_width <= 4 &&
 453             (ai->nports0 + ai->nports1) <= 2) {
 454                 device_printf(sc->dev,
 455                     "PCIe x%d Link, expect reduced performance\n",
 456                     sc->link_width);
 457         }
 458 #endif
 459         touch_bars(dev);
 460         pci_enable_busmaster(dev);
 461         /*
 462          * Allocate the registers and make them available to the driver.
 463          * The registers that we care about for NIC mode are in BAR 0
 464          */
 465         sc->regs_rid = PCIR_BAR(0);
 466         if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 467             &sc->regs_rid, RF_ACTIVE)) == NULL) {
 468                 device_printf(dev, "Cannot allocate BAR region 0\n");
 469                 return (ENXIO);
 470         }
 471         sc->udbs_rid = PCIR_BAR(2);
 472         if ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 473            &sc->udbs_rid, RF_ACTIVE)) == NULL) {
 474                 device_printf(dev, "Cannot allocate BAR region 1\n");
 475                 error = ENXIO;
 476                 goto out;
 477        }
 478
 479         snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
 480             device_get_unit(dev));
 481         ADAPTER_LOCK_INIT(sc, sc->lockbuf);
 482
 483         snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
 484             device_get_unit(dev));
 485         snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
 486             device_get_unit(dev));
 487         snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
 488             device_get_unit(dev));
 489
 490         MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
 491         MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
 492         MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
 493
 494         sc->bt = rman_get_bustag(sc->regs_res);
 495         sc->bh = rman_get_bushandle(sc->regs_res);
 496         sc->mmio_len = rman_get_size(sc->regs_res);
 497
 498         if (t3_prep_adapter(sc, ai, 1) < 0) {
 499                 printf("prep adapter failed\n");
 500                 error = ENODEV;
 501                 goto out;
 502         }
 503         /* Allocate the BAR for doing MSI-X.  If it succeeds, try to allocate
 504          * enough messages for the queue sets.  If that fails, try falling
 505          * back to MSI.  If that fails, then try falling back to the legacy
 506          * interrupt pin model.
 507          */
 508 #ifdef MSI_SUPPORTED
 509
 510         sc->msix_regs_rid = 0x20;
 511         if ((msi_allowed >= 2) &&
 512             (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 513             &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
 514
 515                 msi_needed = sc->msi_count = SGE_MSIX_COUNT;
 516
 517                 if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
 518                     (sc->msi_count != msi_needed)) {
 519                         device_printf(dev, "msix allocation failed - msi_count = %d"
 520                             " msi_needed=%d will try msi err=%d\n", sc->msi_count,
 521                             msi_needed, error);
 522                         sc->msi_count = 0;
 523                         pci_release_msi(dev);
 524                         bus_release_resource(dev, SYS_RES_MEMORY,
 525                             sc->msix_regs_rid, sc->msix_regs_res);
 526                         sc->msix_regs_res = NULL;
 527                 } else {
 528                         sc->flags |= USING_MSIX;
 529                         sc->cxgb_intr = t3_intr_msix;
 530                 }
 531         }
 532
 533         if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
 534                 sc->msi_count = 1;
 535                 if (pci_alloc_msi(dev, &sc->msi_count)) {
 536                         device_printf(dev, "alloc msi failed - will try INTx\n");
 537                         sc->msi_count = 0;
 538                         pci_release_msi(dev);
 539                 } else {
 540                         sc->flags |= USING_MSI;
 541                         sc->irq_rid = 1;
 542                         sc->cxgb_intr = t3_intr_msi;
 543                 }
 544         }
 545 #endif
 546         if (sc->msi_count == 0) {
 547                 device_printf(dev, "using line interrupts\n");
 548                 sc->irq_rid = 0;
 549                 sc->cxgb_intr = t3b_intr;
 550         }
 551
 552         if ((sc->flags & USING_MSIX) && !singleq)
 553                 port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
 554
 555         /* Create a private taskqueue thread for handling driver events */
 556 #ifdef TASKQUEUE_CURRENT
 557         sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
 558             taskqueue_thread_enqueue, &sc->tq);
 559 #else
 560         sc->tq = taskqueue_create_fast("cxgb_taskq", M_NOWAIT,
 561             taskqueue_thread_enqueue, &sc->tq);
 562 #endif
 563         if (sc->tq == NULL) {
 564                 device_printf(dev, "failed to allocate controller task queue\n");
 565                 goto out;
 566         }
 567
 568         taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
 569             device_get_nameunit(dev));
 570         TASK_INIT(&sc->ext_intr_task, 0, cxgb_ext_intr_handler, sc);
 571         TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
 572
 573
 574         /* Create a periodic callout for checking adapter status */
 575         callout_init(&sc->cxgb_tick_ch, TRUE);
 576
 577         if ((t3_check_fw_version(sc, &must_load) != 0 && must_load) || force_fw_update) {
 578                 /*
 579                  * Warn user that a firmware update will be attempted in init.
 580                  */
 581                 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
 582                     FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 583                 sc->flags &= ~FW_UPTODATE;
 584         } else {
 585                 sc->flags |= FW_UPTODATE;
 586         }
 587
 588         if (t3_check_tpsram_version(sc, &must_load) != 0 && must_load) {
 589                 /*
 590                  * Warn user that a firmware update will be attempted in init.
 591                  */
 592                 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
 593                     t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 594                 sc->flags &= ~TPS_UPTODATE;
 595         } else {
 596                 sc->flags |= TPS_UPTODATE;
 597         }
 598
 599         /*
 600          * Create a child device for each MAC.  The ethernet attachment
 601          * will be done in these children.
 602          */
 603         for (i = 0; i < (sc)->params.nports; i++) {
 604                 struct port_info *pi;
 605
 606                 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
 607                         device_printf(dev, "failed to add child port\n");
 608                         error = EINVAL;
 609                         goto out;
 610                 }
 611                 pi = &sc->port[i];
 612                 pi->adapter = sc;
 613                 pi->nqsets = port_qsets;
 614                 pi->first_qset = i*port_qsets;
 615                 pi->port_id = i;
 616                 pi->tx_chan = i >= ai->nports0;
 617                 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
 618                 sc->rxpkt_map[pi->txpkt_intf] = i;
 619                 sc->port[i].tx_chan = i >= ai->nports0;
 620                 sc->portdev[i] = child;
 621                 device_set_softc(child, pi);
 622         }
 623         if ((error = bus_generic_attach(dev)) != 0)
 624                 goto out;
 625
 626         /*
 627          * XXX need to poll for link status
 628          */
 629         sc->params.stats_update_period = 1;
 630
 631         /* initialize sge private state */
 632         t3_sge_init_adapter(sc);
 633
 634         t3_led_ready(sc);
 635
 636         cxgb_offload_init();
 637         if (is_offload(sc)) {
 638                 setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
 639                 cxgb_adapter_ofld(sc);
 640         }
 641         error = t3_get_fw_version(sc, &vers);
 642         if (error)
 643                 goto out;
 644
 645         snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
 646             G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
 647             G_FW_VERSION_MICRO(vers));
 648
 649         device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
 650         callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
 651         t3_add_attach_sysctls(sc);
 652 out:
 653         if (error)
 654                 cxgb_free(sc);
 655
 656         return (error);
 657 }
 658
 659 static int
 660 cxgb_controller_detach(device_t dev)
 661 {
 662         struct adapter *sc;
 663
 664         sc = device_get_softc(dev);
 665
 666         cxgb_free(sc);
 667
 668         return (0);
 669 }
 670
 671 static void
 672 cxgb_free(struct adapter *sc)
 673 {
 674         int i;
 675
 676         ADAPTER_LOCK(sc);
 677         sc->flags |= CXGB_SHUTDOWN;
 678         ADAPTER_UNLOCK(sc);
 679         cxgb_pcpu_shutdown_threads(sc);
 680         ADAPTER_LOCK(sc);
 681
 682 /*
 683  * drops the lock
 684  */
 685         cxgb_down_locked(sc);
 686
 687 #ifdef MSI_SUPPORTED
 688         if (sc->flags & (USING_MSI | USING_MSIX)) {
 689                 device_printf(sc->dev, "releasing msi message(s)\n");
 690                 pci_release_msi(sc->dev);
 691         } else {
 692                 device_printf(sc->dev, "no msi message to release\n");
 693         }
 694 #endif
 695         if (sc->msix_regs_res != NULL) {
 696                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
 697                     sc->msix_regs_res);
 698         }
 699
 700         t3_sge_deinit_sw(sc);
 701         /*
 702          * Wait for last callout
 703          */
 704
 705         DELAY(hz*100);
 706
 707         for (i = 0; i < (sc)->params.nports; ++i) {
 708                 if (sc->portdev[i] != NULL)
 709                         device_delete_child(sc->dev, sc->portdev[i]);
 710         }
 711
 712         bus_generic_detach(sc->dev);
 713         if (sc->tq != NULL) {
 714                 taskqueue_free(sc->tq);
 715                 sc->tq = NULL;
 716         }
 717
 718         if (is_offload(sc)) {
 719                 cxgb_adapter_unofld(sc);
 720                 if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT))
 721                         offload_close(&sc->tdev);
 722                 else
 723                         printf("cxgb_free: DEVMAP_BIT not set\n");
 724         } else
 725                 printf("not offloading set\n");
 726 #ifdef notyet
 727         if (sc->flags & CXGB_OFLD_INIT)
 728                 cxgb_offload_deactivate(sc);
 729 #endif
 730         free(sc->filters, M_DEVBUF);
 731         t3_sge_free(sc);
 732
 733         cxgb_offload_exit();
 734
 735         if (sc->udbs_res != NULL)
 736                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
 737                     sc->udbs_res);
 738
 739         if (sc->regs_res != NULL)
 740                 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
 741                     sc->regs_res);
 742
 743         MTX_DESTROY(&sc->mdio_lock);
 744         MTX_DESTROY(&sc->sge.reg_lock);
 745         MTX_DESTROY(&sc->elmer_lock);
 746         ADAPTER_LOCK_DEINIT(sc);
 747 }
 748
 749 /**
 750  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 751  *      @sc: the controller softc
 752  *
 753  *      Determines how many sets of SGE queues to use and initializes them.
 754  *      We support multiple queue sets per port if we have MSI-X, otherwise
 755  *      just one queue set per port.
 756  */
 757 static int
 758 setup_sge_qsets(adapter_t *sc)
 759 {
 760         int i, j, err, irq_idx = 0, qset_idx = 0;
 761         u_int ntxq = SGE_TXQ_PER_SET;
 762
 763         if ((err = t3_sge_alloc(sc)) != 0) {
 764                 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
 765                 return (err);
 766         }
 767
 768         if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
 769                 irq_idx = -1;
 770
 771         for (i = 0; i < (sc)->params.nports; i++) {
 772                 struct port_info *pi = &sc->port[i];
 773
 774                 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
 775                         err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
 776                             (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
 777                             &sc->params.sge.qset[qset_idx], ntxq, pi);
 778                         if (err) {
 779                                 t3_free_sge_resources(sc);
 780                                 device_printf(sc->dev, "t3_sge_alloc_qset failed with %d\n",
 781                                     err);
 782                                 return (err);
 783                         }
 784                 }
 785         }
 786
 787         return (0);
 788 }
 789
 790 static void
 791 cxgb_teardown_msix(adapter_t *sc)
 792 {
 793         int i, nqsets;
 794
 795         for (nqsets = i = 0; i < (sc)->params.nports; i++)
 796                 nqsets += sc->port[i].nqsets;
 797
 798         for (i = 0; i < nqsets; i++) {
 799                 if (sc->msix_intr_tag[i] != NULL) {
 800                         bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
 801                             sc->msix_intr_tag[i]);
 802                         sc->msix_intr_tag[i] = NULL;
 803                 }
 804                 if (sc->msix_irq_res[i] != NULL) {
 805                         bus_release_resource(sc->dev, SYS_RES_IRQ,
 806                             sc->msix_irq_rid[i], sc->msix_irq_res[i]);
 807                         sc->msix_irq_res[i] = NULL;
 808                 }
 809         }
 810 }
 811
 812 static int
 813 cxgb_setup_msix(adapter_t *sc, int msix_count)
 814 {
 815         int i, j, k, nqsets, rid;
 816
 817         /* The first message indicates link changes and error conditions */
 818         sc->irq_rid = 1;
 819         if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
 820            &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
 821                 device_printf(sc->dev, "Cannot allocate msix interrupt\n");
 822                 return (EINVAL);
 823         }
 824
 825         if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
 826 #ifdef INTR_FILTERS
 827                 NULL,
 828 #endif
 829                 cxgb_async_intr, sc, &sc->intr_tag)) {
 830                 device_printf(sc->dev, "Cannot set up interrupt\n");
 831                 return (EINVAL);
 832         }
 833         for (i = k = 0; i < (sc)->params.nports; i++) {
 834                 nqsets = sc->port[i].nqsets;
 835                 for (j = 0; j < nqsets; j++, k++) {
 836                         struct sge_qset *qs = &sc->sge.qs[k];
 837
 838                         rid = k + 2;
 839                         if (cxgb_debug)
 840                                 printf("rid=%d ", rid);
 841                         if ((sc->msix_irq_res[k] = bus_alloc_resource_any(
 842                             sc->dev, SYS_RES_IRQ, &rid,
 843                             RF_SHAREABLE | RF_ACTIVE)) == NULL) {
 844                                 device_printf(sc->dev, "Cannot allocate "
 845                                     "interrupt for message %d\n", rid);
 846                                 return (EINVAL);
 847                         }
 848                         sc->msix_irq_rid[k] = rid;
 849                         if (bus_setup_intr(sc->dev, sc->msix_irq_res[k],
 850                                 INTR_MPSAFE|INTR_TYPE_NET,
 851 #ifdef INTR_FILTERS
 852                                 NULL,
 853 #endif
 854                                 t3_intr_msix, qs, &sc->msix_intr_tag[k])) {
 855                                 device_printf(sc->dev, "Cannot set up "
 856                                     "interrupt for message %d\n", rid);
 857                                 return (EINVAL);
 858                         }
 859 #ifdef IFNET_MULTIQUEUE
 860                         if (singleq == 0) {
 861                                 int vector = rman_get_start(sc->msix_irq_res[k]);
 862                                 if (bootverbose)
 863                                         device_printf(sc->dev, "binding vector=%d to cpu=%d\n", vector, k % mp_ncpus);
 864                                 intr_bind(vector, k % mp_ncpus);
 865                         }
 866 #endif
 867                 }
 868         }
 869
 870         return (0);
 871 }
 872
 873 static int
 874 cxgb_port_probe(device_t dev)
 875 {
 876         struct port_info *p;
 877         char buf[80];
 878         const char *desc;
 879
 880         p = device_get_softc(dev);
 881         desc = p->phy.desc;
 882         snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
 883         device_set_desc_copy(dev, buf);
 884         return (0);
 885 }
 886
 887
 888 static int
 889 cxgb_makedev(struct port_info *pi)
 890 {
 891
 892         pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
 893             UID_ROOT, GID_WHEEL, 0600, if_name(pi->ifp));
 894
 895         if (pi->port_cdev == NULL)
 896                 return (ENOMEM);
 897
 898         pi->port_cdev->si_drv1 = (void *)pi;
 899
 900         return (0);
 901 }
 902
 903
 904 #ifdef TSO_SUPPORTED
 905 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU)
 906 /* Don't enable TSO6 yet */
 907 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4 | IFCAP_JUMBO_MTU)
 908 #else
 909 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_JUMBO_MTU)
 910 /* Don't enable TSO6 yet */
 911 #define CXGB_CAP_ENABLE (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM |  IFCAP_JUMBO_MTU)
 912 #define IFCAP_TSO4 0x0
 913 #define IFCAP_TSO6 0x0
 914 #define CSUM_TSO   0x0
 915 #endif
 916
 917
 918 static int
 919 cxgb_port_attach(device_t dev)
 920 {
 921         struct port_info *p;
 922         struct ifnet *ifp;
 923         int err, media_flags;
 924         struct adapter *sc;
 925
 926
 927         p = device_get_softc(dev);
 928         sc = p->adapter;
 929         snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
 930             device_get_unit(device_get_parent(dev)), p->port_id);
 931         PORT_LOCK_INIT(p, p->lockbuf);
 932
 933         /* Allocate an ifnet object and set it up */
 934         ifp = p->ifp = if_alloc(IFT_ETHER);
 935         if (ifp == NULL) {
 936                 device_printf(dev, "Cannot allocate ifnet\n");
 937                 return (ENOMEM);
 938         }
 939
 940         /*
 941          * Note that there is currently no watchdog timer.
 942          */
 943         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 944         ifp->if_init = cxgb_init;
 945         ifp->if_softc = p;
 946         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 947         ifp->if_ioctl = cxgb_ioctl;
 948         ifp->if_start = cxgb_start;
 949
 950 #if 0
 951 #ifdef IFNET_MULTIQUEUE
 952         ifp->if_flags |= IFF_MULTIQ;
 953         ifp->if_mq_start = cxgb_pcpu_start;
 954 #endif
 955 #endif
 956         ifp->if_timer = 0;      /* Disable ifnet watchdog */
 957         ifp->if_watchdog = NULL;
 958
 959         ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
 960         IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
 961         IFQ_SET_READY(&ifp->if_snd);
 962
 963         ifp->if_hwassist = ifp->if_capabilities = ifp->if_capenable = 0;
 964         ifp->if_capabilities |= CXGB_CAP;
 965         ifp->if_capenable |= CXGB_CAP_ENABLE;
 966         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO);
 967         /*
 968          * disable TSO on 4-port - it isn't supported by the firmware yet
 969          */
 970         if (p->adapter->params.nports > 2) {
 971                 ifp->if_capabilities &= ~(IFCAP_TSO4 | IFCAP_TSO6);
 972                 ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TSO6);
 973                 ifp->if_hwassist &= ~CSUM_TSO;
 974         }
 975
 976         ether_ifattach(ifp, p->hw_addr);
 977         /*
 978          * Only default to jumbo frames on 10GigE
 979          */
 980         if (p->adapter->params.nports <= 2)
 981                 ifp->if_mtu = 9000;
 982         if ((err = cxgb_makedev(p)) != 0) {
 983                 printf("makedev failed %d\n", err);
 984                 return (err);
 985         }
 986         ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
 987             cxgb_media_status);
 988
 989         if (!strcmp(p->phy.desc,        "10GBASE-CX4")) {
 990                 media_flags = IFM_ETHER | IFM_10G_CX4 | IFM_FDX;
 991         } else if (!strcmp(p->phy.desc, "10GBASE-SR")) {
 992                 media_flags = IFM_ETHER | IFM_10G_SR | IFM_FDX;
 993         } else if (!strcmp(p->phy.desc, "10GBASE-R")) {
 994                 media_flags = IFM_ETHER | IFM_10G_LR | IFM_FDX;
 995         } else if (!strcmp(p->phy.desc, "10/100/1000BASE-T")) {
 996                 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T, 0, NULL);
 997                 ifmedia_add(&p->media, IFM_ETHER | IFM_10_T | IFM_FDX,
 998                             0, NULL);
 999                 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX,
1000                             0, NULL);
1001                 ifmedia_add(&p->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1002                             0, NULL);
1003                 ifmedia_add(&p->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
1004                             0, NULL);
1005                 media_flags = 0;
1006         } else {
1007                 printf("unsupported media type %s\n", p->phy.desc);
1008                 return (ENXIO);
1009         }
1010         if (media_flags) {
1011                 ifmedia_add(&p->media, media_flags, 0, NULL);
1012                 ifmedia_set(&p->media, media_flags);
1013         } else {
1014                 ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1015                 ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO);
1016         }
1017
1018
1019         snprintf(p->taskqbuf, TASKQ_NAME_LEN, "cxgb_port_taskq%d", p->port_id);
1020 #ifdef TASKQUEUE_CURRENT
1021         /* Create a port for handling TX without starvation */
1022         p->tq = taskqueue_create(p->taskqbuf, M_NOWAIT,
1023             taskqueue_thread_enqueue, &p->tq);
1024 #else
1025         /* Create a port for handling TX without starvation */
1026         p->tq = taskqueue_create_fast(p->taskqbuf, M_NOWAIT,
1027             taskqueue_thread_enqueue, &p->tq);
1028 #endif
1029         /* Get the latest mac address, User can use a LAA */
1030         bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN);
1031         t3_sge_init_port(p);
1032 #if defined(LINK_ATTACH)
1033         cxgb_link_start(p);
1034         t3_link_changed(sc, p->port_id);
1035 #endif
1036         return (0);
1037 }
1038
1039 static int
1040 cxgb_port_detach(device_t dev)
1041 {
1042         struct port_info *p;
1043
1044         p = device_get_softc(dev);
1045
1046         PORT_LOCK(p);
1047         if (p->ifp->if_drv_flags & IFF_DRV_RUNNING)
1048                 cxgb_stop_locked(p);
1049         PORT_UNLOCK(p);
1050
1051         if (p->tq != NULL) {
1052                 taskqueue_drain(p->tq, &p->start_task);
1053                 taskqueue_free(p->tq);
1054                 p->tq = NULL;
1055         }
1056
1057         ether_ifdetach(p->ifp);
1058         printf("waiting for callout to stop ...");
1059         DELAY(1000000);
1060         printf("done\n");
1061         /*
1062          * the lock may be acquired in ifdetach
1063          */
1064         PORT_LOCK_DEINIT(p);
1065         if_free(p->ifp);
1066
1067         if (p->port_cdev != NULL)
1068                 destroy_dev(p->port_cdev);
1069
1070         return (0);
1071 }
1072
1073 void
1074 t3_fatal_err(struct adapter *sc)
1075 {
1076         u_int fw_status[4];
1077
1078         if (sc->flags & FULL_INIT_DONE) {
1079                 t3_sge_stop(sc);
1080                 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1081                 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1082                 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1083                 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1084                 t3_intr_disable(sc);
1085         }
1086         device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1087         if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1088                 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1089                     fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1090 }
1091
1092 int
1093 t3_os_find_pci_capability(adapter_t *sc, int cap)
1094 {
1095         device_t dev;
1096         struct pci_devinfo *dinfo;
1097         pcicfgregs *cfg;
1098         uint32_t status;
1099         uint8_t ptr;
1100
1101         dev = sc->dev;
1102         dinfo = device_get_ivars(dev);
1103         cfg = &dinfo->cfg;
1104
1105         status = pci_read_config(dev, PCIR_STATUS, 2);
1106         if (!(status & PCIM_STATUS_CAPPRESENT))
1107                 return (0);
1108
1109         switch (cfg->hdrtype & PCIM_HDRTYPE) {
1110         case 0:
1111         case 1:
1112                 ptr = PCIR_CAP_PTR;
1113                 break;
1114         case 2:
1115                 ptr = PCIR_CAP_PTR_2;
1116                 break;
1117         default:
1118                 return (0);
1119                 break;
1120         }
1121         ptr = pci_read_config(dev, ptr, 1);
1122
1123         while (ptr != 0) {
1124                 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1125                         return (ptr);
1126                 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1127         }
1128
1129         return (0);
1130 }
1131
1132 int
1133 t3_os_pci_save_state(struct adapter *sc)
1134 {
1135         device_t dev;
1136         struct pci_devinfo *dinfo;
1137
1138         dev = sc->dev;
1139         dinfo = device_get_ivars(dev);
1140
1141         pci_cfg_save(dev, dinfo, 0);
1142         return (0);
1143 }
1144
1145 int
1146 t3_os_pci_restore_state(struct adapter *sc)
1147 {
1148         device_t dev;
1149         struct pci_devinfo *dinfo;
1150
1151         dev = sc->dev;
1152         dinfo = device_get_ivars(dev);
1153
1154         pci_cfg_restore(dev, dinfo);
1155         return (0);
1156 }
1157
1158 /**
1159  *      t3_os_link_changed - handle link status changes
1160  *      @adapter: the adapter associated with the link change
1161  *      @port_id: the port index whose limk status has changed
1162  *      @link_status: the new status of the link
1163  *      @speed: the new speed setting
1164  *      @duplex: the new duplex setting
1165  *      @fc: the new flow-control setting
1166  *
1167  *      This is the OS-dependent handler for link status changes.  The OS
1168  *      neutral handler takes care of most of the processing for these events,
1169  *      then calls this handler for any OS-specific processing.
1170  */
1171 void
1172 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1173      int duplex, int fc)
1174 {
1175         struct port_info *pi = &adapter->port[port_id];
1176         struct cmac *mac = &adapter->port[port_id].mac;
1177
1178         if (link_status) {
1179                 DELAY(10);
1180                 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1181                         /* Clear errors created by MAC enable */
1182                         t3_set_reg_field(adapter,
1183                                          A_XGM_STAT_CTRL + pi->mac.offset,
1184                                          F_CLRSTATS, 1);
1185                 if_link_state_change(pi->ifp, LINK_STATE_UP);
1186
1187         } else {
1188                 pi->phy.ops->power_down(&pi->phy, 1);
1189                 t3_mac_disable(mac, MAC_DIRECTION_RX);
1190                 t3_link_start(&pi->phy, mac, &pi->link_config);
1191                 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1192                 if_link_state_change(pi->ifp, LINK_STATE_DOWN);
1193         }
1194 }
1195
1196 /*
1197  * Interrupt-context handler for external (PHY) interrupts.
1198  */
1199 void
1200 t3_os_ext_intr_handler(adapter_t *sc)
1201 {
1202         if (cxgb_debug)
1203                 printf("t3_os_ext_intr_handler\n");
1204         /*
1205          * Schedule a task to handle external interrupts as they may be slow
1206          * and we use a mutex to protect MDIO registers.  We disable PHY
1207          * interrupts in the meantime and let the task reenable them when
1208          * it's done.
1209          */
1210         ADAPTER_LOCK(sc);
1211         if (sc->slow_intr_mask) {
1212                 sc->slow_intr_mask &= ~F_T3DBG;
1213                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
1214                 taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
1215         }
1216         ADAPTER_UNLOCK(sc);
1217 }
1218
1219 void
1220 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1221 {
1222
1223         /*
1224          * The ifnet might not be allocated before this gets called,
1225          * as this is called early on in attach by t3_prep_adapter
1226          * save the address off in the port structure
1227          */
1228         if (cxgb_debug)
1229                 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1230         bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1231 }
1232
1233 /**
1234  *      link_start - enable a port
1235  *      @p: the port to enable
1236  *
1237  *      Performs the MAC and PHY actions needed to enable a port.
1238  */
1239 static void
1240 cxgb_link_start(struct port_info *p)
1241 {
1242         struct ifnet *ifp;
1243         struct t3_rx_mode rm;
1244         struct cmac *mac = &p->mac;
1245
1246         ifp = p->ifp;
1247
1248         t3_init_rx_mode(&rm, p);
1249         if (!mac->multiport)
1250                 t3_mac_reset(mac);
1251         t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1252         t3_mac_set_address(mac, 0, p->hw_addr);
1253         t3_mac_set_rx_mode(mac, &rm);
1254         t3_link_start(&p->phy, mac, &p->link_config);
1255         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1256 }
1257
1258
1259 static int
1260 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1261                               unsigned long n)
1262 {
1263         int attempts = 5;
1264
1265         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1266                 if (!--attempts)
1267                         return (ETIMEDOUT);
1268                 t3_os_sleep(10);
1269         }
1270         return 0;
1271 }
1272
1273 static int
1274 init_tp_parity(struct adapter *adap)
1275 {
1276         int i;
1277         struct mbuf *m;
1278         struct cpl_set_tcb_field *greq;
1279         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1280
1281         t3_tp_set_offload_mode(adap, 1);
1282
1283         for (i = 0; i < 16; i++) {
1284                 struct cpl_smt_write_req *req;
1285
1286                 m = m_gethdr(M_WAITOK, MT_DATA);
1287                 req = mtod(m, struct cpl_smt_write_req *);
1288                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1289                 memset(req, 0, sizeof(*req));
1290                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1291                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1292                 req->iff = i;
1293                 t3_mgmt_tx(adap, m);
1294         }
1295
1296         for (i = 0; i < 2048; i++) {
1297                 struct cpl_l2t_write_req *req;
1298
1299                 m = m_gethdr(M_WAITOK, MT_DATA);
1300                 req = mtod(m, struct cpl_l2t_write_req *);
1301                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1302                 memset(req, 0, sizeof(*req));
1303                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1304                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1305                 req->params = htonl(V_L2T_W_IDX(i));
1306                 t3_mgmt_tx(adap, m);
1307         }
1308
1309         for (i = 0; i < 2048; i++) {
1310                 struct cpl_rte_write_req *req;
1311
1312                 m = m_gethdr(M_WAITOK, MT_DATA);
1313                 req = mtod(m, struct cpl_rte_write_req *);
1314                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1315                 memset(req, 0, sizeof(*req));
1316                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1317                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1318                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1319                 t3_mgmt_tx(adap, m);
1320         }
1321
1322         m = m_gethdr(M_WAITOK, MT_DATA);
1323         greq = mtod(m, struct cpl_set_tcb_field *);
1324         m->m_len = m->m_pkthdr.len = sizeof(*greq);
1325         memset(greq, 0, sizeof(*greq));
1326         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1327         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1328         greq->mask = htobe64(1);
1329         t3_mgmt_tx(adap, m);
1330
1331         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1332         t3_tp_set_offload_mode(adap, 0);
1333         return (i);
1334 }
1335
1336 /**
1337  *      setup_rss - configure Receive Side Steering (per-queue connection demux)
1338  *      @adap: the adapter
1339  *
1340  *      Sets up RSS to distribute packets to multiple receive queues.  We
1341  *      configure the RSS CPU lookup table to distribute to the number of HW
1342  *      receive queues, and the response queue lookup table to narrow that
1343  *      down to the response queues actually configured for each port.
1344  *      We always configure the RSS mapping for two ports since the mapping
1345  *      table has plenty of entries.
1346  */
1347 static void
1348 setup_rss(adapter_t *adap)
1349 {
1350         int i;
1351         u_int nq[2];
1352         uint8_t cpus[SGE_QSETS + 1];
1353         uint16_t rspq_map[RSS_TABLE_SIZE];
1354
1355         for (i = 0; i < SGE_QSETS; ++i)
1356                 cpus[i] = i;
1357         cpus[SGE_QSETS] = 0xff;
1358
1359         nq[0] = nq[1] = 0;
1360         for_each_port(adap, i) {
1361                 const struct port_info *pi = adap2pinfo(adap, i);
1362
1363                 nq[pi->tx_chan] += pi->nqsets;
1364         }
1365         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1366                 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1367                 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1368         }
1369         /* Calculate the reverse RSS map table */
1370         for (i = 0; i < RSS_TABLE_SIZE; ++i)
1371                 if (adap->rrss_map[rspq_map[i]] == 0xff)
1372                         adap->rrss_map[rspq_map[i]] = i;
1373
1374         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1375                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1376                       F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1377                       cpus, rspq_map);
1378
1379 }
1380
1381 /*
1382  * Sends an mbuf to an offload queue driver
1383  * after dealing with any active network taps.
1384  */
1385 static inline int
1386 offload_tx(struct t3cdev *tdev, struct mbuf *m)
1387 {
1388         int ret;
1389
1390         ret = t3_offload_tx(tdev, m);
1391         return (ret);
1392 }
1393
1394 static int
1395 write_smt_entry(struct adapter *adapter, int idx)
1396 {
1397         struct port_info *pi = &adapter->port[idx];
1398         struct cpl_smt_write_req *req;
1399         struct mbuf *m;
1400
1401         if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
1402                 return (ENOMEM);
1403
1404         req = mtod(m, struct cpl_smt_write_req *);
1405         m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req);
1406
1407         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1408         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
1409         req->mtu_idx = NMTUS - 1;  /* should be 0 but there's a T3 bug */
1410         req->iff = idx;
1411         memset(req->src_mac1, 0, sizeof(req->src_mac1));
1412         memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
1413
1414         m_set_priority(m, 1);
1415
1416         offload_tx(&adapter->tdev, m);
1417
1418         return (0);
1419 }
1420
1421 static int
1422 init_smt(struct adapter *adapter)
1423 {
1424         int i;
1425
1426         for_each_port(adapter, i)
1427                 write_smt_entry(adapter, i);
1428         return 0;
1429 }
1430
1431 static void
1432 init_port_mtus(adapter_t *adapter)
1433 {
1434         unsigned int mtus = adapter->port[0].ifp->if_mtu;
1435
1436         if (adapter->port[1].ifp)
1437                 mtus |= adapter->port[1].ifp->if_mtu << 16;
1438         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
1439 }
1440
1441 static void
1442 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1443                               int hi, int port)
1444 {
1445         struct mbuf *m;
1446         struct mngt_pktsched_wr *req;
1447
1448         m = m_gethdr(M_DONTWAIT, MT_DATA);
1449         if (m) {
1450                 req = mtod(m, struct mngt_pktsched_wr *);
1451                 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1452                 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1453                 req->sched = sched;
1454                 req->idx = qidx;
1455                 req->min = lo;
1456                 req->max = hi;
1457                 req->binding = port;
1458                 m->m_len = m->m_pkthdr.len = sizeof(*req);
1459                 t3_mgmt_tx(adap, m);
1460         }
1461 }
1462
1463 static void
1464 bind_qsets(adapter_t *sc)
1465 {
1466         int i, j;
1467
1468         cxgb_pcpu_startup_threads(sc);
1469         for (i = 0; i < (sc)->params.nports; ++i) {
1470                 const struct port_info *pi = adap2pinfo(sc, i);
1471
1472                 for (j = 0; j < pi->nqsets; ++j) {
1473                         send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1474                                           -1, pi->tx_chan);
1475
1476                 }
1477         }
1478 }
1479
1480 static void
1481 update_tpeeprom(struct adapter *adap)
1482 {
1483 #ifdef FIRMWARE_LATEST
1484         const struct firmware *tpeeprom;
1485 #else
1486         struct firmware *tpeeprom;
1487 #endif
1488
1489         uint32_t version;
1490         unsigned int major, minor;
1491         int ret, len;
1492         char rev;
1493
1494         t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1495
1496         major = G_TP_VERSION_MAJOR(version);
1497         minor = G_TP_VERSION_MINOR(version);
1498         if (major == TP_VERSION_MAJOR  && minor == TP_VERSION_MINOR)
1499                 return;
1500
1501         rev = t3rev2char(adap);
1502
1503         tpeeprom = firmware_get(TPEEPROM_NAME);
1504         if (tpeeprom == NULL) {
1505                 device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n",
1506                     TPEEPROM_NAME);
1507                 return;
1508         }
1509
1510         len = tpeeprom->datasize - 4;
1511
1512         ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1513         if (ret)
1514                 goto release_tpeeprom;
1515
1516         if (len != TP_SRAM_LEN) {
1517                 device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", TPEEPROM_NAME, len, TP_SRAM_LEN);
1518                 return;
1519         }
1520
1521         ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1522             TP_SRAM_OFFSET);
1523
1524         if (!ret) {
1525                 device_printf(adap->dev,
1526                         "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1527                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1528         } else
1529                 device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n");
1530
1531 release_tpeeprom:
1532         firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1533
1534         return;
1535 }
1536
1537 static int
1538 update_tpsram(struct adapter *adap)
1539 {
1540 #ifdef FIRMWARE_LATEST
1541         const struct firmware *tpsram;
1542 #else
1543         struct firmware *tpsram;
1544 #endif
1545         int ret;
1546         char rev;
1547
1548         rev = t3rev2char(adap);
1549         if (!rev)
1550                 return 0;
1551
1552         update_tpeeprom(adap);
1553
1554         tpsram = firmware_get(TPSRAM_NAME);
1555         if (tpsram == NULL){
1556                 device_printf(adap->dev, "could not load TP SRAM\n");
1557                 return (EINVAL);
1558         } else
1559                 device_printf(adap->dev, "updating TP SRAM\n");
1560
1561         ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1562         if (ret)
1563                 goto release_tpsram;
1564
1565         ret = t3_set_proto_sram(adap, tpsram->data);
1566         if (ret)
1567                 device_printf(adap->dev, "loading protocol SRAM failed\n");
1568
1569 release_tpsram:
1570         firmware_put(tpsram, FIRMWARE_UNLOAD);
1571
1572         return ret;
1573 }
1574
1575 /**
1576  *      cxgb_up - enable the adapter
1577  *      @adap: adapter being enabled
1578  *
1579  *      Called when the first port is enabled, this function performs the
1580  *      actions necessary to make an adapter operational, such as completing
1581  *      the initialization of HW modules, and enabling interrupts.
1582  *
1583  */
1584 static int
1585 cxgb_up(struct adapter *sc)
1586 {
1587         int err = 0;
1588
1589         if ((sc->flags & FULL_INIT_DONE) == 0) {
1590
1591                 if ((sc->flags & FW_UPTODATE) == 0)
1592                         if ((err = upgrade_fw(sc)))
1593                                 goto out;
1594                 if ((sc->flags & TPS_UPTODATE) == 0)
1595                         if ((err = update_tpsram(sc)))
1596                                 goto out;
1597                 err = t3_init_hw(sc, 0);
1598                 if (err)
1599                         goto out;
1600
1601                 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1602                 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1603
1604                 err = setup_sge_qsets(sc);
1605                 if (err)
1606                         goto out;
1607
1608                 setup_rss(sc);
1609                 t3_add_configured_sysctls(sc);
1610                 sc->flags |= FULL_INIT_DONE;
1611         }
1612
1613         t3_intr_clear(sc);
1614
1615         /* If it's MSI or INTx, allocate a single interrupt for everything */
1616         if ((sc->flags & USING_MSIX) == 0) {
1617                 if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
1618                    &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
1619                         device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n",
1620                             sc->irq_rid);
1621                         err = EINVAL;
1622                         goto out;
1623                 }
1624                 device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
1625
1626                 if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
1627 #ifdef INTR_FILTERS
1628                         NULL,
1629 #endif
1630                         sc->cxgb_intr, sc, &sc->intr_tag)) {
1631                         device_printf(sc->dev, "Cannot set up interrupt\n");
1632                         err = EINVAL;
1633                         goto irq_err;
1634                 }
1635         } else {
1636                 cxgb_setup_msix(sc, sc->msi_count);
1637         }
1638
1639         t3_sge_start(sc);
1640         t3_intr_enable(sc);
1641
1642         if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1643             is_offload(sc) && init_tp_parity(sc) == 0)
1644                 sc->flags |= TP_PARITY_INIT;
1645
1646         if (sc->flags & TP_PARITY_INIT) {
1647                 t3_write_reg(sc, A_TP_INT_CAUSE,
1648                                 F_CMCACHEPERR | F_ARPLUTPERR);
1649                 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1650         }
1651
1652
1653         if (!(sc->flags & QUEUES_BOUND)) {
1654                 bind_qsets(sc);
1655                 sc->flags |= QUEUES_BOUND;
1656         }
1657 out:
1658         return (err);
1659 irq_err:
1660         CH_ERR(sc, "request_irq failed, err %d\n", err);
1661         goto out;
1662 }
1663
1664
1665 /*
1666  * Release resources when all the ports and offloading have been stopped.
1667  */
1668 static void
1669 cxgb_down_locked(struct adapter *sc)
1670 {
1671
1672         t3_sge_stop(sc);
1673         t3_intr_disable(sc);
1674
1675         if (sc->intr_tag != NULL) {
1676                 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
1677                 sc->intr_tag = NULL;
1678         }
1679         if (sc->irq_res != NULL) {
1680                 device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
1681                     sc->irq_rid, sc->irq_res);
1682                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
1683                     sc->irq_res);
1684                 sc->irq_res = NULL;
1685         }
1686
1687         if (sc->flags & USING_MSIX)
1688                 cxgb_teardown_msix(sc);
1689
1690         callout_stop(&sc->cxgb_tick_ch);
1691         callout_stop(&sc->sge_timer_ch);
1692         callout_drain(&sc->cxgb_tick_ch);
1693         callout_drain(&sc->sge_timer_ch);
1694
1695         if (sc->tq != NULL) {
1696                 printf("draining slow intr\n");
1697
1698                 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1699                         printf("draining ext intr\n");
1700                 taskqueue_drain(sc->tq, &sc->ext_intr_task);
1701                 printf("draining tick task\n");
1702                 taskqueue_drain(sc->tq, &sc->tick_task);
1703         }
1704         ADAPTER_UNLOCK(sc);
1705 }
1706
1707 static int
1708 offload_open(struct port_info *pi)
1709 {
1710         struct adapter *adapter = pi->adapter;
1711         struct t3cdev *tdev = &adapter->tdev;
1712 #ifdef notyet
1713             T3CDEV(pi->ifp);
1714 #endif
1715         int adap_up = adapter->open_device_map & PORT_MASK;
1716         int err = 0;
1717
1718         CTR1(KTR_CXGB, "device_map=0x%x", adapter->open_device_map);
1719         if (atomic_cmpset_int(&adapter->open_device_map,
1720                 (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)),
1721                 (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0)
1722                 return (0);
1723
1724
1725         if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1726                 printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", adapter->open_device_map);
1727         ADAPTER_LOCK(pi->adapter);
1728         if (!adap_up)
1729                 err = cxgb_up(adapter);
1730         ADAPTER_UNLOCK(pi->adapter);
1731         if (err)
1732                 return (err);
1733
1734         t3_tp_set_offload_mode(adapter, 1);
1735         tdev->lldev = pi->ifp;
1736
1737         init_port_mtus(adapter);
1738         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1739                      adapter->params.b_wnd,
1740                      adapter->params.rev == 0 ?
1741                        adapter->port[0].ifp->if_mtu : 0xffff);
1742         init_smt(adapter);
1743
1744         /* Call back all registered clients */
1745         cxgb_add_clients(tdev);
1746
1747
1748         /* restore them in case the offload module has changed them */
1749         if (err) {
1750                 t3_tp_set_offload_mode(adapter, 0);
1751                 clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1752                 cxgb_set_dummy_ops(tdev);
1753         }
1754         return (err);
1755 }
1756
1757 static int
1758 offload_close(struct t3cdev *tdev)
1759 {
1760         struct adapter *adapter = tdev2adap(tdev);
1761
1762         if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
1763                 return (0);
1764
1765         /* Call back all registered clients */
1766         cxgb_remove_clients(tdev);
1767
1768         tdev->lldev = NULL;
1769         cxgb_set_dummy_ops(tdev);
1770         t3_tp_set_offload_mode(adapter, 0);
1771         clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
1772
1773         ADAPTER_LOCK(adapter);
1774         if (!adapter->open_device_map)
1775                 cxgb_down_locked(adapter);
1776         else
1777                 ADAPTER_UNLOCK(adapter);
1778         return (0);
1779 }
1780
1781
1782 static void
1783 cxgb_init(void *arg)
1784 {
1785         struct port_info *p = arg;
1786
1787         PORT_LOCK(p);
1788         cxgb_init_locked(p);
1789         PORT_UNLOCK(p);
1790 }
1791
1792 static void
1793 cxgb_init_locked(struct port_info *p)
1794 {
1795         struct ifnet *ifp;
1796         adapter_t *sc = p->adapter;
1797         int err;
1798
1799         PORT_LOCK_ASSERT_OWNED(p);
1800         ifp = p->ifp;
1801
1802         ADAPTER_LOCK(p->adapter);
1803         if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) {
1804                 ADAPTER_UNLOCK(p->adapter);
1805                 cxgb_stop_locked(p);
1806                 return;
1807         }
1808         if (p->adapter->open_device_map == 0) {
1809                 t3_intr_clear(sc);
1810         }
1811         setbit(&p->adapter->open_device_map, p->port_id);
1812         ADAPTER_UNLOCK(p->adapter);
1813
1814         if (is_offload(sc) && !ofld_disable) {
1815                 err = offload_open(p);
1816                 if (err)
1817                         log(LOG_WARNING,
1818                             "Could not initialize offload capabilities\n");
1819         }
1820 #if !defined(LINK_ATTACH)
1821         cxgb_link_start(p);
1822         t3_link_changed(sc, p->port_id);
1823 #endif
1824         ifp->if_baudrate = p->link_config.speed * 1000000;
1825
1826         device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id);
1827         t3_port_intr_enable(sc, p->port_id);
1828
1829         t3_sge_reset_adapter(sc);
1830
1831         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1832         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1833 }
1834
1835 static void
1836 cxgb_set_rxmode(struct port_info *p)
1837 {
1838         struct t3_rx_mode rm;
1839         struct cmac *mac = &p->mac;
1840
1841         t3_init_rx_mode(&rm, p);
1842         mtx_lock(&p->adapter->mdio_lock);
1843         t3_mac_set_rx_mode(mac, &rm);
1844         mtx_unlock(&p->adapter->mdio_lock);
1845 }
1846
1847 static void
1848 cxgb_stop_locked(struct port_info *pi)
1849 {
1850         struct ifnet *ifp;
1851
1852         PORT_LOCK_ASSERT_OWNED(pi);
1853         ADAPTER_LOCK_ASSERT_NOTOWNED(pi->adapter);
1854
1855         ifp = pi->ifp;
1856         t3_port_intr_disable(pi->adapter, pi->port_id);
1857         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1858
1859         /* disable pause frames */
1860         t3_set_reg_field(pi->adapter, A_XGM_TX_CFG + pi->mac.offset,
1861                          F_TXPAUSEEN, 0);
1862
1863         /* Reset RX FIFO HWM */
1864         t3_set_reg_field(pi->adapter, A_XGM_RXFIFO_CFG +  pi->mac.offset,
1865                          V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1866
1867
1868         ADAPTER_LOCK(pi->adapter);
1869         clrbit(&pi->adapter->open_device_map, pi->port_id);
1870
1871         if (pi->adapter->open_device_map == 0) {
1872                 cxgb_down_locked(pi->adapter);
1873         } else
1874                 ADAPTER_UNLOCK(pi->adapter);
1875
1876 #if !defined(LINK_ATTACH)
1877         DELAY(100);
1878
1879         /* Wait for TXFIFO empty */
1880         t3_wait_op_done(pi->adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
1881                         F_TXFIFO_EMPTY, 1, 20, 5);
1882
1883         DELAY(100);
1884         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1885
1886         pi->phy.ops->power_down(&pi->phy, 1);
1887 #endif
1888
1889 }
1890
1891 static int
1892 cxgb_set_mtu(struct port_info *p, int mtu)
1893 {
1894         struct ifnet *ifp = p->ifp;
1895         int error = 0;
1896
1897         if ((mtu < ETHERMIN) || (mtu > ETHER_MAX_LEN_JUMBO))
1898                 error = EINVAL;
1899         else if (ifp->if_mtu != mtu) {
1900                 PORT_LOCK(p);
1901                 ifp->if_mtu = mtu;
1902                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1903                         cxgb_stop_locked(p);
1904                         cxgb_init_locked(p);
1905                 }
1906                 PORT_UNLOCK(p);
1907         }
1908         return (error);
1909 }
1910
1911 static int
1912 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1913 {
1914         struct port_info *p = ifp->if_softc;
1915         struct ifaddr *ifa = (struct ifaddr *)data;
1916         struct ifreq *ifr = (struct ifreq *)data;
1917         int flags, error = 0;
1918         uint32_t mask;
1919
1920         /*
1921          * XXX need to check that we aren't in the middle of an unload
1922          */
1923         switch (command) {
1924         case SIOCSIFMTU:
1925                 error = cxgb_set_mtu(p, ifr->ifr_mtu);
1926                 break;
1927         case SIOCSIFADDR:
1928                 if (ifa->ifa_addr->sa_family == AF_INET) {
1929                         ifp->if_flags |= IFF_UP;
1930                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1931                                 PORT_LOCK(p);
1932                                 cxgb_init_locked(p);
1933                                 PORT_UNLOCK(p);
1934                         }
1935                         arp_ifinit(ifp, ifa);
1936                 } else
1937                         error = ether_ioctl(ifp, command, data);
1938                 break;
1939         case SIOCSIFFLAGS:
1940                 PORT_LOCK(p);
1941                 if (ifp->if_flags & IFF_UP) {
1942                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1943                                 flags = p->if_flags;
1944                                 if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1945                                     ((ifp->if_flags ^ flags) & IFF_ALLMULTI))
1946                                         cxgb_set_rxmode(p);
1947                         } else
1948                                 cxgb_init_locked(p);
1949                         p->if_flags = ifp->if_flags;
1950                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1951                         cxgb_stop_locked(p);
1952
1953                 PORT_UNLOCK(p);
1954                 break;
1955         case SIOCADDMULTI:
1956         case SIOCDELMULTI:
1957                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1958                         cxgb_set_rxmode(p);
1959                 }
1960                 break;
1961         case SIOCSIFMEDIA:
1962         case SIOCGIFMEDIA:
1963                 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
1964                 break;
1965         case SIOCSIFCAP:
1966                 PORT_LOCK(p);
1967                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1968                 if (mask & IFCAP_TXCSUM) {
1969                         if (IFCAP_TXCSUM & ifp->if_capenable) {
1970                                 ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4);
1971                                 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP
1972                                     | CSUM_TSO);
1973                         } else {
1974                                 ifp->if_capenable |= IFCAP_TXCSUM;
1975                                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1976                         }
1977                 } else if (mask & IFCAP_RXCSUM) {
1978                         if (IFCAP_RXCSUM & ifp->if_capenable) {
1979                                 ifp->if_capenable &= ~IFCAP_RXCSUM;
1980                         } else {
1981                                 ifp->if_capenable |= IFCAP_RXCSUM;
1982                         }
1983                 }
1984                 if (mask & IFCAP_TSO4) {
1985                         if (IFCAP_TSO4 & ifp->if_capenable) {
1986                                 ifp->if_capenable &= ~IFCAP_TSO4;
1987                                 ifp->if_hwassist &= ~CSUM_TSO;
1988                         } else if (IFCAP_TXCSUM & ifp->if_capenable) {
1989                                 ifp->if_capenable |= IFCAP_TSO4;
1990                                 ifp->if_hwassist |= CSUM_TSO;
1991                         } else {
1992                                 if (cxgb_debug)
1993                                         printf("cxgb requires tx checksum offload"
1994                                             " be enabled to use TSO\n");
1995                                 error = EINVAL;
1996                         }
1997                 }
1998                 PORT_UNLOCK(p);
1999                 break;
2000         default:
2001                 error = ether_ioctl(ifp, command, data);
2002                 break;
2003         }
2004         return (error);
2005 }
2006
2007 static int
2008 cxgb_media_change(struct ifnet *ifp)
2009 {
2010         if_printf(ifp, "media change not supported\n");
2011         return (ENXIO);
2012 }
2013
2014 static void
2015 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2016 {
2017         struct port_info *p = ifp->if_softc;
2018
2019         ifmr->ifm_status = IFM_AVALID;
2020         ifmr->ifm_active = IFM_ETHER;
2021
2022         if (!p->link_config.link_ok)
2023                 return;
2024
2025         ifmr->ifm_status |= IFM_ACTIVE;
2026
2027         switch (p->link_config.speed) {
2028         case 10:
2029                 ifmr->ifm_active |= IFM_10_T;
2030                 break;
2031         case 100:
2032                 ifmr->ifm_active |= IFM_100_TX;
2033                         break;
2034         case 1000:
2035                 ifmr->ifm_active |= IFM_1000_T;
2036                 break;
2037         }
2038
2039         if (p->link_config.duplex)
2040                 ifmr->ifm_active |= IFM_FDX;
2041         else
2042                 ifmr->ifm_active |= IFM_HDX;
2043 }
2044
2045 static void
2046 cxgb_async_intr(void *data)
2047 {
2048         adapter_t *sc = data;
2049
2050         if (cxgb_debug)
2051                 device_printf(sc->dev, "cxgb_async_intr\n");
2052         /*
2053          * May need to sleep - defer to taskqueue
2054          */
2055         taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2056 }
2057
2058 static void
2059 cxgb_ext_intr_handler(void *arg, int count)
2060 {
2061         adapter_t *sc = (adapter_t *)arg;
2062
2063         if (cxgb_debug)
2064                 printf("cxgb_ext_intr_handler\n");
2065
2066         t3_phy_intr_handler(sc);
2067
2068         /* Now reenable external interrupts */
2069         ADAPTER_LOCK(sc);
2070         if (sc->slow_intr_mask) {
2071                 sc->slow_intr_mask |= F_T3DBG;
2072                 t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
2073                 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
2074         }
2075         ADAPTER_UNLOCK(sc);
2076 }
2077
2078 static void
2079 check_link_status(adapter_t *sc)
2080 {
2081         int i;
2082
2083         for (i = 0; i < (sc)->params.nports; ++i) {
2084                 struct port_info *p = &sc->port[i];
2085
2086                 if (!(p->phy.caps & SUPPORTED_IRQ))
2087                         t3_link_changed(sc, i);
2088                 p->ifp->if_baudrate = p->link_config.speed * 1000000;
2089         }
2090 }
2091
2092 static void
2093 check_t3b2_mac(struct adapter *adapter)
2094 {
2095         int i;
2096
2097         if(adapter->flags & CXGB_SHUTDOWN)
2098                 return;
2099
2100         for_each_port(adapter, i) {
2101                 struct port_info *p = &adapter->port[i];
2102                 struct ifnet *ifp = p->ifp;
2103                 int status;
2104
2105                 if(adapter->flags & CXGB_SHUTDOWN)
2106                         return;
2107
2108                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2109                         continue;
2110
2111                 status = 0;
2112                 PORT_LOCK(p);
2113                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
2114                         status = t3b2_mac_watchdog_task(&p->mac);
2115                 if (status == 1)
2116                         p->mac.stats.num_toggled++;
2117                 else if (status == 2) {
2118                         struct cmac *mac = &p->mac;
2119
2120                         t3_mac_set_mtu(mac, ifp->if_mtu + ETHER_HDR_LEN
2121                             + ETHER_VLAN_ENCAP_LEN);
2122                         t3_mac_set_address(mac, 0, p->hw_addr);
2123                         cxgb_set_rxmode(p);
2124                         t3_link_start(&p->phy, mac, &p->link_config);
2125                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2126                         t3_port_intr_enable(adapter, p->port_id);
2127                         p->mac.stats.num_resets++;
2128                 }
2129                 PORT_UNLOCK(p);
2130         }
2131 }
2132
2133 static void
2134 cxgb_tick(void *arg)
2135 {
2136         adapter_t *sc = (adapter_t *)arg;
2137
2138         if(sc->flags & CXGB_SHUTDOWN)
2139                 return;
2140
2141         taskqueue_enqueue(sc->tq, &sc->tick_task);
2142         callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2143 }
2144
2145 static void
2146 cxgb_tick_handler(void *arg, int count)
2147 {
2148         adapter_t *sc = (adapter_t *)arg;
2149         const struct adapter_params *p = &sc->params;
2150
2151         if(sc->flags & CXGB_SHUTDOWN)
2152                 return;
2153
2154         ADAPTER_LOCK(sc);
2155         if (p->linkpoll_period)
2156                 check_link_status(sc);
2157
2158         /*
2159          * adapter lock can currently only be acquired after the
2160          * port lock
2161          */
2162         ADAPTER_UNLOCK(sc);
2163
2164         if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2165                 check_t3b2_mac(sc);
2166 }
2167
2168 static void
2169 touch_bars(device_t dev)
2170 {
2171         /*
2172          * Don't enable yet
2173          */
2174 #if !defined(__LP64__) && 0
2175         u32 v;
2176
2177         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2178         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2179         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2180         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2181         pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2182         pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2183 #endif
2184 }
2185
2186 static int
2187 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2188 {
2189         uint8_t *buf;
2190         int err = 0;
2191         u32 aligned_offset, aligned_len, *p;
2192         struct adapter *adapter = pi->adapter;
2193
2194
2195         aligned_offset = offset & ~3;
2196         aligned_len = (len + (offset & 3) + 3) & ~3;
2197
2198         if (aligned_offset != offset || aligned_len != len) {
2199                 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2200                 if (!buf)
2201                         return (ENOMEM);
2202                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2203                 if (!err && aligned_len > 4)
2204                         err = t3_seeprom_read(adapter,
2205                                               aligned_offset + aligned_len - 4,
2206                                               (u32 *)&buf[aligned_len - 4]);
2207                 if (err)
2208                         goto out;
2209                 memcpy(buf + (offset & 3), data, len);
2210         } else
2211                 buf = (uint8_t *)(uintptr_t)data;
2212
2213         err = t3_seeprom_wp(adapter, 0);
2214         if (err)
2215                 goto out;
2216
2217         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2218                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2219                 aligned_offset += 4;
2220         }
2221
2222         if (!err)
2223                 err = t3_seeprom_wp(adapter, 1);
2224 out:
2225         if (buf != data)
2226                 free(buf, M_DEVBUF);
2227         return err;
2228 }
2229
2230
2231 static int
2232 in_range(int val, int lo, int hi)
2233 {
2234         return val < 0 || (val <= hi && val >= lo);
2235 }
2236
2237 static int
2238 cxgb_extension_open(struct cdev *dev, int flags, int fmp, d_thread_t *td)
2239 {
2240        return (0);
2241 }
2242
2243 static int
2244 cxgb_extension_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
2245 {
2246        return (0);
2247 }
2248
2249 static int
2250 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2251     int fflag, struct thread *td)
2252 {
2253         int mmd, error = 0;
2254         struct port_info *pi = dev->si_drv1;
2255         adapter_t *sc = pi->adapter;
2256
2257 #ifdef PRIV_SUPPORTED
2258         if (priv_check(td, PRIV_DRIVER)) {
2259                 if (cxgb_debug)
2260                         printf("user does not have access to privileged ioctls\n");
2261                 return (EPERM);
2262         }
2263 #else
2264         if (suser(td)) {
2265                 if (cxgb_debug)
2266                         printf("user does not have access to privileged ioctls\n");
2267                 return (EPERM);
2268         }
2269 #endif
2270
2271         switch (cmd) {
2272         case SIOCGMIIREG: {
2273                 uint32_t val;
2274                 struct cphy *phy = &pi->phy;
2275                 struct mii_data *mid = (struct mii_data *)data;
2276
2277                 if (!phy->mdio_read)
2278                         return (EOPNOTSUPP);
2279                 if (is_10G(sc)) {
2280                         mmd = mid->phy_id >> 8;
2281                         if (!mmd)
2282                                 mmd = MDIO_DEV_PCS;
2283                         else if (mmd > MDIO_DEV_XGXS)
2284                                 return (EINVAL);
2285
2286                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2287                                              mid->reg_num, &val);
2288                 } else
2289                         error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2290                                              mid->reg_num & 0x1f, &val);
2291                 if (error == 0)
2292                         mid->val_out = val;
2293                 break;
2294         }
2295         case SIOCSMIIREG: {
2296                 struct cphy *phy = &pi->phy;
2297                 struct mii_data *mid = (struct mii_data *)data;
2298
2299                 if (!phy->mdio_write)
2300                         return (EOPNOTSUPP);
2301                 if (is_10G(sc)) {
2302                         mmd = mid->phy_id >> 8;
2303                         if (!mmd)
2304                                 mmd = MDIO_DEV_PCS;
2305                         else if (mmd > MDIO_DEV_XGXS)
2306                                 return (EINVAL);
2307
2308                         error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2309                                               mmd, mid->reg_num, mid->val_in);
2310                 } else
2311                         error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2312                                               mid->reg_num & 0x1f,
2313                                               mid->val_in);
2314                 break;
2315         }
2316         case CHELSIO_SETREG: {
2317                 struct ch_reg *edata = (struct ch_reg *)data;
2318                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2319                         return (EFAULT);
2320                 t3_write_reg(sc, edata->addr, edata->val);
2321                 break;
2322         }
2323         case CHELSIO_GETREG: {
2324                 struct ch_reg *edata = (struct ch_reg *)data;
2325                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2326                         return (EFAULT);
2327                 edata->val = t3_read_reg(sc, edata->addr);
2328                 break;
2329         }
2330         case CHELSIO_GET_SGE_CONTEXT: {
2331                 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2332                 mtx_lock_spin(&sc->sge.reg_lock);
2333                 switch (ecntxt->cntxt_type) {
2334                 case CNTXT_TYPE_EGRESS:
2335                         error = t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2336                             ecntxt->data);
2337                         break;
2338                 case CNTXT_TYPE_FL:
2339                         error = t3_sge_read_fl(sc, ecntxt->cntxt_id,
2340                             ecntxt->data);
2341                         break;
2342                 case CNTXT_TYPE_RSP:
2343                         error = t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2344                             ecntxt->data);
2345                         break;
2346                 case CNTXT_TYPE_CQ:
2347                         error = t3_sge_read_cq(sc, ecntxt->cntxt_id,
2348                             ecntxt->data);
2349                         break;
2350                 default:
2351                         error = EINVAL;
2352                         break;
2353                 }
2354                 mtx_unlock_spin(&sc->sge.reg_lock);
2355                 break;
2356         }
2357         case CHELSIO_GET_SGE_DESC: {
2358                 struct ch_desc *edesc = (struct ch_desc *)data;
2359                 int ret;
2360                 if (edesc->queue_num >= SGE_QSETS * 6)
2361                         return (EINVAL);
2362                 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2363                     edesc->queue_num % 6, edesc->idx, edesc->data);
2364                 if (ret < 0)
2365                         return (EINVAL);
2366                 edesc->size = ret;
2367                 break;
2368         }
2369         case CHELSIO_SET_QSET_PARAMS: {
2370                 struct qset_params *q;
2371                 struct ch_qset_params *t = (struct ch_qset_params *)data;
2372                 int i;
2373
2374                 if (t->qset_idx >= SGE_QSETS)
2375                         return (EINVAL);
2376                 if (!in_range(t->intr_lat, 0, M_NEWTIMER) ||
2377                     !in_range(t->cong_thres, 0, 255) ||
2378                     !in_range(t->txq_size[0], MIN_TXQ_ENTRIES,
2379                               MAX_TXQ_ENTRIES) ||
2380                     !in_range(t->txq_size[1], MIN_TXQ_ENTRIES,
2381                               MAX_TXQ_ENTRIES) ||
2382                     !in_range(t->txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2383                               MAX_CTRL_TXQ_ENTRIES) ||
2384                     !in_range(t->fl_size[0], MIN_FL_ENTRIES, MAX_RX_BUFFERS) ||
2385                     !in_range(t->fl_size[1], MIN_FL_ENTRIES,
2386                               MAX_RX_JUMBO_BUFFERS) ||
2387                     !in_range(t->rspq_size, MIN_RSPQ_ENTRIES, MAX_RSPQ_ENTRIES))
2388                         return (EINVAL);
2389
2390                 if ((sc->flags & FULL_INIT_DONE) && t->lro > 0)
2391                         for_each_port(sc, i) {
2392                                 pi = adap2pinfo(sc, i);
2393                                 if (t->qset_idx >= pi->first_qset &&
2394                                     t->qset_idx < pi->first_qset + pi->nqsets
2395 #if 0
2396                                         && !pi->rx_csum_offload
2397 #endif
2398                                         )
2399                                         return -EINVAL;
2400                         }
2401                 if ((sc->flags & FULL_INIT_DONE) &&
2402                     (t->rspq_size >= 0 || t->fl_size[0] >= 0 ||
2403                      t->fl_size[1] >= 0 || t->txq_size[0] >= 0 ||
2404                      t->txq_size[1] >= 0 || t->txq_size[2] >= 0 ||
2405                      t->polling >= 0 || t->cong_thres >= 0))
2406                         return (EBUSY);
2407
2408                 q = &sc->params.sge.qset[t->qset_idx];
2409
2410                 if (t->rspq_size >= 0)
2411                         q->rspq_size = t->rspq_size;
2412                 if (t->fl_size[0] >= 0)
2413                         q->fl_size = t->fl_size[0];
2414                 if (t->fl_size[1] >= 0)
2415                         q->jumbo_size = t->fl_size[1];
2416                 if (t->txq_size[0] >= 0)
2417                         q->txq_size[0] = t->txq_size[0];
2418                 if (t->txq_size[1] >= 0)
2419                         q->txq_size[1] = t->txq_size[1];
2420                 if (t->txq_size[2] >= 0)
2421                         q->txq_size[2] = t->txq_size[2];
2422                 if (t->cong_thres >= 0)
2423                         q->cong_thres = t->cong_thres;
2424                 if (t->intr_lat >= 0) {
2425                         struct sge_qset *qs = &sc->sge.qs[t->qset_idx];
2426
2427                         q->coalesce_nsecs = t->intr_lat*1000;
2428                         t3_update_qset_coalesce(qs, q);
2429                 }
2430                 break;
2431         }
2432         case CHELSIO_GET_QSET_PARAMS: {
2433                 struct qset_params *q;
2434                 struct ch_qset_params *t = (struct ch_qset_params *)data;
2435
2436                 if (t->qset_idx >= SGE_QSETS)
2437                         return (EINVAL);
2438
2439                 q = &(sc)->params.sge.qset[t->qset_idx];
2440                 t->rspq_size   = q->rspq_size;
2441                 t->txq_size[0] = q->txq_size[0];
2442                 t->txq_size[1] = q->txq_size[1];
2443                 t->txq_size[2] = q->txq_size[2];
2444                 t->fl_size[0]  = q->fl_size;
2445                 t->fl_size[1]  = q->jumbo_size;
2446                 t->polling     = q->polling;
2447                 t->intr_lat    = q->coalesce_nsecs / 1000;
2448                 t->cong_thres  = q->cong_thres;
2449                 break;
2450         }
2451         case CHELSIO_SET_QSET_NUM: {
2452                 struct ch_reg *edata = (struct ch_reg *)data;
2453                 unsigned int port_idx = pi->port_id;
2454
2455                 if (sc->flags & FULL_INIT_DONE)
2456                         return (EBUSY);
2457                 if (edata->val < 1 ||
2458                     (edata->val > 1 && !(sc->flags & USING_MSIX)))
2459                         return (EINVAL);
2460                 if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS)
2461                         return (EINVAL);
2462                 sc->port[port_idx].nqsets = edata->val;
2463                 sc->port[0].first_qset = 0;
2464                 /*
2465                  * XXX hardcode ourselves to 2 ports just like LEEENUX
2466                  */
2467                 sc->port[1].first_qset = sc->port[0].nqsets;
2468                 break;
2469         }
2470         case CHELSIO_GET_QSET_NUM: {
2471                 struct ch_reg *edata = (struct ch_reg *)data;
2472                 edata->val = pi->nqsets;
2473                 break;
2474         }
2475 #ifdef notyet
2476         case CHELSIO_LOAD_FW:
2477         case CHELSIO_GET_PM:
2478         case CHELSIO_SET_PM:
2479                 return (EOPNOTSUPP);
2480                 break;
2481 #endif
2482         case CHELSIO_SETMTUTAB: {
2483                 struct ch_mtus *m = (struct ch_mtus *)data;
2484                 int i;
2485
2486                 if (!is_offload(sc))
2487                         return (EOPNOTSUPP);
2488                 if (offload_running(sc))
2489                         return (EBUSY);
2490                 if (m->nmtus != NMTUS)
2491                         return (EINVAL);
2492                 if (m->mtus[0] < 81)         /* accommodate SACK */
2493                         return (EINVAL);
2494
2495                 /*
2496                  * MTUs must be in ascending order
2497                  */
2498                 for (i = 1; i < NMTUS; ++i)
2499                         if (m->mtus[i] < m->mtus[i - 1])
2500                                 return (EINVAL);
2501
2502                 memcpy(sc->params.mtus, m->mtus,
2503                        sizeof(sc->params.mtus));
2504                 break;
2505         }
2506         case CHELSIO_GETMTUTAB: {
2507                 struct ch_mtus *m = (struct ch_mtus *)data;
2508
2509                 if (!is_offload(sc))
2510                         return (EOPNOTSUPP);
2511
2512                 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2513                 m->nmtus = NMTUS;
2514                 break;
2515         }
2516         case CHELSIO_DEVUP:
2517                 if (!is_offload(sc))
2518                         return (EOPNOTSUPP);
2519                 return offload_open(pi);
2520                 break;
2521         case CHELSIO_GET_MEM: {
2522                 struct ch_mem_range *t = (struct ch_mem_range *)data;
2523                 struct mc7 *mem;
2524                 uint8_t *useraddr;
2525                 u64 buf[32];
2526
2527                 if (!is_offload(sc))
2528                         return (EOPNOTSUPP);
2529                 if (!(sc->flags & FULL_INIT_DONE))
2530                         return (EIO);         /* need the memory controllers */
2531                 if ((t->addr & 0x7) || (t->len & 0x7))
2532                         return (EINVAL);
2533                 if (t->mem_id == MEM_CM)
2534                         mem = &sc->cm;
2535                 else if (t->mem_id == MEM_PMRX)
2536                         mem = &sc->pmrx;
2537                 else if (t->mem_id == MEM_PMTX)
2538                         mem = &sc->pmtx;
2539                 else
2540                         return (EINVAL);
2541
2542                 /*
2543                  * Version scheme:
2544                  * bits 0..9: chip version
2545                  * bits 10..15: chip revision
2546                  */
2547                 t->version = 3 | (sc->params.rev << 10);
2548
2549                 /*
2550                  * Read 256 bytes at a time as len can be large and we don't
2551                  * want to use huge intermediate buffers.
2552                  */
2553                 useraddr = (uint8_t *)t->buf;
2554                 while (t->len) {
2555                         unsigned int chunk = min(t->len, sizeof(buf));
2556
2557                         error = t3_mc7_bd_read(mem, t->addr / 8, chunk / 8, buf);
2558                         if (error)
2559                                 return (-error);
2560                         if (copyout(buf, useraddr, chunk))
2561                                 return (EFAULT);
2562                         useraddr += chunk;
2563                         t->addr += chunk;
2564                         t->len -= chunk;
2565                 }
2566                 break;
2567         }
2568         case CHELSIO_READ_TCAM_WORD: {
2569                 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2570
2571                 if (!is_offload(sc))
2572                         return (EOPNOTSUPP);
2573                 if (!(sc->flags & FULL_INIT_DONE))
2574                         return (EIO);         /* need MC5 */
2575                 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2576                 break;
2577         }
2578         case CHELSIO_SET_TRACE_FILTER: {
2579                 struct ch_trace *t = (struct ch_trace *)data;
2580                 const struct trace_params *tp;
2581
2582                 tp = (const struct trace_params *)&t->sip;
2583                 if (t->config_tx)
2584                         t3_config_trace_filter(sc, tp, 0, t->invert_match,
2585                                                t->trace_tx);
2586                 if (t->config_rx)
2587                         t3_config_trace_filter(sc, tp, 1, t->invert_match,
2588                                                t->trace_rx);
2589                 break;
2590         }
2591         case CHELSIO_SET_PKTSCHED: {
2592                 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2593                 if (sc->open_device_map == 0)
2594                         return (EAGAIN);
2595                 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2596                     p->binding);
2597                 break;
2598         }
2599         case CHELSIO_IFCONF_GETREGS: {
2600                 struct ifconf_regs *regs = (struct ifconf_regs *)data;
2601                 int reglen = cxgb_get_regs_len();
2602                 uint8_t *buf = malloc(REGDUMP_SIZE, M_DEVBUF, M_NOWAIT);
2603                 if (buf == NULL) {
2604                         return (ENOMEM);
2605                 } if (regs->len > reglen)
2606                         regs->len = reglen;
2607                 else if (regs->len < reglen) {
2608                         error = E2BIG;
2609                         goto done;
2610                 }
2611                 cxgb_get_regs(sc, regs, buf);
2612                 error = copyout(buf, regs->data, reglen);
2613
2614                 done:
2615                 free(buf, M_DEVBUF);
2616
2617                 break;
2618         }
2619         case CHELSIO_SET_HW_SCHED: {
2620                 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2621                 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2622
2623                 if ((sc->flags & FULL_INIT_DONE) == 0)
2624                         return (EAGAIN);       /* need TP to be initialized */
2625                 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2626                     !in_range(t->channel, 0, 1) ||
2627                     !in_range(t->kbps, 0, 10000000) ||
2628                     !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2629                     !in_range(t->flow_ipg, 0,
2630                               dack_ticks_to_usec(sc, 0x7ff)))
2631                         return (EINVAL);
2632
2633                 if (t->kbps >= 0) {
2634                         error = t3_config_sched(sc, t->kbps, t->sched);
2635                         if (error < 0)
2636                                 return (-error);
2637                 }
2638                 if (t->class_ipg >= 0)
2639                         t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2640                 if (t->flow_ipg >= 0) {
2641                         t->flow_ipg *= 1000;     /* us -> ns */
2642                         t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2643                 }
2644                 if (t->mode >= 0) {
2645                         int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2646
2647                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2648                                          bit, t->mode ? bit : 0);
2649                 }
2650                 if (t->channel >= 0)
2651                         t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2652                                          1 << t->sched, t->channel << t->sched);
2653                 break;
2654         }
2655         default:
2656                 return (EOPNOTSUPP);
2657                 break;
2658         }
2659
2660         return (error);
2661 }
2662
2663 static __inline void
2664 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
2665     unsigned int end)
2666 {
2667         uint32_t *p = (uint32_t *)buf + start;
2668
2669         for ( ; start <= end; start += sizeof(uint32_t))
2670                 *p++ = t3_read_reg(ap, start);
2671 }
2672
2673 #define T3_REGMAP_SIZE (3 * 1024)
2674 static int
2675 cxgb_get_regs_len(void)
2676 {
2677         return T3_REGMAP_SIZE;
2678 }
2679 #undef T3_REGMAP_SIZE
2680
2681 static void
2682 cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf)
2683 {
2684
2685         /*
2686          * Version scheme:
2687          * bits 0..9: chip version
2688          * bits 10..15: chip revision
2689          * bit 31: set for PCIe cards
2690          */
2691         regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
2692
2693         /*
2694          * We skip the MAC statistics registers because they are clear-on-read.
2695          * Also reading multi-register stats would need to synchronize with the
2696          * periodic mac stats accumulation.  Hard to justify the complexity.
2697          */
2698         memset(buf, 0, REGDUMP_SIZE);
2699         reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
2700         reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
2701         reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
2702         reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
2703         reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
2704         reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
2705                        XGM_REG(A_XGM_SERDES_STAT3, 1));
2706         reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
2707                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
2708 }
2709
2710
2711 MODULE_DEPEND(if_cxgb, cxgb_t3fw, 1, 1, 1);